use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::atomic::{AtomicU64, Ordering};
use anyhow::{bail, Context, Result};
use serde::Deserialize;
const TEST_OMIT: &str = "*_test.py";
const SUPPORT_OMIT: &str = "*conftest.py";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Thresholds {
pub fail_under: u8,
pub branch: bool,
}
#[derive(Debug, Clone, Deserialize)]
pub struct CoverageReport {
pub totals: Totals,
#[serde(default)]
pub files: BTreeMap<String, FileCoverage>,
}
#[derive(Debug, Clone, Default, Deserialize)]
pub struct FileCoverage {
#[serde(default)]
pub executed_lines: Vec<u64>,
#[serde(default)]
pub missing_lines: Vec<u64>,
#[serde(default)]
pub excluded_lines: Vec<u64>,
#[serde(default)]
pub missing_branches: Vec<Vec<i64>>,
}
#[derive(Debug, Clone, Deserialize)]
pub struct Totals {
pub percent_covered: f64,
#[serde(default)]
pub num_branches: u64,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Outcome {
Pass,
Fail(String),
}
pub fn parse_report(json: &str) -> Result<CoverageReport> {
serde_json::from_str(json).context("parsing coverage.py JSON report")
}
pub fn evaluate(report: &CoverageReport, thresholds: Thresholds) -> Outcome {
if thresholds.branch && report.totals.num_branches == 0 {
return Outcome::Fail(
"branch coverage is required but the report measured no branches".to_string(),
);
}
let actual = report.totals.percent_covered;
let required = f64::from(thresholds.fail_under);
if actual + 1e-9 >= required {
Outcome::Pass
} else {
Outcome::Fail(format!(
"coverage {actual:.2}% is below the required {}%",
thresholds.fail_under
))
}
}
pub const BASELINE_PATH: &str = "coverage-baseline.json";
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Baseline {
#[serde(default)]
pub python: Option<PythonBaseline>,
}
#[derive(Debug, Clone, Copy, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PythonBaseline {
pub percent_covered: f64,
}
pub fn read_baseline(root: &Path) -> Result<Option<Baseline>> {
let path = root.join(BASELINE_PATH);
if !path.exists() {
return Ok(None);
}
let contents = std::fs::read_to_string(&path)
.with_context(|| format!("reading coverage baseline `{}`", path.display()))?;
let baseline = serde_json::from_str(&contents)
.with_context(|| format!("parsing coverage baseline `{}`", path.display()))?;
Ok(Some(baseline))
}
pub fn evaluate_ratchet(percent: f64, baseline: Option<f64>) -> Outcome {
match baseline {
Some(required) if percent + 1e-9 < required => Outcome::Fail(format!(
"coverage {percent:.2}% regressed below the recorded baseline {required:.2}%"
)),
_ => Outcome::Pass,
}
}
pub fn measure(root: &Path, thresholds: Thresholds, omit: &[String]) -> Result<Outcome> {
Ok(evaluate(&measure_report(root, omit)?, thresholds))
}
pub fn measure_report(root: &Path, omit: &[String]) -> Result<CoverageReport> {
run_coverage(root, omit, false)
}
pub fn measure_patch_report(root: &Path, omit: &[String]) -> Result<CoverageReport> {
run_coverage(root, omit, true)
}
struct DataFile(PathBuf);
impl DataFile {
fn new() -> Self {
static COUNTER: AtomicU64 = AtomicU64::new(0);
let name = format!(
"testing-conventions-{}-{}.coverage",
std::process::id(),
COUNTER.fetch_add(1, Ordering::Relaxed),
);
DataFile(std::env::temp_dir().join(name))
}
}
impl Drop for DataFile {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.0);
}
}
fn run_coverage(root: &Path, omit: &[String], include_all_sources: bool) -> Result<CoverageReport> {
let data = DataFile::new();
let omit = build_omit(omit);
let mut command = Command::new("coverage");
command
.current_dir(root)
.args(["run", "--branch"])
.arg(format!("--omit={omit}"));
if include_all_sources {
command.arg("--source=.");
}
let run = command
.args(["-m", "pytest", "-q", "-p", "no:cacheprovider", "."])
.env("COVERAGE_FILE", &data.0)
.env("PYTHONDONTWRITEBYTECODE", "1")
.output()
.context("running `coverage run -m pytest` (is coverage.py installed?)")?;
if !run.status.success() {
bail!(
"the unit suite did not run cleanly under coverage in `{}`:\n{}{}",
root.display(),
String::from_utf8_lossy(&run.stdout),
String::from_utf8_lossy(&run.stderr),
);
}
let json = Command::new("coverage")
.current_dir(root)
.args(["json", "-o", "-"])
.env("COVERAGE_FILE", &data.0)
.output()
.context("running `coverage json`")?;
if !json.status.success() {
bail!(
"`coverage json` failed:\n{}",
String::from_utf8_lossy(&json.stderr),
);
}
parse_report(&String::from_utf8_lossy(&json.stdout))
}
fn build_omit(omit: &[String]) -> String {
[TEST_OMIT.to_string(), SUPPORT_OMIT.to_string()]
.into_iter()
.chain(omit.iter().cloned())
.collect::<Vec<_>>()
.join(",")
}
const TS_INCLUDE: &str = "**/*.{ts,tsx,mts,cts}";
const TS_TEST_EXCLUDE: &str = "**/*.test.*";
const TS_DECL_EXCLUDE: &str = "**/*.d.{ts,mts,cts}";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TypeScriptThresholds {
pub lines: u8,
pub branches: u8,
pub functions: u8,
pub statements: u8,
}
#[derive(Debug, Clone, Copy, Deserialize)]
pub struct VitestReport {
pub total: VitestTotals,
}
#[derive(Debug, Clone, Copy, Deserialize)]
pub struct VitestTotals {
pub lines: VitestMetric,
pub branches: VitestMetric,
pub functions: VitestMetric,
pub statements: VitestMetric,
}
#[derive(Debug, Clone, Copy, Deserialize)]
pub struct VitestMetric {
#[serde(deserialize_with = "deserialize_pct")]
pub pct: Option<f64>,
pub total: u64,
}
fn deserialize_pct<'de, D>(deserializer: D) -> std::result::Result<Option<f64>, D::Error>
where
D: serde::Deserializer<'de>,
{
struct PctVisitor;
impl serde::de::Visitor<'_> for PctVisitor {
type Value = Option<f64>;
fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.write_str("a coverage percent number or the string \"Unknown\"")
}
fn visit_f64<E>(self, value: f64) -> std::result::Result<Self::Value, E> {
Ok(Some(value))
}
fn visit_u64<E>(self, value: u64) -> std::result::Result<Self::Value, E> {
Ok(Some(value as f64))
}
fn visit_str<E>(self, _value: &str) -> std::result::Result<Self::Value, E> {
Ok(None)
}
}
deserializer.deserialize_any(PctVisitor)
}
pub fn parse_vitest_report(json: &str) -> Result<VitestReport> {
serde_json::from_str(json).context("parsing vitest coverage-summary JSON report")
}
pub fn evaluate_typescript(report: &VitestReport, thresholds: TypeScriptThresholds) -> Outcome {
let total = &report.total;
if total.lines.total == 0 {
return Outcome::Fail(
"the unit suite measured no code — check the path and that the suite runs".to_string(),
);
}
let checks = [
("lines", total.lines, thresholds.lines),
("branches", total.branches, thresholds.branches),
("functions", total.functions, thresholds.functions),
("statements", total.statements, thresholds.statements),
];
let mut shortfalls = Vec::new();
for (name, metric, required) in checks {
let actual = metric.pct.unwrap_or(100.0);
if actual + 1e-9 < f64::from(required) {
shortfalls.push(format!("{name} {actual:.2}% < {required}%"));
}
}
if shortfalls.is_empty() {
Outcome::Pass
} else {
Outcome::Fail(format!(
"coverage below thresholds: {}",
shortfalls.join(", ")
))
}
}
pub fn measure_typescript(
root: &Path,
thresholds: TypeScriptThresholds,
exclude: &[String],
) -> Result<Outcome> {
let report = run_vitest(root, exclude)?;
Ok(evaluate_typescript(&report, thresholds))
}
struct ReportDir(PathBuf);
impl ReportDir {
fn new() -> Self {
static COUNTER: AtomicU64 = AtomicU64::new(0);
let name = format!(
"testing-conventions-vitest-{}-{}",
std::process::id(),
COUNTER.fetch_add(1, Ordering::Relaxed),
);
ReportDir(std::env::temp_dir().join(name))
}
fn summary(&self) -> PathBuf {
self.0.join("coverage-summary.json")
}
}
impl Drop for ReportDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn run_vitest(root: &Path, exclude: &[String]) -> Result<VitestReport> {
let reports = ReportDir::new();
let mut command = Command::new("npx");
command
.current_dir(root)
.args(["--yes", "vitest", "run", "--no-cache"])
.args([
"--coverage.enabled",
"--coverage.provider=v8",
"--coverage.reporter=json-summary",
"--coverage.all=true",
])
.arg(format!(
"--coverage.reportsDirectory={}",
reports.0.display()
))
.arg(format!("--coverage.include={TS_INCLUDE}"))
.arg(format!("--coverage.exclude={TS_TEST_EXCLUDE}"))
.arg(format!("--coverage.exclude={TS_DECL_EXCLUDE}"));
for path in exclude {
command.arg(format!("--coverage.exclude={path}"));
}
let run = command.env("CI", "1").output().context(
"running `npx vitest run --coverage` (are vitest and @vitest/coverage-v8 installed?)",
)?;
if !run.status.success() {
bail!(
"the unit suite did not run cleanly under vitest in `{}`:\n{}{}",
root.display(),
String::from_utf8_lossy(&run.stdout),
String::from_utf8_lossy(&run.stderr),
);
}
let summary = reports.summary();
let json = std::fs::read_to_string(&summary).with_context(|| {
format!(
"reading vitest coverage summary `{}` (did the run produce a json-summary report?)",
summary.display()
)
})?;
parse_vitest_report(&json)
}
#[cfg(test)]
mod tests {
use super::*;
fn report(percent_covered: f64, num_branches: u64) -> CoverageReport {
CoverageReport {
totals: Totals {
percent_covered,
num_branches,
},
files: BTreeMap::new(),
}
}
#[test]
fn passes_when_total_meets_the_floor() {
assert_eq!(
evaluate(
&report(100.0, 12),
Thresholds {
fail_under: 100,
branch: true
}
),
Outcome::Pass
);
}
#[test]
fn fails_when_total_is_below_the_floor() {
assert!(matches!(
evaluate(
&report(80.0, 12),
Thresholds {
fail_under: 100,
branch: true
}
),
Outcome::Fail(_)
));
}
#[test]
fn fails_when_branch_required_but_unmeasured() {
assert!(matches!(
evaluate(
&report(100.0, 0),
Thresholds {
fail_under: 90,
branch: true
}
),
Outcome::Fail(_)
));
}
#[test]
fn parses_a_coverage_py_report() {
let json = r#"{"totals":{"percent_covered":91.5,"num_branches":8,"covered_lines":91}}"#;
let report = parse_report(json).expect("valid coverage.py json");
assert_eq!(report.totals.percent_covered, 91.5);
assert_eq!(report.totals.num_branches, 8);
}
#[test]
fn parses_the_per_file_block_for_patch_coverage() {
let json = r#"{
"files": {
"widget.py": {
"executed_lines": [1, 2, 3, 4, 6],
"summary": {"percent_covered": 85.0},
"missing_lines": [5],
"excluded_lines": [],
"missing_branches": [[4, 5]]
}
},
"totals": {"percent_covered": 85.0, "num_branches": 4}
}"#;
let report = parse_report(json).expect("valid coverage.py json with files");
let widget = report.files.get("widget.py").expect("widget.py is present");
assert_eq!(widget.missing_lines, vec![5]);
assert_eq!(widget.missing_branches, vec![vec![4, 5]]);
assert_eq!(report.totals.percent_covered, 85.0);
}
#[test]
fn a_report_without_a_files_block_parses_with_an_empty_map() {
let report = parse_report(r#"{"totals":{"percent_covered":100.0,"num_branches":2}}"#)
.expect("valid coverage.py json");
assert!(report.files.is_empty());
}
#[test]
fn omit_is_the_test_and_support_globs_when_nothing_is_exempt() {
assert_eq!(build_omit(&[]), "*_test.py,*conftest.py");
}
#[test]
fn omit_folds_in_the_exempt_paths_after_the_test_glob() {
let exempt = vec!["pkg/gen.py".to_string(), "shim.py".to_string()];
assert_eq!(
build_omit(&exempt),
"*_test.py,*conftest.py,pkg/gen.py,shim.py"
);
}
#[test]
fn ratchet_passes_when_coverage_holds_at_the_baseline() {
assert_eq!(evaluate_ratchet(100.0, Some(100.0)), Outcome::Pass);
}
#[test]
fn ratchet_passes_when_coverage_improves_over_the_baseline() {
assert_eq!(evaluate_ratchet(92.0, Some(85.0)), Outcome::Pass);
}
#[test]
fn ratchet_fails_on_a_drop_below_the_baseline() {
assert!(matches!(
evaluate_ratchet(86.0, Some(90.0)),
Outcome::Fail(message) if message.contains("regressed") && message.contains("90")
));
}
#[test]
fn ratchet_is_vacuous_without_a_recorded_baseline() {
assert_eq!(evaluate_ratchet(10.0, None), Outcome::Pass);
}
#[test]
fn ratchet_tolerates_float_noise_at_the_baseline() {
assert_eq!(evaluate_ratchet(99.999_999_999, Some(100.0)), Outcome::Pass);
}
static BASELINE_COUNTER: AtomicU64 = AtomicU64::new(0);
struct TempDir(PathBuf);
impl TempDir {
fn new() -> Self {
let dir = std::env::temp_dir().join(format!(
"tc-baseline-{}-{}",
std::process::id(),
BASELINE_COUNTER.fetch_add(1, Ordering::Relaxed),
));
std::fs::create_dir_all(&dir).unwrap();
TempDir(dir)
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
#[test]
fn read_baseline_is_none_when_the_file_is_absent() {
let dir = TempDir::new();
assert!(read_baseline(&dir.0).unwrap().is_none());
}
#[test]
fn read_baseline_parses_the_recorded_python_total() {
let dir = TempDir::new();
std::fs::write(
dir.0.join(BASELINE_PATH),
r#"{"python":{"percent_covered":91.5}}"#,
)
.unwrap();
let baseline = read_baseline(&dir.0)
.unwrap()
.expect("a baseline file is present");
assert_eq!(baseline.python.unwrap().percent_covered, 91.5);
}
#[test]
fn read_baseline_errors_on_a_malformed_file() {
let dir = TempDir::new();
std::fs::write(dir.0.join(BASELINE_PATH), "{ not json").unwrap();
assert!(read_baseline(&dir.0).is_err());
}
fn metric(pct: f64) -> VitestMetric {
VitestMetric {
pct: Some(pct),
total: 10,
}
}
fn ts_report(lines: f64, branches: f64, functions: f64, statements: f64) -> VitestReport {
VitestReport {
total: VitestTotals {
lines: metric(lines),
branches: metric(branches),
functions: metric(functions),
statements: metric(statements),
},
}
}
const TS_FULL: TypeScriptThresholds = TypeScriptThresholds {
lines: 100,
branches: 100,
functions: 100,
statements: 100,
};
const TS_MID: TypeScriptThresholds = TypeScriptThresholds {
lines: 80,
branches: 75,
functions: 80,
statements: 80,
};
#[test]
fn typescript_passes_when_every_metric_meets_its_floor() {
assert_eq!(
evaluate_typescript(&ts_report(100.0, 100.0, 100.0, 100.0), TS_FULL),
Outcome::Pass
);
}
#[test]
fn typescript_fails_on_the_one_metric_below_its_floor() {
let outcome = evaluate_typescript(&ts_report(100.0, 66.66, 100.0, 100.0), TS_MID);
assert!(
matches!(&outcome, Outcome::Fail(message) if message.contains("branches") && !message.contains("lines")),
"got: {outcome:?}"
);
}
#[test]
fn typescript_fail_message_names_every_metric_below() {
let outcome = evaluate_typescript(&ts_report(70.0, 70.0, 70.0, 70.0), TS_MID);
assert!(
matches!(&outcome, Outcome::Fail(message)
if message.contains("lines")
&& message.contains("branches")
&& message.contains("functions")
&& message.contains("statements")),
"got: {outcome:?}"
);
}
#[test]
fn typescript_tolerates_float_noise_at_the_floor() {
assert_eq!(
evaluate_typescript(&ts_report(99.999_999_999, 100.0, 100.0, 100.0), TS_FULL),
Outcome::Pass
);
}
#[test]
fn typescript_empty_denominator_metric_is_vacuously_satisfied() {
let report = VitestReport {
total: VitestTotals {
lines: metric(100.0),
branches: VitestMetric {
pct: None,
total: 0,
},
functions: metric(100.0),
statements: metric(100.0),
},
};
assert_eq!(evaluate_typescript(&report, TS_FULL), Outcome::Pass);
}
#[test]
fn typescript_fails_a_vacuous_run_that_measured_no_code() {
let nothing = VitestMetric {
pct: None,
total: 0,
};
let report = VitestReport {
total: VitestTotals {
lines: nothing,
branches: nothing,
functions: nothing,
statements: nothing,
},
};
let outcome = evaluate_typescript(&report, TS_MID);
assert!(
matches!(&outcome, Outcome::Fail(message) if message.contains("measured no code")),
"got: {outcome:?}"
);
}
#[test]
fn parses_a_vitest_summary_report() {
let json = r#"{
"total": {
"lines": {"total": 5, "covered": 4, "skipped": 0, "pct": 80},
"statements": {"total": 5, "covered": 4, "skipped": 0, "pct": 80},
"functions": {"total": 2, "covered": 2, "skipped": 0, "pct": 100},
"branches": {"total": 3, "covered": 2, "skipped": 0, "pct": 66.66},
"branchesTrue": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"}
},
"/abs/widget.ts": {
"lines": {"total": 5, "covered": 4, "skipped": 0, "pct": 80}
}
}"#;
let report = parse_vitest_report(json).expect("valid vitest json-summary");
assert_eq!(report.total.lines.pct, Some(80.0));
assert_eq!(report.total.branches.pct, Some(66.66));
assert_eq!(report.total.functions.total, 2);
}
#[test]
fn parses_an_unknown_pct_as_unmeasured() {
let json = r#"{"total": {
"lines": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"},
"statements": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"},
"functions": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"},
"branches": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"}
}}"#;
let report = parse_vitest_report(json).expect("valid vitest json-summary");
assert_eq!(report.total.lines.pct, None);
assert_eq!(report.total.lines.total, 0);
}
#[test]
fn a_pct_that_is_neither_number_nor_string_is_a_parse_error() {
let json = r#"{"total":{
"lines": {"total": 1, "covered": 1, "skipped": 0, "pct": true},
"statements": {"total": 1, "covered": 1, "skipped": 0, "pct": 100},
"functions": {"total": 1, "covered": 1, "skipped": 0, "pct": 100},
"branches": {"total": 1, "covered": 1, "skipped": 0, "pct": 100}
}}"#;
assert!(parse_vitest_report(json).is_err());
}
}