1use std::collections::BTreeMap;
24use std::path::{Path, PathBuf};
25use std::process::Command;
26use std::sync::atomic::{AtomicU64, Ordering};
27
28use anyhow::{bail, Context, Result};
29use serde::Deserialize;
30
31const TEST_OMIT: &str = "*_test.py";
34
35const SUPPORT_OMIT: &str = "*conftest.py";
39
40#[derive(Debug, Clone, Copy, PartialEq, Eq)]
42pub struct Thresholds {
43 pub fail_under: u8,
45 pub branch: bool,
47}
48
49#[derive(Debug, Clone, Deserialize)]
54pub struct CoverageReport {
55 pub totals: Totals,
56 #[serde(default)]
60 pub files: BTreeMap<String, FileCoverage>,
61}
62
63#[derive(Debug, Clone, Default, Deserialize)]
67pub struct FileCoverage {
68 #[serde(default)]
70 pub executed_lines: Vec<u64>,
71 #[serde(default)]
74 pub missing_lines: Vec<u64>,
75 #[serde(default)]
77 pub excluded_lines: Vec<u64>,
78 #[serde(default)]
82 pub missing_branches: Vec<Vec<i64>>,
83}
84
85#[derive(Debug, Clone, Deserialize)]
87pub struct Totals {
88 pub percent_covered: f64,
90 #[serde(default)]
92 pub num_branches: u64,
93}
94
95#[derive(Debug, Clone, PartialEq)]
97pub enum Outcome {
98 Pass,
100 Fail(String),
102}
103
104pub fn parse_report(json: &str) -> Result<CoverageReport> {
106 serde_json::from_str(json).context("parsing coverage.py JSON report")
107}
108
109pub fn evaluate(report: &CoverageReport, thresholds: Thresholds) -> Outcome {
114 if thresholds.branch && report.totals.num_branches == 0 {
115 return Outcome::Fail(
116 "branch coverage is required but the report measured no branches".to_string(),
117 );
118 }
119 let actual = report.totals.percent_covered;
120 let required = f64::from(thresholds.fail_under);
121 if actual + 1e-9 >= required {
124 Outcome::Pass
125 } else {
126 Outcome::Fail(format!(
127 "coverage {actual:.2}% is below the required {}%",
128 thresholds.fail_under
129 ))
130 }
131}
132
133pub const BASELINE_PATH: &str = "coverage-baseline.json";
148
149#[derive(Debug, Clone, Default, Deserialize)]
154#[serde(deny_unknown_fields)]
155pub struct Baseline {
156 #[serde(default)]
158 pub python: Option<PythonBaseline>,
159}
160
161#[derive(Debug, Clone, Copy, Deserialize)]
163#[serde(deny_unknown_fields)]
164pub struct PythonBaseline {
165 pub percent_covered: f64,
167}
168
169pub fn read_baseline(root: &Path) -> Result<Option<Baseline>> {
173 let path = root.join(BASELINE_PATH);
174 if !path.exists() {
175 return Ok(None);
176 }
177 let contents = std::fs::read_to_string(&path)
178 .with_context(|| format!("reading coverage baseline `{}`", path.display()))?;
179 let baseline = serde_json::from_str(&contents)
180 .with_context(|| format!("parsing coverage baseline `{}`", path.display()))?;
181 Ok(Some(baseline))
182}
183
184pub fn evaluate_ratchet(percent: f64, baseline: Option<f64>) -> Outcome {
189 match baseline {
190 Some(required) if percent + 1e-9 < required => Outcome::Fail(format!(
191 "coverage {percent:.2}% regressed below the recorded baseline {required:.2}%"
192 )),
193 _ => Outcome::Pass,
194 }
195}
196
197pub fn measure(root: &Path, thresholds: Thresholds, omit: &[String]) -> Result<Outcome> {
206 Ok(evaluate(&measure_report(root, omit)?, thresholds))
207}
208
209pub fn measure_report(root: &Path, omit: &[String]) -> Result<CoverageReport> {
213 run_coverage(root, omit, false)
214}
215
216pub fn measure_patch_report(root: &Path, omit: &[String]) -> Result<CoverageReport> {
223 run_coverage(root, omit, true)
224}
225
226struct DataFile(PathBuf);
230
231impl DataFile {
232 fn new() -> Self {
233 static COUNTER: AtomicU64 = AtomicU64::new(0);
234 let name = format!(
235 "testing-conventions-{}-{}.coverage",
236 std::process::id(),
237 COUNTER.fetch_add(1, Ordering::Relaxed),
238 );
239 DataFile(std::env::temp_dir().join(name))
240 }
241}
242
243impl Drop for DataFile {
244 fn drop(&mut self) {
245 let _ = std::fs::remove_file(&self.0);
246 }
247}
248
249fn run_coverage(root: &Path, omit: &[String], include_all_sources: bool) -> Result<CoverageReport> {
256 let data = DataFile::new();
257 let omit = build_omit(omit);
258
259 let mut command = Command::new("coverage");
263 command
264 .current_dir(root)
265 .args(["run", "--branch"])
266 .arg(format!("--omit={omit}"));
267 if include_all_sources {
268 command.arg("--source=.");
269 }
270 let run = command
271 .args(["-m", "pytest", "-q", "-p", "no:cacheprovider", "."])
272 .env("COVERAGE_FILE", &data.0)
273 .env("PYTHONDONTWRITEBYTECODE", "1")
274 .output()
275 .context("running `coverage run -m pytest` (is coverage.py installed?)")?;
276 if !run.status.success() {
277 bail!(
278 "the unit suite did not run cleanly under coverage in `{}`:\n{}{}",
279 root.display(),
280 String::from_utf8_lossy(&run.stdout),
281 String::from_utf8_lossy(&run.stderr),
282 );
283 }
284
285 let json = Command::new("coverage")
287 .current_dir(root)
288 .args(["json", "-o", "-"])
289 .env("COVERAGE_FILE", &data.0)
290 .output()
291 .context("running `coverage json`")?;
292 if !json.status.success() {
293 bail!(
294 "`coverage json` failed:\n{}",
295 String::from_utf8_lossy(&json.stderr),
296 );
297 }
298
299 parse_report(&String::from_utf8_lossy(&json.stdout))
300}
301
302fn build_omit(omit: &[String]) -> String {
309 [TEST_OMIT.to_string(), SUPPORT_OMIT.to_string()]
310 .into_iter()
311 .chain(omit.iter().cloned())
312 .collect::<Vec<_>>()
313 .join(",")
314}
315
316const TS_INCLUDE: &str = "**/*.{ts,tsx,mts,cts}";
330const TS_TEST_EXCLUDE: &str = "**/*.test.*";
334const TS_DECL_EXCLUDE: &str = "**/*.d.{ts,mts,cts}";
335
336#[derive(Debug, Clone, Copy, PartialEq, Eq)]
339pub struct TypeScriptThresholds {
340 pub lines: u8,
341 pub branches: u8,
342 pub functions: u8,
343 pub statements: u8,
344}
345
346#[derive(Debug, Clone, Copy, Deserialize)]
349pub struct VitestReport {
350 pub total: VitestTotals,
351}
352
353#[derive(Debug, Clone, Copy, Deserialize)]
356pub struct VitestTotals {
357 pub lines: VitestMetric,
358 pub branches: VitestMetric,
359 pub functions: VitestMetric,
360 pub statements: VitestMetric,
361}
362
363#[derive(Debug, Clone, Copy, Deserialize)]
366pub struct VitestMetric {
367 #[serde(deserialize_with = "deserialize_pct")]
370 pub pct: Option<f64>,
371 pub total: u64,
373}
374
375fn deserialize_pct<'de, D>(deserializer: D) -> std::result::Result<Option<f64>, D::Error>
379where
380 D: serde::Deserializer<'de>,
381{
382 struct PctVisitor;
383 impl serde::de::Visitor<'_> for PctVisitor {
384 type Value = Option<f64>;
385
386 fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
387 f.write_str("a coverage percent number or the string \"Unknown\"")
388 }
389
390 fn visit_f64<E>(self, value: f64) -> std::result::Result<Self::Value, E> {
391 Ok(Some(value))
392 }
393
394 fn visit_u64<E>(self, value: u64) -> std::result::Result<Self::Value, E> {
397 Ok(Some(value as f64))
398 }
399
400 fn visit_str<E>(self, _value: &str) -> std::result::Result<Self::Value, E> {
403 Ok(None)
404 }
405 }
406 deserializer.deserialize_any(PctVisitor)
407}
408
409pub fn parse_vitest_report(json: &str) -> Result<VitestReport> {
411 serde_json::from_str(json).context("parsing vitest coverage-summary JSON report")
412}
413
414pub fn evaluate_typescript(report: &VitestReport, thresholds: TypeScriptThresholds) -> Outcome {
423 let total = &report.total;
424 if total.lines.total == 0 {
428 return Outcome::Fail(
429 "the unit suite measured no code — check the path and that the suite runs".to_string(),
430 );
431 }
432 let checks = [
433 ("lines", total.lines, thresholds.lines),
434 ("branches", total.branches, thresholds.branches),
435 ("functions", total.functions, thresholds.functions),
436 ("statements", total.statements, thresholds.statements),
437 ];
438 let mut shortfalls = Vec::new();
439 for (name, metric, required) in checks {
440 let actual = metric.pct.unwrap_or(100.0);
443 if actual + 1e-9 < f64::from(required) {
446 shortfalls.push(format!("{name} {actual:.2}% < {required}%"));
447 }
448 }
449 if shortfalls.is_empty() {
450 Outcome::Pass
451 } else {
452 Outcome::Fail(format!(
453 "coverage below thresholds: {}",
454 shortfalls.join(", ")
455 ))
456 }
457}
458
459pub fn measure_typescript(
469 root: &Path,
470 thresholds: TypeScriptThresholds,
471 exclude: &[String],
472) -> Result<Outcome> {
473 let report = run_vitest(root, exclude)?;
474 Ok(evaluate_typescript(&report, thresholds))
475}
476
477struct ReportDir(PathBuf);
481
482impl ReportDir {
483 fn new() -> Self {
484 static COUNTER: AtomicU64 = AtomicU64::new(0);
485 let name = format!(
486 "testing-conventions-vitest-{}-{}",
487 std::process::id(),
488 COUNTER.fetch_add(1, Ordering::Relaxed),
489 );
490 ReportDir(std::env::temp_dir().join(name))
491 }
492
493 fn summary(&self) -> PathBuf {
495 self.0.join("coverage-summary.json")
496 }
497}
498
499impl Drop for ReportDir {
500 fn drop(&mut self) {
501 let _ = std::fs::remove_dir_all(&self.0);
502 }
503}
504
505fn run_vitest(root: &Path, exclude: &[String]) -> Result<VitestReport> {
507 let reports = ReportDir::new();
508
509 let mut command = Command::new("npx");
516 command
517 .current_dir(root)
518 .args(["--yes", "vitest", "run", "--no-cache"])
519 .args([
520 "--coverage.enabled",
521 "--coverage.provider=v8",
522 "--coverage.reporter=json-summary",
523 "--coverage.all=true",
524 ])
525 .arg(format!(
526 "--coverage.reportsDirectory={}",
527 reports.0.display()
528 ))
529 .arg(format!("--coverage.include={TS_INCLUDE}"))
530 .arg(format!("--coverage.exclude={TS_TEST_EXCLUDE}"))
531 .arg(format!("--coverage.exclude={TS_DECL_EXCLUDE}"));
532 for path in exclude {
533 command.arg(format!("--coverage.exclude={path}"));
534 }
535 let run = command.env("CI", "1").output().context(
537 "running `npx vitest run --coverage` (are vitest and @vitest/coverage-v8 installed?)",
538 )?;
539 if !run.status.success() {
540 bail!(
541 "the unit suite did not run cleanly under vitest in `{}`:\n{}{}",
542 root.display(),
543 String::from_utf8_lossy(&run.stdout),
544 String::from_utf8_lossy(&run.stderr),
545 );
546 }
547
548 let summary = reports.summary();
549 let json = std::fs::read_to_string(&summary).with_context(|| {
550 format!(
551 "reading vitest coverage summary `{}` (did the run produce a json-summary report?)",
552 summary.display()
553 )
554 })?;
555 parse_vitest_report(&json)
556}
557
558#[cfg(test)]
559mod tests {
560 use super::*;
561
562 fn report(percent_covered: f64, num_branches: u64) -> CoverageReport {
563 CoverageReport {
564 totals: Totals {
565 percent_covered,
566 num_branches,
567 },
568 files: BTreeMap::new(),
569 }
570 }
571
572 #[test]
573 fn passes_when_total_meets_the_floor() {
574 assert_eq!(
575 evaluate(
576 &report(100.0, 12),
577 Thresholds {
578 fail_under: 100,
579 branch: true
580 }
581 ),
582 Outcome::Pass
583 );
584 }
585
586 #[test]
587 fn fails_when_total_is_below_the_floor() {
588 assert!(matches!(
589 evaluate(
590 &report(80.0, 12),
591 Thresholds {
592 fail_under: 100,
593 branch: true
594 }
595 ),
596 Outcome::Fail(_)
597 ));
598 }
599
600 #[test]
601 fn fails_when_branch_required_but_unmeasured() {
602 assert!(matches!(
604 evaluate(
605 &report(100.0, 0),
606 Thresholds {
607 fail_under: 90,
608 branch: true
609 }
610 ),
611 Outcome::Fail(_)
612 ));
613 }
614
615 #[test]
616 fn parses_a_coverage_py_report() {
617 let json = r#"{"totals":{"percent_covered":91.5,"num_branches":8,"covered_lines":91}}"#;
618 let report = parse_report(json).expect("valid coverage.py json");
619 assert_eq!(report.totals.percent_covered, 91.5);
620 assert_eq!(report.totals.num_branches, 8);
621 }
622
623 #[test]
624 fn parses_the_per_file_block_for_patch_coverage() {
625 let json = r#"{
628 "files": {
629 "widget.py": {
630 "executed_lines": [1, 2, 3, 4, 6],
631 "summary": {"percent_covered": 85.0},
632 "missing_lines": [5],
633 "excluded_lines": [],
634 "missing_branches": [[4, 5]]
635 }
636 },
637 "totals": {"percent_covered": 85.0, "num_branches": 4}
638 }"#;
639 let report = parse_report(json).expect("valid coverage.py json with files");
640 let widget = report.files.get("widget.py").expect("widget.py is present");
641 assert_eq!(widget.missing_lines, vec![5]);
642 assert_eq!(widget.missing_branches, vec![vec![4, 5]]);
643 assert_eq!(report.totals.percent_covered, 85.0);
645 }
646
647 #[test]
648 fn a_report_without_a_files_block_parses_with_an_empty_map() {
649 let report = parse_report(r#"{"totals":{"percent_covered":100.0,"num_branches":2}}"#)
651 .expect("valid coverage.py json");
652 assert!(report.files.is_empty());
653 }
654
655 #[test]
656 fn omit_is_the_test_and_support_globs_when_nothing_is_exempt() {
657 assert_eq!(build_omit(&[]), "*_test.py,*conftest.py");
658 }
659
660 #[test]
661 fn omit_folds_in_the_exempt_paths_after_the_test_glob() {
662 let exempt = vec!["pkg/gen.py".to_string(), "shim.py".to_string()];
664 assert_eq!(
665 build_omit(&exempt),
666 "*_test.py,*conftest.py,pkg/gen.py,shim.py"
667 );
668 }
669
670 #[test]
673 fn ratchet_passes_when_coverage_holds_at_the_baseline() {
674 assert_eq!(evaluate_ratchet(100.0, Some(100.0)), Outcome::Pass);
675 }
676
677 #[test]
678 fn ratchet_passes_when_coverage_improves_over_the_baseline() {
679 assert_eq!(evaluate_ratchet(92.0, Some(85.0)), Outcome::Pass);
680 }
681
682 #[test]
683 fn ratchet_fails_on_a_drop_below_the_baseline() {
684 assert!(matches!(
685 evaluate_ratchet(86.0, Some(90.0)),
686 Outcome::Fail(message) if message.contains("regressed") && message.contains("90")
687 ));
688 }
689
690 #[test]
691 fn ratchet_is_vacuous_without_a_recorded_baseline() {
692 assert_eq!(evaluate_ratchet(10.0, None), Outcome::Pass);
693 }
694
695 #[test]
696 fn ratchet_tolerates_float_noise_at_the_baseline() {
697 assert_eq!(evaluate_ratchet(99.999_999_999, Some(100.0)), Outcome::Pass);
698 }
699
700 static BASELINE_COUNTER: AtomicU64 = AtomicU64::new(0);
701
702 struct TempDir(PathBuf);
705
706 impl TempDir {
707 fn new() -> Self {
708 let dir = std::env::temp_dir().join(format!(
709 "tc-baseline-{}-{}",
710 std::process::id(),
711 BASELINE_COUNTER.fetch_add(1, Ordering::Relaxed),
712 ));
713 std::fs::create_dir_all(&dir).unwrap();
714 TempDir(dir)
715 }
716 }
717
718 impl Drop for TempDir {
719 fn drop(&mut self) {
720 let _ = std::fs::remove_dir_all(&self.0);
721 }
722 }
723
724 #[test]
725 fn read_baseline_is_none_when_the_file_is_absent() {
726 let dir = TempDir::new();
727 assert!(read_baseline(&dir.0).unwrap().is_none());
728 }
729
730 #[test]
731 fn read_baseline_parses_the_recorded_python_total() {
732 let dir = TempDir::new();
733 std::fs::write(
734 dir.0.join(BASELINE_PATH),
735 r#"{"python":{"percent_covered":91.5}}"#,
736 )
737 .unwrap();
738 let baseline = read_baseline(&dir.0)
739 .unwrap()
740 .expect("a baseline file is present");
741 assert_eq!(baseline.python.unwrap().percent_covered, 91.5);
742 }
743
744 #[test]
745 fn read_baseline_errors_on_a_malformed_file() {
746 let dir = TempDir::new();
747 std::fs::write(dir.0.join(BASELINE_PATH), "{ not json").unwrap();
748 assert!(read_baseline(&dir.0).is_err());
749 }
750
751 fn metric(pct: f64) -> VitestMetric {
754 VitestMetric {
755 pct: Some(pct),
756 total: 10,
757 }
758 }
759
760 fn ts_report(lines: f64, branches: f64, functions: f64, statements: f64) -> VitestReport {
761 VitestReport {
762 total: VitestTotals {
763 lines: metric(lines),
764 branches: metric(branches),
765 functions: metric(functions),
766 statements: metric(statements),
767 },
768 }
769 }
770
771 const TS_FULL: TypeScriptThresholds = TypeScriptThresholds {
772 lines: 100,
773 branches: 100,
774 functions: 100,
775 statements: 100,
776 };
777 const TS_MID: TypeScriptThresholds = TypeScriptThresholds {
778 lines: 80,
779 branches: 75,
780 functions: 80,
781 statements: 80,
782 };
783
784 #[test]
785 fn typescript_passes_when_every_metric_meets_its_floor() {
786 assert_eq!(
787 evaluate_typescript(&ts_report(100.0, 100.0, 100.0, 100.0), TS_FULL),
788 Outcome::Pass
789 );
790 }
791
792 #[test]
793 fn typescript_fails_on_the_one_metric_below_its_floor() {
794 let outcome = evaluate_typescript(&ts_report(100.0, 66.66, 100.0, 100.0), TS_MID);
798 assert!(
799 matches!(&outcome, Outcome::Fail(message) if message.contains("branches") && !message.contains("lines")),
800 "got: {outcome:?}"
801 );
802 }
803
804 #[test]
805 fn typescript_fail_message_names_every_metric_below() {
806 let outcome = evaluate_typescript(&ts_report(70.0, 70.0, 70.0, 70.0), TS_MID);
807 assert!(
808 matches!(&outcome, Outcome::Fail(message)
809 if message.contains("lines")
810 && message.contains("branches")
811 && message.contains("functions")
812 && message.contains("statements")),
813 "got: {outcome:?}"
814 );
815 }
816
817 #[test]
818 fn typescript_tolerates_float_noise_at_the_floor() {
819 assert_eq!(
821 evaluate_typescript(&ts_report(99.999_999_999, 100.0, 100.0, 100.0), TS_FULL),
822 Outcome::Pass
823 );
824 }
825
826 #[test]
827 fn typescript_empty_denominator_metric_is_vacuously_satisfied() {
828 let report = VitestReport {
831 total: VitestTotals {
832 lines: metric(100.0),
833 branches: VitestMetric {
834 pct: None,
835 total: 0,
836 },
837 functions: metric(100.0),
838 statements: metric(100.0),
839 },
840 };
841 assert_eq!(evaluate_typescript(&report, TS_FULL), Outcome::Pass);
842 }
843
844 #[test]
845 fn typescript_fails_a_vacuous_run_that_measured_no_code() {
846 let nothing = VitestMetric {
849 pct: None,
850 total: 0,
851 };
852 let report = VitestReport {
853 total: VitestTotals {
854 lines: nothing,
855 branches: nothing,
856 functions: nothing,
857 statements: nothing,
858 },
859 };
860 let outcome = evaluate_typescript(&report, TS_MID);
861 assert!(
862 matches!(&outcome, Outcome::Fail(message) if message.contains("measured no code")),
863 "got: {outcome:?}"
864 );
865 }
866
867 #[test]
868 fn parses_a_vitest_summary_report() {
869 let json = r#"{
872 "total": {
873 "lines": {"total": 5, "covered": 4, "skipped": 0, "pct": 80},
874 "statements": {"total": 5, "covered": 4, "skipped": 0, "pct": 80},
875 "functions": {"total": 2, "covered": 2, "skipped": 0, "pct": 100},
876 "branches": {"total": 3, "covered": 2, "skipped": 0, "pct": 66.66},
877 "branchesTrue": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"}
878 },
879 "/abs/widget.ts": {
880 "lines": {"total": 5, "covered": 4, "skipped": 0, "pct": 80}
881 }
882 }"#;
883 let report = parse_vitest_report(json).expect("valid vitest json-summary");
884 assert_eq!(report.total.lines.pct, Some(80.0));
886 assert_eq!(report.total.branches.pct, Some(66.66));
887 assert_eq!(report.total.functions.total, 2);
888 }
889
890 #[test]
891 fn parses_an_unknown_pct_as_unmeasured() {
892 let json = r#"{"total": {
893 "lines": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"},
894 "statements": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"},
895 "functions": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"},
896 "branches": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"}
897 }}"#;
898 let report = parse_vitest_report(json).expect("valid vitest json-summary");
899 assert_eq!(report.total.lines.pct, None);
900 assert_eq!(report.total.lines.total, 0);
901 }
902
903 #[test]
904 fn a_pct_that_is_neither_number_nor_string_is_a_parse_error() {
905 let json = r#"{"total":{
908 "lines": {"total": 1, "covered": 1, "skipped": 0, "pct": true},
909 "statements": {"total": 1, "covered": 1, "skipped": 0, "pct": 100},
910 "functions": {"total": 1, "covered": 1, "skipped": 0, "pct": 100},
911 "branches": {"total": 1, "covered": 1, "skipped": 0, "pct": 100}
912 }}"#;
913 assert!(parse_vitest_report(json).is_err());
914 }
915}