1use std::path::{Path, PathBuf};
24use std::process::Command;
25use std::sync::atomic::{AtomicU64, Ordering};
26
27use anyhow::{bail, Context, Result};
28use serde::Deserialize;
29
30const TEST_OMIT: &str = "*_test.py";
33
34const SUPPORT_OMIT: &str = "*conftest.py";
38
39#[derive(Debug, Clone, Copy, PartialEq, Eq)]
41pub struct Thresholds {
42 pub fail_under: u8,
44 pub branch: bool,
46}
47
48#[derive(Debug, Clone, Deserialize)]
51pub struct CoverageReport {
52 pub totals: Totals,
53}
54
55#[derive(Debug, Clone, Deserialize)]
57pub struct Totals {
58 pub percent_covered: f64,
60 #[serde(default)]
62 pub num_branches: u64,
63}
64
65#[derive(Debug, Clone, PartialEq)]
67pub enum Outcome {
68 Pass,
70 Fail(String),
72}
73
74pub fn parse_report(json: &str) -> Result<CoverageReport> {
76 serde_json::from_str(json).context("parsing coverage.py JSON report")
77}
78
79pub fn evaluate(report: &CoverageReport, thresholds: Thresholds) -> Outcome {
84 if thresholds.branch && report.totals.num_branches == 0 {
85 return Outcome::Fail(
86 "branch coverage is required but the report measured no branches".to_string(),
87 );
88 }
89 let actual = report.totals.percent_covered;
90 let required = f64::from(thresholds.fail_under);
91 if actual + 1e-9 >= required {
94 Outcome::Pass
95 } else {
96 Outcome::Fail(format!(
97 "coverage {actual:.2}% is below the required {}%",
98 thresholds.fail_under
99 ))
100 }
101}
102
103pub const BASELINE_PATH: &str = "coverage-baseline.json";
118
119#[derive(Debug, Clone, Default, Deserialize)]
124#[serde(deny_unknown_fields)]
125pub struct Baseline {
126 #[serde(default)]
128 pub python: Option<PythonBaseline>,
129}
130
131#[derive(Debug, Clone, Copy, Deserialize)]
133#[serde(deny_unknown_fields)]
134pub struct PythonBaseline {
135 pub percent_covered: f64,
137}
138
139pub fn read_baseline(root: &Path) -> Result<Option<Baseline>> {
143 let path = root.join(BASELINE_PATH);
144 if !path.exists() {
145 return Ok(None);
146 }
147 let contents = std::fs::read_to_string(&path)
148 .with_context(|| format!("reading coverage baseline `{}`", path.display()))?;
149 let baseline = serde_json::from_str(&contents)
150 .with_context(|| format!("parsing coverage baseline `{}`", path.display()))?;
151 Ok(Some(baseline))
152}
153
154pub fn evaluate_ratchet(percent: f64, baseline: Option<f64>) -> Outcome {
159 match baseline {
160 Some(required) if percent + 1e-9 < required => Outcome::Fail(format!(
161 "coverage {percent:.2}% regressed below the recorded baseline {required:.2}%"
162 )),
163 _ => Outcome::Pass,
164 }
165}
166
167pub fn measure(root: &Path, thresholds: Thresholds, omit: &[String]) -> Result<Outcome> {
176 Ok(evaluate(&measure_report(root, omit)?, thresholds))
177}
178
179pub fn measure_report(root: &Path, omit: &[String]) -> Result<CoverageReport> {
183 run_coverage(root, omit)
184}
185
186struct DataFile(PathBuf);
190
191impl DataFile {
192 fn new() -> Self {
193 static COUNTER: AtomicU64 = AtomicU64::new(0);
194 let name = format!(
195 "testing-conventions-{}-{}.coverage",
196 std::process::id(),
197 COUNTER.fetch_add(1, Ordering::Relaxed),
198 );
199 DataFile(std::env::temp_dir().join(name))
200 }
201}
202
203impl Drop for DataFile {
204 fn drop(&mut self) {
205 let _ = std::fs::remove_file(&self.0);
206 }
207}
208
209fn run_coverage(root: &Path, omit: &[String]) -> Result<CoverageReport> {
211 let data = DataFile::new();
212 let omit = build_omit(omit);
213
214 let run = Command::new("coverage")
218 .current_dir(root)
219 .args(["run", "--branch"])
220 .arg(format!("--omit={omit}"))
221 .args(["-m", "pytest", "-q", "-p", "no:cacheprovider", "."])
222 .env("COVERAGE_FILE", &data.0)
223 .env("PYTHONDONTWRITEBYTECODE", "1")
224 .output()
225 .context("running `coverage run -m pytest` (is coverage.py installed?)")?;
226 if !run.status.success() {
227 bail!(
228 "the unit suite did not run cleanly under coverage in `{}`:\n{}{}",
229 root.display(),
230 String::from_utf8_lossy(&run.stdout),
231 String::from_utf8_lossy(&run.stderr),
232 );
233 }
234
235 let json = Command::new("coverage")
237 .current_dir(root)
238 .args(["json", "-o", "-"])
239 .env("COVERAGE_FILE", &data.0)
240 .output()
241 .context("running `coverage json`")?;
242 if !json.status.success() {
243 bail!(
244 "`coverage json` failed:\n{}",
245 String::from_utf8_lossy(&json.stderr),
246 );
247 }
248
249 parse_report(&String::from_utf8_lossy(&json.stdout))
250}
251
252fn build_omit(omit: &[String]) -> String {
259 [TEST_OMIT.to_string(), SUPPORT_OMIT.to_string()]
260 .into_iter()
261 .chain(omit.iter().cloned())
262 .collect::<Vec<_>>()
263 .join(",")
264}
265
266const TS_INCLUDE: &str = "**/*.{ts,tsx,mts,cts}";
280const TS_TEST_EXCLUDE: &str = "**/*.test.*";
284const TS_DECL_EXCLUDE: &str = "**/*.d.{ts,mts,cts}";
285
286#[derive(Debug, Clone, Copy, PartialEq, Eq)]
289pub struct TypeScriptThresholds {
290 pub lines: u8,
291 pub branches: u8,
292 pub functions: u8,
293 pub statements: u8,
294}
295
296#[derive(Debug, Clone, Copy, Deserialize)]
299pub struct VitestReport {
300 pub total: VitestTotals,
301}
302
303#[derive(Debug, Clone, Copy, Deserialize)]
306pub struct VitestTotals {
307 pub lines: VitestMetric,
308 pub branches: VitestMetric,
309 pub functions: VitestMetric,
310 pub statements: VitestMetric,
311}
312
313#[derive(Debug, Clone, Copy, Deserialize)]
316pub struct VitestMetric {
317 #[serde(deserialize_with = "deserialize_pct")]
320 pub pct: Option<f64>,
321 pub total: u64,
323}
324
325fn deserialize_pct<'de, D>(deserializer: D) -> std::result::Result<Option<f64>, D::Error>
329where
330 D: serde::Deserializer<'de>,
331{
332 struct PctVisitor;
333 impl serde::de::Visitor<'_> for PctVisitor {
334 type Value = Option<f64>;
335
336 fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
337 f.write_str("a coverage percent number or the string \"Unknown\"")
338 }
339
340 fn visit_f64<E>(self, value: f64) -> std::result::Result<Self::Value, E> {
341 Ok(Some(value))
342 }
343
344 fn visit_u64<E>(self, value: u64) -> std::result::Result<Self::Value, E> {
347 Ok(Some(value as f64))
348 }
349
350 fn visit_str<E>(self, _value: &str) -> std::result::Result<Self::Value, E> {
353 Ok(None)
354 }
355 }
356 deserializer.deserialize_any(PctVisitor)
357}
358
359pub fn parse_vitest_report(json: &str) -> Result<VitestReport> {
361 serde_json::from_str(json).context("parsing vitest coverage-summary JSON report")
362}
363
364pub fn evaluate_typescript(report: &VitestReport, thresholds: TypeScriptThresholds) -> Outcome {
373 let total = &report.total;
374 if total.lines.total == 0 {
378 return Outcome::Fail(
379 "the unit suite measured no code — check the path and that the suite runs".to_string(),
380 );
381 }
382 let checks = [
383 ("lines", total.lines, thresholds.lines),
384 ("branches", total.branches, thresholds.branches),
385 ("functions", total.functions, thresholds.functions),
386 ("statements", total.statements, thresholds.statements),
387 ];
388 let mut shortfalls = Vec::new();
389 for (name, metric, required) in checks {
390 let actual = metric.pct.unwrap_or(100.0);
393 if actual + 1e-9 < f64::from(required) {
396 shortfalls.push(format!("{name} {actual:.2}% < {required}%"));
397 }
398 }
399 if shortfalls.is_empty() {
400 Outcome::Pass
401 } else {
402 Outcome::Fail(format!(
403 "coverage below thresholds: {}",
404 shortfalls.join(", ")
405 ))
406 }
407}
408
409pub fn measure_typescript(
419 root: &Path,
420 thresholds: TypeScriptThresholds,
421 exclude: &[String],
422) -> Result<Outcome> {
423 let report = run_vitest(root, exclude)?;
424 Ok(evaluate_typescript(&report, thresholds))
425}
426
427struct ReportDir(PathBuf);
431
432impl ReportDir {
433 fn new() -> Self {
434 static COUNTER: AtomicU64 = AtomicU64::new(0);
435 let name = format!(
436 "testing-conventions-vitest-{}-{}",
437 std::process::id(),
438 COUNTER.fetch_add(1, Ordering::Relaxed),
439 );
440 ReportDir(std::env::temp_dir().join(name))
441 }
442
443 fn summary(&self) -> PathBuf {
445 self.0.join("coverage-summary.json")
446 }
447}
448
449impl Drop for ReportDir {
450 fn drop(&mut self) {
451 let _ = std::fs::remove_dir_all(&self.0);
452 }
453}
454
455fn run_vitest(root: &Path, exclude: &[String]) -> Result<VitestReport> {
457 let reports = ReportDir::new();
458
459 let mut command = Command::new("npx");
466 command
467 .current_dir(root)
468 .args(["--yes", "vitest", "run", "--no-cache"])
469 .args([
470 "--coverage.enabled",
471 "--coverage.provider=v8",
472 "--coverage.reporter=json-summary",
473 "--coverage.all=true",
474 ])
475 .arg(format!(
476 "--coverage.reportsDirectory={}",
477 reports.0.display()
478 ))
479 .arg(format!("--coverage.include={TS_INCLUDE}"))
480 .arg(format!("--coverage.exclude={TS_TEST_EXCLUDE}"))
481 .arg(format!("--coverage.exclude={TS_DECL_EXCLUDE}"));
482 for path in exclude {
483 command.arg(format!("--coverage.exclude={path}"));
484 }
485 let run = command.env("CI", "1").output().context(
487 "running `npx vitest run --coverage` (are vitest and @vitest/coverage-v8 installed?)",
488 )?;
489 if !run.status.success() {
490 bail!(
491 "the unit suite did not run cleanly under vitest in `{}`:\n{}{}",
492 root.display(),
493 String::from_utf8_lossy(&run.stdout),
494 String::from_utf8_lossy(&run.stderr),
495 );
496 }
497
498 let summary = reports.summary();
499 let json = std::fs::read_to_string(&summary).with_context(|| {
500 format!(
501 "reading vitest coverage summary `{}` (did the run produce a json-summary report?)",
502 summary.display()
503 )
504 })?;
505 parse_vitest_report(&json)
506}
507
508#[cfg(test)]
509mod tests {
510 use super::*;
511
512 fn report(percent_covered: f64, num_branches: u64) -> CoverageReport {
513 CoverageReport {
514 totals: Totals {
515 percent_covered,
516 num_branches,
517 },
518 }
519 }
520
521 #[test]
522 fn passes_when_total_meets_the_floor() {
523 assert_eq!(
524 evaluate(
525 &report(100.0, 12),
526 Thresholds {
527 fail_under: 100,
528 branch: true
529 }
530 ),
531 Outcome::Pass
532 );
533 }
534
535 #[test]
536 fn fails_when_total_is_below_the_floor() {
537 assert!(matches!(
538 evaluate(
539 &report(80.0, 12),
540 Thresholds {
541 fail_under: 100,
542 branch: true
543 }
544 ),
545 Outcome::Fail(_)
546 ));
547 }
548
549 #[test]
550 fn fails_when_branch_required_but_unmeasured() {
551 assert!(matches!(
553 evaluate(
554 &report(100.0, 0),
555 Thresholds {
556 fail_under: 90,
557 branch: true
558 }
559 ),
560 Outcome::Fail(_)
561 ));
562 }
563
564 #[test]
565 fn parses_a_coverage_py_report() {
566 let json = r#"{"totals":{"percent_covered":91.5,"num_branches":8,"covered_lines":91}}"#;
567 let report = parse_report(json).expect("valid coverage.py json");
568 assert_eq!(report.totals.percent_covered, 91.5);
569 assert_eq!(report.totals.num_branches, 8);
570 }
571
572 #[test]
573 fn omit_is_the_test_and_support_globs_when_nothing_is_exempt() {
574 assert_eq!(build_omit(&[]), "*_test.py,*conftest.py");
575 }
576
577 #[test]
578 fn omit_folds_in_the_exempt_paths_after_the_test_glob() {
579 let exempt = vec!["pkg/gen.py".to_string(), "shim.py".to_string()];
581 assert_eq!(
582 build_omit(&exempt),
583 "*_test.py,*conftest.py,pkg/gen.py,shim.py"
584 );
585 }
586
587 #[test]
590 fn ratchet_passes_when_coverage_holds_at_the_baseline() {
591 assert_eq!(evaluate_ratchet(100.0, Some(100.0)), Outcome::Pass);
592 }
593
594 #[test]
595 fn ratchet_passes_when_coverage_improves_over_the_baseline() {
596 assert_eq!(evaluate_ratchet(92.0, Some(85.0)), Outcome::Pass);
597 }
598
599 #[test]
600 fn ratchet_fails_on_a_drop_below_the_baseline() {
601 assert!(matches!(
602 evaluate_ratchet(86.0, Some(90.0)),
603 Outcome::Fail(message) if message.contains("regressed") && message.contains("90")
604 ));
605 }
606
607 #[test]
608 fn ratchet_is_vacuous_without_a_recorded_baseline() {
609 assert_eq!(evaluate_ratchet(10.0, None), Outcome::Pass);
610 }
611
612 #[test]
613 fn ratchet_tolerates_float_noise_at_the_baseline() {
614 assert_eq!(evaluate_ratchet(99.999_999_999, Some(100.0)), Outcome::Pass);
615 }
616
617 static BASELINE_COUNTER: AtomicU64 = AtomicU64::new(0);
618
619 struct TempDir(PathBuf);
622
623 impl TempDir {
624 fn new() -> Self {
625 let dir = std::env::temp_dir().join(format!(
626 "tc-baseline-{}-{}",
627 std::process::id(),
628 BASELINE_COUNTER.fetch_add(1, Ordering::Relaxed),
629 ));
630 std::fs::create_dir_all(&dir).unwrap();
631 TempDir(dir)
632 }
633 }
634
635 impl Drop for TempDir {
636 fn drop(&mut self) {
637 let _ = std::fs::remove_dir_all(&self.0);
638 }
639 }
640
641 #[test]
642 fn read_baseline_is_none_when_the_file_is_absent() {
643 let dir = TempDir::new();
644 assert!(read_baseline(&dir.0).unwrap().is_none());
645 }
646
647 #[test]
648 fn read_baseline_parses_the_recorded_python_total() {
649 let dir = TempDir::new();
650 std::fs::write(
651 dir.0.join(BASELINE_PATH),
652 r#"{"python":{"percent_covered":91.5}}"#,
653 )
654 .unwrap();
655 let baseline = read_baseline(&dir.0)
656 .unwrap()
657 .expect("a baseline file is present");
658 assert_eq!(baseline.python.unwrap().percent_covered, 91.5);
659 }
660
661 #[test]
662 fn read_baseline_errors_on_a_malformed_file() {
663 let dir = TempDir::new();
664 std::fs::write(dir.0.join(BASELINE_PATH), "{ not json").unwrap();
665 assert!(read_baseline(&dir.0).is_err());
666 }
667
668 fn metric(pct: f64) -> VitestMetric {
671 VitestMetric {
672 pct: Some(pct),
673 total: 10,
674 }
675 }
676
677 fn ts_report(lines: f64, branches: f64, functions: f64, statements: f64) -> VitestReport {
678 VitestReport {
679 total: VitestTotals {
680 lines: metric(lines),
681 branches: metric(branches),
682 functions: metric(functions),
683 statements: metric(statements),
684 },
685 }
686 }
687
688 const TS_FULL: TypeScriptThresholds = TypeScriptThresholds {
689 lines: 100,
690 branches: 100,
691 functions: 100,
692 statements: 100,
693 };
694 const TS_MID: TypeScriptThresholds = TypeScriptThresholds {
695 lines: 80,
696 branches: 75,
697 functions: 80,
698 statements: 80,
699 };
700
701 #[test]
702 fn typescript_passes_when_every_metric_meets_its_floor() {
703 assert_eq!(
704 evaluate_typescript(&ts_report(100.0, 100.0, 100.0, 100.0), TS_FULL),
705 Outcome::Pass
706 );
707 }
708
709 #[test]
710 fn typescript_fails_on_the_one_metric_below_its_floor() {
711 let outcome = evaluate_typescript(&ts_report(100.0, 66.66, 100.0, 100.0), TS_MID);
715 assert!(
716 matches!(&outcome, Outcome::Fail(message) if message.contains("branches") && !message.contains("lines")),
717 "got: {outcome:?}"
718 );
719 }
720
721 #[test]
722 fn typescript_fail_message_names_every_metric_below() {
723 let outcome = evaluate_typescript(&ts_report(70.0, 70.0, 70.0, 70.0), TS_MID);
724 assert!(
725 matches!(&outcome, Outcome::Fail(message)
726 if message.contains("lines")
727 && message.contains("branches")
728 && message.contains("functions")
729 && message.contains("statements")),
730 "got: {outcome:?}"
731 );
732 }
733
734 #[test]
735 fn typescript_tolerates_float_noise_at_the_floor() {
736 assert_eq!(
738 evaluate_typescript(&ts_report(99.999_999_999, 100.0, 100.0, 100.0), TS_FULL),
739 Outcome::Pass
740 );
741 }
742
743 #[test]
744 fn typescript_empty_denominator_metric_is_vacuously_satisfied() {
745 let report = VitestReport {
748 total: VitestTotals {
749 lines: metric(100.0),
750 branches: VitestMetric {
751 pct: None,
752 total: 0,
753 },
754 functions: metric(100.0),
755 statements: metric(100.0),
756 },
757 };
758 assert_eq!(evaluate_typescript(&report, TS_FULL), Outcome::Pass);
759 }
760
761 #[test]
762 fn typescript_fails_a_vacuous_run_that_measured_no_code() {
763 let nothing = VitestMetric {
766 pct: None,
767 total: 0,
768 };
769 let report = VitestReport {
770 total: VitestTotals {
771 lines: nothing,
772 branches: nothing,
773 functions: nothing,
774 statements: nothing,
775 },
776 };
777 let outcome = evaluate_typescript(&report, TS_MID);
778 assert!(
779 matches!(&outcome, Outcome::Fail(message) if message.contains("measured no code")),
780 "got: {outcome:?}"
781 );
782 }
783
784 #[test]
785 fn parses_a_vitest_summary_report() {
786 let json = r#"{
789 "total": {
790 "lines": {"total": 5, "covered": 4, "skipped": 0, "pct": 80},
791 "statements": {"total": 5, "covered": 4, "skipped": 0, "pct": 80},
792 "functions": {"total": 2, "covered": 2, "skipped": 0, "pct": 100},
793 "branches": {"total": 3, "covered": 2, "skipped": 0, "pct": 66.66},
794 "branchesTrue": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"}
795 },
796 "/abs/widget.ts": {
797 "lines": {"total": 5, "covered": 4, "skipped": 0, "pct": 80}
798 }
799 }"#;
800 let report = parse_vitest_report(json).expect("valid vitest json-summary");
801 assert_eq!(report.total.lines.pct, Some(80.0));
803 assert_eq!(report.total.branches.pct, Some(66.66));
804 assert_eq!(report.total.functions.total, 2);
805 }
806
807 #[test]
808 fn parses_an_unknown_pct_as_unmeasured() {
809 let json = r#"{"total": {
810 "lines": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"},
811 "statements": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"},
812 "functions": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"},
813 "branches": {"total": 0, "covered": 0, "skipped": 0, "pct": "Unknown"}
814 }}"#;
815 let report = parse_vitest_report(json).expect("valid vitest json-summary");
816 assert_eq!(report.total.lines.pct, None);
817 assert_eq!(report.total.lines.total, 0);
818 }
819
820 #[test]
821 fn a_pct_that_is_neither_number_nor_string_is_a_parse_error() {
822 let json = r#"{"total":{
825 "lines": {"total": 1, "covered": 1, "skipped": 0, "pct": true},
826 "statements": {"total": 1, "covered": 1, "skipped": 0, "pct": 100},
827 "functions": {"total": 1, "covered": 1, "skipped": 0, "pct": 100},
828 "branches": {"total": 1, "covered": 1, "skipped": 0, "pct": 100}
829 }}"#;
830 assert!(parse_vitest_report(json).is_err());
831 }
832}