1pub mod co_change;
2pub mod colocated_test;
3pub mod config;
4pub mod coverage;
5pub mod e2e;
6pub mod isolation;
7pub mod lint;
8pub mod mutation;
9pub mod packaging;
10pub mod patch_coverage;
11pub mod ts;
12pub mod violation;
13pub mod workflow;
14
15use std::path::{Path, PathBuf};
16
17use clap::{CommandFactory, Parser, Subcommand};
18
19#[derive(Parser, Debug)]
20#[command(
21 name = "testing-conventions",
22 version,
23 about = "Enforce testing conventions in libraries (Python, TypeScript, and Rust).",
24 long_about = None,
25)]
26pub struct Cli {
27 #[command(subcommand)]
28 command: Option<Command>,
29}
30
31#[derive(Subcommand, Debug)]
32enum Command {
33 Check,
35 Unit {
37 #[command(subcommand)]
38 rule: UnitRule,
39 },
40 Integration {
42 #[command(subcommand)]
43 rule: IntegrationRule,
44 },
45 Packaging {
47 path: PathBuf,
49 #[arg(long, value_enum)]
51 language: colocated_test::Language,
52 },
53 #[command(hide = true)]
58 Workflow {
59 path: PathBuf,
61 },
62 E2e {
64 #[command(subcommand)]
65 command: E2eCommand,
66 },
67}
68
69#[derive(Subcommand, Debug)]
71enum UnitRule {
72 ColocatedTest {
78 path: PathBuf,
80 #[arg(long, value_enum)]
82 language: colocated_test::Language,
83 #[arg(long)]
89 base: Option<String>,
90 #[arg(long, default_value = "testing-conventions.toml")]
93 config: PathBuf,
94 },
95 Coverage {
100 path: PathBuf,
102 #[arg(long, value_enum)]
104 language: colocated_test::Language,
105 #[arg(long)]
111 base: Option<String>,
112 #[arg(long, default_value = "testing-conventions.toml")]
117 config: PathBuf,
118 },
119 Lint {
121 path: PathBuf,
123 #[arg(long, value_enum)]
125 language: isolation::Language,
126 #[arg(long, default_value = "testing-conventions.toml")]
129 config: PathBuf,
130 },
131 Mutation {
137 path: PathBuf,
139 #[arg(long, value_enum)]
141 language: colocated_test::Language,
142 #[arg(long)]
146 base: Option<String>,
147 #[arg(long, default_value = "testing-conventions.toml")]
150 config: PathBuf,
151 },
152}
153
154#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
158pub enum IntegrationLintLanguage {
159 #[value(name = "python")]
161 Python,
162 #[value(name = "typescript")]
164 TypeScript,
165 #[value(name = "rust")]
167 Rust,
168}
169
170#[derive(Subcommand, Debug)]
173enum IntegrationRule {
174 Lint {
176 path: PathBuf,
178 #[arg(long, value_enum)]
180 language: IntegrationLintLanguage,
181 #[arg(long, default_value = "testing-conventions.toml")]
184 config: PathBuf,
185 },
186}
187
188#[derive(Subcommand, Debug)]
191enum E2eCommand {
192 Attest {
194 command: String,
196 },
197 Verify,
199}
200
201pub fn run<I, T>(args: I) -> anyhow::Result<i32>
202where
203 I: IntoIterator<Item = T>,
204 T: Into<std::ffi::OsString> + Clone,
205{
206 let cli = Cli::try_parse_from(args)?;
207 match cli.command {
208 Some(Command::Check) | None => Ok(0),
212 Some(Command::Unit { rule }) => match rule {
213 UnitRule::ColocatedTest {
214 path,
215 language,
216 base,
217 config,
218 } => run_unit_colocated_test(&path, language, base.as_deref(), &config),
219 UnitRule::Coverage {
220 path,
221 language,
222 base,
223 config,
224 } => run_unit_coverage(&path, language, base.as_deref(), &config),
225 UnitRule::Lint {
226 path,
227 language,
228 config,
229 } => run_unit_lint(&path, language, &config),
230 UnitRule::Mutation {
231 path,
232 language,
233 base,
234 config,
235 } => run_unit_mutation(&path, language, base.as_deref(), &config),
236 },
237 Some(Command::Integration { rule }) => match rule {
238 IntegrationRule::Lint {
239 path,
240 language,
241 config,
242 } => run_integration_lint(&path, language, &config),
243 },
244 Some(Command::Packaging { path, language }) => run_packaging(&path, language),
245 Some(Command::Workflow { path }) => run_workflow(&path),
246 Some(Command::E2e { command }) => match command {
247 E2eCommand::Attest { command } => run_e2e_attest(&command),
248 E2eCommand::Verify => run_e2e_verify(),
249 },
250 }
251}
252
253pub fn command() -> clap::Command {
257 Cli::command()
258}
259
260fn run_unit_colocated_test(
273 root: &Path,
274 language: colocated_test::Language,
275 base: Option<&str>,
276 config_path: &Path,
277) -> anyhow::Result<i32> {
278 if base.is_some() && language == colocated_test::Language::Rust {
281 anyhow::bail!(
282 "`unit colocated-test --base` supports `--language python` / `typescript`; Rust \
283 units are inline `#[cfg(test)]` in the same file, so a sibling test can't go stale"
284 );
285 }
286 let presence_clean = report_colocated_presence(root, language, config_path)?;
287 let co_change_clean = match base {
288 Some(base) => report_co_change(root, base, language, config_path)?,
289 None => true,
290 };
291 Ok(if presence_clean && co_change_clean {
292 0
293 } else {
294 1
295 })
296}
297
298fn report_colocated_presence(
304 root: &Path,
305 language: colocated_test::Language,
306 config_path: &Path,
307) -> anyhow::Result<bool> {
308 let exempt = colocated_test_exemptions(root, language, config_path)?;
309 let orphans = match language {
310 colocated_test::Language::Rust => colocated_test::missing_inline_tests(root, &exempt)?,
313 _ => colocated_test::missing_unit_tests(root, language, &exempt)?,
314 };
315 if orphans.is_empty() {
316 return Ok(true);
317 }
318 let (label, summary) = match language {
319 colocated_test::Language::Rust => (
320 "missing inline `#[cfg(test)]` tests",
321 "source file(s) with testable code but no inline `#[cfg(test)]` module \
322 (add an inline test module, or an `exempt` entry with a reason)",
323 ),
324 _ => (
325 "missing colocated unit test",
326 "source file(s) missing a colocated unit test \
327 (add a colocated test, or an `exempt` entry with a reason)",
328 ),
329 };
330 for orphan in &orphans {
331 eprintln!("{label}: {}", orphan.display());
332 }
333 eprintln!("error: {} {summary}", orphans.len());
334 Ok(false)
335}
336
337fn colocated_test_exemptions(
341 root: &Path,
342 language: colocated_test::Language,
343 config_path: &Path,
344) -> anyhow::Result<std::collections::BTreeSet<String>> {
345 if !config_path.exists() {
346 return Ok(std::collections::BTreeSet::new());
347 }
348 let config = config::load_config(config_path)?;
349 config::resolve_exempt(
350 root,
351 config.exemptions(language),
352 config::Rule::ColocatedTest,
353 )
354}
355
356fn report_co_change(
366 root: &Path,
367 base: &str,
368 language: colocated_test::Language,
369 config_path: &Path,
370) -> anyhow::Result<bool> {
371 let exempt = co_change_exemptions(root, language, config_path)?;
372 let stale = co_change::stale_sources(root, base, language, &exempt)?;
373 if stale.is_empty() {
374 return Ok(true);
375 }
376 for source in &stale {
377 eprintln!(
378 "source changed without its colocated test: {}",
379 source.display()
380 );
381 }
382 eprintln!(
383 "error: {} source file(s) changed without their colocated test co-changing \
384 (update the test, or add an `exempt` entry with a reason)",
385 stale.len()
386 );
387 Ok(false)
388}
389
390fn co_change_exemptions(
394 root: &Path,
395 language: colocated_test::Language,
396 config_path: &Path,
397) -> anyhow::Result<std::collections::BTreeSet<String>> {
398 if !config_path.exists() {
399 return Ok(std::collections::BTreeSet::new());
400 }
401 let config = config::load_config(config_path)?;
402 config::resolve_exempt(root, config.exemptions(language), config::Rule::CoChange)
403}
404
405fn split_scopes(
410 scopes: std::collections::BTreeMap<String, config::LineScope>,
411) -> (
412 Vec<String>,
413 std::collections::BTreeMap<String, std::collections::BTreeSet<u32>>,
414) {
415 let mut whole_file = Vec::new();
416 let mut line_scoped = std::collections::BTreeMap::new();
417 for (path, scope) in scopes {
418 match scope {
419 config::LineScope::WholeFile => whole_file.push(path),
420 config::LineScope::Lines(lines) => {
421 line_scoped.insert(path, lines);
422 }
423 }
424 }
425 (whole_file, line_scoped)
426}
427
428fn run_unit_coverage(
446 root: &Path,
447 language: colocated_test::Language,
448 base: Option<&str>,
449 config_path: &Path,
450) -> anyhow::Result<i32> {
451 let config = if config_path.exists() {
452 config::load_config(config_path)?
453 } else {
454 config::Config::default()
455 };
456 let outcome = match language {
457 colocated_test::Language::Python => {
458 let python = config.python.unwrap_or_default();
459 let coverage = python.coverage.unwrap_or_default();
460 let thresholds = coverage::Thresholds {
461 fail_under: coverage.fail_under,
462 branch: coverage.branch,
463 };
464 let (omit, exempt_lines) = split_scopes(config::resolve_exempt_scoped(
465 root,
466 &python.exempt,
467 config::Rule::Coverage,
468 )?);
469 match base {
470 Some(base) => {
471 patch_coverage::measure(root, base, thresholds, &omit, &exempt_lines)?
472 }
473 None if exempt_lines.is_empty() => coverage::measure(root, thresholds, &omit)?,
474 None => {
475 patch_coverage::measure_line_exempt(root, thresholds, &omit, &exempt_lines)?
476 }
477 }
478 }
479 colocated_test::Language::TypeScript => {
480 let typescript = config.typescript.unwrap_or_default();
481 let coverage = typescript.coverage.unwrap_or_default();
482 let thresholds = coverage::TypeScriptThresholds {
483 lines: coverage.lines,
484 branches: coverage.branches,
485 functions: coverage.functions,
486 statements: coverage.statements,
487 };
488 let (exclude, exempt_lines) = split_scopes(config::resolve_exempt_scoped(
489 root,
490 &typescript.exempt,
491 config::Rule::Coverage,
492 )?);
493 match base {
494 Some(base) => patch_coverage::measure_typescript(
495 root,
496 base,
497 thresholds,
498 &exclude,
499 &exempt_lines,
500 )?,
501 None if exempt_lines.is_empty() => {
502 coverage::measure_typescript(root, thresholds, &exclude)?
503 }
504 None => patch_coverage::measure_line_exempt_typescript(
505 root,
506 thresholds,
507 &exclude,
508 &exempt_lines,
509 )?,
510 }
511 }
512 colocated_test::Language::Rust => {
513 let rust = config.rust.unwrap_or_default();
514 let coverage = rust.coverage.unwrap_or_default();
519 let thresholds = coverage::RustThresholds {
520 regions: coverage.regions,
521 lines: coverage.lines,
522 };
523 let (ignore, exempt_lines) = split_scopes(config::resolve_exempt_scoped(
524 root,
525 &rust.exempt,
526 config::Rule::Coverage,
527 )?);
528 match base {
529 Some(base) => {
530 patch_coverage::measure_rust(root, base, thresholds, &ignore, &exempt_lines)?
531 }
532 None if exempt_lines.is_empty() => {
533 coverage::measure_rust(root, thresholds, &ignore)?
534 }
535 None => patch_coverage::measure_line_exempt_rust(
536 root,
537 thresholds,
538 &ignore,
539 &exempt_lines,
540 )?,
541 }
542 }
543 };
544 match outcome {
545 coverage::Outcome::Pass => Ok(0),
546 coverage::Outcome::Fail(reason) => {
547 eprintln!("error: coverage check failed — {reason}");
548 Ok(1)
549 }
550 }
551}
552
553fn run_unit_mutation(
564 root: &Path,
565 language: colocated_test::Language,
566 base: Option<&str>,
567 config_path: &Path,
568) -> anyhow::Result<i32> {
569 let config = if config_path.exists() {
570 config::load_config(config_path)?
571 } else {
572 config::Config::default()
573 };
574 let survivors = match language {
575 colocated_test::Language::Rust => {
576 let rust = config.rust.unwrap_or_default();
577 let (exempt, exempt_lines) = split_scopes(config::resolve_exempt_scoped(
578 root,
579 &rust.exempt,
580 config::Rule::Mutation,
581 )?);
582 mutation::measure_rust(root, &exempt, &exempt_lines, base)?
583 }
584 colocated_test::Language::TypeScript => {
585 let typescript = config.typescript.unwrap_or_default();
586 let (exempt, exempt_lines) = split_scopes(config::resolve_exempt_scoped(
587 root,
588 &typescript.exempt,
589 config::Rule::Mutation,
590 )?);
591 mutation::measure_typescript(root, &exempt, &exempt_lines, base)?
592 }
593 colocated_test::Language::Python => {
594 let python = config.python.unwrap_or_default();
595 let (exempt, exempt_lines) = split_scopes(config::resolve_exempt_scoped(
596 root,
597 &python.exempt,
598 config::Rule::Mutation,
599 )?);
600 mutation::measure_python(root, &exempt, &exempt_lines, base)?
601 }
602 };
603 if survivors.is_empty() {
604 println!("unit mutation: no surviving mutants — every mutation was caught");
605 return Ok(0);
606 }
607
608 eprintln!(
609 "error: {} unexplained surviving mutant(s) — kill each with an assertion, or lift an \
610 equivalent/defensive one with a reason-required `[[<language>.exempt]] rules = [\"mutation\"]`:",
611 survivors.len()
612 );
613 for survivor in &survivors {
614 eprintln!(
615 " {}:{}: {}",
616 survivor.file, survivor.line, survivor.description
617 );
618 }
619 Ok(1)
620}
621
622fn run_unit_lint(
627 root: &Path,
628 language: isolation::Language,
629 config_path: &Path,
630) -> anyhow::Result<i32> {
631 let (raw, select): (Vec<lint::Violation>, ExemptSelect) = match language {
632 isolation::Language::Rust => (isolation::find_violations(root)?, |c| c.rust_exemptions()),
633 isolation::Language::TypeScript => (ts::find_unit_violations(root)?, |c| {
634 c.exemptions(colocated_test::Language::TypeScript)
635 }),
636 isolation::Language::Python => (lint::find_unit_isolation_violations(root)?, |c| {
637 c.exemptions(colocated_test::Language::Python)
638 }),
639 };
640 let violations = apply_waivers(raw, root, config_path, select)?;
641 if violations.is_empty() {
642 return Ok(0);
643 }
644 for v in &violations {
645 eprintln!(
646 "{}:{}: {} — {}",
647 v.file.display(),
648 v.line,
649 v.rule,
650 v.message
651 );
652 }
653 eprintln!("error: {} isolation violation(s)", violations.len());
654 Ok(1)
655}
656
657fn run_integration_lint(
661 root: &Path,
662 language: IntegrationLintLanguage,
663 config_path: &Path,
664) -> anyhow::Result<i32> {
665 let (raw, select): (Vec<lint::Violation>, ExemptSelect) = match language {
666 IntegrationLintLanguage::Python => (lint::find_violations(root)?, |c| {
667 c.exemptions(colocated_test::Language::Python)
668 }),
669 IntegrationLintLanguage::TypeScript => (ts::find_integration_violations(root)?, |c| {
670 c.exemptions(colocated_test::Language::TypeScript)
671 }),
672 IntegrationLintLanguage::Rust => (isolation::find_integration_violations(root)?, |c| {
673 c.rust_exemptions()
674 }),
675 };
676 let violations = apply_waivers(raw, root, config_path, select)?;
677 if violations.is_empty() {
678 return Ok(0);
679 }
680 for v in &violations {
681 eprintln!(
682 "{}:{}: {} — {}",
683 v.file.display(),
684 v.line,
685 v.rule,
686 v.message
687 );
688 }
689 eprintln!("error: {} lint violation(s)", violations.len());
690 Ok(1)
691}
692
693type ExemptSelect = fn(&config::Config) -> &[config::Exemption];
696
697fn apply_waivers(
704 violations: Vec<lint::Violation>,
705 root: &Path,
706 config_path: &Path,
707 exemptions: ExemptSelect,
708) -> anyhow::Result<Vec<lint::Violation>> {
709 use std::collections::hash_map::Entry;
710
711 if !config_path.exists() {
712 return Ok(violations);
713 }
714 let config = config::load_config(config_path)?;
715 let exempt = exemptions(&config);
716 let mut resolved: std::collections::HashMap<config::Rule, std::collections::BTreeSet<String>> =
718 std::collections::HashMap::new();
719 let mut kept = Vec::new();
720 for violation in violations {
721 let waived = match config::Rule::from_id(violation.rule) {
722 Some(rule) => {
723 let exempt_paths = match resolved.entry(rule) {
724 Entry::Occupied(entry) => entry.into_mut(),
725 Entry::Vacant(entry) => {
726 entry.insert(config::resolve_exempt(root, exempt, rule)?)
727 }
728 };
729 violation
730 .file
731 .strip_prefix(root)
732 .ok()
733 .map(|rel| rel.to_string_lossy().replace('\\', "/"))
734 .is_some_and(|rel| exempt_paths.contains(&rel))
735 }
736 None => false,
737 };
738 if !waived {
739 kept.push(violation);
740 }
741 }
742 Ok(kept)
743}
744
745fn run_packaging(artifact: &Path, language: colocated_test::Language) -> anyhow::Result<i32> {
754 let globs = match language {
755 colocated_test::Language::Python => vec!["*_test.py".to_string()],
756 colocated_test::Language::TypeScript => vec!["*.test.*".to_string()],
757 colocated_test::Language::Rust => vec!["tests/".to_string()],
760 };
761 let offenders = packaging::inspect(artifact, &globs)?;
762 if offenders.is_empty() {
763 return Ok(0);
764 }
765 for offender in &offenders {
766 eprintln!("test file in built artifact: {}", offender.display());
767 }
768 eprintln!(
769 "error: {} test file(s) present in the built artifact \
770 (they must be excluded from packaging)",
771 offenders.len()
772 );
773 Ok(1)
774}
775
776fn run_workflow(path: &Path) -> anyhow::Result<i32> {
781 let violations = workflow::check(path, &command())?;
782 if violations.is_empty() {
783 return Ok(0);
784 }
785 for v in &violations {
786 eprintln!(
787 "{}:{}: {} — {}",
788 v.file.display(),
789 v.line,
790 v.rule,
791 v.message
792 );
793 }
794 eprintln!(
795 "error: {} workflow invocation(s) name a subcommand this binary no longer exposes",
796 violations.len()
797 );
798 Ok(1)
799}
800
801fn run_e2e_attest(command: &str) -> anyhow::Result<i32> {
805 let repo = std::env::current_dir()?;
806 let attestation = e2e::attest(&repo, command)?;
807 println!(
808 "e2e attestation recorded for commit {} (command exited {})",
809 attestation.commit, attestation.exit_code
810 );
811 Ok(0)
812}
813
814fn run_e2e_verify() -> anyhow::Result<i32> {
818 let repo = std::env::current_dir()?;
819 match e2e::verify(&repo)? {
820 e2e::Verification::Fresh => Ok(0),
821 e2e::Verification::Missing => {
822 eprintln!(
823 "e2e attestation missing — run `testing-conventions e2e attest '<your e2e command>'`"
824 );
825 Ok(1)
826 }
827 e2e::Verification::Stale { attested, latest } => {
828 eprintln!(
829 "e2e attestation out of date: attested {}, latest code commit {} — \
830 run `testing-conventions e2e attest '<your e2e command>'`",
831 &attested[..attested.len().min(7)],
832 &latest[..latest.len().min(7)]
833 );
834 Ok(1)
835 }
836 }
837}
838
839#[cfg(test)]
840mod tests {
841 use super::*;
842
843 #[test]
844 fn no_args_returns_ok_zero() {
845 assert_eq!(run(["testing-conventions"]).unwrap(), 0);
846 }
847
848 #[test]
849 fn check_returns_ok_zero() {
850 assert_eq!(run(["testing-conventions", "check"]).unwrap(), 0);
851 }
852
853 #[test]
854 fn unknown_flag_errors() {
855 assert!(run(["testing-conventions", "--bogus"]).is_err());
856 }
857
858 #[test]
859 fn help_flag_returns_clap_display_help() {
860 let err = run(["testing-conventions", "--help"]).expect_err("--help should bubble");
861 let clap_err = err
862 .downcast_ref::<clap::Error>()
863 .expect("error should be a clap::Error");
864 assert_eq!(clap_err.kind(), clap::error::ErrorKind::DisplayHelp);
865 }
866
867 #[test]
868 fn version_flag_returns_clap_display_version() {
869 let err = run(["testing-conventions", "--version"]).expect_err("--version should bubble");
870 let clap_err = err
871 .downcast_ref::<clap::Error>()
872 .expect("error should be a clap::Error");
873 assert_eq!(clap_err.kind(), clap::error::ErrorKind::DisplayVersion);
874 }
875}