1pub mod agents;
2pub mod changelog;
3pub mod co_change;
4pub mod colocated_test;
5pub mod config;
6pub mod coverage;
7pub mod e2e;
8pub mod entrypoint;
9mod git_diff;
10mod git_path;
11mod integration_lint;
12pub mod isolation;
13pub mod lint;
14pub mod mutation;
15pub mod one_function;
16pub mod packaging;
17pub mod patch_coverage;
18mod rust_patch_coverage;
19mod subcommand_walk;
20mod subprocess_seam;
21pub mod tiers;
22pub mod ts;
23mod unit_coverage;
24mod unit_lint;
25mod unit_mutation;
26mod unit_one_function;
27pub mod violation;
28mod waivers;
29mod walk;
30pub mod workflow;
31pub mod workflow_lint;
32
33use std::path::{Path, PathBuf};
34
35use clap::{CommandFactory, Parser, Subcommand};
36
37#[derive(Parser, Debug)]
38#[command(
39 name = "testing-conventions",
40 version,
41 about = "Enforce testing conventions in libraries (Python, TypeScript, and Rust).",
42 long_about = None,
43)]
44pub struct Cli {
45 #[command(subcommand)]
46 command: Option<Command>,
47}
48
49#[derive(Subcommand, Debug)]
50enum Command {
51 Install {
56 #[arg(default_value = "AGENTS.md")]
58 path: PathBuf,
59 },
60 Unit {
62 #[command(subcommand)]
63 rule: UnitRule,
64 },
65 Integration {
67 #[command(subcommand)]
68 rule: IntegrationRule,
69 },
70 Packaging {
72 path: PathBuf,
75 #[arg(long, value_enum)]
78 language: Option<colocated_test::Language>,
79 },
80 #[command(hide = true)]
85 Workflow {
86 path: PathBuf,
88 },
89 E2e {
91 #[command(subcommand)]
92 command: E2eCommand,
93 },
94 #[command(args_conflicts_with_subcommands = true)]
98 WorkflowLint {
99 #[command(subcommand)]
100 command: Option<WorkflowLintCommand>,
101 #[arg(default_value = ".github")]
103 path: PathBuf,
104 },
105 Changelog {
108 #[arg(long)]
110 base: String,
111 #[arg(default_value = ".")]
113 path: PathBuf,
114 },
115}
116
117#[derive(Subcommand, Debug)]
118enum UnitRule {
119 ColocatedTest {
125 path: PathBuf,
127 #[arg(long, value_enum)]
129 language: colocated_test::Language,
130 #[arg(long)]
136 base: Option<String>,
137 #[arg(long, default_value = "testing-conventions.toml")]
140 config: PathBuf,
141 },
142 Coverage {
147 path: PathBuf,
149 #[arg(long, value_enum)]
151 language: colocated_test::Language,
152 #[arg(long)]
158 base: Option<String>,
159 #[arg(long, default_value = "testing-conventions.toml")]
164 config: PathBuf,
165 },
166 OneFunctionPerFile {
170 path: PathBuf,
172 #[arg(long, value_enum)]
174 language: colocated_test::Language,
175 #[arg(long, default_value = "testing-conventions.toml")]
180 config: PathBuf,
181 },
182 Lint {
184 path: PathBuf,
186 #[arg(long, value_enum)]
188 language: isolation::Language,
189 #[arg(long, default_value = "testing-conventions.toml")]
192 config: PathBuf,
193 },
194 Mutation {
200 path: PathBuf,
202 #[arg(long, value_enum)]
204 language: colocated_test::Language,
205 #[arg(long)]
209 base: Option<String>,
210 #[arg(long, default_value = "testing-conventions.toml")]
213 config: PathBuf,
214 #[arg(long = "ts-mutation-adapter", hide = true)]
218 ts_adapter: Option<PathBuf>,
219 },
220}
221
222#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
224pub enum IntegrationLintLanguage {
225 #[value(name = "python")]
227 Python,
228 #[value(name = "typescript")]
230 TypeScript,
231 #[value(name = "rust")]
233 Rust,
234}
235
236#[derive(Subcommand, Debug)]
237enum IntegrationRule {
238 Lint {
240 path: PathBuf,
242 #[arg(long, value_enum)]
244 language: IntegrationLintLanguage,
245 #[arg(long, default_value = "testing-conventions.toml")]
248 config: PathBuf,
249 },
250}
251
252#[derive(Subcommand, Debug)]
253enum WorkflowLintCommand {
254 Scope {
257 #[arg(long)]
259 base: Option<String>,
260 #[arg(default_value = ".github")]
262 path: PathBuf,
263 },
264}
265
266#[derive(Subcommand, Debug)]
267enum E2eCommand {
268 Attest {
272 command: String,
274 },
275 Verify {
277 #[arg(default_value = ".")]
280 path: PathBuf,
281 #[arg(long)]
284 scope: Option<PathBuf>,
285 #[arg(long)]
292 base: Option<String>,
293 #[arg(long = "extra-scope")]
298 extra_scope: Vec<PathBuf>,
299 #[arg(long = "exclude")]
303 exclude: Vec<PathBuf>,
304 #[arg(long)]
307 branch: Option<String>,
308 },
309 Slug {
312 branch: Option<String>,
314 },
315}
316
317pub fn run<I, T>(args: I) -> anyhow::Result<i32>
318where
319 I: IntoIterator<Item = T>,
320 T: Into<std::ffi::OsString> + Clone,
321{
322 eprintln!("testing-conventions {}", env!("CARGO_PKG_VERSION"));
325 let cli = Cli::try_parse_from(args)?;
326 match cli.command {
327 None => Ok(0),
328 Some(Command::Unit { rule }) => match rule {
329 UnitRule::ColocatedTest {
330 path,
331 language,
332 base,
333 config,
334 } => run_unit_colocated_test(&path, language, base.as_deref(), &config),
335 UnitRule::Coverage {
336 path,
337 language,
338 base,
339 config,
340 } => unit_coverage::run(&path, language, base.as_deref(), &config),
341 UnitRule::OneFunctionPerFile {
342 path,
343 language,
344 config,
345 } => unit_one_function::run(&path, language, &config),
346 UnitRule::Lint {
347 path,
348 language,
349 config,
350 } => unit_lint::run(&path, language, &config),
351 UnitRule::Mutation {
352 path,
353 language,
354 base,
355 config,
356 ts_adapter,
357 } => unit_mutation::run(
358 &path,
359 language,
360 base.as_deref(),
361 &config,
362 ts_adapter.as_deref(),
363 ),
364 },
365 Some(Command::Integration { rule }) => match rule {
366 IntegrationRule::Lint {
367 path,
368 language,
369 config,
370 } => integration_lint::run(&path, language, &config),
371 },
372 Some(Command::Packaging { path, language }) => run_packaging(&path, language),
373 Some(Command::Changelog { base, path }) => changelog::run(&path, &base),
374 Some(Command::Workflow { path }) => run_workflow(&path),
375 Some(Command::WorkflowLint { command, path }) => match command {
376 None => run_workflow_lint(&path),
377 Some(WorkflowLintCommand::Scope { base, path }) => {
378 run_workflow_lint_scope(&path, base.as_deref())
379 }
380 },
381 Some(Command::E2e { command }) => match command {
382 E2eCommand::Attest { command } => run_e2e_attest(&command),
383 E2eCommand::Verify {
384 path,
385 scope,
386 base,
387 extra_scope,
388 exclude,
389 branch,
390 } => run_e2e_verify(
391 &path,
392 scope.as_deref(),
393 base.as_deref(),
394 &extra_scope,
395 &exclude,
396 branch.as_deref(),
397 ),
398 E2eCommand::Slug { branch } => run_e2e_slug(branch.as_deref()),
399 },
400 Some(Command::Install { path }) => {
401 agents::install(&path)?;
402 Ok(0)
403 }
404 }
405}
406
407pub fn command() -> clap::Command {
409 Cli::command()
410}
411
412fn run_unit_colocated_test(
415 root: &Path,
416 language: colocated_test::Language,
417 base: Option<&str>,
418 config_path: &Path,
419) -> anyhow::Result<i32> {
420 if base.is_some() && language == colocated_test::Language::Rust {
421 anyhow::bail!(
422 "`unit colocated-test --base` supports `--language python` / `typescript`; Rust \
423 units are inline `#[cfg(test)]` in the same file, so a sibling test can't go stale"
424 );
425 }
426 let presence_clean = report_colocated_presence(root, language, config_path)?;
427 let co_change_clean = match base {
428 Some(base) => report_co_change(root, base, language, config_path)?,
429 None => true,
430 };
431 Ok(if presence_clean && co_change_clean {
432 0
433 } else {
434 1
435 })
436}
437
438fn report_colocated_presence(
441 root: &Path,
442 language: colocated_test::Language,
443 config_path: &Path,
444) -> anyhow::Result<bool> {
445 let exempt = colocated_test_exemptions(root, language, config_path)?;
446 let orphans = match language {
447 colocated_test::Language::Rust => colocated_test::missing_inline_tests(root, &exempt)?,
448 _ => colocated_test::missing_unit_tests(root, language, &exempt)?,
449 };
450 if orphans.is_empty() {
451 return Ok(true);
452 }
453 let (label, summary) = orphan_wording(language);
454 for orphan in &orphans {
455 eprintln!("{label}: {}", orphan.display());
456 }
457 eprintln!("error: {} {summary}", orphans.len());
458 Ok(false)
459}
460
461fn orphan_wording(language: colocated_test::Language) -> (&'static str, &'static str) {
465 match language {
466 colocated_test::Language::Rust => (
467 "missing inline `#[cfg(test)]` tests",
468 "source file(s) with testable code but no inline `#[cfg(test)]` module \
469 (add an inline test module, or an `exempt` entry with a reason)",
470 ),
471 _ => (
472 "missing colocated unit test",
473 "source file(s) missing a colocated unit test \
474 (add a colocated test, or an `exempt` entry with a reason)",
475 ),
476 }
477}
478
479fn colocated_test_exemptions(
481 root: &Path,
482 language: colocated_test::Language,
483 config_path: &Path,
484) -> anyhow::Result<std::collections::BTreeSet<String>> {
485 if !config_path.exists() {
486 return Ok(std::collections::BTreeSet::new());
487 }
488 let config = config::load_config(config_path)?;
489 config::resolve_exempt(
490 root,
491 config.exemptions(language),
492 config::Rule::ColocatedTest,
493 )
494}
495
496fn report_co_change(
499 root: &Path,
500 base: &str,
501 language: colocated_test::Language,
502 config_path: &Path,
503) -> anyhow::Result<bool> {
504 let exempt = co_change_exemptions(root, language, config_path)?;
505 let stale = co_change::stale_sources(root, base, language, &exempt)?;
506 if stale.is_empty() {
507 return Ok(true);
508 }
509 for source in &stale {
510 eprintln!(
511 "source changed without its colocated test: {}",
512 source.display()
513 );
514 }
515 eprintln!(
516 "error: {} source file(s) changed without their colocated test co-changing \
517 (update the test, or add an `exempt` entry with a reason)",
518 stale.len()
519 );
520 Ok(false)
521}
522
523fn co_change_exemptions(
525 root: &Path,
526 language: colocated_test::Language,
527 config_path: &Path,
528) -> anyhow::Result<std::collections::BTreeSet<String>> {
529 if !config_path.exists() {
530 return Ok(std::collections::BTreeSet::new());
531 }
532 let config = config::load_config(config_path)?;
533 config::resolve_exempt(root, config.exemptions(language), config::Rule::CoChange)
534}
535
536fn split_scopes(
538 scopes: std::collections::BTreeMap<String, config::LineScope>,
539) -> (
540 Vec<String>,
541 std::collections::BTreeMap<String, std::collections::BTreeSet<u32>>,
542) {
543 let mut whole_file = Vec::new();
544 let mut line_scoped = std::collections::BTreeMap::new();
545 for (path, scope) in scopes {
546 match scope {
547 config::LineScope::WholeFile => whole_file.push(path),
548 config::LineScope::Lines(lines) => {
549 line_scoped.insert(path, lines);
550 }
551 }
552 }
553 (whole_file, line_scoped)
554}
555
556fn run_packaging(path: &Path, language: Option<colocated_test::Language>) -> anyhow::Result<i32> {
560 let distributions = distributions_at(path, language)?;
561 let mut shipped = 0;
562 for distribution in &distributions {
563 shipped += report_shipped_test_files(distribution)?;
564 }
565 if shipped > 0 {
566 eprintln!(
567 "error: {shipped} test file(s) present in the built distribution(s) \
568 (they must be excluded from packaging)"
569 );
570 return Ok(1);
571 }
572 println!(
573 "checked {} built distribution(s); no test files shipped",
574 distributions.len()
575 );
576 Ok(0)
577}
578
579fn distributions_at(
583 path: &Path,
584 language: Option<colocated_test::Language>,
585) -> anyhow::Result<Vec<packaging::Distribution>> {
586 let distributions = match language {
587 Some(language) => vec![packaging::Distribution {
588 path: path.to_path_buf(),
589 language,
590 }],
591 None => packaging::discover(path)?,
592 };
593 if distributions.is_empty() {
594 anyhow::bail!(
595 "no recognized built distribution (`.whl`, `.tar.gz`, `.tgz`, `.crate`) at `{}`",
596 path.display()
597 );
598 }
599 Ok(distributions)
600}
601
602fn report_shipped_test_files(distribution: &packaging::Distribution) -> anyhow::Result<usize> {
604 let globs = match distribution.language {
605 colocated_test::Language::Python => vec!["*_test.py".to_string()],
606 colocated_test::Language::TypeScript => vec!["*.test.*".to_string()],
607 colocated_test::Language::Rust => vec!["tests/".to_string()],
609 };
610 let offenders = packaging::inspect(&distribution.path, &globs)?;
611 for offender in &offenders {
612 eprintln!(
613 "test file in built artifact `{}`: {}",
614 distribution.path.display(),
615 offender.display()
616 );
617 }
618 Ok(offenders.len())
619}
620
621fn run_workflow(path: &Path) -> anyhow::Result<i32> {
624 let violations = workflow::check(path, &command())?;
625 if violations.is_empty() {
626 return Ok(0);
627 }
628 for v in &violations {
629 eprintln!(
630 "{}:{}: {} — {}",
631 v.file.display(),
632 v.line,
633 v.rule,
634 v.message
635 );
636 }
637 eprintln!(
638 "error: {} workflow invocation(s) name a subcommand this binary no longer exposes",
639 violations.len()
640 );
641 Ok(1)
642}
643
644fn run_workflow_lint_scope(path: &Path, base: Option<&str>) -> anyhow::Result<i32> {
645 let in_scope = workflow_lint::in_scope(Path::new("."), path, base).unwrap_or_else(|err| {
646 eprintln!("workflow-lint scope: {err:#}; scoping the whole tree in");
647 true
648 });
649 println!("{in_scope}");
650 Ok(0)
651}
652
653fn run_workflow_lint(path: &Path) -> anyhow::Result<i32> {
654 let findings = workflow_lint::scan(path)?;
655 if findings.is_empty() {
656 return Ok(0);
657 }
658 for f in &findings {
659 eprintln!("{}", inline_logic_finding(f));
660 }
661 eprintln!(
662 "error: {} step(s) encode logic in CI YAML, where nothing tests it",
663 findings.len()
664 );
665 Ok(1)
666}
667
668fn inline_logic_finding(finding: &workflow_lint::Finding) -> String {
670 format!(
671 "{}:{}: {} step `{}` encodes logic inline ({}) — move it into a tested package in \
672 this repository's own language, invoked as a one-line `run:`",
673 finding.file.display(),
674 finding.line,
675 finding.kind,
676 finding.step,
677 finding.reasons.join("; ")
678 )
679}
680
681fn run_e2e_attest(command: &str) -> anyhow::Result<i32> {
684 let repo = std::env::current_dir()?;
685 let attestation = e2e::attest(&repo, command)?;
686 if attestation.exit_code != 0 {
687 eprintln!(
688 "e2e command `{command}` exited {}; a receipt records a run that passed — \
689 fix the failure and attest again",
690 attestation.exit_code
691 );
692 return Ok(attestation.exit_code);
693 }
694 println!(
695 "e2e receipt recorded for branch {} at {}/{}.json",
696 attestation.branch,
697 e2e::RECEIPTS_DIR,
698 e2e::branch_slug(&attestation.branch),
699 );
700 Ok(0)
701}
702
703fn run_e2e_verify(
707 path: &Path,
708 scope: Option<&Path>,
709 base: Option<&str>,
710 extra_scopes: &[PathBuf],
711 excludes: &[PathBuf],
712 branch: Option<&str>,
713) -> anyhow::Result<i32> {
714 match e2e::verify_extra_scoped(
715 path,
716 scope.unwrap_or(path),
717 base,
718 extra_scopes,
719 excludes,
720 branch,
721 )? {
722 e2e::Verification::Fresh => Ok(0),
723 e2e::Verification::Missing => {
724 eprintln!(
725 "no e2e receipt answers this change — run \
726 `testing-conventions e2e attest '<your e2e command>'`; the command is \
727 your judgment: the full suite, a targeted subset, or a no-op"
728 );
729 Ok(1)
730 }
731 }
732}
733
734fn run_e2e_slug(branch: Option<&str>) -> anyhow::Result<i32> {
736 let slug = match branch {
737 Some(name) => e2e::branch_slug(name),
738 None => {
739 let repo = std::env::current_dir()?;
740 e2e::branch_slug(&e2e::current_branch(&repo)?)
741 }
742 };
743 println!("{slug}");
744 Ok(0)
745}
746
747#[cfg(test)]
748mod tests {
749 use std::sync::atomic::{AtomicU64, Ordering};
750
751 use super::*;
752
753 fn scratch(slug: &str, files: &[(&str, &str)]) -> PathBuf {
755 static COUNTER: AtomicU64 = AtomicU64::new(0);
756 let root = std::env::temp_dir().join(format!(
757 "tc-lib-{}-{}-{}",
758 slug,
759 std::process::id(),
760 COUNTER.fetch_add(1, Ordering::Relaxed),
761 ));
762 let _ = std::fs::remove_dir_all(&root);
763 std::fs::create_dir_all(&root).unwrap();
764 for (rel, contents) in files {
765 let full = root.join(rel);
766 std::fs::create_dir_all(full.parent().unwrap()).unwrap();
767 std::fs::write(full, contents).unwrap();
768 }
769 root
770 }
771
772 #[test]
773 fn a_workflow_encoding_logic_fails_the_lint_and_a_wiring_one_passes() {
774 let workflow =
775 |body: &str| format!("on: push\njobs:\n j:\n steps:\n - run: |\n{body}");
776 let logic = scratch(
777 "workflow-lint-logic",
778 &[(
779 "ci.yml",
780 &workflow(" for f in *.txt; do\n echo $f\n done\n"),
781 )],
782 );
783 assert_eq!(run_workflow_lint(&logic.join("ci.yml")).unwrap(), 1);
784
785 let wiring = scratch(
786 "workflow-lint-wiring",
787 &[("ci.yml", &workflow(" echo hello\n"))],
788 );
789 assert_eq!(run_workflow_lint(&wiring.join("ci.yml")).unwrap(), 0);
790 }
791
792 #[test]
793 fn an_inline_logic_finding_names_the_step_its_location_and_every_reason() {
794 let rendered = inline_logic_finding(&workflow_lint::Finding {
795 file: PathBuf::from(".github/workflows/ci.yml"),
796 line: 42,
797 step: "Publish".to_string(),
798 kind: "run",
799 reasons: vec!["for loop".to_string(), "sed".to_string()],
800 });
801 assert!(
802 rendered.starts_with(".github/workflows/ci.yml:42: run step `Publish` encodes logic inline (for loop; sed)"),
803 "got: {rendered}"
804 );
805 assert!(rendered.contains("one-line `run:`"), "got: {rendered}");
806 }
807
808 #[test]
809 fn a_directory_holding_no_distribution_is_an_error_rather_than_a_clean_run() {
810 let root = scratch("packaging-empty", &[("README.md", "hi\n")]);
812 assert!(run_packaging(&root, None).is_err());
813 }
814
815 #[test]
816 fn a_rust_file_with_no_inline_tests_makes_the_presence_report_unclean() {
817 let root = scratch(
818 "presence-orphan",
819 &[("src/widget.rs", "pub fn f() -> u8 { 1 }\n")],
820 );
821 let config = root.join("testing-conventions.toml");
822 assert!(
823 !report_colocated_presence(&root, colocated_test::Language::Rust, &config).unwrap()
824 );
825 }
826
827 #[test]
828 fn a_rust_file_carrying_inline_tests_makes_the_presence_report_clean() {
829 let root = scratch(
830 "presence-clean",
831 &[(
832 "src/widget.rs",
833 "pub fn f() -> u8 { 1 }\n#[cfg(test)]\nmod tests { #[test] fn t() {} }\n",
834 )],
835 );
836 let config = root.join("testing-conventions.toml");
837 assert!(report_colocated_presence(&root, colocated_test::Language::Rust, &config).unwrap());
838 }
839
840 #[test]
841 fn rust_is_told_about_inline_test_modules_and_the_others_about_sibling_files() {
842 let (label, summary) = orphan_wording(colocated_test::Language::Rust);
843 assert!(label.contains("inline"), "got: {label}");
844 assert!(summary.contains("inline test module"), "got: {summary}");
845
846 let (label, summary) = orphan_wording(colocated_test::Language::Python);
847 assert!(label.contains("colocated unit test"), "got: {label}");
848 assert!(summary.contains("add a colocated test"), "got: {summary}");
849 }
850
851 #[test]
852 fn a_named_language_makes_the_path_itself_the_distribution() {
853 let found = distributions_at(
854 Path::new("/tmp/pkg.whl"),
855 Some(colocated_test::Language::Python),
856 )
857 .unwrap();
858 assert_eq!(found.len(), 1);
859 assert_eq!(found[0].path, Path::new("/tmp/pkg.whl"));
860 }
861
862 #[test]
863 fn a_search_that_finds_nothing_is_an_error_not_a_clean_run() {
864 let dir = std::env::temp_dir().join(format!(
865 "testing-conventions-no-distributions-{}",
866 std::process::id()
867 ));
868 std::fs::create_dir_all(&dir).unwrap();
869 let err = distributions_at(&dir, None).unwrap_err();
870 assert!(
871 err.to_string().contains("no recognized built distribution"),
872 "got: {err}"
873 );
874 }
875
876 #[test]
877 fn no_args_returns_ok_zero() {
878 assert_eq!(run(["testing-conventions"]).unwrap(), 0);
879 }
880
881 #[test]
882 fn unknown_flag_errors() {
883 assert!(run(["testing-conventions", "--bogus"]).is_err());
884 }
885
886 #[test]
887 fn split_scopes_separates_whole_file_paths_from_line_sets() {
888 let mut scopes = std::collections::BTreeMap::new();
889 scopes.insert("shim.py".to_string(), config::LineScope::WholeFile);
890 scopes.insert(
891 "widget.py".to_string(),
892 config::LineScope::Lines(std::collections::BTreeSet::from([3])),
893 );
894 let (whole_file, line_scoped) = split_scopes(scopes);
895 assert_eq!(whole_file, vec!["shim.py".to_string()]);
896 assert_eq!(line_scoped.len(), 1);
897 assert_eq!(
898 line_scoped["widget.py"],
899 std::collections::BTreeSet::from([3])
900 );
901 }
902
903 #[test]
904 fn help_flag_returns_clap_display_help() {
905 let err = run(["testing-conventions", "--help"]).expect_err("--help should bubble");
906 let clap_err = err
907 .downcast_ref::<clap::Error>()
908 .expect("error should be a clap::Error");
909 assert_eq!(clap_err.kind(), clap::error::ErrorKind::DisplayHelp);
910 }
911
912 #[test]
913 fn version_flag_returns_clap_display_version() {
914 let err = run(["testing-conventions", "--version"]).expect_err("--version should bubble");
915 let clap_err = err
916 .downcast_ref::<clap::Error>()
917 .expect("error should be a clap::Error");
918 assert_eq!(clap_err.kind(), clap::error::ErrorKind::DisplayVersion);
919 }
920}