1pub mod agents;
2pub mod changelog;
3pub mod co_change;
4pub mod colocated_test;
5pub mod config;
6pub mod coverage;
7pub mod e2e;
8pub mod entrypoint;
9mod git_diff;
10mod git_path;
11mod integration_lint;
12pub mod isolation;
13pub mod lint;
14pub mod mutation;
15pub mod one_function;
16pub mod packaging;
17pub mod patch_coverage;
18mod python_imports;
19mod relocation;
20mod rust_patch_coverage;
21mod subcommand_walk;
22mod subprocess_seam;
23pub mod tiers;
24pub mod ts;
25mod ts_imports;
26mod unit_coverage;
27mod unit_lint;
28mod unit_mutation;
29mod unit_one_function;
30pub mod violation;
31mod waivers;
32mod walk;
33pub mod workflow;
34pub mod workflow_lint;
35
36use std::path::{Path, PathBuf};
37
38use clap::{CommandFactory, Parser, Subcommand};
39
40#[derive(Parser, Debug)]
41#[command(
42 name = "testing-conventions",
43 version,
44 about = "Enforce testing conventions in libraries (Python, TypeScript, and Rust).",
45 long_about = None,
46)]
47pub struct Cli {
48 #[command(subcommand)]
49 command: Option<Command>,
50}
51
52#[derive(Subcommand, Debug)]
53enum Command {
54 Install {
59 #[arg(default_value = "AGENTS.md")]
61 path: PathBuf,
62 },
63 Unit {
65 #[command(subcommand)]
66 rule: UnitRule,
67 },
68 Integration {
70 #[command(subcommand)]
71 rule: IntegrationRule,
72 },
73 Packaging {
75 path: PathBuf,
78 #[arg(long, value_enum)]
81 language: Option<colocated_test::Language>,
82 },
83 #[command(hide = true)]
88 Workflow {
89 path: PathBuf,
91 },
92 E2e {
94 #[command(subcommand)]
95 command: E2eCommand,
96 },
97 #[command(args_conflicts_with_subcommands = true)]
101 WorkflowLint {
102 #[command(subcommand)]
103 command: Option<WorkflowLintCommand>,
104 #[arg(default_value = ".github")]
106 path: PathBuf,
107 },
108 Changelog {
111 #[arg(long)]
113 base: String,
114 #[arg(default_value = ".")]
116 path: PathBuf,
117 },
118}
119
120#[derive(Subcommand, Debug)]
121enum UnitRule {
122 ColocatedTest {
128 path: PathBuf,
130 #[arg(long, value_enum)]
132 language: colocated_test::Language,
133 #[arg(long)]
139 base: Option<String>,
140 #[arg(long, default_value = "testing-conventions.toml")]
143 config: PathBuf,
144 },
145 Coverage {
150 path: PathBuf,
152 #[arg(long, value_enum)]
154 language: colocated_test::Language,
155 #[arg(long)]
161 base: Option<String>,
162 #[arg(long, default_value = "testing-conventions.toml")]
167 config: PathBuf,
168 },
169 OneFunctionPerFile {
173 path: PathBuf,
175 #[arg(long, value_enum)]
177 language: colocated_test::Language,
178 #[arg(long, default_value = "testing-conventions.toml")]
183 config: PathBuf,
184 },
185 Lint {
187 path: PathBuf,
189 #[arg(long, value_enum)]
191 language: isolation::Language,
192 #[arg(long, default_value = "testing-conventions.toml")]
195 config: PathBuf,
196 },
197 Mutation {
203 path: PathBuf,
205 #[arg(long, value_enum)]
207 language: colocated_test::Language,
208 #[arg(long)]
212 base: Option<String>,
213 #[arg(long, default_value = "testing-conventions.toml")]
216 config: PathBuf,
217 #[arg(long = "ts-mutation-adapter", hide = true)]
221 ts_adapter: Option<PathBuf>,
222 },
223}
224
225#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
227pub enum IntegrationLintLanguage {
228 #[value(name = "python")]
230 Python,
231 #[value(name = "typescript")]
233 TypeScript,
234 #[value(name = "rust")]
236 Rust,
237}
238
239#[derive(Subcommand, Debug)]
240enum IntegrationRule {
241 Lint {
243 path: PathBuf,
245 #[arg(long, value_enum)]
247 language: IntegrationLintLanguage,
248 #[arg(long, default_value = "testing-conventions.toml")]
251 config: PathBuf,
252 },
253}
254
255#[derive(Subcommand, Debug)]
256enum WorkflowLintCommand {
257 Scope {
260 #[arg(long)]
262 base: Option<String>,
263 #[arg(default_value = ".github")]
265 path: PathBuf,
266 },
267}
268
269#[derive(Subcommand, Debug)]
270enum E2eCommand {
271 Attest {
275 command: String,
277 },
278 Verify {
280 #[arg(default_value = ".")]
283 path: PathBuf,
284 #[arg(long)]
287 scope: Option<PathBuf>,
288 #[arg(long)]
295 base: Option<String>,
296 #[arg(long = "extra-scope")]
301 extra_scope: Vec<PathBuf>,
302 #[arg(long = "exclude")]
306 exclude: Vec<PathBuf>,
307 #[arg(long)]
310 branch: Option<String>,
311 },
312 Slug {
315 branch: Option<String>,
317 },
318}
319
320pub fn run<I, T>(args: I) -> anyhow::Result<i32>
321where
322 I: IntoIterator<Item = T>,
323 T: Into<std::ffi::OsString> + Clone,
324{
325 eprintln!("testing-conventions {}", env!("CARGO_PKG_VERSION"));
328 let cli = Cli::try_parse_from(args)?;
329 match cli.command {
330 None => Ok(0),
331 Some(Command::Unit { rule }) => match rule {
332 UnitRule::ColocatedTest {
333 path,
334 language,
335 base,
336 config,
337 } => run_unit_colocated_test(&path, language, base.as_deref(), &config),
338 UnitRule::Coverage {
339 path,
340 language,
341 base,
342 config,
343 } => unit_coverage::run(&path, language, base.as_deref(), &config),
344 UnitRule::OneFunctionPerFile {
345 path,
346 language,
347 config,
348 } => unit_one_function::run(&path, language, &config),
349 UnitRule::Lint {
350 path,
351 language,
352 config,
353 } => unit_lint::run(&path, language, &config),
354 UnitRule::Mutation {
355 path,
356 language,
357 base,
358 config,
359 ts_adapter,
360 } => unit_mutation::run(
361 &path,
362 language,
363 base.as_deref(),
364 &config,
365 ts_adapter.as_deref(),
366 ),
367 },
368 Some(Command::Integration { rule }) => match rule {
369 IntegrationRule::Lint {
370 path,
371 language,
372 config,
373 } => integration_lint::run(&path, language, &config),
374 },
375 Some(Command::Packaging { path, language }) => run_packaging(&path, language),
376 Some(Command::Changelog { base, path }) => changelog::run(&path, &base),
377 Some(Command::Workflow { path }) => run_workflow(&path),
378 Some(Command::WorkflowLint { command, path }) => match command {
379 None => run_workflow_lint(&path),
380 Some(WorkflowLintCommand::Scope { base, path }) => {
381 run_workflow_lint_scope(&path, base.as_deref())
382 }
383 },
384 Some(Command::E2e { command }) => match command {
385 E2eCommand::Attest { command } => run_e2e_attest(&command),
386 E2eCommand::Verify {
387 path,
388 scope,
389 base,
390 extra_scope,
391 exclude,
392 branch,
393 } => run_e2e_verify(
394 &path,
395 scope.as_deref(),
396 base.as_deref(),
397 &extra_scope,
398 &exclude,
399 branch.as_deref(),
400 ),
401 E2eCommand::Slug { branch } => run_e2e_slug(branch.as_deref()),
402 },
403 Some(Command::Install { path }) => {
404 agents::install(&path)?;
405 Ok(0)
406 }
407 }
408}
409
410pub fn command() -> clap::Command {
412 Cli::command()
413}
414
415fn run_unit_colocated_test(
418 root: &Path,
419 language: colocated_test::Language,
420 base: Option<&str>,
421 config_path: &Path,
422) -> anyhow::Result<i32> {
423 if base.is_some() && language == colocated_test::Language::Rust {
424 anyhow::bail!(
425 "`unit colocated-test --base` supports `--language python` / `typescript`; Rust \
426 units are inline `#[cfg(test)]` in the same file, so a sibling test can't go stale"
427 );
428 }
429 let presence_clean = report_colocated_presence(root, language, config_path)?;
430 let co_change_clean = match base {
431 Some(base) => report_co_change(root, base, language, config_path)?,
432 None => true,
433 };
434 Ok(if presence_clean && co_change_clean {
435 0
436 } else {
437 1
438 })
439}
440
441fn report_colocated_presence(
444 root: &Path,
445 language: colocated_test::Language,
446 config_path: &Path,
447) -> anyhow::Result<bool> {
448 let exempt = colocated_test_exemptions(root, language, config_path)?;
449 let orphans = match language {
450 colocated_test::Language::Rust => colocated_test::missing_inline_tests(root, &exempt)?,
451 _ => colocated_test::missing_unit_tests(root, language, &exempt)?,
452 };
453 if orphans.is_empty() {
454 return Ok(true);
455 }
456 let (label, summary) = orphan_wording(language);
457 for orphan in &orphans {
458 eprintln!("{label}: {}", orphan.display());
459 }
460 eprintln!("error: {} {summary}", orphans.len());
461 Ok(false)
462}
463
464fn orphan_wording(language: colocated_test::Language) -> (&'static str, &'static str) {
468 match language {
469 colocated_test::Language::Rust => (
470 "missing inline `#[cfg(test)]` tests",
471 "source file(s) with testable code but no inline `#[cfg(test)]` module \
472 (add an inline test module, or an `exempt` entry with a reason)",
473 ),
474 _ => (
475 "missing colocated unit test",
476 "source file(s) missing a colocated unit test \
477 (add a colocated test, or an `exempt` entry with a reason)",
478 ),
479 }
480}
481
482fn colocated_test_exemptions(
484 root: &Path,
485 language: colocated_test::Language,
486 config_path: &Path,
487) -> anyhow::Result<std::collections::BTreeSet<String>> {
488 if !config_path.exists() {
489 return Ok(std::collections::BTreeSet::new());
490 }
491 let config = config::load_config(config_path)?;
492 config::resolve_exempt(
493 root,
494 config.exemptions(language),
495 config::Rule::ColocatedTest,
496 )
497}
498
499fn report_co_change(
502 root: &Path,
503 base: &str,
504 language: colocated_test::Language,
505 config_path: &Path,
506) -> anyhow::Result<bool> {
507 let exempt = co_change_exemptions(root, language, config_path)?;
508 let stale = co_change::stale_sources(root, base, language, &exempt)?;
509 if stale.is_empty() {
510 return Ok(true);
511 }
512 for source in &stale {
513 eprintln!(
514 "source changed without its colocated test: {}",
515 source.display()
516 );
517 }
518 eprintln!(
519 "error: {} source file(s) changed without their colocated test co-changing \
520 (update the test, or add an `exempt` entry with a reason)",
521 stale.len()
522 );
523 Ok(false)
524}
525
526fn co_change_exemptions(
528 root: &Path,
529 language: colocated_test::Language,
530 config_path: &Path,
531) -> anyhow::Result<std::collections::BTreeSet<String>> {
532 if !config_path.exists() {
533 return Ok(std::collections::BTreeSet::new());
534 }
535 let config = config::load_config(config_path)?;
536 config::resolve_exempt(root, config.exemptions(language), config::Rule::CoChange)
537}
538
539fn split_scopes(
541 scopes: std::collections::BTreeMap<String, config::LineScope>,
542) -> (
543 Vec<String>,
544 std::collections::BTreeMap<String, std::collections::BTreeSet<u32>>,
545) {
546 let mut whole_file = Vec::new();
547 let mut line_scoped = std::collections::BTreeMap::new();
548 for (path, scope) in scopes {
549 match scope {
550 config::LineScope::WholeFile => whole_file.push(path),
551 config::LineScope::Lines(lines) => {
552 line_scoped.insert(path, lines);
553 }
554 }
555 }
556 (whole_file, line_scoped)
557}
558
559fn run_packaging(path: &Path, language: Option<colocated_test::Language>) -> anyhow::Result<i32> {
563 let distributions = distributions_at(path, language)?;
564 let mut shipped = 0;
565 for distribution in &distributions {
566 shipped += report_shipped_test_files(distribution)?;
567 }
568 if shipped > 0 {
569 eprintln!(
570 "error: {shipped} test file(s) present in the built distribution(s) \
571 (they must be excluded from packaging)"
572 );
573 return Ok(1);
574 }
575 println!(
576 "checked {} built distribution(s); no test files shipped",
577 distributions.len()
578 );
579 Ok(0)
580}
581
582fn distributions_at(
586 path: &Path,
587 language: Option<colocated_test::Language>,
588) -> anyhow::Result<Vec<packaging::Distribution>> {
589 let distributions = match language {
590 Some(language) => vec![packaging::Distribution {
591 path: path.to_path_buf(),
592 language,
593 }],
594 None => packaging::discover(path)?,
595 };
596 if distributions.is_empty() {
597 anyhow::bail!(
598 "no recognized built distribution (`.whl`, `.tar.gz`, `.tgz`, `.crate`) at `{}`",
599 path.display()
600 );
601 }
602 Ok(distributions)
603}
604
605fn report_shipped_test_files(distribution: &packaging::Distribution) -> anyhow::Result<usize> {
607 let globs = match distribution.language {
608 colocated_test::Language::Python => vec!["*_test.py".to_string()],
609 colocated_test::Language::TypeScript => vec!["*.test.*".to_string()],
610 colocated_test::Language::Rust => vec!["tests/".to_string()],
612 };
613 let offenders = packaging::inspect(&distribution.path, &globs)?;
614 for offender in &offenders {
615 eprintln!(
616 "test file in built artifact `{}`: {}",
617 distribution.path.display(),
618 offender.display()
619 );
620 }
621 Ok(offenders.len())
622}
623
624fn run_workflow(path: &Path) -> anyhow::Result<i32> {
627 let violations = workflow::check(path, &command())?;
628 if violations.is_empty() {
629 return Ok(0);
630 }
631 for v in &violations {
632 eprintln!(
633 "{}:{}: {} — {}",
634 v.file.display(),
635 v.line,
636 v.rule,
637 v.message
638 );
639 }
640 eprintln!(
641 "error: {} workflow invocation(s) name a subcommand this binary no longer exposes",
642 violations.len()
643 );
644 Ok(1)
645}
646
647fn run_workflow_lint_scope(path: &Path, base: Option<&str>) -> anyhow::Result<i32> {
648 let in_scope = workflow_lint::in_scope(Path::new("."), path, base).unwrap_or_else(|err| {
649 eprintln!("workflow-lint scope: {err:#}; scoping the whole tree in");
650 true
651 });
652 println!("{in_scope}");
653 Ok(0)
654}
655
656fn run_workflow_lint(path: &Path) -> anyhow::Result<i32> {
657 let findings = workflow_lint::scan(path)?;
658 if findings.is_empty() {
659 return Ok(0);
660 }
661 for f in &findings {
662 eprintln!("{}", inline_logic_finding(f));
663 }
664 eprintln!(
665 "error: {} step(s) encode logic in CI YAML, where nothing tests it",
666 findings.len()
667 );
668 Ok(1)
669}
670
671fn inline_logic_finding(finding: &workflow_lint::Finding) -> String {
673 format!(
674 "{}:{}: {} step `{}` encodes logic inline ({}) — move it into a tested package in \
675 this repository's own language, invoked as a one-line `run:`",
676 finding.file.display(),
677 finding.line,
678 finding.kind,
679 finding.step,
680 finding.reasons.join("; ")
681 )
682}
683
684fn run_e2e_attest(command: &str) -> anyhow::Result<i32> {
687 let repo = std::env::current_dir()?;
688 let attestation = e2e::attest(&repo, command)?;
689 if attestation.exit_code != 0 {
690 eprintln!(
691 "e2e command `{command}` exited {}; a receipt records a run that passed — \
692 fix the failure and attest again",
693 attestation.exit_code
694 );
695 return Ok(attestation.exit_code);
696 }
697 println!(
698 "e2e receipt recorded for branch {} at {}/{}.json",
699 attestation.branch,
700 e2e::RECEIPTS_DIR,
701 e2e::branch_slug(&attestation.branch),
702 );
703 Ok(0)
704}
705
706fn run_e2e_verify(
710 path: &Path,
711 scope: Option<&Path>,
712 base: Option<&str>,
713 extra_scopes: &[PathBuf],
714 excludes: &[PathBuf],
715 branch: Option<&str>,
716) -> anyhow::Result<i32> {
717 match e2e::verify_extra_scoped(
718 path,
719 scope.unwrap_or(path),
720 base,
721 extra_scopes,
722 excludes,
723 branch,
724 )? {
725 e2e::Verification::Fresh => Ok(0),
726 e2e::Verification::Missing => {
727 eprintln!(
728 "no e2e receipt answers this change — run \
729 `testing-conventions e2e attest '<your e2e command>'`; the command is \
730 your judgment: the full suite, a targeted subset, or a no-op"
731 );
732 Ok(1)
733 }
734 }
735}
736
737fn run_e2e_slug(branch: Option<&str>) -> anyhow::Result<i32> {
739 let slug = match branch {
740 Some(name) => e2e::branch_slug(name),
741 None => {
742 let repo = std::env::current_dir()?;
743 e2e::branch_slug(&e2e::current_branch(&repo)?)
744 }
745 };
746 println!("{slug}");
747 Ok(0)
748}
749
750#[cfg(test)]
751mod tests {
752 use std::sync::atomic::{AtomicU64, Ordering};
753
754 use super::*;
755
756 fn scratch(slug: &str, files: &[(&str, &str)]) -> PathBuf {
758 static COUNTER: AtomicU64 = AtomicU64::new(0);
759 let root = std::env::temp_dir().join(format!(
760 "tc-lib-{}-{}-{}",
761 slug,
762 std::process::id(),
763 COUNTER.fetch_add(1, Ordering::Relaxed),
764 ));
765 let _ = std::fs::remove_dir_all(&root);
766 std::fs::create_dir_all(&root).unwrap();
767 for (rel, contents) in files {
768 let full = root.join(rel);
769 std::fs::create_dir_all(full.parent().unwrap()).unwrap();
770 std::fs::write(full, contents).unwrap();
771 }
772 root
773 }
774
775 #[test]
776 fn a_workflow_encoding_logic_fails_the_lint_and_a_wiring_one_passes() {
777 let workflow =
778 |body: &str| format!("on: push\njobs:\n j:\n steps:\n - run: |\n{body}");
779 let logic = scratch(
780 "workflow-lint-logic",
781 &[(
782 "ci.yml",
783 &workflow(" for f in *.txt; do\n echo $f\n done\n"),
784 )],
785 );
786 assert_eq!(run_workflow_lint(&logic.join("ci.yml")).unwrap(), 1);
787
788 let wiring = scratch(
789 "workflow-lint-wiring",
790 &[("ci.yml", &workflow(" echo hello\n"))],
791 );
792 assert_eq!(run_workflow_lint(&wiring.join("ci.yml")).unwrap(), 0);
793 }
794
795 #[test]
796 fn an_inline_logic_finding_names_the_step_its_location_and_every_reason() {
797 let rendered = inline_logic_finding(&workflow_lint::Finding {
798 file: PathBuf::from(".github/workflows/ci.yml"),
799 line: 42,
800 step: "Publish".to_string(),
801 kind: "run",
802 reasons: vec!["for loop".to_string(), "sed".to_string()],
803 });
804 assert!(
805 rendered.starts_with(".github/workflows/ci.yml:42: run step `Publish` encodes logic inline (for loop; sed)"),
806 "got: {rendered}"
807 );
808 assert!(rendered.contains("one-line `run:`"), "got: {rendered}");
809 }
810
811 #[test]
812 fn a_directory_holding_no_distribution_is_an_error_rather_than_a_clean_run() {
813 let root = scratch("packaging-empty", &[("README.md", "hi\n")]);
815 assert!(run_packaging(&root, None).is_err());
816 }
817
818 #[test]
819 fn a_rust_file_with_no_inline_tests_makes_the_presence_report_unclean() {
820 let root = scratch(
821 "presence-orphan",
822 &[("src/widget.rs", "pub fn f() -> u8 { 1 }\n")],
823 );
824 let config = root.join("testing-conventions.toml");
825 assert!(
826 !report_colocated_presence(&root, colocated_test::Language::Rust, &config).unwrap()
827 );
828 }
829
830 #[test]
831 fn a_rust_file_carrying_inline_tests_makes_the_presence_report_clean() {
832 let root = scratch(
833 "presence-clean",
834 &[(
835 "src/widget.rs",
836 "pub fn f() -> u8 { 1 }\n#[cfg(test)]\nmod tests { #[test] fn t() {} }\n",
837 )],
838 );
839 let config = root.join("testing-conventions.toml");
840 assert!(report_colocated_presence(&root, colocated_test::Language::Rust, &config).unwrap());
841 }
842
843 #[test]
844 fn rust_is_told_about_inline_test_modules_and_the_others_about_sibling_files() {
845 let (label, summary) = orphan_wording(colocated_test::Language::Rust);
846 assert!(label.contains("inline"), "got: {label}");
847 assert!(summary.contains("inline test module"), "got: {summary}");
848
849 let (label, summary) = orphan_wording(colocated_test::Language::Python);
850 assert!(label.contains("colocated unit test"), "got: {label}");
851 assert!(summary.contains("add a colocated test"), "got: {summary}");
852 }
853
854 #[test]
855 fn a_named_language_makes_the_path_itself_the_distribution() {
856 let found = distributions_at(
857 Path::new("/tmp/pkg.whl"),
858 Some(colocated_test::Language::Python),
859 )
860 .unwrap();
861 assert_eq!(found.len(), 1);
862 assert_eq!(found[0].path, Path::new("/tmp/pkg.whl"));
863 }
864
865 #[test]
866 fn a_search_that_finds_nothing_is_an_error_not_a_clean_run() {
867 let dir = std::env::temp_dir().join(format!(
868 "testing-conventions-no-distributions-{}",
869 std::process::id()
870 ));
871 std::fs::create_dir_all(&dir).unwrap();
872 let err = distributions_at(&dir, None).unwrap_err();
873 assert!(
874 err.to_string().contains("no recognized built distribution"),
875 "got: {err}"
876 );
877 }
878
879 #[test]
880 fn no_args_returns_ok_zero() {
881 assert_eq!(run(["testing-conventions"]).unwrap(), 0);
882 }
883
884 #[test]
885 fn unknown_flag_errors() {
886 assert!(run(["testing-conventions", "--bogus"]).is_err());
887 }
888
889 #[test]
890 fn split_scopes_separates_whole_file_paths_from_line_sets() {
891 let mut scopes = std::collections::BTreeMap::new();
892 scopes.insert("shim.py".to_string(), config::LineScope::WholeFile);
893 scopes.insert(
894 "widget.py".to_string(),
895 config::LineScope::Lines(std::collections::BTreeSet::from([3])),
896 );
897 let (whole_file, line_scoped) = split_scopes(scopes);
898 assert_eq!(whole_file, vec!["shim.py".to_string()]);
899 assert_eq!(line_scoped.len(), 1);
900 assert_eq!(
901 line_scoped["widget.py"],
902 std::collections::BTreeSet::from([3])
903 );
904 }
905
906 #[test]
907 fn help_flag_returns_clap_display_help() {
908 let err = run(["testing-conventions", "--help"]).expect_err("--help should bubble");
909 let clap_err = err
910 .downcast_ref::<clap::Error>()
911 .expect("error should be a clap::Error");
912 assert_eq!(clap_err.kind(), clap::error::ErrorKind::DisplayHelp);
913 }
914
915 #[test]
916 fn version_flag_returns_clap_display_version() {
917 let err = run(["testing-conventions", "--version"]).expect_err("--version should bubble");
918 let clap_err = err
919 .downcast_ref::<clap::Error>()
920 .expect("error should be a clap::Error");
921 assert_eq!(clap_err.kind(), clap::error::ErrorKind::DisplayVersion);
922 }
923}