pub mod co_change;
pub mod colocated_test;
pub mod config;
pub mod coverage;
pub mod e2e;
pub mod isolation;
pub mod lint;
pub mod packaging;
pub mod patch_coverage;
pub mod ts;
pub mod violation;
pub mod workflow;
use std::path::{Path, PathBuf};
use clap::{CommandFactory, Parser, Subcommand};
#[derive(Parser, Debug)]
#[command(
name = "testing-conventions",
version,
about = "Enforce testing conventions in libraries (Python, TypeScript, and Rust).",
long_about = None,
)]
pub struct Cli {
#[command(subcommand)]
command: Option<Command>,
}
#[derive(Subcommand, Debug)]
enum Command {
Check,
Unit {
#[command(subcommand)]
rule: UnitRule,
},
Integration {
#[command(subcommand)]
rule: IntegrationRule,
},
Packaging {
path: PathBuf,
#[arg(long, value_enum)]
language: colocated_test::Language,
},
Workflow {
path: PathBuf,
},
E2e {
#[command(subcommand)]
command: E2eCommand,
},
}
#[derive(Subcommand, Debug)]
enum UnitRule {
ColocatedTest {
path: PathBuf,
#[arg(long, value_enum)]
language: colocated_test::Language,
#[arg(long)]
base: Option<String>,
#[arg(long, default_value = "testing-conventions.toml")]
config: PathBuf,
},
Coverage {
path: PathBuf,
#[arg(long, value_enum)]
language: colocated_test::Language,
#[arg(long)]
base: Option<String>,
#[arg(long, default_value = "testing-conventions.toml")]
config: PathBuf,
},
Lint {
path: PathBuf,
#[arg(long, value_enum)]
language: isolation::Language,
#[arg(long, default_value = "testing-conventions.toml")]
config: PathBuf,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
pub enum IntegrationLintLanguage {
#[value(name = "python")]
Python,
#[value(name = "typescript")]
TypeScript,
#[value(name = "rust")]
Rust,
}
#[derive(Subcommand, Debug)]
enum IntegrationRule {
Lint {
path: PathBuf,
#[arg(long, value_enum)]
language: IntegrationLintLanguage,
#[arg(long, default_value = "testing-conventions.toml")]
config: PathBuf,
},
}
#[derive(Subcommand, Debug)]
enum E2eCommand {
Attest {
command: String,
},
Verify,
}
pub fn run<I, T>(args: I) -> anyhow::Result<i32>
where
I: IntoIterator<Item = T>,
T: Into<std::ffi::OsString> + Clone,
{
let cli = Cli::try_parse_from(args)?;
match cli.command {
Some(Command::Check) | None => Ok(0),
Some(Command::Unit { rule }) => match rule {
UnitRule::ColocatedTest {
path,
language,
base,
config,
} => run_unit_colocated_test(&path, language, base.as_deref(), &config),
UnitRule::Coverage {
path,
language,
base,
config,
} => run_unit_coverage(&path, language, base.as_deref(), &config),
UnitRule::Lint {
path,
language,
config,
} => run_unit_lint(&path, language, &config),
},
Some(Command::Integration { rule }) => match rule {
IntegrationRule::Lint {
path,
language,
config,
} => run_integration_lint(&path, language, &config),
},
Some(Command::Packaging { path, language }) => run_packaging(&path, language),
Some(Command::Workflow { path }) => run_workflow(&path),
Some(Command::E2e { command }) => match command {
E2eCommand::Attest { command } => run_e2e_attest(&command),
E2eCommand::Verify => run_e2e_verify(),
},
}
}
pub fn command() -> clap::Command {
Cli::command()
}
fn run_unit_colocated_test(
root: &Path,
language: colocated_test::Language,
base: Option<&str>,
config_path: &Path,
) -> anyhow::Result<i32> {
if base.is_some() && language == colocated_test::Language::Rust {
anyhow::bail!(
"`unit colocated-test --base` supports `--language python` / `typescript`; Rust \
units are inline `#[cfg(test)]` in the same file, so a sibling test can't go stale"
);
}
let presence_clean = report_colocated_presence(root, language, config_path)?;
let co_change_clean = match base {
Some(base) => report_co_change(root, base, language, config_path)?,
None => true,
};
Ok(if presence_clean && co_change_clean {
0
} else {
1
})
}
fn report_colocated_presence(
root: &Path,
language: colocated_test::Language,
config_path: &Path,
) -> anyhow::Result<bool> {
let exempt = colocated_test_exemptions(root, language, config_path)?;
let orphans = match language {
colocated_test::Language::Rust => colocated_test::missing_inline_tests(root, &exempt)?,
_ => colocated_test::missing_unit_tests(root, language, &exempt)?,
};
if orphans.is_empty() {
return Ok(true);
}
let (label, summary) = match language {
colocated_test::Language::Rust => (
"missing inline `#[cfg(test)]` tests",
"source file(s) with testable code but no inline `#[cfg(test)]` module \
(add an inline test module, or an `exempt` entry with a reason)",
),
_ => (
"missing colocated unit test",
"source file(s) missing a colocated unit test \
(add a colocated test, or an `exempt` entry with a reason)",
),
};
for orphan in &orphans {
eprintln!("{label}: {}", orphan.display());
}
eprintln!("error: {} {summary}", orphans.len());
Ok(false)
}
fn colocated_test_exemptions(
root: &Path,
language: colocated_test::Language,
config_path: &Path,
) -> anyhow::Result<std::collections::BTreeSet<String>> {
if !config_path.exists() {
return Ok(std::collections::BTreeSet::new());
}
let config = config::load_config(config_path)?;
config::resolve_exempt(
root,
config.exemptions(language),
config::Rule::ColocatedTest,
)
}
fn report_co_change(
root: &Path,
base: &str,
language: colocated_test::Language,
config_path: &Path,
) -> anyhow::Result<bool> {
let exempt = co_change_exemptions(root, language, config_path)?;
let stale = co_change::stale_sources(root, base, language, &exempt)?;
if stale.is_empty() {
return Ok(true);
}
for source in &stale {
eprintln!(
"source changed without its colocated test: {}",
source.display()
);
}
eprintln!(
"error: {} source file(s) changed without their colocated test co-changing \
(update the test, or add an `exempt` entry with a reason)",
stale.len()
);
Ok(false)
}
fn co_change_exemptions(
root: &Path,
language: colocated_test::Language,
config_path: &Path,
) -> anyhow::Result<std::collections::BTreeSet<String>> {
if !config_path.exists() {
return Ok(std::collections::BTreeSet::new());
}
let config = config::load_config(config_path)?;
config::resolve_exempt(root, config.exemptions(language), config::Rule::CoChange)
}
fn run_unit_coverage(
root: &Path,
language: colocated_test::Language,
base: Option<&str>,
config_path: &Path,
) -> anyhow::Result<i32> {
let config = if config_path.exists() {
config::load_config(config_path)?
} else {
config::Config::default()
};
let outcome = match language {
colocated_test::Language::Python => {
let python = config.python.unwrap_or_default();
let coverage = python.coverage.unwrap_or_default();
let thresholds = coverage::Thresholds {
fail_under: coverage.fail_under,
branch: coverage.branch,
};
let omit: Vec<String> =
config::resolve_exempt(root, &python.exempt, config::Rule::Coverage)?
.into_iter()
.collect();
match base {
Some(base) => patch_coverage::measure(root, base, thresholds, &omit)?,
None => coverage::measure(root, thresholds, &omit)?,
}
}
colocated_test::Language::TypeScript => {
let typescript = config.typescript.unwrap_or_default();
let coverage = typescript.coverage.unwrap_or_default();
let thresholds = coverage::TypeScriptThresholds {
lines: coverage.lines,
branches: coverage.branches,
functions: coverage.functions,
statements: coverage.statements,
};
let exclude: Vec<String> =
config::resolve_exempt(root, &typescript.exempt, config::Rule::Coverage)?
.into_iter()
.collect();
match base {
Some(base) => patch_coverage::measure_typescript(root, base, thresholds, &exclude)?,
None => coverage::measure_typescript(root, thresholds, &exclude)?,
}
}
colocated_test::Language::Rust => {
let rust = config.rust.unwrap_or_default();
let coverage = rust.coverage.ok_or_else(|| {
anyhow::anyhow!(
"Rust coverage needs a `[rust].coverage` table (regions / lines) in `{}` — \
there is no zero-config default floor for Rust yet",
config_path.display()
)
})?;
let thresholds = coverage::RustThresholds {
regions: coverage.regions,
lines: coverage.lines,
};
let ignore: Vec<String> =
config::resolve_exempt(root, &rust.exempt, config::Rule::Coverage)?
.into_iter()
.collect();
match base {
Some(base) => patch_coverage::measure_rust(root, base, thresholds, &ignore)?,
None => coverage::measure_rust(root, thresholds, &ignore)?,
}
}
};
match outcome {
coverage::Outcome::Pass => Ok(0),
coverage::Outcome::Fail(reason) => {
eprintln!("error: coverage check failed — {reason}");
Ok(1)
}
}
}
fn run_unit_lint(
root: &Path,
language: isolation::Language,
config_path: &Path,
) -> anyhow::Result<i32> {
let (raw, select): (Vec<lint::Violation>, ExemptSelect) = match language {
isolation::Language::Rust => (isolation::find_violations(root)?, |c| c.rust_exemptions()),
isolation::Language::TypeScript => (ts::find_unit_violations(root)?, |c| {
c.exemptions(colocated_test::Language::TypeScript)
}),
isolation::Language::Python => (lint::find_unit_isolation_violations(root)?, |c| {
c.exemptions(colocated_test::Language::Python)
}),
};
let violations = apply_waivers(raw, root, config_path, select)?;
if violations.is_empty() {
return Ok(0);
}
for v in &violations {
eprintln!(
"{}:{}: {} — {}",
v.file.display(),
v.line,
v.rule,
v.message
);
}
eprintln!("error: {} isolation violation(s)", violations.len());
Ok(1)
}
fn run_integration_lint(
root: &Path,
language: IntegrationLintLanguage,
config_path: &Path,
) -> anyhow::Result<i32> {
let (raw, select): (Vec<lint::Violation>, ExemptSelect) = match language {
IntegrationLintLanguage::Python => (lint::find_violations(root)?, |c| {
c.exemptions(colocated_test::Language::Python)
}),
IntegrationLintLanguage::TypeScript => (ts::find_integration_violations(root)?, |c| {
c.exemptions(colocated_test::Language::TypeScript)
}),
IntegrationLintLanguage::Rust => (isolation::find_integration_violations(root)?, |c| {
c.rust_exemptions()
}),
};
let violations = apply_waivers(raw, root, config_path, select)?;
if violations.is_empty() {
return Ok(0);
}
for v in &violations {
eprintln!(
"{}:{}: {} — {}",
v.file.display(),
v.line,
v.rule,
v.message
);
}
eprintln!("error: {} lint violation(s)", violations.len());
Ok(1)
}
type ExemptSelect = fn(&config::Config) -> &[config::Exemption];
fn apply_waivers(
violations: Vec<lint::Violation>,
root: &Path,
config_path: &Path,
exemptions: ExemptSelect,
) -> anyhow::Result<Vec<lint::Violation>> {
use std::collections::hash_map::Entry;
if !config_path.exists() {
return Ok(violations);
}
let config = config::load_config(config_path)?;
let exempt = exemptions(&config);
let mut resolved: std::collections::HashMap<config::Rule, std::collections::BTreeSet<String>> =
std::collections::HashMap::new();
let mut kept = Vec::new();
for violation in violations {
let waived = match config::Rule::from_id(violation.rule) {
Some(rule) => {
let exempt_paths = match resolved.entry(rule) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
entry.insert(config::resolve_exempt(root, exempt, rule)?)
}
};
violation
.file
.strip_prefix(root)
.ok()
.map(|rel| rel.to_string_lossy().replace('\\', "/"))
.is_some_and(|rel| exempt_paths.contains(&rel))
}
None => false,
};
if !waived {
kept.push(violation);
}
}
Ok(kept)
}
fn run_packaging(artifact: &Path, language: colocated_test::Language) -> anyhow::Result<i32> {
let globs = match language {
colocated_test::Language::Python => vec!["*_test.py".to_string()],
colocated_test::Language::TypeScript => vec!["*.test.*".to_string()],
colocated_test::Language::Rust => vec!["tests/".to_string()],
};
let offenders = packaging::inspect(artifact, &globs)?;
if offenders.is_empty() {
return Ok(0);
}
for offender in &offenders {
eprintln!("test file in built artifact: {}", offender.display());
}
eprintln!(
"error: {} test file(s) present in the built artifact \
(they must be excluded from packaging)",
offenders.len()
);
Ok(1)
}
fn run_workflow(path: &Path) -> anyhow::Result<i32> {
let violations = workflow::check(path, &command())?;
if violations.is_empty() {
return Ok(0);
}
for v in &violations {
eprintln!(
"{}:{}: {} — {}",
v.file.display(),
v.line,
v.rule,
v.message
);
}
eprintln!(
"error: {} workflow invocation(s) name a subcommand this binary no longer exposes",
violations.len()
);
Ok(1)
}
fn run_e2e_attest(command: &str) -> anyhow::Result<i32> {
let repo = std::env::current_dir()?;
let attestation = e2e::attest(&repo, command)?;
println!(
"e2e attestation recorded for commit {} (command exited {})",
attestation.commit, attestation.exit_code
);
Ok(0)
}
fn run_e2e_verify() -> anyhow::Result<i32> {
let repo = std::env::current_dir()?;
match e2e::verify(&repo)? {
e2e::Verification::Fresh => Ok(0),
e2e::Verification::Missing => {
eprintln!(
"e2e attestation missing — run `testing-conventions e2e attest '<your e2e command>'`"
);
Ok(1)
}
e2e::Verification::Stale { attested, latest } => {
eprintln!(
"e2e attestation out of date: attested {}, latest code commit {} — \
run `testing-conventions e2e attest '<your e2e command>'`",
&attested[..attested.len().min(7)],
&latest[..latest.len().min(7)]
);
Ok(1)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn no_args_returns_ok_zero() {
assert_eq!(run(["testing-conventions"]).unwrap(), 0);
}
#[test]
fn check_returns_ok_zero() {
assert_eq!(run(["testing-conventions", "check"]).unwrap(), 0);
}
#[test]
fn unknown_flag_errors() {
assert!(run(["testing-conventions", "--bogus"]).is_err());
}
#[test]
fn help_flag_returns_clap_display_help() {
let err = run(["testing-conventions", "--help"]).expect_err("--help should bubble");
let clap_err = err
.downcast_ref::<clap::Error>()
.expect("error should be a clap::Error");
assert_eq!(clap_err.kind(), clap::error::ErrorKind::DisplayHelp);
}
#[test]
fn version_flag_returns_clap_display_version() {
let err = run(["testing-conventions", "--version"]).expect_err("--version should bubble");
let clap_err = err
.downcast_ref::<clap::Error>()
.expect("error should be a clap::Error");
assert_eq!(clap_err.kind(), clap::error::ErrorKind::DisplayVersion);
}
#[test]
fn unit_coverage_rust_requires_a_coverage_table() {
let err = run([
"testing-conventions",
"unit",
"coverage",
"pkg",
"--language",
"rust",
])
.unwrap_err();
assert!(err.to_string().contains("[rust].coverage"), "got: {err}");
}
}