testing-conventions 0.0.94

//! E2E tests for the Rust mutation rule: drive the built CLI binary
//! end-to-end (no mocks) against the fixture crates and assert the exit code.
//! Requires only a cargo toolchain — the tool provisions cargo-mutants itself.
//!
//! The gate is **on by default and binary**: an un-exempted surviving mutant fails the
//! run, and the only way to pass with a survivor present is a reason-required
//! `mutation` exemption. The fixtures are the standard pair: `killed` (every mutant
//! caught) and `survivors` (a coverage-passing but assertion-light suite whose mutants
//! all survive).

mod common;

use std::path::PathBuf;
use std::process::Command;

use common::{tested_count, GitRepo, ENGINE_NOT_RUN, NOTHING_TESTED};

fn fixtures() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/unit_mutation")
}

/// Exit code of `testing-conventions unit mutation --language rust [--config <cfg>] <crate>`.
fn unit_mutation_exit(crate_name: &str, config: Option<&str>) -> i32 {
    let mut command = Command::new(env!("CARGO_BIN_EXE_testing-conventions"));
    command.args(["unit", "mutation", "--language", "rust"]);
    if let Some(config) = config {
        command.arg("--config").arg(fixtures().join(config));
    }
    command
        .arg(fixtures().join("rust").join(crate_name))
        .status()
        .expect("the built binary should run")
        .code()
        .expect("the process should exit with a code")
}

#[test]
fn killed_crate_passes_and_states_the_tested_count() {
    // Every mutant is caught, so the crate clears the gate — and the success line states
    // how many mutants the engine judged, the evidence telling this pass apart from an
    // engine-skipped one.
    let out = Command::new(env!("CARGO_BIN_EXE_testing-conventions"))
        .args(["unit", "mutation", "--language", "rust"])
        .arg(fixtures().join("rust").join("killed"))
        .output()
        .expect("the built binary should run");
    let stdout = String::from_utf8_lossy(&out.stdout);
    let stderr = String::from_utf8_lossy(&out.stderr);
    assert_eq!(
        out.status.code(),
        Some(0),
        "every mutant is caught; stderr: {stderr}"
    );
    assert!(
        tested_count(&stdout) > 0,
        "the engine ran, so the count is non-zero; got: {stdout}"
    );
}

#[test]
fn a_diff_without_crate_changes_reports_the_engine_not_run() {
    // The diff touches nothing under the crate (only a top-level note), so the run is
    // skipped — and the output says the engine never ran, distinct from the all-killed
    // success, keeping the vacuous pass visible in the job log. The exit code stays 0:
    // an empty diff owes no run.
    let repo = GitRepo::new("rust-vacuous");
    repo.write(
        "crate/Cargo.toml",
        "[package]\nname = \"tc_mut_vacuous\"\nversion = \"0.0.0\"\nedition = \"2021\"\n\n[workspace]\n",
    );
    repo.write(
        "crate/src/lib.rs",
        "pub fn add(a: i32, b: i32) -> i32 {\n    a + b\n}\n",
    );
    repo.write("notes.md", "before\n");
    repo.commit("baseline");
    let base = repo.head();
    repo.write("notes.md", "before\nafter\n");
    repo.commit("tweak a top-level note, not the crate");

    let out = Command::new(env!("CARGO_BIN_EXE_testing-conventions"))
        .args(["unit", "mutation", "--language", "rust"])
        .args(["--base", &base])
        .arg(repo.path().join("crate"))
        .output()
        .expect("the built binary should run");
    let stdout = String::from_utf8_lossy(&out.stdout);
    let stderr = String::from_utf8_lossy(&out.stderr);
    assert_eq!(
        out.status.code(),
        Some(0),
        "an empty crate-relative diff passes; stderr: {stderr}"
    );
    assert!(
        stdout.contains(ENGINE_NOT_RUN),
        "the skip is stated; got: {stdout}"
    );
    assert!(
        !stdout.contains("every mutation was caught"),
        "an engine-skipped pass never claims mutants were caught; got: {stdout}"
    );
}

#[test]
fn a_source_change_without_mutant_sites_reports_nothing_tested() {
    // The diff touches Rust source, but only a const's value — no function body — so
    // the engine runs and produces no mutants to judge. The pass states exactly that,
    // never the all-caught line: a zero-mutant run claiming "every mutation was caught"
    // reads the same as a filter that silently dropped real mutants. The const sits
    // after the test module so the changed line borders no mutatable function.
    let repo = GitRepo::new("rust-no-sites");
    repo.write(
        "crate/Cargo.toml",
        "[package]\nname = \"tc_mut_no_sites\"\nversion = \"0.0.0\"\nedition = \"2021\"\n\n[workspace]\n",
    );
    let lib = |answer: &str| {
        format!(
            "pub fn add(a: i32, b: i32) -> i32 {{\n    a + b\n}}\n\n#[cfg(test)]\nmod tests {{\n    use super::*;\n    #[test]\n    fn adds() {{\n        assert_eq!(add(2, 3), 5);\n        assert_eq!(add(10, 1), 11);\n    }}\n}}\n\npub const ANSWER: i32 = {answer};\n"
        )
    };
    repo.write("crate/src/lib.rs", &lib("41"));
    repo.commit("baseline: fully-tested add and a const");
    let base = repo.head();
    repo.write("crate/src/lib.rs", &lib("42"));
    repo.commit("correct the const, touch no function");

    let out = Command::new(env!("CARGO_BIN_EXE_testing-conventions"))
        .args(["unit", "mutation", "--language", "rust"])
        .args(["--base", &base])
        .arg(repo.path().join("crate"))
        .output()
        .expect("the built binary should run");
    let stdout = String::from_utf8_lossy(&out.stdout);
    let stderr = String::from_utf8_lossy(&out.stderr);
    assert_eq!(
        out.status.code(),
        Some(0),
        "a change with no mutant sites passes; stdout: {stdout}; stderr: {stderr}"
    );
    assert!(
        stdout.contains(NOTHING_TESTED),
        "the zero-mutant run says the engine found nothing to test; got: {stdout}"
    );
    assert!(
        !stdout.contains("every mutation was caught"),
        "a run that judged no mutants never claims mutants were caught; got: {stdout}"
    );
}

#[test]
fn base_states_a_nonzero_count_for_a_caught_change_in_a_workspace_member_crate() {
    // The crate is a workspace member and the change adds a fully-tested function, so
    // every mutant on the changed lines is caught — and the success line proves the
    // engine tested the member's mutants by stating a non-zero count. A rebase
    // regression that filtered every mutant out would report a zero-mutant run instead
    // of this counted pass.
    let repo = GitRepo::new("rust-member-caught");
    repo.write(
        "Cargo.toml",
        "[workspace]\nmembers = [\"member\"]\nresolver = \"2\"\n",
    );
    repo.write(
        "member/Cargo.toml",
        "[package]\nname = \"tc_mut_member_caught\"\nversion = \"0.0.0\"\nedition = \"2021\"\n",
    );
    repo.write(
        "member/src/lib.rs",
        "pub fn add(a: i32, b: i32) -> i32 {\n    a + b\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    #[test]\n    fn adds() {\n        assert_eq!(add(2, 3), 5);\n        assert_eq!(add(10, 1), 11);\n    }\n}\n",
    );
    repo.commit("baseline: fully-tested add in a workspace member");
    let base = repo.head();
    repo.write(
        "member/src/lib.rs",
        "pub fn add(a: i32, b: i32) -> i32 {\n    a + b\n}\n\npub fn total(a: i32, b: i32) -> i32 {\n    a + b\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    #[test]\n    fn adds() {\n        assert_eq!(add(2, 3), 5);\n        assert_eq!(add(10, 1), 11);\n    }\n    #[test]\n    fn totals() {\n        assert_eq!(total(2, 3), 5);\n        assert_eq!(total(10, 1), 11);\n    }\n}\n",
    );
    repo.commit("add a fully-tested total");

    let out = Command::new(env!("CARGO_BIN_EXE_testing-conventions"))
        .args(["unit", "mutation", "--language", "rust"])
        .args(["--base", &base])
        .arg(repo.path().join("member"))
        .output()
        .expect("the built binary should run");
    let stdout = String::from_utf8_lossy(&out.stdout);
    let stderr = String::from_utf8_lossy(&out.stderr);
    assert_eq!(
        out.status.code(),
        Some(0),
        "every mutant on the changed lines is caught; stdout: {stdout}; stderr: {stderr}"
    );
    assert!(
        tested_count(&stdout) > 0,
        "the engine tested the member's mutants, so the count is non-zero; got: {stdout}"
    );
}

#[test]
fn base_fails_on_a_survivor_in_a_workspace_member_crate() {
    // The crate is a member of a cargo workspace rooted at the repo root — a monorepo
    // consumer's layout. The changed lines add an assertion-light function, so the
    // diff-scoped gate goes red and names the survivor by its scan-path-relative file:
    // cargo-mutants addresses files relative to the workspace root, and a run that
    // never rebases the diff filters every mutant out and passes with `0 mutant(s)
    // tested` — a false green.
    let repo = GitRepo::new("rust-workspace-member");
    repo.write(
        "Cargo.toml",
        "[workspace]\nmembers = [\"member\"]\nresolver = \"2\"\n",
    );
    repo.write(
        "member/Cargo.toml",
        "[package]\nname = \"tc_mut_ws_member\"\nversion = \"0.0.0\"\nedition = \"2021\"\n",
    );
    repo.write(
        "member/src/lib.rs",
        "pub fn add(a: i32, b: i32) -> i32 {\n    a + b\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    #[test]\n    fn adds() {\n        assert_eq!(add(2, 3), 5);\n        assert_eq!(add(10, 1), 11);\n    }\n}\n",
    );
    repo.commit("baseline: fully-tested add in a workspace member");
    let base = repo.head();
    repo.write(
        "member/src/lib.rs",
        "pub fn add(a: i32, b: i32) -> i32 {\n    a + b\n}\n\npub fn is_positive(n: i32) -> bool {\n    n > 0\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    #[test]\n    fn adds() {\n        assert_eq!(add(2, 3), 5);\n        assert_eq!(add(10, 1), 11);\n    }\n    #[test]\n    fn runs_is_positive() {\n        let _ = is_positive(1);\n    }\n}\n",
    );
    repo.commit("add an assertion-light is_positive");

    let out = Command::new(env!("CARGO_BIN_EXE_testing-conventions"))
        .args(["unit", "mutation", "--language", "rust"])
        .args(["--base", &base])
        .arg(repo.path().join("member"))
        .output()
        .expect("the built binary should run");
    let stdout = String::from_utf8_lossy(&out.stdout);
    let stderr = String::from_utf8_lossy(&out.stderr);
    assert_eq!(
        out.status.code(),
        Some(1),
        "the added weak function's survivors fail the gate; stdout: {stdout}; stderr: {stderr}"
    );
    assert!(
        stderr.contains("surviving mutant") && stderr.contains("src/lib.rs"),
        "the failure names the survivor by its scan-path-relative file; stderr: {stderr}"
    );
}

#[test]
fn survivors_fail_the_gate_by_default() {
    // The gate is on by default and binary: an un-exempted surviving mutant fails the
    // run, no config required.
    assert_eq!(unit_mutation_exit("survivors", None), 1);
}

#[test]
fn an_exempted_survivor_passes_the_gate() {
    // The survivor's file carries a `mutation` exemption, so the gate clears it (an
    // equivalent / deliberately-defensive mutation, lifted with a reason) — the only
    // way to pass with a survivor present.
    assert_eq!(
        unit_mutation_exit("survivors", Some("mutation_exempt.toml")),
        0
    );
}