vyre-conform 0.1.0

Conformance suite for vyre backends — proves byte-identical output to CPU reference
Documentation
//! Meta-mutation harness — the self-level analogue of H6.
//!
//! The op-level [`crate::verify::harnesses::mutation`] gate asks: for a given test
//! and a catalog of source mutations, which mutations survive the test?
//! If any mutation survives, the test is weak and the merge is rejected.
//!
//! This harness asks the same question one level up: for a given meta-test
//! set and a catalog of [`MetaMutation`]s, which meta-mutations survive?
//! Surviving meta-mutations are findings — the meta-test set is weak.
//!
//! [`MetaMutation`]: crate::meta::mutation::MetaMutation

use std::collections::BTreeSet;
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::Duration;

use crate::meta::adversary::{matches_signature, AdversaryRef};
use crate::meta::component::ComponentSpec;
use crate::meta::mutation::{self, MetaMutation, MetaMutationClass};

/// Outcome of a single `cargo test` invocation.
#[derive(Debug, Clone)]
pub enum TestOutcome {
    /// All tests passed.
    Pass,
    /// At least one test failed; carries captured stdout and stderr.
    Fail {
        /// Captured standard output from the test run.
        stdout: String,
        /// Captured standard error from the test run.
        stderr: String,
    },
    /// The test process did not finish within the allowed wall-clock time.
    Hung,
}

/// Score a slice of adversaries against the captured test `output`.
///
/// Returns `(killed, survived)` ids. The partitioning is exact:
/// `killed.len() + survived.len() == adversaries.len()`.
#[must_use]
#[inline]
pub fn score_adversaries(output: &str, adversaries: &[AdversaryRef]) -> (Vec<String>, Vec<String>) {
    let mut killed = Vec::new();
    let mut survived = Vec::new();
    for &adv in adversaries {
        if matches_signature(
            output,
            adv.expected_failure_signature,
            adv.signature_match_policy,
        ) {
            killed.push(adv.id.to_string());
        } else {
            survived.push(adv.id.to_string());
        }
    }
    (killed, survived)
}

/// Structured report returned by the meta-mutation gate.
#[derive(Debug, Clone)]
pub struct MetaGateReport {
    /// Qualified name of the [`ComponentSpec`] this report covers.
    pub component: String,
    /// The source file that holds the component, rooted at the
    /// `vyre-conform` crate root.
    pub source_file: PathBuf,
    /// Meta-mutations attempted during this probe.
    pub mutations_attempted: Vec<MetaMutation>,
    /// Meta-mutations the test set caught (the test failed when the
    /// mutation was applied — good).
    pub mutations_killed: Vec<MetaMutation>,
    /// Meta-mutations the test set did NOT catch (the test still passed
    /// when the mutation was applied — findings).
    pub mutations_survived: Vec<MetaMutation>,
    /// Adversaries the meta-test set caught — each one a named
    /// [`BrokenComponent`] whose expected failure signature was produced.
    pub adversaries_killed: Vec<String>,
    /// Adversaries the meta-test set did NOT catch. Each entry is a
    /// finding; the meta-gate emits [`MetaStructuredFeedback`] for each.
    pub adversaries_survived: Vec<String>,
    /// Wall-clock duration of the probe, measured end-to-end.
    pub duration: Duration,
    /// One [`MetaStructuredFeedback`] per surviving mutation or adversary.
    pub feedback: Vec<MetaStructuredFeedback>,
}

/// Actionable feedback for one surviving meta-mutation or adversary.
#[derive(Debug, Clone)]
pub struct MetaStructuredFeedback {
    /// The surviving mutation or the missed adversary's id.
    pub survivor: String,
    /// Actionable hint.
    pub hint: String,
}

/// Probe a component's meta-test set with every applicable meta-mutation
/// and every adversary in its corpus.
#[must_use]
#[inline]
pub fn meta_mutation_probe(
    component: &ComponentSpec,
    crate_root: &Path,
    meta_test_fn_name: &str,
    classes: &[MetaMutationClass],
) -> MetaGateReport {
    let start_time = std::time::Instant::now();
    let source_file = component_source_path(component);
    let full_source_path = crate_root.join(&source_file);
    let _source_guard = acquire_file_guard(&full_source_path.with_extension("meta.lock"))
        .unwrap_or_else(|err| panic!("Fix: could not lock component source for mutation: {err}"));

    let original_source = fs::read_to_string(&full_source_path).unwrap_or_else(|e| {
        panic!(
            "Fix: could not read component source {}: {}",
            full_source_path.display(),
            e
        )
    });

    let mutations = mutation::discover(component.name, &original_source);
    let mut mutations_attempted = Vec::new();
    let mut mutations_killed = Vec::new();
    let mut mutations_survived = Vec::new();

    let mut adversaries_killed_set = BTreeSet::new();

    for mutation in mutations {
        let class = mutation::class_of(mutation);
        if !classes.contains(&class) {
            continue;
        }

        mutations_attempted.push(mutation);

        // Apply mutation
        mutation::apply(crate_root, mutation).expect("Fix: failed to apply mutation");

        // Run the meta-test. We assume the test suite catches the mutation
        // if the test Fails (exit code != 0).
        let outcome = run_cargo_test(crate_root, meta_test_fn_name);
        let killed = !matches!(outcome, TestOutcome::Pass);
        let output = match &outcome {
            TestOutcome::Fail { stdout, stderr } => format!("{stdout}{stderr}"),
            TestOutcome::Hung | TestOutcome::Pass => String::new(),
        };
        let (killed_ids, _) = score_adversaries(&output, component.adversaries);
        for id in killed_ids {
            adversaries_killed_set.insert(id);
        }

        // Restore immediately to minimize broken state
        atomic_write(&full_source_path, original_source.as_bytes())
            .expect("Fix: failed to restore source atomically");

        if killed {
            mutations_killed.push(mutation);
        } else {
            mutations_survived.push(mutation);
        }
    }

    let all_adversary_ids: BTreeSet<String> = component
        .adversaries
        .iter()
        .map(|a| a.id.to_string())
        .collect();
    let adversaries_killed: Vec<String> = adversaries_killed_set.iter().cloned().collect();
    let adversaries_survived: Vec<String> = all_adversary_ids
        .difference(&adversaries_killed_set)
        .cloned()
        .collect();

    let mut feedback = Vec::new();
    for &survivor in &mutations_survived {
        feedback.push(MetaStructuredFeedback {
            survivor: format!("{:?}", survivor),
            hint: format!(
                "Your meta-test set passed when I applied mutation {:?} to {}. \
                 Add a meta-test that asserts the component's strictness against this corruption.",
                survivor, component.name
            ),
        });
    }
    for adv_id in &adversaries_survived {
        feedback.push(MetaStructuredFeedback {
            survivor: adv_id.clone(),
            hint: format!(
                "Your meta-test set failed to catch adversary {} for component {}. \
                 Add a meta-test that detects this corruption.",
                adv_id, component.name
            ),
        });
    }

    let report = MetaGateReport {
        component: component.name.to_string(),
        source_file,
        mutations_attempted,
        mutations_killed,
        mutations_survived: mutations_survived.clone(),
        adversaries_killed,
        adversaries_survived,
        duration: start_time.elapsed(),
        feedback,
    };

    if !mutations_survived.is_empty() {
        file_meta_findings(crate_root, &report);
    }

    report
}

fn run_cargo_test(crate_root: &Path, test_name: &str) -> TestOutcome {
    let child = Command::new("cargo")
        .arg("test")
        .arg("--offline")
        .arg("--quiet")
        .arg(test_name)
        .current_dir(crate_root)
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()
        .expect("Fix: failed to execute cargo test");

    let pid = child.id();
    let timeout = Duration::from_secs(5 * 60);

    let (tx, rx) = std::sync::mpsc::channel();
    std::thread::spawn(move || {
        let output = child.wait_with_output();
        let _ = tx.send(output);
    });

    let start = std::time::Instant::now();
    loop {
        match rx.try_recv() {
            Ok(Ok(output)) => {
                let stdout = String::from_utf8_lossy(&output.stdout).into_owned();
                let stderr = String::from_utf8_lossy(&output.stderr).into_owned();
                let combined = format!("{stdout}{stderr}");
                if output.status.success() && cargo_ran_zero_tests(&combined) {
                    return TestOutcome::Fail {
                        stdout,
                        stderr: format!(
                            "{stderr}\nmeta harness filter `{test_name}` matched zero tests. Fix: set ComponentSpec::test_suite_filter to a committed test function that exercises this component.\n"
                        ),
                    };
                }
                return if output.status.success() {
                    TestOutcome::Pass
                } else {
                    TestOutcome::Fail { stdout, stderr }
                };
            }
            Ok(Err(e)) => panic!("Fix: failed to wait for cargo test: {}", e),
            Err(std::sync::mpsc::TryRecvError::Disconnected) => {
                panic!("Fix: cargo test watcher thread disconnected unexpectedly");
            }
            Err(std::sync::mpsc::TryRecvError::Empty) => {
                if start.elapsed() >= timeout {
                    kill_pid(pid);
                    return TestOutcome::Hung;
                }
                std::thread::sleep(Duration::from_millis(100));
            }
        }
    }
}

fn cargo_ran_zero_tests(output: &str) -> bool {
    output.lines().any(|line| line.trim() == "running 0 tests")
}

#[cfg(unix)]
fn kill_pid(pid: u32) {
    let _ = Command::new("kill").arg("-9").arg(pid.to_string()).status();
}

#[cfg(windows)]
fn kill_pid(pid: u32) {
    let _ = Command::new("taskkill")
        .args(["/F", "/PID", &pid.to_string()])
        .status();
}

#[cfg(not(any(unix, windows)))]
fn kill_pid(_pid: u32) {
    // Fallback for unsupported platforms: cannot force-kill the child
    // from outside the process. The watcher thread will simply return
    // TestOutcome::Hung and leave the child running.
}

fn file_meta_findings(crate_root: &Path, report: &MetaGateReport) {
    let date = "2026-04-12";
    let filename = format!("META_FINDINGS_{}.md", date);
    let path = crate_root.join(filename);
    let _guard = acquire_file_guard(&path.with_extension("lock"))
        .unwrap_or_else(|err| panic!("Fix: could not lock meta-findings file: {err}"));

    let mut content = if path.exists() {
        fs::read_to_string(&path).unwrap_or_default()
    } else {
        format!("# Meta-Conform Findings - {}\n\n", date)
    };

    content.push_str(&format!("## Component: {}\n", report.component));
    content.push_str(&format!("Source: `{}`\n\n", report.source_file.display()));
    content.push_str("| Mutation | Status | Hint |\n");
    content.push_str("| --- | --- | --- |\n");

    for m in &report.mutations_survived {
        content.push_str(&format!(
            "| `{:?}` | SURVIVED | Add test for this corruption |\n",
            m
        ));
    }
    content.push('\n');

    atomic_write(&path, content.as_bytes()).expect("Fix: failed to write meta-findings atomically");
}

/// Return the source file path for a component.
fn component_source_path(component: &ComponentSpec) -> PathBuf {
    PathBuf::from(component.source_file)
}

struct FileGuard {
    path: PathBuf,
}

impl Drop for FileGuard {
    fn drop(&mut self) {
        let _ = fs::remove_file(&self.path);
    }
}

fn acquire_file_guard(path: &Path) -> std::io::Result<FileGuard> {
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent)?;
    }
    loop {
        match fs::OpenOptions::new()
            .write(true)
            .create_new(true)
            .open(path)
        {
            Ok(_) => {
                return Ok(FileGuard {
                    path: path.to_path_buf(),
                });
            }
            Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => {
                std::thread::sleep(Duration::from_millis(25));
            }
            Err(err) => return Err(err),
        }
    }
}

fn atomic_write(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
    if let Some(parent) = path.parent() {
        fs::create_dir_all(parent)?;
    }
    let tmp = unique_temp_path(path);
    let mut file = fs::OpenOptions::new()
        .write(true)
        .create_new(true)
        .open(&tmp)?;
    if let Err(err) = file
        .write_all(bytes)
        .and_then(|()| file.sync_all())
        .and_then(|()| fs::rename(&tmp, path))
    {
        let _ = fs::remove_file(&tmp);
        return Err(err);
    }
    Ok(())
}

fn unique_temp_path(path: &Path) -> PathBuf {
    let pid = std::process::id();
    let thread_id = format!("{:?}", std::thread::current().id())
        .chars()
        .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '_' })
        .collect::<String>();
    let nanos = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .map_or(0, |duration| duration.as_nanos());
    let file_name = path
        .file_name()
        .and_then(|name| name.to_str())
        .unwrap_or("meta-harness");
    path.with_file_name(format!("{file_name}.tmp.{pid}.{thread_id}.{nanos}"))
}