axiom_engine 0.5.0

//! Agentic self-evaluation harness — capability as a *measured number*.
//!
//! The point of this whole pillar is that AXIOM's coding ability should be
//! reported, not asserted. [`run_eval`] materializes a set of seeded broken-repo
//! fixtures, runs the real autonomous [`crate::solve`] loop on each (localize →
//! deterministic Poly-JIT repair → verify, all reversible), and reports the
//! fraction it drives green.
//!
//! The built-in fixtures ([`builtin_cases`]) are deliberately **deterministic
//! and offline**: each failure is one the Poly-JIT layer can repair without any
//! model or network, so the score is reproducible in CI and means exactly what
//! it says — "the autonomous loop fixed N of M broken repos end-to-end."

use std::path::{Path, PathBuf};

use crate::agentic::{
    agentic_loop, agentic_loop_with_holdout, AttemptMemory, EditSet, ProposeContext, Proposer,
};
use crate::inference::InferencePipeline;
use crate::solve::{posix_shell, run_verify, run_verify_capture, solve, SolveOptions};

/// A seeded broken project: files to materialize and the verify command that
/// must be driven to green. The fix is reachable by the deterministic repair
/// layer, so no LLM is required.
#[derive(Debug, Clone)]
pub struct EvalCase {
    pub name: String,
    /// (relative path, contents) written under a fresh temp project root.
    pub files: Vec<(String, String)>,
    /// Verify program + args, run with the project root as working dir.
    pub command: String,
    pub args: Vec<String>,
    /// When non-empty, this case exercises the **agentic multi-file** path: a
    /// deterministic scripted proposer applies these (relative path, contents)
    /// as one atomic [`EditSet`] through [`agentic_loop`]. Used for fixtures
    /// where no single-file repair passes the verifier, so the single-file
    /// `solve` path cannot solve them — only a coordinated multi-file edit can.
    pub agentic_fix: Vec<(String, String)>,
    /// Optional held-out verifier (program, args). When set on an agentic
    /// case, a candidate edit-set is committed only if it passes BOTH the
    /// train `command` and this held-out command
    /// ([`crate::agentic::agentic_loop_with_holdout`]) — the anti-overfitting
    /// split: a fix shaped against the train verifier that fails held-out
    /// checks is rolled back instead of trusted.
    pub holdout: Option<(String, Vec<String>)>,
}

impl EvalCase {
    fn new(name: &str, files: &[(&str, &str)], command: &str, args: &[&str]) -> Self {
        EvalCase {
            name: name.to_string(),
            files: files.iter().map(|(p, c)| (p.to_string(), c.to_string())).collect(),
            command: command.to_string(),
            args: args.iter().map(|s| s.to_string()).collect(),
            agentic_fix: Vec::new(),
            holdout: None,
        }
    }

    /// Attach a held-out verifier: commits require passing both the train
    /// command and this one. Meaningful only on agentic cases.
    fn with_holdout(mut self, command: &str, args: &[&str]) -> Self {
        self.holdout = Some((command.to_string(), args.iter().map(|s| s.to_string()).collect()));
        self
    }

    /// A case solved via the agentic multi-file transaction path: `fix` is the
    /// coordinated edit-set a deterministic proposer applies atomically.
    fn new_agentic(
        name: &str,
        files: &[(&str, &str)],
        command: &str,
        args: &[&str],
        fix: &[(&str, &str)],
    ) -> Self {
        let mut c = Self::new(name, files, command, args);
        c.agentic_fix = fix.iter().map(|(p, c)| (p.to_string(), c.to_string())).collect();
        c
    }
}

/// A deterministic [`Proposer`] (no LLM) that yields one predefined edit-set on
/// its first call, then gives up. Lets the eval certify the agentic loop's
/// atomic multi-file apply/verify/commit machinery offline and reproducibly.
struct ScriptedProposer {
    once: Option<EditSet>,
}

impl Proposer for ScriptedProposer {
    fn propose(&mut self, _ctx: &ProposeContext) -> Option<EditSet> {
        self.once.take()
    }
}

/// Result of one case.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CaseResult {
    pub name: String,
    pub solved: bool,
}

/// Aggregate evaluation report.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct EvalReport {
    pub results: Vec<CaseResult>,
}

impl EvalReport {
    pub fn total(&self) -> usize {
        self.results.len()
    }
    pub fn solved(&self) -> usize {
        self.results.iter().filter(|r| r.solved).count()
    }
    /// Success rate in [0, 1]; an empty suite scores 0.
    pub fn score(&self) -> f64 {
        if self.results.is_empty() {
            0.0
        } else {
            self.solved() as f64 / self.total() as f64
        }
    }
    /// Human-readable one-liner per case plus the headline score.
    pub fn summary(&self) -> String {
        let mut s = String::new();
        for r in &self.results {
            s.push_str(&format!(
                "  [{}] {}\n",
                if r.solved { "PASS" } else { "FAIL" },
                r.name
            ));
        }
        s.push_str(&format!(
            "score: {}/{} = {:.0}%",
            self.solved(),
            self.total(),
            self.score() * 100.0
        ));
        s
    }
}

/// The built-in deterministic fixture suite. Each case's failure is repairable
/// by the Poly-JIT layer (e.g. an `exit 1` flip or a fixture marker), so the
/// whole suite is offline and reproducible.
pub fn builtin_cases() -> Vec<EvalCase> {
    vec![
        // 1. A script that fails with a self-naming frame; Poly-JIT flips exit 1.
        EvalCase::new(
            "shell-exit-flip",
            &[(
                "run.sh",
                "#!/bin/sh\necho 'run.sh:2:1: boom' >&2\nexit 1\n",
            )],
            &posix_shell(),
            &["run.sh"],
        ),
        // 2. Poly-JIT fixture marker repair.
        EvalCase::new(
            "fixture-marker",
            &[(
                "check.sh",
                "#!/bin/sh\necho 'check.sh:1:1: AXIOM_POLYJIT_FIXTURE_FAIL'\nexit 1\n",
            )],
            &posix_shell(),
            &["check.sh"],
        ),
        // 3. Multi-file: a passing helper cited first, the failing file second;
        //    the loop must localize both and repair the right one.
        EvalCase::new(
            "multi-file-pick-failing",
            &[
                ("helper.sh", "#!/bin/sh\necho 'helper.sh:1:1: note'\nexit 0\n"),
                ("main.sh", "#!/bin/sh\necho 'main.sh:3:1: boom' >&2\nexit 1\n"),
            ],
            &posix_shell(),
            &["-c", "sh helper.sh; sh main.sh"],
        ),
        // 4. Rust-format localization (`--> file:line`) + a second deterministic
        //    repair pattern (the `assert_eq!(1, 2)` -> `(1, 1)` flip). The grep
        //    guard fails while the bad assert is present and passes once flipped.
        EvalCase::new(
            "rust-assert-flip",
            &[("widget.rs", "pub fn f() { assert_eq!(1, 2); }\n")],
            &posix_shell(),
            &[
                "-c",
                "if grep -q 'assert_eq!(1, 2)' widget.rs; then \
                 echo 'error[E0001]'; echo '  --> widget.rs:1:14'; exit 1; else exit 0; fi",
            ],
        ),
        // 5. Python-traceback localization (`File \"x.py\", line N`) + fixture
        //    marker repair (FAIL -> PASS).
        EvalCase::new(
            "python-frame-localize",
            &[("core.py", "# AXIOM_POLYJIT_FIXTURE_FAIL\n")],
            &posix_shell(),
            &[
                "-c",
                "if grep -q AXIOM_POLYJIT_FIXTURE_FAIL core.py; then \
                 echo '  File \"core.py\", line 1, in <module>'; exit 1; else exit 0; fi",
            ],
        ),
        // 6. JS stack-frame localization (`at fn (file:line:col)`) + exit flip.
        EvalCase::new(
            "js-stack-localize",
            &[("index.js", "// index.js\nprocess.exit 1\n")],
            &posix_shell(),
            &[
                "-c",
                "if grep -q 'exit 1' index.js; then \
                 echo '    at Object.<anonymous> (index.js:2:9)'; exit 1; else exit 0; fi",
            ],
        ),
        // 7. Go-format localization (`./pkg/file.go:line:col:`) + exit flip.
        EvalCase::new(
            "go-frame-localize",
            &[("server.go", "package main\n// exit 1 placeholder\n")],
            &posix_shell(),
            &[
                "-c",
                "if grep -q 'exit 1' server.go; then \
                 echo './server.go:2:4: build failed'; exit 1; else exit 0; fi",
            ],
        ),
        // 8. C/C++ compiler diagnostic format (`file.c:line:col: error:`) +
        //    fixture-marker repair. Exercises the inline `path.ext:line:col`
        //    localizer against a `.c` extension.
        EvalCase::new(
            "c-compiler-localize",
            &[(
                "calc.c",
                "/* AXIOM_POLYJIT_FIXTURE_FAIL */\nint calc(void) { return 0; }\n",
            )],
            &posix_shell(),
            &[
                "-c",
                "if grep -q AXIOM_POLYJIT_FIXTURE_FAIL calc.c; then \
                 echo 'calc.c:1:4: error: build gate failed'; exit 1; else exit 0; fi",
            ],
        ),
        // 9. TypeScript diagnostic format (`file.ts:line:col - error TSxxxx`) +
        //    marker repair. Exercises the `.ts` extension through the same
        //    inline `path.ext:line:col` localizer.
        EvalCase::new(
            "ts-frame-localize",
            &[(
                "api.ts",
                "// AXIOM_POLYJIT_FIXTURE_FAIL\nexport const ok = true;\n",
            )],
            &posix_shell(),
            &[
                "-c",
                "if grep -q AXIOM_POLYJIT_FIXTURE_FAIL api.ts; then \
                 echo 'api.ts:1:4: error TS0001: gate failed'; exit 1; else exit 0; fi",
            ],
        ),
        // 10. Java-flavoured `File.java:line:` frame + marker repair.
        EvalCase::new(
            "java-frame-localize",
            &[("Gate.java", "// AXIOM_POLYJIT_FIXTURE_FAIL\nclass Gate {}\n")],
            &posix_shell(),
            &[
                "-c",
                "if grep -q AXIOM_POLYJIT_FIXTURE_FAIL Gate.java; then \
                 echo 'Gate.java:1: error: gate failed'; exit 1; else exit 0; fi",
            ],
        ),
        // 11. Ruby error format (`file.rb:line: ... (RuntimeError)`) + marker repair.
        EvalCase::new(
            "ruby-frame-localize",
            &[("task.rb", "# AXIOM_POLYJIT_FIXTURE_FAIL\nputs 'ok'\n")],
            &posix_shell(),
            &[
                "-c",
                "if grep -q AXIOM_POLYJIT_FIXTURE_FAIL task.rb; then \
                 echo 'task.rb:1: gate failed (RuntimeError)'; exit 1; else exit 0; fi",
            ],
        ),
        // 12. Nested-path resolution (`src/util/gate.sh:line`) + exit-1 flip.
        //     The fault is in a subdirectory, so the localizer must resolve a
        //     relative multi-segment path under the sandbox root, not just a
        //     root-level file.
        EvalCase::new(
            "nested-path-localize",
            &[(
                "src/util/gate.sh",
                "#!/bin/sh\necho placeholder\nexit 1\n",
            )],
            &posix_shell(),
            &[
                "-c",
                "if grep -q 'exit 1' src/util/gate.sh; then \
                 echo 'src/util/gate.sh:3:1: nonzero exit'; exit 1; else exit 0; fi",
            ],
        ),
        // 13. Coordinated multi-file: the verifier requires BOTH scripts to exit
        //    0, but each is independently broken. No single-file repair passes,
        //    so the single-file solve path cannot solve it — only the agentic
        //    loop's atomic multi-file transaction (fix both, verify, commit) can.
        EvalCase::new_agentic(
            "agentic-multi-file-both-broken",
            &[
                ("a.sh", "#!/bin/sh\nexit 1\n"),
                ("b.sh", "#!/bin/sh\nexit 1\n"),
            ],
            &posix_shell(),
            &["-c", "sh a.sh && sh b.sh"],
            // Coordinated fix: both files must go green together.
            &[
                ("a.sh", "#!/bin/sh\nexit 0\n"),
                ("b.sh", "#!/bin/sh\nexit 0\n"),
            ],
        ),
        // 14. Held-out verification split: the fix must pass the train verifier
        //    (a.sh) AND the held-out verifier (b.sh) before it is committed.
        //    A train-only fix (patching a.sh alone) would be the classic
        //    test-overfit patch — the holdout gate rejects it; the scripted
        //    fix repairs both, so the case solves only through the
        //    agentic_loop_with_holdout path.
        EvalCase::new_agentic(
            "agentic-holdout-split",
            &[
                ("a.sh", "#!/bin/sh\nexit 1\n"),
                ("b.sh", "#!/bin/sh\nexit 1\n"),
            ],
            &posix_shell(),
            &["-c", "sh a.sh"],
            &[
                ("a.sh", "#!/bin/sh\nexit 0\n"),
                ("b.sh", "#!/bin/sh\nexit 0\n"),
            ],
        )
        .with_holdout(&posix_shell(), &["-c", "sh b.sh"]),
    ]
}

/// Materialize `cases` into fresh temp roots, run the autonomous loop on each,
/// and collect pass/fail. Each case gets its own throwaway directory which is
/// removed afterward. `solve` is run with no caller-named source, so the loop
/// must localize the fault itself — exactly the production path.
pub fn run_eval(pipeline: &InferencePipeline, cases: &[EvalCase]) -> EvalReport {
    let mut results = Vec::with_capacity(cases.len());
    for case in cases {
        let solved = run_one(pipeline, case).unwrap_or(false);
        results.push(CaseResult {
            name: case.name.clone(),
            solved,
        });
    }
    EvalReport { results }
}

fn run_one(pipeline: &InferencePipeline, case: &EvalCase) -> std::io::Result<bool> {
    let root = unique_root(&case.name);
    std::fs::create_dir_all(&root)?;
    for (rel, content) in &case.files {
        // EvalCase is public; never let a case write outside its sandbox root via
        // an absolute path or a `..` escape.
        if !is_safe_rel(rel) {
            return Err(std::io::Error::new(
                std::io::ErrorKind::InvalidInput,
                format!("eval case path escapes the sandbox root: {rel}"),
            ));
        }
        let path = root.join(rel);
        if let Some(parent) = path.parent() {
            std::fs::create_dir_all(parent)?;
        }
        std::fs::write(&path, content)?;
    }
    let solved = if case.agentic_fix.is_empty() {
        // Single-file repair path: the autonomous solve loop must localize and
        // fix the fault itself (no caller-named source).
        let opts = SolveOptions {
            max_rounds: 2,
            max_restarts: 1,
            anchor: Some(root.clone()),
            source_path: None,       // force autonomous localization
            patch_memory_path: None, // isolate the eval from real fleet memory
            ..SolveOptions::default()
        };
        solve(pipeline, &case.command, &case.args, &opts)
            .map(|r| r.solved)
            .unwrap_or(false)
    } else {
        solve_agentic_case(case, &root)
    };
    let _ = std::fs::remove_dir_all(&root);
    Ok(solved)
}

/// Run a coordinated multi-file case through the agentic loop with a scripted
/// (deterministic, no-LLM) proposer that applies the case's `agentic_fix` as one
/// atomic edit-set. Certifies the transaction apply/verify/commit machinery on a
/// fixture the single-file path cannot solve.
fn solve_agentic_case(case: &EvalCase, root: &Path) -> bool {
    let mut edits = EditSet::new();
    for (rel, content) in &case.agentic_fix {
        // `agentic_fix` is public too: a committed transaction writes these
        // paths, so an absolute/`..` escape must be refused (treat as unsolved)
        // rather than allowed to overwrite files outside the sandbox root.
        if !is_safe_rel(rel) {
            return false;
        }
        edits = edits.with(root.join(rel), content.clone());
    }
    let mut proposer = ScriptedProposer { once: Some(edits) };
    let mut memory = AttemptMemory::new();
    let working_dir = Some(root);
    let outcome = match &case.holdout {
        // Held-out split: commit only when the candidate passes both the
        // train verifier and the held-out verifier (anti-overfitting gate).
        Some((holdout_cmd, holdout_args)) => agentic_loop_with_holdout(
            &format!("eval:{}", case.name),
            3,
            &mut memory,
            &mut proposer,
            || run_verify(&case.command, &case.args, working_dir),
            || run_verify(holdout_cmd, holdout_args, working_dir),
            || run_verify_capture(&case.command, &case.args, working_dir).1,
        ),
        None => agentic_loop(
            &format!("eval:{}", case.name),
            3,
            &mut memory,
            &mut proposer,
            || run_verify(&case.command, &case.args, working_dir),
            || run_verify_capture(&case.command, &case.args, working_dir).1,
        ),
    };
    outcome.solved
}

/// A case-supplied relative path is safe to materialize/commit under the sandbox
/// root only if it is relative and contains no `..` traversal. Applied to both
/// `EvalCase::files` (seed) and `EvalCase::agentic_fix` (committed edits), since
/// both are public and a committed transaction writes the latter to disk.
fn is_safe_rel(rel: &str) -> bool {
    let p = Path::new(rel);
    // `has_root()` catches rooted-but-not-absolute forms Windows treats as
    // relative (`/etc/passwd`, `\x`); prefix components catch drive/UNC paths
    // (`C:x` is drive-relative yet neither absolute nor rooted on Windows).
    !p.is_absolute()
        && !p.has_root()
        && !p.components().any(|c| {
            matches!(
                c,
                std::path::Component::ParentDir | std::path::Component::Prefix(_)
            )
        })
}

fn unique_root(name: &str) -> PathBuf {
    let n = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap()
        .as_nanos();
    let safe: String = name
        .chars()
        .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
        .collect();
    std::env::temp_dir().join(format!("axiom_eval_{safe}_{n}"))
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::config::AxiomConfig;
    use candle_core::Device;

    fn tiny_pipeline() -> InferencePipeline {
        let cfg = AxiomConfig {
            d_model: 16,
            n_layers: 2,
            vocab_size: 64,
            lr_inner: 1e-3,
            norm_eps: 1e-6,
        };
        InferencePipeline::new(cfg, Device::Cpu).expect("tiny pipeline")
    }

    #[test]
    fn builtin_suite_is_fully_solved_by_the_deterministic_loop() {
        // The headline guarantee: the autonomous loop repairs every built-in
        // fixture end-to-end with no LLM — a real, reproducible capability score.
        let report = run_eval(&tiny_pipeline(), &builtin_cases());
        assert_eq!(
            report.solved(),
            report.total(),
            "every deterministic fixture should be auto-repaired:\n{}",
            report.summary()
        );
        assert!((report.score() - 1.0).abs() < f64::EPSILON);
    }

    #[test]
    fn unsafe_case_paths_are_rejected() {
        // Both seed files and committed agentic-fix paths must reject escapes.
        assert!(is_safe_rel("src/a.rs"));
        assert!(is_safe_rel("a.sh"));
        assert!(!is_safe_rel("/etc/passwd"));
        assert!(!is_safe_rel("../escape.rs"));
        assert!(!is_safe_rel("src/../../escape.rs"));

        // An agentic case whose fix escapes the sandbox is refused (unsolved),
        // never committed outside the root.
        let case = EvalCase::new_agentic(
            "escape-attempt",
            &[("a.sh", "#!/bin/sh\nexit 1\n")],
            &posix_shell(),
            &["-c", "exit 1"],
            &[("../../evil.sh", "pwned")],
        );
        let root = unique_root(&case.name);
        std::fs::create_dir_all(&root).unwrap();
        assert!(!solve_agentic_case(&case, &root), "escaping fix must be refused");
        let _ = std::fs::remove_dir_all(&root);
    }

    #[test]
    fn report_score_math() {
        let report = EvalReport {
            results: vec![
                CaseResult { name: "a".into(), solved: true },
                CaseResult { name: "b".into(), solved: false },
            ],
        };
        assert_eq!(report.total(), 2);
        assert_eq!(report.solved(), 1);
        assert!((report.score() - 0.5).abs() < f64::EPSILON);
        assert!(EvalReport { results: vec![] }.score() == 0.0);
    }
}