lean_ctx/core/eval_ab/

mod.rs

//! Deterministic with/without output-quality eval (#232).
//!
//! Proves — reproducibly and with a signature — whether putting lean-ctx in front of a model
//! changes the *quality of its answers*, not just the token count. The design separates the two
//! sources of variance:
//!
//! * **Context** is deterministic. Both the baseline ("raw dump") and the lean-ctx
//!   ("retrieve + compress") window are assembled byte-for-byte reproducibly and digested.
//! * **The model** is the only stochastic part. It is pinned (`temperature = 0`, fixed `seed`)
//!   and, for CI, replaced by [`model::RecordedRunner`] replaying captured real responses, so a
//!   run is byte-identical everywhere.
//!
//! The pipeline per task is: [`conditions::assemble`] → [`model::ModelRunner`] →
//! [`scorers::score_task`]. Results become a paired [`report::AbReport`], which a
//! [`artifact::SignedAbReportV1`] turns into a portable, verifiable attestation.

pub mod artifact;
pub mod conditions;
pub mod footprint;
pub mod model;
pub mod report;
pub mod scorers;
pub mod suite;

use anyhow::Result;

use conditions::{Condition, DEFAULT_BUDGET_TOKENS, assemble};
use model::{ModelRequest, ModelRunner};
use report::{AbReport, PairRecord, ReportConfig};
use scorers::score_task;
use suite::EvalSuite;

/// Shared hex SHA-256 used across the eval modules for context/answer/fingerprint digests.
pub(crate) fn sha256_hex(bytes: &[u8]) -> String {
    use sha2::{Digest, Sha256};
    let mut hasher = Sha256::new();
    hasher.update(bytes);
    crate::core::agent_identity::hex_encode(&hasher.finalize())
}

/// Identical framing for both conditions — only the CONTEXT block differs between A and B.
const SYSTEM_PROMPT: &str = "You are a precise engineering assistant. Answer using only the provided CONTEXT. \
If the context does not contain the answer, say so. Be concise and correct.";

/// Configuration for one A/B run.
#[derive(Debug, Clone, Copy)]
pub struct AbRunConfig {
    /// Token budget enforced identically on both conditions.
    pub budget_tokens: usize,
    /// Statistics + gate configuration.
    pub report: ReportConfig,
}

impl Default for AbRunConfig {
    fn default() -> Self {
        Self {
            budget_tokens: DEFAULT_BUDGET_TOKENS,
            report: ReportConfig::default(),
        }
    }
}

/// Builds the user turn from a context window + the task prompt.
fn build_request(context: &str, prompt: &str) -> ModelRequest {
    ModelRequest {
        system: SYSTEM_PROMPT.to_string(),
        user: format!("CONTEXT:\n{context}\n\nTASK:\n{prompt}"),
    }
}

/// Runs every task in `suite` under both conditions through `runner`, scoring each answer, and
/// assembles the paired report. The model is the only non-deterministic input.
pub fn run_ab(
    suite: &EvalSuite,
    suite_name: &str,
    runner: &dyn ModelRunner,
    cfg: &AbRunConfig,
) -> Result<AbReport> {
    let mut records = Vec::with_capacity(suite.tasks.len());
    for task in &suite.tasks {
        let workspace = task.workspace_path(&suite.dir);

        let base_ctx = assemble(
            Condition::Baseline,
            &workspace,
            task.query(),
            cfg.budget_tokens,
        )?;
        let lean_ctx = assemble(
            Condition::LeanCtx,
            &workspace,
            task.query(),
            cfg.budget_tokens,
        )?;

        let base_resp = runner.run(&build_request(&base_ctx.text, &task.prompt))?;
        let lean_resp = runner.run(&build_request(&lean_ctx.text, &task.prompt))?;

        let base_score = score_task(task, &base_resp.text, &workspace)?;
        let lean_score = score_task(task, &lean_resp.text, &workspace)?;

        records.push(PairRecord {
            task_id: task.id.clone(),
            domain: task.domain.label().to_string(),
            baseline_value: base_score.value,
            lean_ctx_value: lean_score.value,
            baseline_passed: base_score.passed,
            lean_ctx_passed: lean_score.passed,
            baseline_tokens: base_ctx.tokens,
            lean_ctx_tokens: lean_ctx.tokens,
            baseline_context_digest: base_ctx.digest,
            lean_ctx_context_digest: lean_ctx.digest,
            baseline_answer_digest: base_resp.digest(),
            lean_ctx_answer_digest: lean_resp.digest(),
        });
    }

    Ok(AbReport::build(
        suite_name,
        cfg.budget_tokens,
        runner.fingerprint().clone(),
        records,
        cfg.report,
    ))
}

#[cfg(test)]
mod tests {
    use super::*;
    use model::{ModelFingerprint, ModelParams, ModelResponse, RecordedRunner, Recording};
    use std::path::PathBuf;

    /// Builds a workspace where one file holds the answer and another is noise.
    fn workspace(dir: &std::path::Path) {
        std::fs::write(
            dir.join("answer.md"),
            "Consolidation persists artifacts to bm25, graph, knowledge and session stores.",
        )
        .unwrap();
        std::fs::write(
            dir.join("noise.md"),
            "Completely unrelated notes about weather, cats, and lunch plans for the week.",
        )
        .unwrap();
    }

    #[test]
    fn full_pipeline_runs_and_scores_deterministically() {
        let root = tempfile::tempdir().unwrap();
        let ws = root.path().join("corpus");
        std::fs::create_dir_all(&ws).unwrap();
        workspace(&ws);

        let raw = r#"{"id":"t1","domain":"qa","prompt":"Which stores does consolidation persist to?","workspace":"corpus","answers":["bm25 graph knowledge session"]}"#;
        let suite = EvalSuite::parse(raw, root.path().to_path_buf()).unwrap();
        let task = &suite.tasks[0];

        // Pre-compute the exact requests so we can record canned answers (replay scaffolding).
        let cfg = AbRunConfig::default();
        let base_ctx = assemble(Condition::Baseline, &ws, task.query(), cfg.budget_tokens).unwrap();
        let lean_ctx = assemble(Condition::LeanCtx, &ws, task.query(), cfg.budget_tokens).unwrap();
        let base_req = build_request(&base_ctx.text, &task.prompt);
        let lean_req = build_request(&lean_ctx.text, &task.prompt);

        let fp = ModelFingerprint {
            provider: model::PROVIDER_RECORDED.into(),
            endpoint: "test".into(),
            params: ModelParams {
                model: "fixture".into(),
                ..ModelParams::default()
            },
        };
        let mut rec = Recording::new(fp);
        rec.entries
            .insert(base_req.key(), ModelResponse::new("I don't know."));
        rec.entries.insert(
            lean_req.key(),
            ModelResponse::new("bm25, graph, knowledge and session"),
        );
        let runner = RecordedRunner::new(rec);

        let report = run_ab(&suite, "fixture-suite", &runner, &cfg).unwrap();
        assert_eq!(report.records.len(), 1);
        assert!(
            report.stats.lean_ctx_mean > report.stats.baseline_mean,
            "lean-ctx answer should outscore the baseline: {:?}",
            report.stats
        );

        // Determinism: a second identical run yields the same evidence digest.
        let report2 = run_ab(&suite, "fixture-suite", &runner, &cfg).unwrap();
        assert_eq!(
            artifact::determinism_digest(&report),
            artifact::determinism_digest(&report2)
        );
    }

    #[test]
    fn run_ab_propagates_recorded_miss() {
        let root = tempfile::tempdir().unwrap();
        let ws = root.path().join("corpus");
        std::fs::create_dir_all(&ws).unwrap();
        workspace(&ws);
        let raw = r#"{"id":"t1","domain":"qa","prompt":"q","workspace":"corpus","answers":["x"]}"#;
        let suite = EvalSuite::parse(raw, root.path().to_path_buf()).unwrap();

        let fp = ModelFingerprint {
            provider: model::PROVIDER_RECORDED.into(),
            endpoint: "test".into(),
            params: ModelParams::default(),
        };
        let runner = RecordedRunner::new(Recording::new(fp));
        // Empty recording → first request misses → run errors (no silent fallback).
        assert!(run_ab(&suite, "s", &runner, &AbRunConfig::default()).is_err());
        let _ = PathBuf::new();
    }
}

#[cfg(test)]
mod accuracy_suite_tests {
    //! Guards the committed accuracy suite (`rust/eval/accuracy-suite.ndjson`, #730)
    //! in-process so a corpus/answer drift fails in `cargo test` — i.e. during
    //! `dev-install` — not only when someone runs the live gate.

    use super::*;
    use std::path::Path;
    use suite::Domain;

    fn load_accuracy_suite() -> EvalSuite {
        let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("eval/accuracy-suite.ndjson");
        EvalSuite::load(&path).expect("committed accuracy suite must load + validate")
    }

    /// The suite covers all three TTC-relevant shapes (needle, long-context QA, code).
    #[test]
    fn accuracy_suite_has_all_three_shapes() {
        let suite = load_accuracy_suite();
        let qa = suite
            .tasks
            .iter()
            .filter(|t| t.domain == Domain::Qa)
            .count();
        let code = suite
            .tasks
            .iter()
            .filter(|t| t.domain == Domain::Code)
            .count();
        assert!(qa >= 2, "need needle + long-context QA, got {qa}");
        assert!(code >= 1, "need a code-edit task, got {code}");
    }

    /// Model-free accuracy floor (#730): for every QA task, lean-ctx's own
    /// retrieve+compress context must still CONTAIN a gold answer at the default
    /// budget — compression preserves the answer-bearing signal. We reuse the SQuAD
    /// containment scorer over the assembled context itself, so this is the
    /// deterministic lower bound of the "compressed ≥ raw" claim with no live model.
    #[test]
    fn lean_ctx_compression_preserves_every_qa_answer() {
        let suite = load_accuracy_suite();
        let budget = AbRunConfig::default().budget_tokens;
        let qa = suite.tasks.iter().filter(|t| t.domain == Domain::Qa);
        for task in qa {
            let ws = task.workspace_path(&suite.dir);
            let ctx = assemble(Condition::LeanCtx, &ws, task.query(), budget)
                .unwrap_or_else(|e| panic!("assemble {}: {e:#}", task.id));
            let score = score_task(task, &ctx.text, &ws)
                .unwrap_or_else(|e| panic!("score {}: {e:#}", task.id));
            assert!(
                score.passed,
                "lean-ctx compression dropped the answer for '{}' ({})",
                task.id, score.detail
            );
        }
    }

    /// #942: the dedicated `json_crush` condition must clear the same accuracy
    /// floor (the gold answer survives the lossless array crush) while packing the
    /// answer in strictly fewer tokens than the raw baseline — proving the crush is
    /// a real, answer-preserving saving on a redundant JSON payload, model-free.
    #[test]
    fn json_crush_condition_preserves_answer_and_beats_baseline() {
        let suite = load_accuracy_suite();
        let budget = AbRunConfig::default().budget_tokens;
        let task = suite
            .tasks
            .iter()
            .find(|t| t.id == "jsonqa-operator-clearance")
            .expect("json-qa fixture present");
        let ws = task.workspace_path(&suite.dir);

        let crushed = assemble(Condition::JsonCrush, &ws, task.query(), budget)
            .unwrap_or_else(|e| panic!("assemble json_crush: {e:#}"));
        let baseline = assemble(Condition::Baseline, &ws, task.query(), budget)
            .unwrap_or_else(|e| panic!("assemble baseline: {e:#}"));

        let score = score_task(task, &crushed.text, &ws)
            .unwrap_or_else(|e| panic!("score {}: {e:#}", task.id));
        assert!(
            score.passed,
            "json_crush dropped the answer for '{}' ({})",
            task.id, score.detail
        );
        assert!(
            crushed.tokens < baseline.tokens,
            "json_crush ({}) must beat the raw baseline ({}) on a redundant array",
            crushed.tokens,
            baseline.tokens
        );
    }

    /// The code-edit task must be genuinely solvable: a correct reference solution
    /// passes the committed unit test and the shipped failing stub does not. Proves
    /// the harness end-to-end (sandbox copy + `test_cmd`) with zero model calls.
    #[test]
    fn code_task_is_solvable_and_stub_fails() {
        let suite = load_accuracy_suite();
        let task = suite
            .tasks
            .iter()
            .find(|t| t.domain == Domain::Code)
            .expect("code task present");
        let ws = task.workspace_path(&suite.dir);

        let reference = "factorial() { n=$1; [ \"$n\" -le 1 ] && { echo 1; return; }; \
             r=1; i=2; while [ \"$i\" -le \"$n\" ]; do r=$((r * i)); i=$((i + 1)); done; echo \"$r\"; }";
        let good = score_task(task, reference, &ws).unwrap();
        assert!(good.passed, "reference solution must pass: {}", good.detail);

        let stub = "factorial() { echo 0; }";
        let bad = score_task(task, stub, &ws).unwrap();
        assert!(!bad.passed, "wrong solution must fail the unit test");
    }
}