areev 1.12.2

//! `areev eval` — the §7.4 gating edge, through the real binary.
//!
//! This verb is load-bearing for Rule E1: a `code_revision` recommendation is
//! pinned to the evalset it was judged against and may be applied only through
//! the gating edge that judged it. That makes "did the gate actually run, and
//! what did it record" a governance question, not a convenience — and it had
//! no test.
//!
//! Every case runs a host command with the case input on stdin, so `cat` is a
//! perfect echo tool: `expect.equals` against the input passes, anything else
//! fails, with no fixture binary and no model key.

use std::process::Command;
use tempfile::TempDir;

fn areev(args: &[&str]) -> (bool, String, String) {
    let out = Command::new(env!("CARGO_BIN_EXE_areev")).args(args).output().expect("spawn areev");
    (
        out.status.success(),
        String::from_utf8_lossy(&out.stdout).to_string(),
        String::from_utf8_lossy(&out.stderr).to_string(),
    )
}

/// Pull the trailing hash out of `evalset 'NAME' stored: <hash> (N cases)`.
fn stored_hash(out: &str) -> String {
    let h = out
        .split_whitespace()
        .find(|t| t.len() == 64 && t.chars().all(|c| c.is_ascii_hexdigit()))
        .unwrap_or_else(|| panic!("no hash in: {out}"));
    h.to_string()
}

fn write_cases(dir: &TempDir, name: &str, json: &str) -> String {
    let p = dir.path().join(name);
    std::fs::write(&p, json).unwrap();
    p.to_str().unwrap().to_string()
}

#[test]
fn an_evalset_is_validated_before_it_is_stored() {
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("e.db").to_str().unwrap().to_string();

    let good = write_cases(
        &dir,
        "cases.json",
        r#"[{"name": "echo", "input": {"q": 1}, "expect": {"equals": {"q": 1}}}]"#,
    );

    // Both arguments are required, and the error says the whole invocation
    // rather than naming one missing flag at a time.
    let (ok, _out, err) = areev(&["eval", "create", "--db", &db, "--name", "gate"]);
    assert!(!ok, "create without --cases must fail");
    assert!(err.contains("--cases"), "{err}");

    let (ok, _out, err) = areev(&["eval", "create", "--db", &db, "--cases", &good]);
    assert!(!ok, "create without --name must fail");
    assert!(err.contains("--name"), "{err}");

    // An evalset with no cases is a gate that passes everything — refused.
    let empty = write_cases(&dir, "empty.json", "[]");
    let (ok, _out, err) =
        areev(&["eval", "create", "--db", &db, "--name", "g", "--cases", &empty]);
    assert!(!ok, "an empty evalset must be refused");
    assert!(err.contains("at least one case"), "{err}");

    // A case missing its expectation is the dangerous shape: it would score as
    // a failure forever, or (worse) as a pass against nothing.
    let bad = write_cases(&dir, "bad.json", r#"[{"name": "x", "input": {}}]"#);
    let (ok, _out, err) =
        areev(&["eval", "create", "--db", &db, "--name", "g", "--cases", &bad]);
    assert!(!ok, "a case without an expectation must be refused");
    assert!(err.contains("expect"), "{err}");

    let notjson = write_cases(&dir, "notjson.json", "{ not json");
    let (ok, _out, err) =
        areev(&["eval", "create", "--db", &db, "--name", "g", "--cases", &notjson]);
    assert!(!ok, "a malformed cases file must be refused");
    assert!(err.contains("JSON"), "{err}");

    let (ok, out, err) =
        areev(&["eval", "create", "--db", &db, "--name", "gate", "--cases", &good]);
    assert!(ok, "create refused: {err}");
    assert_eq!(stored_hash(&out).len(), 64);
    assert!(out.contains("1 cases"), "{out}");

    let (ok, _out, err) = areev(&["eval", "bless", "--db", &db]);
    assert!(!ok, "an unknown eval subcommand must fail");
    assert!(err.contains("create") && err.contains("run"), "{err}");
}

#[test]
fn a_gate_run_records_its_result_and_fails_the_command_when_a_case_fails() {
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("e.db").to_str().unwrap().to_string();

    // Two cases against a perfect echo: one expects the echo (passes), one
    // expects something else (fails). A gate that cannot fail is not a gate.
    let cases = write_cases(
        &dir,
        "cases.json",
        r#"[
            {"name": "echoes", "input": {"q": 1}, "expect": {"equals": {"q": 1}}},
            {"name": "does-not", "input": {"q": 2}, "expect": {"equals": {"q": 99}}}
        ]"#,
    );
    let (ok, out, err) =
        areev(&["eval", "create", "--db", &db, "--name", "gate", "--cases", &cases]);
    assert!(ok, "create refused: {err}");
    let hash = stored_hash(&out);

    let (ok, _out, err) = areev(&["eval", "run", "--db", &db, "--evalset", &hash]);
    assert!(!ok, "run without --tool-cmd must fail");
    assert!(err.contains("--tool-cmd"), "{err}");

    let (ok, _out, _err) = areev(&[
        "eval", "run", "--db", &db, "--evalset", "not-a-hash", "--tool-cmd", "cat",
    ]);
    assert!(!ok, "run against a malformed evalset hash must fail");

    // A real grain that is not an evalset must be refused rather than scored
    // as zero cases.
    let (ok, out, err) = areev(&["add", "a", "b", "c", "--db", &db, "--ns", "agent:harness"]);
    assert!(ok, "{err}");
    let plain = stored_hash(&out);
    let (ok, _out, err) = areev(&[
        "eval", "run", "--db", &db, "--evalset", &plain, "--tool-cmd", "cat",
    ]);
    assert!(!ok, "a non-evalset grain must be refused");
    assert!(err.contains("evalset"), "{err}");

    // The real run: one case passes, one fails, and the command exits
    // non-zero *because* a case failed — that non-zero is what a CI gate reads.
    let (ok, out, err) =
        areev(&["eval", "run", "--db", &db, "--evalset", &hash, "--tool-cmd", "cat"]);
    assert!(!ok, "a run with a failing case must exit non-zero");
    assert!(err.contains("1 case(s) failed"), "{err}");

    let report: serde_json::Value = serde_json::from_str(out.trim()).expect("run report is JSON");
    assert_eq!(report["passed"], 1);
    assert_eq!(report["failed"], 1);
    assert_eq!(report["evalset"], hash.as_str());
    let run_id = report["run_id"].as_str().unwrap().to_string();
    assert!(run_id.starts_with("eval-"), "{run_id}");

    let rows = report["cases"].as_array().unwrap();
    assert_eq!(rows.len(), 2);
    assert_eq!(rows[0]["case"], "echoes");
    assert_eq!(rows[0]["ok"], true);
    assert_eq!(rows[1]["case"], "does-not");
    assert_eq!(rows[1]["ok"], false);

    // What the run cost rides beside what it scored: one effect per case the
    // tool command ran, and the wall time they took — integers, so the Verify
    // gate's cost bound can read them fail-closed. No model, no tokens.
    assert_eq!(report["effects"], 2, "{report}");
    assert!(report["wall_ms"].as_u64().is_some(), "{report}");
    assert!(report["input_tokens"].is_null() && report["output_tokens"].is_null(), "{report}");
    let (ok, out, err) = areev(&[
        "cal", r#"RECALL facts WHERE relation = "mg:eval_run""#, "--db", &db, "--ns", "agent:harness",
    ]);
    assert!(ok, "{err}");
    let recalled: serde_json::Value = serde_json::from_str(&out).unwrap();
    let summary: serde_json::Value = recalled["grains"]
        .as_array()
        .unwrap()
        .iter()
        .find_map(|g| serde_json::from_str(g["fields"]["object"].as_str()?).ok())
        .expect("the mg:eval_run Fact carries the summary");
    assert_eq!(summary["effects"], 2, "{summary}");
    assert!(summary["wall_ms"].as_u64().is_some(), "{summary}");

    // The gate is evidence, not just an exit code: every case is journaled
    // under the run id, so `run-trace` can show what the gate actually saw.
    let (ok, out, err) = areev(&["run-trace", "--db", &db, "--ns", "agent:harness", "--run-id", &run_id]);
    assert!(ok, "run-trace failed: {err}");
    assert!(
        out.contains("eval:echoes") || out.contains("Tool"),
        "the gate run must be journaled: {out}"
    );
}

#[test]
fn reacceptance_compares_a_rerun_against_a_recorded_baseline() {
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("e.db").to_str().unwrap().to_string();

    // An all-passing evalset, so a baseline run is a clean 100%.
    let cases = write_cases(
        &dir,
        "cases.json",
        r#"[
            {"name": "one", "input": {"q": 1}, "expect": {"equals": {"q": 1}}},
            {"name": "two", "input": {"q": 2}, "expect": {"equals": {"q": 2}}}
        ]"#,
    );
    let (ok, out, err) =
        areev(&["eval", "create", "--db", &db, "--name", "gate", "--cases", &cases]);
    assert!(ok, "{err}");
    let hash = stored_hash(&out);

    let (ok, out, err) =
        areev(&["eval", "run", "--db", &db, "--evalset", &hash, "--tool-cmd", "cat"]);
    assert!(ok, "the baseline run should pass every case: {err}");
    let baseline: serde_json::Value = serde_json::from_str(out.trim()).unwrap();
    assert_eq!(baseline["failed"], 0);
    let baseline_run = baseline["run_id"].as_str().unwrap().to_string();

    // Re-run the SAME evalset against a degraded tool — the model-swap shape.
    // `head -c1` truncates the echo, so both cases now miss.
    let (ok, out, _err) = areev(&[
        "eval", "run", "--db", &db, "--evalset", &hash,
        "--tool-cmd", "head -c1", "--baseline", &baseline_run,
    ]);
    assert!(!ok, "a regressed re-run must exit non-zero");
    let report: serde_json::Value = serde_json::from_str(out.trim()).unwrap();
    assert_eq!(report["failed"], 2);

    // The comparison must be *recorded*, not just printed: "we re-ran
    // acceptance after the swap" has to be evidence someone can query later.
    let re = &report["reacceptance"];
    assert!(!re.is_null(), "a --baseline run must report a comparison: {report}");
    assert_eq!(re["baseline_run"], baseline_run.as_str());
    assert_eq!(re["baseline_pass_rate"], 100.0);
    assert_eq!(re["pass_rate"], 0.0);
    assert_eq!(re["tolerance_points"], 0.0);
    assert_eq!(
        re["accepted"], false,
        "a drop from 100% to 0% must not pass at the default zero tolerance: {re}"
    );

    // A tolerance wide enough to swallow the drop accepts it — the knob has to
    // actually move, or it is decoration.
    let (_ok, out, _err) = areev(&[
        "eval", "run", "--db", &db, "--evalset", &hash,
        "--tool-cmd", "head -c1", "--baseline", &baseline_run, "--tolerance", "100",
    ]);
    let report: serde_json::Value = serde_json::from_str(out.trim()).unwrap();
    assert_eq!(
        report["reacceptance"]["accepted"], true,
        "a 100-point tolerance must accept the same drop: {report}"
    );
    assert_eq!(report["reacceptance"]["tolerance_points"], 100.0);

    // The boundary is the Verify gate's own rule (`is_regression`): a drop of
    // exactly the tolerance is accepted, a hair past it is not. Pinned here
    // so the two readers of "did it get worse" cannot drift apart.
    let (_ok, out, _err) = areev(&[
        "eval", "run", "--db", &db, "--evalset", &hash,
        "--tool-cmd", "head -c1", "--baseline", &baseline_run, "--tolerance", "99.9",
    ]);
    let report: serde_json::Value = serde_json::from_str(out.trim()).unwrap();
    assert_eq!(report["reacceptance"]["accepted"], false, "99.9 points does not cover a 100-point drop: {report}");

    // A tolerance that is not a number of points is refused, not read as zero.
    let (ok, _out, err) = areev(&[
        "eval", "run", "--db", &db, "--evalset", &hash,
        "--tool-cmd", "cat", "--baseline", &baseline_run, "--tolerance", "wide",
    ]);
    assert!(!ok && err.contains("--tolerance"), "{err}");
}

// ── `--model`: grading through the ToolCallLlm seam (the adapter gate) ────

/// Serve `count` canned OpenAI-compatible chat completions on a loopback
/// listener — no live keys, no new dependencies (the areev-llm fixture
/// pattern). Each response carries `usage`, which the seam requires.
fn canned_openai_server(count: usize, content: &str, with_usage: bool) -> String {
    use std::io::{Read, Write};
    use std::net::TcpListener;
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
    let addr = listener.local_addr().unwrap();
    let mut body = serde_json::json!({
        "choices": [{"message": {"role": "assistant", "content": content},
                     "finish_reason": "stop"}],
        "usage": {"prompt_tokens": 10, "completion_tokens": 2}
    });
    if !with_usage {
        body.as_object_mut().unwrap().remove("usage");
    }
    let body = body.to_string();
    std::thread::spawn(move || {
        for _ in 0..count {
            let Ok((mut stream, _)) = listener.accept() else { return };
            // Drain headers + Content-Length body bytes, then answer.
            let mut buf = Vec::new();
            let mut tmp = [0u8; 4096];
            let header_end = loop {
                let Ok(n) = stream.read(&mut tmp) else { return };
                buf.extend_from_slice(&tmp[..n]);
                if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
                    break pos + 4;
                }
            };
            let content_length: usize = String::from_utf8_lossy(&buf[..header_end])
                .lines()
                .find_map(|l| {
                    let (k, v) = l.split_once(':')?;
                    k.eq_ignore_ascii_case("content-length").then(|| v.trim().parse().ok())?
                })
                .unwrap_or(0);
            while buf.len() < header_end + content_length {
                let Ok(n) = stream.read(&mut tmp) else { return };
                buf.extend_from_slice(&tmp[..n]);
            }
            let resp = format!(
                "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
                 Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
                body.len()
            );
            let _ = stream.write_all(resp.as_bytes());
        }
    });
    format!("http://{addr}")
}

fn areev_env(args: &[&str], envs: &[(&str, &str)]) -> (bool, String, String) {
    let mut cmd = Command::new(env!("CARGO_BIN_EXE_areev"));
    cmd.args(args);
    for (k, v) in envs {
        cmd.env(k, v);
    }
    let out = cmd.output().expect("spawn areev");
    (
        out.status.success(),
        String::from_utf8_lossy(&out.stdout).to_string(),
        String::from_utf8_lossy(&out.stderr).to_string(),
    )
}

#[test]
fn tool_cmd_and_model_are_one_of() {
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("e.db").to_str().unwrap().to_string();
    let cases = write_cases(
        &dir,
        "cases.json",
        r#"[{"name": "x", "input": "q", "expect": {"contains": "a"}}]"#,
    );
    let (ok, out, err) =
        areev(&["eval", "create", "--db", &db, "--name", "gate", "--cases", &cases]);
    assert!(ok, "{err}");
    let hash = stored_hash(&out);

    let (ok, _out, err) = areev(&[
        "eval", "run", "--db", &db, "--evalset", &hash,
        "--tool-cmd", "cat", "--model", "openai-compat:x",
    ]);
    assert!(!ok, "both executors must be refused");
    assert!(err.contains("not both"), "{err}");

    let (ok, _out, err) = areev(&["eval", "run", "--db", &db, "--evalset", &hash]);
    assert!(!ok, "no executor must be refused");
    assert!(err.contains("--tool-cmd") && err.contains("--model"), "{err}");
}

#[test]
fn a_model_grades_the_gate_through_an_openai_compatible_endpoint() {
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("e.db").to_str().unwrap().to_string();

    // Two prompt cases against a model that always answers "k8s": one
    // passes, one fails — a gate that cannot fail is not a gate. The second
    // case's messages-array shape exercises the conversation mapping.
    let cases = write_cases(
        &dir,
        "cases.json",
        r#"[
            {"name": "target", "input": "what is the deploy target?",
             "expect": {"contains": "k8s"}},
            {"name": "cloud",
             "input": [{"role": "system", "content": "answer tersely"},
                        {"role": "user", "content": "which cloud?"}],
             "expect": {"contains": "aws"}}
        ]"#,
    );
    let (ok, out, err) =
        areev(&["eval", "create", "--db", &db, "--name", "gate", "--cases", &cases]);
    assert!(ok, "{err}");
    let hash = stored_hash(&out);

    let base = canned_openai_server(2, "k8s", true);
    let (ok, out, err) = areev_env(
        &[
            "eval", "run", "--db", &db, "--evalset", &hash,
            "--model", "openai-compat:fake-adapter", "--base-url", &base, "--key-env", "FAKE_KEY",
        ],
        &[("FAKE_KEY", "dummy")],
    );
    assert!(!ok, "a run with a failing case must exit non-zero: {err}");
    let report: serde_json::Value = serde_json::from_str(out.trim()).expect("report is JSON");
    assert_eq!(report["passed"], 1, "{out}");
    assert_eq!(report["failed"], 1, "{out}");

    // The recorded edge names WHICH model was judged — part of the evidence.
    let (ok, out, err) = areev(&[
        "cal",
        r#"RECALL facts WHERE relation = "mg:eval_run""#,
        "--db", &db, "--ns", "agent:harness",
    ]);
    assert!(ok, "{err}");
    assert!(
        out.contains("openai-compat:fake-adapter"),
        "the summary must record the graded model: {out}"
    );
    // …and what the model path spent: the provider's usage summed over the
    // cases (10 + 2 per canned reply, two cases), beside the effects count.
    let recalled: serde_json::Value = serde_json::from_str(&out).unwrap();
    let summary: serde_json::Value = recalled["grains"]
        .as_array()
        .unwrap()
        .iter()
        .find_map(|g| serde_json::from_str(g["fields"]["object"].as_str()?).ok())
        .unwrap();
    assert_eq!(summary["input_tokens"], 20, "{summary}");
    assert_eq!(summary["output_tokens"], 4, "{summary}");
    assert_eq!(summary["effects"], 2, "{summary}");
    assert_eq!(report["input_tokens"], 20, "{report}");
}

#[test]
fn model_cases_are_prevalidated_and_record_nothing_on_shape_errors() {
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("e.db").to_str().unwrap().to_string();
    // An object input is fine for tool-cmd (any JSON on stdin) but has no
    // meaning as a chat — the model path must refuse it BEFORE any case
    // runs, and record no summary (a partial gate is not a gate).
    let cases = write_cases(
        &dir,
        "cases.json",
        r#"[
            {"name": "ok", "input": "hi", "expect": {"contains": "x"}},
            {"name": "shapeless", "input": {"q": 1}, "expect": {"contains": "x"}}
        ]"#,
    );
    let (ok, out, err) =
        areev(&["eval", "create", "--db", &db, "--name", "gate", "--cases", &cases]);
    assert!(ok, "{err}");
    let hash = stored_hash(&out);

    let (ok, _out, err) = areev_env(
        &[
            "eval", "run", "--db", &db, "--evalset", &hash,
            "--model", "openai-compat:x", "--base-url", "http://127.0.0.1:1", "--key-env", "FAKE_KEY",
        ],
        &[("FAKE_KEY", "dummy")],
    );
    assert!(!ok, "a shapeless model case must refuse the whole run");
    assert!(err.contains("shapeless"), "the refusal names the case: {err}");

    let (ok, out, err) = areev(&[
        "cal", r#"RECALL facts WHERE relation = "mg:eval_run""#, "--db", &db, "--ns", "agent:harness",
    ]);
    assert!(ok, "{err}");
    let payload: serde_json::Value = serde_json::from_str(out.trim()).unwrap();
    assert_eq!(
        payload["grains"].as_array().map(Vec::len),
        Some(0),
        "a refused run records no gating edge: {out}"
    );
}

#[test]
fn a_response_without_usage_fails_the_case() {
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("e.db").to_str().unwrap().to_string();
    let cases = write_cases(
        &dir,
        "cases.json",
        r#"[{"name": "target", "input": "q", "expect": {"contains": "k8s"}}]"#,
    );
    let (ok, out, err) =
        areev(&["eval", "create", "--db", &db, "--name", "gate", "--cases", &cases]);
    assert!(ok, "{err}");
    let hash = stored_hash(&out);

    // The seam requires token usage on every response (budgeted runs). A
    // proxy that strips `usage` must fail the case, not silently pass it.
    let base = canned_openai_server(1, "k8s", false);
    let (ok, out, _err) = areev_env(
        &[
            "eval", "run", "--db", &db, "--evalset", &hash,
            "--model", "openai-compat:x", "--base-url", &base, "--key-env", "FAKE_KEY",
        ],
        &[("FAKE_KEY", "dummy")],
    );
    assert!(!ok, "a usage-less response must fail the gate");
    let report: serde_json::Value = serde_json::from_str(out.trim()).unwrap();
    assert_eq!(report["failed"], 1, "{out}");
}

// ---- `areev tool provenance` chains the blob (#179) -------------------------

/// The forensic question is "what code actually ran", and a Tool grain answers
/// it only by reference. Until now `tool provenance` stopped at the reference:
/// it reported the recommendations and the runs, and left the reader to parse
/// `executor_uri`, find the `.blobs` sidecar and check the digest by hand.
///
/// Seeded in-process because no CLI verb authors a Tool Definition (`ADD` is
/// limited to fact/observation/goal/skill) — the handle is dropped before the
/// binary runs, since one memory means one writer.
#[test]
fn tool_provenance_chains_the_executor_blob() {
    use areev_core::types::{Grain, Tool, ToolKind};

    let dir = TempDir::new().unwrap();
    let db = dir.path().join("p.db").to_str().unwrap().to_string();
    const CODE: &[u8] = b"#!/bin/sh\necho '{\"ok\":true}'\n";

    let (with_blob, no_blob) = {
        let mut m = areev_store::Areev::open(&db).unwrap();
        let uri = m.put_blob(CODE).unwrap();
        let coded = Tool::new("validate_rows")
            .kind(ToolKind::Definition)
            .tool_description("a code-carrying tool")
            .executor_uri(&uri)
            .namespace("ops");
        let plain = Tool::new("ask_human")
            .kind(ToolKind::Definition)
            .tool_description("no code behind it")
            .namespace("ops");
        (m.add(&coded).unwrap().to_hex(), m.add(&plain).unwrap().to_hex())
    };

    let (ok, out, err) = areev(&["tool", "provenance", &with_blob, "--db", &db, "--ns", "ops"]);
    assert!(ok, "{err}");
    let report: serde_json::Value = serde_json::from_str(out.trim()).unwrap();
    let ex = &report["executor"];
    assert!(
        ex["executor_uri"].as_str().unwrap_or_default().starts_with("cas://sha256:"),
        "the chain starts at the content address: {out}"
    );
    assert_eq!(ex["blob_present"], true, "{out}");
    assert_eq!(
        ex["blob_bytes"].as_u64(),
        Some(CODE.len() as u64),
        "the bytes are the code that ran, not an estimate: {out}"
    );

    // A Definition with no executor_uri names no code, so it reports none —
    // rather than an empty "executor" block that reads like a missing blob.
    let (ok, out, err) = areev(&["tool", "provenance", &no_blob, "--db", &db, "--ns", "ops"]);
    assert!(ok, "{err}");
    let report: serde_json::Value = serde_json::from_str(out.trim()).unwrap();
    assert!(report.get("executor").is_none(), "{out}");
}

// ---------------------------------------------------------------------------
// #313 — a grader's field metrics and usage reach the summary
// ---------------------------------------------------------------------------

/// A grader that echoes stdin (so `expect` still scores it) and writes an
/// out-of-band report to `$AREEV_EVAL_REPORT`.
#[cfg(not(windows))]
fn grader(dir: &TempDir, body: &str) -> String {
    let p = dir.path().join("grade.sh");
    std::fs::write(&p, format!("#!/bin/sh\ncat\n{body}\n")).unwrap();
    use std::os::unix::fs::PermissionsExt;
    std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap();
    format!("sh {}", p.to_str().unwrap())
}

#[cfg(not(windows))]
fn run_with_grader(db: &str, evalset: &str, cmd: &str) -> (bool, String, String) {
    areev(&[
        "eval", "run", "--db", db, "--evalset", evalset, "--tool-cmd", cmd, "--format", "json",
    ])
}

#[cfg(not(windows))]
fn seed(dir: &TempDir, db: &str) -> String {
    let cases = write_cases(
        dir,
        "m-cases.json",
        r#"[{"name": "a", "input": {"q": 1}, "expect": {"equals": {"q": 1}}},
            {"name": "b", "input": {"q": 2}, "expect": {"equals": {"q": 2}}}]"#,
    );
    let (ok, out, err) =
        areev(&["eval", "create", "--db", db, "--name", "gate", "--cases", &cases]);
    assert!(ok, "{err}");
    stored_hash(&out)
}

#[cfg(not(windows))]
#[test]
fn a_grader_report_reaches_the_summary_as_means_and_counts() {
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("m.db").to_str().unwrap().to_string();
    let evalset = seed(&dir, &db);
    // unit_ok = 1 on both cases, period_ok = 1 then 0 → mean 0.5.
    let cmd = grader(
        &dir,
        r#"if [ "$AREEV_EVAL_CASE" = "a" ]; then P=1; else P=0; fi
printf '{"metrics":{"unit_ok":1,"period_ok":%s},"usage":{"input_tokens":1200,"output_tokens":300}}' "$P" > "$AREEV_EVAL_REPORT""#,
    );
    let (ok, out, err) = run_with_grader(&db, &evalset, &cmd);
    assert!(ok, "stdout={out} stderr={err}");
    let v: serde_json::Value = serde_json::from_str(&out).expect(&out);
    assert_eq!(v["passed"], 2, "{out}");
    assert_eq!(v["unit_ok"], 1.0, "{out}");
    assert_eq!(v["unit_ok_n"], 2, "{out}");
    assert_eq!(v["period_ok"], 0.5, "a 0/1 field result reads as a ratio: {out}");
    assert_eq!(v["period_ok_n"], 2, "{out}");
    assert_eq!(v["input_tokens"], 2400, "{out}");
    assert_eq!(v["output_tokens"], 600, "{out}");
    // And per case.
    assert_eq!(v["cases"][0]["metrics"]["period_ok"], 1.0, "{out}");
    assert_eq!(v["cases"][1]["metrics"]["period_ok"], 0.0, "{out}");
    assert_eq!(v["cases"][0]["usage"]["input_tokens"], 1200, "{out}");
}

#[cfg(not(windows))]
#[test]
fn a_command_that_writes_no_report_produces_todays_summary() {
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("m.db").to_str().unwrap().to_string();
    let evalset = seed(&dir, &db);
    let (ok, out, err) = run_with_grader(&db, &evalset, "cat");
    assert!(ok, "stdout={out} stderr={err}");
    let v: serde_json::Value = serde_json::from_str(&out).expect(&out);
    assert_eq!(v["passed"], 2);
    assert!(v.get("unit_ok").is_none());
    // The tool-cmd path reports no tokens when nothing reported any.
    assert!(v["input_tokens"].is_null(), "{out}");
    assert!(v["cases"][0].get("metrics").is_none(), "{out}");
}

#[cfg(not(windows))]
#[test]
fn a_malformed_report_fails_the_case_rather_than_reading_as_zero() {
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("m.db").to_str().unwrap().to_string();
    let evalset = seed(&dir, &db);
    // A string where an integer belongs: reading it as 0 would report a
    // model call as free.
    let cmd = grader(
        &dir,
        r#"printf '{"usage":{"input_tokens":"1200"}}' > "$AREEV_EVAL_REPORT""#,
    );
    let (ok, out, _err) = run_with_grader(&db, &evalset, &cmd);
    assert!(!ok, "a malformed report must fail the run: {out}");
    let v: serde_json::Value = serde_json::from_str(&out).expect(&out);
    assert_eq!(v["failed"], 2, "{out}");
}

#[cfg(not(windows))]
#[test]
fn a_metric_colliding_with_a_reserved_key_fails_the_case() {
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("m.db").to_str().unwrap().to_string();
    let evalset = seed(&dir, &db);
    let cmd = grader(
        &dir,
        r#"printf '{"metrics":{"passed":1}}' > "$AREEV_EVAL_REPORT""#,
    );
    let (ok, out, _err) = run_with_grader(&db, &evalset, &cmd);
    assert!(!ok, "a metric named `passed` would overwrite the gate's count: {out}");
    assert!(out.contains("reserved") || out.contains("passed"), "{out}");
}

// ---------------------------------------------------------------------------
// #314 — case content follows its namespace
// ---------------------------------------------------------------------------

#[cfg(not(windows))]
#[test]
fn case_ns_keeps_confidential_case_content_out_of_the_harness() {
    const MARKER: &str = "zz-subscription-doc-zz";
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("m.db").to_str().unwrap().to_string();
    let cases = write_cases(
        &dir,
        "ns-cases.json",
        &format!(
            r#"[{{"name": "a", "input": {{"doc": "{MARKER}"}}, "expect": {{"contains": "{MARKER}"}}}}]"#
        ),
    );
    let (ok, out, err) = areev(&[
        "eval", "create", "--db", &db, "--name", "g", "--cases", &cases, "--case-ns", "deal.alpha",
    ]);
    assert!(ok, "{err}");
    let evalset = stored_hash(&out);
    let (ok, out, err) = areev(&[
        "eval", "run", "--db", &db, "--evalset", &evalset, "--tool-cmd", "cat",
        "--case-ns", "deal.alpha", "--format", "json",
    ]);
    assert!(ok, "stdout={out} stderr={err}");
    let v: serde_json::Value = serde_json::from_str(&out).expect(&out);
    assert_eq!(v["passed"], 1, "{out}");
    assert_eq!(v["case_ns"], "deal.alpha", "the summary says where the content went");

    // The harness holds the numbers and none of the content.
    let (ok, harness, err) = areev(&[
        "cal", "--db", &db,
        r#"RECALL tools WHERE namespace = "agent:harness" LIMIT 500"#,
    ]);
    assert!(ok, "{err}");
    let (_ok2, harness_facts, _e2) = areev(&[
        "cal", "--db", &db,
        r#"RECALL facts WHERE namespace = "agent:harness" LIMIT 500"#,
    ]);
    assert!(
        !harness.contains(MARKER) && !harness_facts.contains(MARKER),
        "no case content in agent:harness: {harness}{harness_facts}"
    );
    // And the namespace holds the cases.
    let (ok, scoped, err) = areev(&[
        "cal", "--db", &db,
        r#"RECALL tools WHERE namespace = "deal.alpha" LIMIT 500"#,
    ]);
    assert!(ok, "{err}");
    assert!(scoped.contains(MARKER), "the cases are in deal.alpha: {scoped}");
}

#[cfg(not(windows))]
#[test]
fn without_the_flag_placement_is_unchanged() {
    const MARKER: &str = "zz-ordinary-zz";
    let dir = TempDir::new().unwrap();
    let db = dir.path().join("m.db").to_str().unwrap().to_string();
    let cases = write_cases(
        &dir,
        "plain.json",
        &format!(
            r#"[{{"name": "a", "input": {{"doc": "{MARKER}"}}, "expect": {{"contains": "{MARKER}"}}}}]"#
        ),
    );
    let (ok, out, err) =
        areev(&["eval", "create", "--db", &db, "--name", "g", "--cases", &cases]);
    assert!(ok, "{err}");
    let evalset = stored_hash(&out);
    let (ok, out, err) = areev(&[
        "eval", "run", "--db", &db, "--evalset", &evalset, "--tool-cmd", "cat", "--format", "json",
    ]);
    assert!(ok, "stdout={out} stderr={err}");
    let v: serde_json::Value = serde_json::from_str(&out).expect(&out);
    assert!(v.get("case_ns").is_none(), "no flag, no key: {out}");
    let (ok, harness, err) = areev(&[
        "cal", "--db", &db,
        r#"RECALL tools WHERE namespace = "agent:harness" LIMIT 500"#,
    ]);
    assert!(ok, "{err}");
    let (_ok2, harness_facts, _e2) = areev(&[
        "cal", "--db", &db,
        r#"RECALL facts WHERE namespace = "agent:harness" LIMIT 500"#,
    ]);
    assert!(
        harness.contains(MARKER) || harness_facts.contains(MARKER),
        "unchanged placement: {harness}{harness_facts}"
    );
}