car-builder 0.35.0

//! LIVE NL→workflow eval harness — the Part 1 acceptance gate of
//! `docs/proposals/h2-builder-discovery-acceptance.md` (spec v2, items 2–4).
//! Ignored by default; hits a real model API and spends tokens.
//!
//!   ANTHROPIC_API_KEY=sk-... cargo test -p car-builder --test live_nl2workflow_eval \
//!       -- --ignored --nocapture
//!
//! ## Eval config (shipped defaults — no test-only knobs)
//!
//! - **Model id is PINNED** to [`PINNED_MODEL`], overridable via the
//!   `CAR_BUILDER_EVAL_MODEL` env var. Changing the pin (or the override) is a
//!   **spec-visible event**: the acceptance numbers are only comparable against
//!   the same pinned model, so a repin must be recorded in the h2 acceptance
//!   doc, not slipped in silently. Routing is explicit (`GenerateRequest.model`)
//!   so a router change can never silently change what the eval measures.
//! - **Temperature 0**, N = [`RUNS`] repeat runs, **worst-run counts reported**
//!   (the spec's determinism discipline — ten cases per band means one flake
//!   flips a band target; worst-of-3 makes the number defensible).
//! - `max_attempts` = 3 (the daemon `builder.build` shipped default), catalog =
//!   the seven commodity built-in tools the engine registers
//!   (`car-engine::agent_basics::entries()`, sourced here from
//!   `car_ir::builtins`) + the engine's unified model registry — the same
//!   authoritative catalog shape `builder.build` assembles on the daemon.
//!
//! ## Scoring (per the acceptance spec)
//!
//! pass = manifest validates via `car-workflow` (`build_workflow`'s own
//! verify+catalog gate) AND `car_verify::verify_workflow_graph` reports no
//! structural defects AND the case's `must_have` assertions hold. A **typed
//! refusal** (the builder returning a typed failure instead of a valid
//! manifest — never a panic) passes only where the case is labeled
//! `refusal_ok: true`, and refusal-based passes are capped at 40% of a band
//! (an always-refuse builder must not ace the adversarial band). An empty
//! goal is refused before any model call, mirroring the daemon
//! `builder.build` "missing 'goal'" guard.
//!
//! Targets asserted (making a live run THE acceptance gate): band 1 ≥ 90%,
//! band 2 ≥ 80%, band 3 ≥ 70% (worst run), repair recovers ≥ 4 of the ≥ 5
//! repair-forcing cases (worst run), median repair iterations ≤ 1 (worst
//! run), zero panics and zero unparseable final outcomes.
//!
//! Everything deterministic — fixture loading, the `must_have` checker, the
//! workflow→graph bridge, outcome classification, the refusal cap, worst-of-N
//! aggregation, repair statistics — is pure and unit-tested below WITHOUT a
//! model: `cargo test -p car-builder --test live_nl2workflow_eval` runs those;
//! only the live gate is `#[ignore]`d.

use car_builder::{build_workflow, BuildRequest, BuildResult, ToolCatalog, ToolInfo};
use car_verify::{verify_workflow_graph, WorkflowEdge, WorkflowGraph};
use car_workflow::{StageStep, Workflow};
use serde::Deserialize;
use std::collections::HashSet;
use std::sync::Arc;

// ---------------------------------------------------------------------------
// Eval config — shipped defaults; every value below is printed by the live run.
// ---------------------------------------------------------------------------

/// The pinned model id (a catalog id from car-inference's builtin catalog).
/// Changing this default — or running with `CAR_BUILDER_EVAL_MODEL` — is a
/// spec-visible event: record it in `docs/proposals/h2-builder-discovery-acceptance.md`.
const PINNED_MODEL: &str = "anthropic/claude-sonnet-4-6:latest";
/// Env override for the model pin (still explicit routing, never the router).
const MODEL_ENV: &str = "CAR_BUILDER_EVAL_MODEL";
/// The API key env the default pin's remote path reads (Anthropic protocol).
const KEY_ENV: &str = "ANTHROPIC_API_KEY";
/// Repeat runs; worst-run counts are what the targets are asserted against.
const RUNS: usize = 3;
/// Determinism discipline: temperature 0.
const TEMPERATURE: f64 = 0.0;
/// Same output budget as the shipped `builder.build` / `car build` call sites.
const MAX_TOKENS: usize = 4096;
/// The daemon `builder.build` shipped default (`default_builder_attempts`).
const MAX_ATTEMPTS: u32 = 3;
/// Refusal-based passes are capped at this fraction of a band (band 3 is the
/// only band where `refusal_ok` labels exist, enforced by the fixture validator).
const REFUSAL_CAP: f64 = 0.40;
/// Repair must recover at least this many of the repair-forcing cases.
const MIN_REPAIR_RECOVERED: usize = 4;
/// Worst-run median repair iterations must not exceed this.
const MAX_MEDIAN_REPAIR_ITERS: f64 = 1.0;
/// Per-band pass-rate targets (band id, minimum worst-run pass rate).
const BAND_TARGETS: &[(&str, f64)] = &[
    ("single_stage", 0.90),
    ("multi_stage", 0.80),
    ("adversarial", 0.70),
];

const FIXTURE: &str = include_str!("../eval/nl2workflow.jsonl");

// ---------------------------------------------------------------------------
// Fixture schema (mirrors tests/eval_fixtures.rs, the shape validator).
// ---------------------------------------------------------------------------

#[derive(Debug, Clone, Deserialize)]
struct EvalCase {
    id: String,
    band: String,
    nl_request: String,
    must_have: MustHave,
    #[serde(default)]
    refusal_ok: Option<bool>,
    #[serde(default)]
    repair_forcing: Option<bool>,
    #[serde(default)]
    #[allow(dead_code)]
    notes: String,
}

#[derive(Debug, Clone, Deserialize)]
struct MustHave {
    /// Inclusive [min, max] stage-count range (top-level stages).
    stage_count: [i64; 2],
    /// Tool names that must be referenced somewhere in the workflow.
    required_tools: Vec<String>,
    /// Substrings that must appear in some edge condition key or operator.
    required_edge_conditions: Vec<String>,
}

fn load_cases() -> Vec<EvalCase> {
    FIXTURE
        .lines()
        .filter(|l| !l.trim().is_empty())
        .enumerate()
        .map(|(i, line)| {
            serde_json::from_str::<EvalCase>(line)
                .unwrap_or_else(|e| panic!("line {} is not a valid eval case: {e}", i + 1))
        })
        .collect()
}

// ---------------------------------------------------------------------------
// Pure harness core — unit-tested without a model.
// ---------------------------------------------------------------------------

/// The classified result of one case.
#[derive(Debug, Clone, PartialEq)]
enum Outcome {
    /// Valid manifest + clean workflow graph + `must_have` holds.
    Pass { attempts: u32 },
    /// Typed refusal / typed failure: no valid manifest was produced, but the
    /// builder returned a well-typed `BuildResult` (never a panic).
    /// `parse_class` marks a final failure whose last issues were
    /// model-output-parse failures (prose instead of a manifest).
    Refusal { attempts: u32, parse_class: bool },
    /// Valid manifest, but graph defects or `must_have` violations.
    Fail {
        attempts: u32,
        violations: Vec<String>,
    },
    /// The case panicked at the harness level (asserted zero).
    Panicked { message: String },
}

impl Outcome {
    fn attempts(&self) -> u32 {
        match self {
            Outcome::Pass { attempts }
            | Outcome::Refusal { attempts, .. }
            | Outcome::Fail { attempts, .. } => *attempts,
            Outcome::Panicked { .. } => 0,
        }
    }

    fn short(&self) -> String {
        match self {
            Outcome::Pass { attempts } => format!("PASS (attempts={attempts})"),
            Outcome::Refusal {
                attempts,
                parse_class,
            } => format!("REFUSAL (attempts={attempts}, parse_class={parse_class})"),
            Outcome::Fail {
                attempts,
                violations,
            } => format!("FAIL (attempts={attempts}): {}", violations.join("; ")),
            Outcome::Panicked { message } => format!("PANIC: {message}"),
        }
    }
}

/// One case + its classified outcome — the unit band scoring works over.
#[derive(Debug, Clone)]
struct CaseRecord {
    /// Kept for the run log / debugging (not read by the scoring math).
    #[allow(dead_code)]
    id: String,
    band: String,
    refusal_ok: bool,
    repair_forcing: bool,
    outcome: Outcome,
}

/// Mirrors the daemon `builder.build` guard: an empty goal is rejected with a
/// typed error before any model call. The harness scores that as a typed
/// pre-generation refusal (attempts = 0).
fn pre_refusal(case: &EvalCase) -> Option<Outcome> {
    if case.nl_request.trim().is_empty() {
        Some(Outcome::Refusal {
            attempts: 0,
            parse_class: false,
        })
    } else {
        None
    }
}

/// A final failure whose issues are parse-class: the model's last output did
/// not parse as a workflow at all (see `car_builder::parse_workflow` and the
/// repair message it feeds back).
fn is_parse_class(issues: &[String]) -> bool {
    issues
        .iter()
        .any(|i| i.contains("did not parse") || i.contains("no JSON workflow object"))
}

/// Bridge a `car-workflow` manifest into the abstract graph
/// `verify_workflow_graph` checks: entry = `start`, terminals = stages with no
/// outgoing edge (a stage with no out-edge ends the run, per the workflow
/// semantics), edges labeled with their first condition when present.
fn to_graph(wf: &Workflow) -> WorkflowGraph {
    let stages: Vec<String> = wf.stages.iter().map(|s| s.id.clone()).collect();
    let with_out: HashSet<&str> = wf.edges.iter().map(|e| e.from.as_str()).collect();
    let terminals: Vec<String> = stages
        .iter()
        .filter(|s| !with_out.contains(s.as_str()))
        .cloned()
        .collect();
    let edges: Vec<WorkflowEdge> = wf
        .edges
        .iter()
        .map(|e| WorkflowEdge {
            from: e.from.clone(),
            to: e.to.clone(),
            condition: e
                .conditions
                .first()
                .map(|c| format!("{} {} {}", c.key, c.operator, c.value)),
        })
        .collect();
    WorkflowGraph {
        entry: wf.start.clone(),
        terminals,
        stages,
        edges,
    }
}

/// Collect every tool name referenced anywhere in a step (recursing through
/// nested bodies and sub-workflows) — the `required_tools` universe.
fn collect_tools_step(step: &StageStep, out: &mut HashSet<String>) {
    match step {
        StageStep::Proposal(ps) => {
            for action in &ps.proposal.actions {
                if let Some(tool) = &action.tool {
                    out.insert(tool.clone());
                }
            }
        }
        StageStep::Pattern(p) => {
            for agent in &p.agents {
                for tool in &agent.tools {
                    out.insert(tool.clone());
                }
            }
        }
        StageStep::Deliver(d) => {
            for sink in &d.sinks {
                for action in &sink.actions {
                    if let Some(tool) = &action.tool {
                        out.insert(tool.clone());
                    }
                }
            }
        }
        StageStep::SubWorkflow(sw) => {
            for stage in &sw.workflow.stages {
                collect_tools_step(&stage.step, out);
            }
        }
        StageStep::LoopUntil(l) => collect_tools_step(&l.body, out),
        StageStep::ForEach(f) => collect_tools_step(&f.body, out),
        StageStep::Approval(_) | StageStep::Dedup(_) => {}
    }
}

fn collect_tools(wf: &Workflow) -> HashSet<String> {
    let mut out = HashSet::new();
    for stage in &wf.stages {
        collect_tools_step(&stage.step, &mut out);
    }
    out
}

/// Check the fixture's semantic `must_have` assertions against a built
/// workflow. Returns violations (empty = holds).
fn must_have_violations(wf: &Workflow, mh: &MustHave) -> Vec<String> {
    let mut violations = Vec::new();

    let n = wf.stages.len() as i64;
    let [min, max] = mh.stage_count;
    if n < min || n > max {
        violations.push(format!(
            "stage count {n} outside required range [{min}, {max}]"
        ));
    }

    let tools = collect_tools(wf);
    for t in &mh.required_tools {
        if !tools.contains(t) {
            violations.push(format!("required tool '{t}' not referenced by any stage"));
        }
    }

    for needle in &mh.required_edge_conditions {
        let hit = wf
            .edges
            .iter()
            .flat_map(|e| e.conditions.iter())
            .any(|c| c.key.contains(needle.as_str()) || c.operator.contains(needle.as_str()));
        if !hit {
            violations.push(format!(
                "no edge condition key/operator contains required '{needle}'"
            ));
        }
    }

    violations
}

/// Classify a finished `BuildResult` against a case's `must_have` assertions.
fn classify(mh: &MustHave, result: &BuildResult) -> Outcome {
    if !result.valid {
        return Outcome::Refusal {
            attempts: result.attempts,
            parse_class: is_parse_class(&result.issues),
        };
    }
    let Some(wf) = result.workflow.as_ref() else {
        return Outcome::Fail {
            attempts: result.attempts,
            violations: vec!["builder contract violation: valid=true but no workflow".into()],
        };
    };
    let mut violations: Vec<String> = verify_workflow_graph(&to_graph(wf))
        .defects
        .iter()
        .map(|d| {
            format!(
                "graph defect {:?} at '{}': {}",
                d.kind, d.subject, d.explanation
            )
        })
        .collect();
    violations.extend(must_have_violations(wf, mh));
    if violations.is_empty() {
        Outcome::Pass {
            attempts: result.attempts,
        }
    } else {
        Outcome::Fail {
            attempts: result.attempts,
            violations,
        }
    }
}

/// Per-band scoring with the refusal cap applied.
#[derive(Debug, Clone, PartialEq)]
struct BandStats {
    band: String,
    total: usize,
    /// Cases counted as passed (direct + capped refusal passes).
    passed: usize,
    direct_passes: usize,
    /// Refusal-based passes actually counted (post-cap).
    refusal_passes: usize,
    /// Refusal-based would-be passes dropped by the cap.
    refusals_capped_out: usize,
    /// Total typed refusals in the band (pass or not).
    refusals: usize,
}

impl BandStats {
    fn rate(&self) -> f64 {
        if self.total == 0 {
            1.0
        } else {
            self.passed as f64 / self.total as f64
        }
    }
}

/// Score one band. A typed refusal passes only where the case is labeled
/// `refusal_ok: true`, and refusal-based passes are capped at
/// `floor(REFUSAL_CAP * total)` — in fixture (case) order, deterministically.
/// The cap is applied uniformly; only the adversarial band carries
/// `refusal_ok` labels (fixture-validator-enforced), so it can only ever fire
/// there.
fn score_band(band: &str, records: &[CaseRecord]) -> BandStats {
    let in_band: Vec<&CaseRecord> = records.iter().filter(|r| r.band == band).collect();
    let total = in_band.len();
    let allowance = (total as f64 * REFUSAL_CAP).floor() as usize;

    let mut direct_passes = 0usize;
    let mut refusal_passes = 0usize;
    let mut refusals_capped_out = 0usize;
    let mut refusals = 0usize;
    for r in &in_band {
        match &r.outcome {
            Outcome::Pass { .. } => direct_passes += 1,
            Outcome::Refusal { .. } => {
                refusals += 1;
                if r.refusal_ok {
                    if refusal_passes < allowance {
                        refusal_passes += 1;
                    } else {
                        refusals_capped_out += 1;
                    }
                }
            }
            Outcome::Fail { .. } | Outcome::Panicked { .. } => {}
        }
    }
    BandStats {
        band: band.to_string(),
        total,
        passed: direct_passes + refusal_passes,
        direct_passes,
        refusal_passes,
        refusals_capped_out,
        refusals,
    }
}

/// Repair statistics for one run.
#[derive(Debug, Clone, PartialEq)]
struct RepairStats {
    /// Number of repair-forcing cases in the fixture set.
    forcing_total: usize,
    /// Repair-forcing cases that ended in a full Pass.
    recovered: usize,
    /// Repair-forcing cases that actually exercised repair (attempts > 1).
    exercised: usize,
    /// Median repair iterations (attempts - 1) across ALL cases.
    median_repair_iters: f64,
}

fn median(mut xs: Vec<u32>) -> f64 {
    if xs.is_empty() {
        return 0.0;
    }
    xs.sort_unstable();
    let n = xs.len();
    if n % 2 == 1 {
        xs[n / 2] as f64
    } else {
        (xs[n / 2 - 1] as f64 + xs[n / 2] as f64) / 2.0
    }
}

fn repair_stats(records: &[CaseRecord]) -> RepairStats {
    let iters: Vec<u32> = records
        .iter()
        .map(|r| r.outcome.attempts().saturating_sub(1))
        .collect();
    let forcing: Vec<&CaseRecord> = records.iter().filter(|r| r.repair_forcing).collect();
    RepairStats {
        forcing_total: forcing.len(),
        recovered: forcing
            .iter()
            .filter(|r| matches!(r.outcome, Outcome::Pass { .. }))
            .count(),
        exercised: forcing.iter().filter(|r| r.outcome.attempts() > 1).count(),
        median_repair_iters: median(iters),
    }
}

/// One full pass over the 30 cases, scored.
#[derive(Debug, Clone)]
struct RunReport {
    bands: Vec<BandStats>,
    repair: RepairStats,
    panics: usize,
    /// Final parse-class failures on cases where a refusal is NOT acceptable —
    /// the spec's "zero unparseable outputs" defect class.
    unparseable: usize,
}

fn score_run(records: &[CaseRecord]) -> RunReport {
    let bands = BAND_TARGETS
        .iter()
        .map(|(band, _)| score_band(band, records))
        .collect();
    let panics = records
        .iter()
        .filter(|r| matches!(r.outcome, Outcome::Panicked { .. }))
        .count();
    let unparseable = records
        .iter()
        .filter(|r| {
            !r.refusal_ok
                && matches!(
                    r.outcome,
                    Outcome::Refusal {
                        parse_class: true,
                        ..
                    }
                )
        })
        .count();
    RunReport {
        bands,
        repair: repair_stats(records),
        panics,
        unparseable,
    }
}

/// Worst-of-N aggregation: per-band worst pass counts, worst repair recovery,
/// worst (highest) median repair iterations, total panics, worst unparseable.
#[derive(Debug, Clone)]
struct WorstReport {
    /// (band, worst passed, total) in `BAND_TARGETS` order.
    band_passed: Vec<(String, usize, usize)>,
    worst_recovered: usize,
    forcing_total: usize,
    worst_median_repair_iters: f64,
    total_panics: usize,
    worst_unparseable: usize,
}

fn worst_of(runs: &[RunReport]) -> WorstReport {
    assert!(!runs.is_empty(), "worst_of needs at least one run");
    let band_passed = (0..runs[0].bands.len())
        .map(|i| {
            let band = runs[0].bands[i].band.clone();
            let total = runs[0].bands[i].total;
            let worst = runs.iter().map(|r| r.bands[i].passed).min().unwrap_or(0);
            (band, worst, total)
        })
        .collect();
    WorstReport {
        band_passed,
        worst_recovered: runs.iter().map(|r| r.repair.recovered).min().unwrap_or(0),
        forcing_total: runs[0].repair.forcing_total,
        worst_median_repair_iters: runs
            .iter()
            .map(|r| r.repair.median_repair_iters)
            .fold(0.0, f64::max),
        total_panics: runs.iter().map(|r| r.panics).sum(),
        worst_unparseable: runs.iter().map(|r| r.unparseable).max().unwrap_or(0),
    }
}

/// The shipped-default eval catalog: the seven commodity built-in tools the
/// engine registers (`car-engine::agent_basics::entries()` — sourced from the
/// same `car_ir::builtins` definitions to avoid a heavy dev-dependency) plus
/// the models known to the inference engine. This mirrors what the daemon's
/// `builder.build` assembles from a real session, so the builder's
/// tool-existence cross-check is meaningful (e.g. the `quantum_teleport`
/// repair-forcing case relies on it).
fn eval_catalog(models: Vec<String>) -> ToolCatalog {
    let tools = [
        car_ir::builtins::read_file(),
        car_ir::builtins::list_dir(),
        car_ir::builtins::find_files(),
        car_ir::builtins::grep_files(),
        car_ir::builtins::calculate(),
        car_ir::builtins::write_file(),
        car_ir::builtins::edit_file(),
    ]
    .into_iter()
    .map(|s| ToolInfo {
        name: s.name,
        description: s.description,
    })
    .collect();
    ToolCatalog {
        agents: Vec::new(),
        tools,
        models,
    }
}

// ---------------------------------------------------------------------------
// The live runner (only reachable from the #[ignore]d test).
// ---------------------------------------------------------------------------

/// Run one case against the real inference engine. Panics inside the build are
/// caught via the task join handle and classified as `Outcome::Panicked`.
async fn run_case_live(
    engine: Arc<car_inference::InferenceEngine>,
    model: &str,
    catalog: &ToolCatalog,
    case: &EvalCase,
) -> Outcome {
    if let Some(outcome) = pre_refusal(case) {
        return outcome;
    }
    let goal = case.nl_request.clone();
    let catalog = catalog.clone();
    let model = model.to_string();
    let must_have = case.must_have.clone();
    let handle = tokio::spawn(async move {
        let req = BuildRequest {
            goal,
            catalog,
            existing: None,
            feedback: None,
            max_attempts: MAX_ATTEMPTS,
        };
        let result = build_workflow(
            |prompt: String| {
                let engine = engine.clone();
                let model = model.clone();
                async move {
                    let greq = car_inference::GenerateRequest {
                        prompt,
                        model: Some(model),
                        params: car_inference::GenerateParams {
                            temperature: TEMPERATURE,
                            max_tokens: MAX_TOKENS,
                            ..Default::default()
                        },
                        context: None,
                        context_stable_prefix: None,
                        tools: None,
                        images: None,
                        messages: None,
                        cache_control: false,
                        response_format: None,
                        intent: None,
                    };
                    engine
                        .generate_tracked(greq)
                        .await
                        .map(|r| r.text)
                        .map_err(|e| e.to_string())
                }
            },
            &req,
        )
        .await;
        classify(&must_have, &result)
    });
    match handle.await {
        Ok(outcome) => outcome,
        Err(e) => Outcome::Panicked {
            message: e.to_string(),
        },
    }
}

fn print_run_report(run_idx: usize, report: &RunReport) {
    eprintln!("\n== run {run_idx}/{RUNS} ==");
    for b in &report.bands {
        eprintln!(
            "  band {:<13} passed {}/{} ({:.0}%)  [direct={} refusal_passes={} refusals={} capped_out={}]",
            b.band,
            b.passed,
            b.total,
            b.rate() * 100.0,
            b.direct_passes,
            b.refusal_passes,
            b.refusals,
            b.refusals_capped_out,
        );
    }
    eprintln!(
        "  repair: recovered {}/{} forcing cases (exercised repair on {}), median repair iterations {:.1}",
        report.repair.recovered,
        report.repair.forcing_total,
        report.repair.exercised,
        report.repair.median_repair_iters,
    );
    eprintln!(
        "  panics={} unparseable_final={}",
        report.panics, report.unparseable
    );
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "hits a real model API; set ANTHROPIC_API_KEY and run with --ignored --nocapture"]
async fn live_nl2workflow_eval() {
    if std::env::var(KEY_ENV).is_err() {
        eprintln!(
            "[SKIP] {KEY_ENV} is not set — the live NL→workflow eval needs a real API key.\n\
             Run it with:\n\
             \n\
             \t{KEY_ENV}=sk-... cargo test -p car-builder --test live_nl2workflow_eval -- --ignored --nocapture\n\
             \n\
             Optional: {MODEL_ENV}=<catalog id> overrides the pinned model\n\
             (default: {PINNED_MODEL}; changing the pin is a spec-visible event —\n\
             record it in docs/proposals/h2-builder-discovery-acceptance.md)."
        );
        return;
    }

    let model = std::env::var(MODEL_ENV).unwrap_or_else(|_| PINNED_MODEL.to_string());
    let engine = Arc::new(car_inference::InferenceEngine::new(
        car_inference::InferenceConfig::default(),
    ));
    let models: Vec<String> = engine
        .list_models_unified()
        .into_iter()
        .map(|m| m.id)
        .collect();
    assert!(
        models.iter().any(|m| m == &model),
        "eval model '{model}' is not in the engine's unified registry — \
         the catalog moved out from under the pin; repin deliberately (spec-visible event)"
    );
    let catalog = eval_catalog(models);
    let cases = load_cases();

    // Full eval config — printed so a run's numbers are reproducible.
    eprintln!("== eval config (shipped defaults; no test-only knobs) ==");
    eprintln!(
        "  resolved model id : {model} (pinned default: {PINNED_MODEL}; override env: {MODEL_ENV})"
    );
    eprintln!("  temperature       : {TEMPERATURE}");
    eprintln!("  max_tokens        : {MAX_TOKENS}");
    eprintln!("  max_attempts      : {MAX_ATTEMPTS} (daemon builder.build default)");
    eprintln!("  repeat runs (N)   : {RUNS} — targets asserted on WORST-run counts");
    eprintln!(
        "  refusal cap       : {:.0}% of a band, refusal_ok-labeled cases only",
        REFUSAL_CAP * 100.0
    );
    eprintln!(
        "  catalog tools     : {}",
        catalog
            .tools
            .iter()
            .map(|t| t.name.as_str())
            .collect::<Vec<_>>()
            .join(", ")
    );
    eprintln!("  cases             : {} (3 bands x 10)", cases.len());
    for (band, target) in BAND_TARGETS {
        eprintln!("  target {:<13}: >= {:.0}%", band, target * 100.0);
    }
    eprintln!(
        "  repair target     : recovers >= {MIN_REPAIR_RECOVERED} repair-forcing cases; median repair iterations <= {MAX_MEDIAN_REPAIR_ITERS}"
    );

    let mut runs: Vec<RunReport> = Vec::with_capacity(RUNS);
    for run_idx in 1..=RUNS {
        let mut records: Vec<CaseRecord> = Vec::with_capacity(cases.len());
        for case in &cases {
            let outcome = run_case_live(engine.clone(), &model, &catalog, case).await;
            eprintln!("[run {run_idx}] {:<24} {}", case.id, outcome.short());
            records.push(CaseRecord {
                id: case.id.clone(),
                band: case.band.clone(),
                refusal_ok: case.refusal_ok == Some(true),
                repair_forcing: case.repair_forcing == Some(true),
                outcome,
            });
        }
        let report = score_run(&records);
        print_run_report(run_idx, &report);
        runs.push(report);
    }

    let worst = worst_of(&runs);
    eprintln!("\n== WORST-OF-{RUNS} (the acceptance numbers) ==");
    for (band, passed, total) in &worst.band_passed {
        eprintln!(
            "  band {:<13} worst {}/{} ({:.0}%)",
            band,
            passed,
            total,
            *passed as f64 / *total as f64 * 100.0
        );
    }
    eprintln!(
        "  repair recovered (worst) : {}/{}",
        worst.worst_recovered, worst.forcing_total
    );
    eprintln!(
        "  median repair iters (worst): {:.1}",
        worst.worst_median_repair_iters
    );
    eprintln!(
        "  panics={} unparseable_final(worst)={}",
        worst.total_panics, worst.worst_unparseable
    );

    // --- The acceptance gate: assert the spec targets on worst-run counts. ---
    assert_eq!(
        worst.total_panics, 0,
        "zero panics required (typed failure only)"
    );
    assert_eq!(
        worst.worst_unparseable, 0,
        "zero unparseable final outputs required on non-refusal cases (typed failure only)"
    );
    for ((band, passed, total), (_, target)) in worst.band_passed.iter().zip(BAND_TARGETS) {
        let rate = *passed as f64 / *total as f64;
        assert!(
            rate >= *target,
            "band '{band}' worst-run pass rate {:.0}% below target {:.0}% ({passed}/{total})",
            rate * 100.0,
            target * 100.0
        );
    }
    assert!(
        worst.worst_recovered >= MIN_REPAIR_RECOVERED,
        "repair loop recovered only {}/{} repair-forcing cases (need >= {MIN_REPAIR_RECOVERED})",
        worst.worst_recovered,
        worst.forcing_total
    );
    assert!(
        worst.worst_median_repair_iters <= MAX_MEDIAN_REPAIR_ITERS,
        "worst-run median repair iterations {:.1} exceeds {MAX_MEDIAN_REPAIR_ITERS}",
        worst.worst_median_repair_iters
    );

    eprintln!("\n[OK] Part 1 acceptance targets met on worst-of-{RUNS} counts with model {model}.");
}

// ---------------------------------------------------------------------------
// Unit tests for the pure harness core — no model, run by default.
// ---------------------------------------------------------------------------

fn wf(json: &str) -> Workflow {
    serde_json::from_str(json).expect("test workflow JSON parses")
}

/// read -> branch: write (on gt) / review; both branch targets terminal.
const BRANCHING_WF: &str = r#"{
    "id":"wf","name":"WF","start":"read",
    "stages":[
        {"id":"read","name":"Read","step":{"type":"proposal","proposal":{"id":"p1","source":"b","actions":[
            {"id":"a1","type":"tool_call","tool":"read_file","parameters":{}}
        ],"context":{}}}},
        {"id":"write","name":"Write","step":{"type":"proposal","proposal":{"id":"p2","source":"b","actions":[
            {"id":"a2","type":"tool_call","tool":"write_file","parameters":{}}
        ],"context":{}}}},
        {"id":"review","name":"Review","step":{"type":"approval","prompt":"ok?","fields":[],"output_key":"approval"}}
    ],
    "edges":[
        {"from":"read","to":"write","conditions":[{"key":"stage.read.total","operator":"gt","value":100}],"label":""},
        {"from":"read","to":"review","conditions":[{"key":"stage.read.total","operator":"lte","value":100}],"label":""}
    ]
}"#;

/// Structurally broken: an unreachable stage disconnected from the entry.
const UNREACHABLE_WF: &str = r#"{
    "id":"wf","name":"WF","start":"a",
    "stages":[
        {"id":"a","name":"A","step":{"type":"approval","prompt":"ok?","fields":[],"output_key":"k"}},
        {"id":"orphan","name":"Orphan","step":{"type":"approval","prompt":"?","fields":[],"output_key":"k2"}}
    ],
    "edges":[]
}"#;

/// Tools nested inside for_each body and a pattern agent.
const NESTED_TOOLS_WF: &str = r#"{
    "id":"wf","name":"WF","start":"fan",
    "stages":[
        {"id":"fan","name":"Fan","step":{"type":"for_each","items_from":"items","body":
            {"type":"proposal","proposal":{"id":"p","source":"b","actions":[
                {"id":"a","type":"tool_call","tool":"calculate","parameters":{}}
            ],"context":{}}},
            "max_concurrent":2}},
        {"id":"team","name":"Team","step":{"type":"pattern","pattern":"pipeline","task":"t","agents":[
            {"name":"g","system_prompt":"s","tools":["grep_files"],"max_turns":3,"metadata":{}}
        ],"config":{}}}
    ],
    "edges":[{"from":"fan","to":"team","conditions":[],"label":""}]
}"#;

fn mh(stage_count: [i64; 2], tools: &[&str], conds: &[&str]) -> MustHave {
    MustHave {
        stage_count,
        required_tools: tools.iter().map(|s| s.to_string()).collect(),
        required_edge_conditions: conds.iter().map(|s| s.to_string()).collect(),
    }
}

fn built(
    workflow: Option<Workflow>,
    valid: bool,
    issues: Vec<String>,
    attempts: u32,
) -> BuildResult {
    BuildResult {
        workflow,
        valid,
        issues,
        warnings: Vec::new(),
        attempts,
        raw: None,
    }
}

fn rec(id: &str, band: &str, refusal_ok: bool, forcing: bool, outcome: Outcome) -> CaseRecord {
    CaseRecord {
        id: id.into(),
        band: band.into(),
        refusal_ok,
        repair_forcing: forcing,
        outcome,
    }
}

#[test]
fn must_have_holds_on_matching_workflow() {
    let w = wf(BRANCHING_WF);
    let violations = must_have_violations(
        &w,
        &mh([3, 5], &["read_file", "write_file"], &["gt", "stage.read"]),
    );
    assert!(
        violations.is_empty(),
        "unexpected violations: {violations:?}"
    );
}

#[test]
fn must_have_flags_stage_count_out_of_range() {
    let w = wf(BRANCHING_WF); // 3 stages
    let violations = must_have_violations(&w, &mh([4, 6], &[], &[]));
    assert_eq!(violations.len(), 1);
    assert!(violations[0].contains("stage count 3"), "{violations:?}");
}

#[test]
fn must_have_flags_missing_tool_and_condition() {
    let w = wf(BRANCHING_WF);
    let violations = must_have_violations(&w, &mh([1, 9], &["grep_files"], &["approval"]));
    assert_eq!(violations.len(), 2, "{violations:?}");
    assert!(violations.iter().any(|v| v.contains("grep_files")));
    assert!(violations.iter().any(|v| v.contains("approval")));
}

#[test]
fn must_have_matches_operator_substring() {
    let w = wf(BRANCHING_WF);
    // "lte" only appears as an operator, not in any key.
    assert!(must_have_violations(&w, &mh([1, 9], &[], &["lte"])).is_empty());
}

#[test]
fn collect_tools_recurses_into_bodies_and_agents() {
    let w = wf(NESTED_TOOLS_WF);
    let tools = collect_tools(&w);
    assert!(tools.contains("calculate"), "for_each body tool");
    assert!(tools.contains("grep_files"), "pattern agent tool");
    assert!(must_have_violations(&w, &mh([1, 9], &["calculate", "grep_files"], &[])).is_empty());
}

#[test]
fn to_graph_derives_terminals_and_verifies_clean() {
    let w = wf(BRANCHING_WF);
    let g = to_graph(&w);
    assert_eq!(g.entry, "read");
    let mut terminals = g.terminals.clone();
    terminals.sort();
    assert_eq!(terminals, vec!["review".to_string(), "write".to_string()]);
    let report = verify_workflow_graph(&g);
    assert!(
        report.sound,
        "expected clean graph, got {:?}",
        report.defects
    );
}

#[test]
fn classify_flags_structural_graph_defect() {
    let w = wf(UNREACHABLE_WF);
    let outcome = classify(&mh([1, 9], &[], &[]), &built(Some(w), true, vec![], 1));
    match outcome {
        Outcome::Fail { violations, .. } => {
            assert!(
                violations.iter().any(|v| v.contains("orphan")),
                "{violations:?}"
            );
        }
        other => panic!("expected Fail, got {other:?}"),
    }
}

#[test]
fn classify_pass_refusal_and_must_have_fail() {
    let w = wf(BRANCHING_WF);
    // Pass: valid + clean graph + must_have holds.
    assert_eq!(
        classify(
            &mh([1, 9], &["read_file"], &["gt"]),
            &built(Some(w.clone()), true, vec![], 2)
        ),
        Outcome::Pass { attempts: 2 }
    );
    // Typed refusal: no valid manifest; parse-class detected from issues.
    assert_eq!(
        classify(
            &mh([1, 9], &[], &[]),
            &built(
                None,
                false,
                vec!["Your output did not parse as a workflow JSON object: x".into()],
                3
            )
        ),
        Outcome::Refusal {
            attempts: 3,
            parse_class: true
        }
    );
    // Non-parse typed failure is a refusal but not parse-class.
    assert_eq!(
        classify(
            &mh([1, 9], &[], &[]),
            &built(
                None,
                false,
                vec!["gate: edge to 'ghost' references unknown stage".into()],
                3
            )
        ),
        Outcome::Refusal {
            attempts: 3,
            parse_class: false
        }
    );
    // Valid manifest that misses must_have → Fail, not refusal.
    assert!(matches!(
        classify(
            &mh([1, 9], &["quantum_teleport"], &[]),
            &built(Some(w), true, vec![], 1)
        ),
        Outcome::Fail { .. }
    ));
}

#[test]
fn refusal_cap_limits_refusal_passes_to_forty_percent() {
    // 10 adversarial cases, all labeled refusal_ok, all refusing:
    // only floor(0.4 * 10) = 4 refusal passes may count.
    let records: Vec<CaseRecord> = (0..10)
        .map(|i| {
            rec(
                &format!("c{i}"),
                "adversarial",
                true,
                false,
                Outcome::Refusal {
                    attempts: 1,
                    parse_class: false,
                },
            )
        })
        .collect();
    let stats = score_band("adversarial", &records);
    assert_eq!(stats.total, 10);
    assert_eq!(stats.passed, 4, "cap must hold passes at 40%");
    assert_eq!(stats.refusal_passes, 4);
    assert_eq!(stats.refusals_capped_out, 6);
    assert_eq!(stats.refusals, 10);
}

#[test]
fn refusal_never_passes_where_not_labeled() {
    let records = vec![
        rec(
            "a",
            "adversarial",
            false,
            false,
            Outcome::Refusal {
                attempts: 3,
                parse_class: false,
            },
        ),
        rec(
            "b",
            "adversarial",
            true,
            false,
            Outcome::Refusal {
                attempts: 1,
                parse_class: false,
            },
        ),
        rec(
            "c",
            "adversarial",
            false,
            false,
            Outcome::Pass { attempts: 1 },
        ),
    ];
    let stats = score_band("adversarial", &records);
    assert_eq!(stats.passed, 2, "one direct pass + one labeled refusal");
    assert_eq!(stats.direct_passes, 1);
    assert_eq!(stats.refusal_passes, 1);
}

#[test]
fn direct_passes_are_never_capped() {
    // All 10 pass outright — the cap only constrains refusal-based passes.
    let records: Vec<CaseRecord> = (0..10)
        .map(|i| {
            rec(
                &format!("c{i}"),
                "adversarial",
                true,
                false,
                Outcome::Pass { attempts: 1 },
            )
        })
        .collect();
    let stats = score_band("adversarial", &records);
    assert_eq!(stats.passed, 10);
    assert_eq!(stats.refusals_capped_out, 0);
}

#[test]
fn repair_stats_count_recovery_and_median() {
    let records = vec![
        rec(
            "f1",
            "multi_stage",
            false,
            true,
            Outcome::Pass { attempts: 2 },
        ),
        rec(
            "f2",
            "multi_stage",
            false,
            true,
            Outcome::Pass { attempts: 3 },
        ),
        rec(
            "f3",
            "multi_stage",
            false,
            true,
            Outcome::Pass { attempts: 1 },
        ),
        rec(
            "f4",
            "multi_stage",
            false,
            true,
            Outcome::Fail {
                attempts: 3,
                violations: vec![],
            },
        ),
        rec(
            "f5",
            "multi_stage",
            false,
            true,
            Outcome::Refusal {
                attempts: 3,
                parse_class: false,
            },
        ),
        rec(
            "p1",
            "single_stage",
            false,
            false,
            Outcome::Pass { attempts: 1 },
        ),
    ];
    let stats = repair_stats(&records);
    assert_eq!(stats.forcing_total, 5);
    assert_eq!(stats.recovered, 3, "only full passes count as recovered");
    assert_eq!(stats.exercised, 4, "attempts > 1 among forcing cases");
    // Repair iterations: [1, 2, 0, 2, 2, 0] → sorted [0,0,1,2,2,2] → median 1.5.
    assert_eq!(stats.median_repair_iters, 1.5);
}

#[test]
fn median_handles_odd_even_and_empty() {
    assert_eq!(median(vec![]), 0.0);
    assert_eq!(median(vec![2]), 2.0);
    assert_eq!(median(vec![0, 1, 2]), 1.0);
    assert_eq!(median(vec![0, 0, 1, 1]), 0.5);
}

#[test]
fn worst_of_takes_per_band_minima_and_median_maximum() {
    // Three synthetic runs over one case per band with varying outcomes.
    let run = |b1_pass: bool, attempts: u32| {
        let records = vec![
            rec(
                "s1",
                "single_stage",
                false,
                false,
                if b1_pass {
                    Outcome::Pass { attempts }
                } else {
                    Outcome::Fail {
                        attempts,
                        violations: vec![],
                    }
                },
            ),
            rec("m1", "multi_stage", false, true, Outcome::Pass { attempts }),
            rec(
                "a1",
                "adversarial",
                true,
                false,
                Outcome::Refusal {
                    attempts: 1,
                    parse_class: false,
                },
            ),
        ];
        score_run(&records)
    };
    let runs = vec![run(true, 1), run(false, 3), run(true, 2)];
    let worst = worst_of(&runs);
    // single_stage worst = 0 (the failing middle run), multi_stage worst = 1.
    assert_eq!(worst.band_passed[0], ("single_stage".into(), 0, 1));
    assert_eq!(worst.band_passed[1], ("multi_stage".into(), 1, 1));
    // adversarial: floor(0.4 * 1) = 0 allowance → the labeled refusal is capped out.
    assert_eq!(worst.band_passed[2], ("adversarial".into(), 0, 1));
    // Worst median = the run with attempts=3 → repair iters median of [2,2,0] = 2.
    assert_eq!(worst.worst_median_repair_iters, 2.0);
    assert_eq!(worst.worst_recovered, 1);
    assert_eq!(worst.total_panics, 0);
}

#[test]
fn score_run_counts_panics_and_unparseable() {
    let records = vec![
        rec(
            "a",
            "single_stage",
            false,
            false,
            Outcome::Panicked {
                message: "boom".into(),
            },
        ),
        // Parse-class refusal on a case where refusal is NOT ok → unparseable defect.
        rec(
            "b",
            "multi_stage",
            false,
            false,
            Outcome::Refusal {
                attempts: 3,
                parse_class: true,
            },
        ),
        // Parse-class refusal where refusal IS ok → the typed-refusal path, not a defect.
        rec(
            "c",
            "adversarial",
            true,
            false,
            Outcome::Refusal {
                attempts: 3,
                parse_class: true,
            },
        ),
    ];
    let report = score_run(&records);
    assert_eq!(report.panics, 1);
    assert_eq!(report.unparseable, 1);
}

#[test]
fn empty_goal_is_pre_refused_without_a_model_call() {
    let cases = load_cases();
    let empty = cases.iter().find(|c| c.id == "b3-empty-request").unwrap();
    assert_eq!(
        pre_refusal(empty),
        Some(Outcome::Refusal {
            attempts: 0,
            parse_class: false
        })
    );
    let nonempty = cases.iter().find(|c| c.id == "b1-read-file").unwrap();
    assert_eq!(pre_refusal(nonempty), None);
}

#[test]
fn eval_catalog_lists_the_seven_commodity_tools() {
    let catalog = eval_catalog(vec!["m1".into()]);
    let names: HashSet<&str> = catalog.tools.iter().map(|t| t.name.as_str()).collect();
    for tool in [
        "read_file",
        "list_dir",
        "find_files",
        "grep_files",
        "calculate",
        "write_file",
        "edit_file",
    ] {
        assert!(names.contains(tool), "missing commodity tool {tool}");
    }
    assert_eq!(catalog.tools.len(), 7);
    assert_eq!(catalog.models, vec!["m1".to_string()]);
}

#[test]
fn fixture_repair_forcing_cases_cover_the_recovery_assertion() {
    // The live gate asserts recovery over these; keep the floor in sync with
    // the fixture validator (tests/eval_fixtures.rs).
    let cases = load_cases();
    let forcing = cases
        .iter()
        .filter(|c| c.repair_forcing == Some(true))
        .count();
    assert!(
        forcing > MIN_REPAIR_RECOVERED,
        "need at least {} repair-forcing cases for a >= {MIN_REPAIR_RECOVERED} recovery target, found {forcing}",
        MIN_REPAIR_RECOVERED + 1
    );
}