klieo_eval/

agent_eval.rs

//! Replay-first agent regression eval (ADR-047).
//!
//! A golden fixture is an ADR-046 [`Capture`]. [`eval_capture_determinism`]
//! replays it
//! through the *current* agent code via the klieo-runlog divergence engine —
//! the LLM and tool doubles are scripted from the recording, so a divergence
//! means the current code made a different decision given the same recorded
//! responses, i.e. a **code-/orchestration-behavior** regression. This is
//! deterministic and CI-stable.
//!
//! It does NOT measure model-quality drift (the LLM is scripted, not live), and
//! recorded latency / token cost are properties of the recording, not the
//! replay — they are reported but are not regression signals until a baseline is
//! re-recorded (live mode, deferred per ADR-047).

use std::sync::Arc;

use klieo_runlog::replay::{scripted_llm_from_runlog, scripted_tools_from_runlog};
use klieo_runlog::{replay_with_divergence, Capture, DivergenceReport, ReproVerdict, RunLogError};

/// Deterministic metrics for one replayed capture.
///
/// `reproduced`, `exact_match`, and `mean_similarity` are the **gated**
/// regression signals (see [`check_regression`]). `recorded_latency_ms` and
/// `recorded_total_tokens` come from the recorded run, not the replay, so they
/// are informational only — they move when a baseline is re-recorded, not when
/// code changes (ADR-047).
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
#[non_exhaustive]
pub struct EvalMetrics {
    /// Whole run reproduced byte-identically (`ReproVerdict::Identical`).
    pub reproduced: bool,
    /// Mean per-step divergence similarity; `1.0` when nothing diverged.
    pub mean_similarity: f64,
    /// The final recorded step reproduced (its output did not diverge) — the
    /// "did we still arrive at the same answer" signal, distinct from
    /// `reproduced` which requires every intermediate step to match.
    pub exact_match: bool,
    /// Summed step latency of the *recorded* run, in milliseconds (informational).
    pub recorded_latency_ms: u64,
    /// Prompt + completion tokens of the *recorded* run (informational).
    pub recorded_total_tokens: u32,
}

/// Replays `capture` against the *recorded* LLM/tool scripts and returns its
/// [`EvalMetrics`]. This measures **replay determinism only** — the scripted
/// doubles feed back the recorded bytes, so a faithful replay always reproduces.
/// It does NOT exercise the current agent logic against fresh model output; for
/// that (regression detection on a live re-drive) use [`eval_capture_live`].
/// Errors if the replay engine fails.
///
/// [`eval_capture_live`]: crate::eval_capture_live
pub async fn eval_capture_determinism(capture: &Capture) -> Result<EvalMetrics, RunLogError> {
    let log = &capture.run_log;
    let llm = Arc::new(scripted_llm_from_runlog("klieo-eval", log));
    let tools = Arc::new(scripted_tools_from_runlog(log));
    let report = replay_with_divergence(log, llm, tools, scripted_ctx()).await?;
    Ok(metrics_from(capture, &report))
}

/// Deprecated alias for [`eval_capture_determinism`].
#[deprecated(since = "2.3.0", note = "renamed to eval_capture_determinism")]
pub async fn eval_capture(capture: &Capture) -> Result<EvalMetrics, RunLogError> {
    eval_capture_determinism(capture).await
}

fn metrics_from(capture: &Capture, report: &DivergenceReport) -> EvalMetrics {
    let reproduced = report.verdict == ReproVerdict::Identical;
    let mean_similarity = if report.divergences.is_empty() {
        1.0
    } else {
        let sum: f64 = report.divergences.iter().map(|d| d.similarity).sum();
        sum / report.divergences.len() as f64
    };
    let steps = &capture.run_log.steps;
    let exact_match = match steps.len() {
        0 => true,
        n => {
            let last = (n - 1) as u32;
            report.divergences.iter().all(|d| d.step_index != last)
        }
    };
    let recorded_latency_ms = steps.iter().map(|s| s.latency.as_millis() as u64).sum();
    let recorded_total_tokens =
        capture.run_log.tokens.prompt_tokens + capture.run_log.tokens.completion_tokens;
    EvalMetrics {
        reproduced,
        mean_similarity,
        exact_match,
        recorded_latency_ms,
        recorded_total_tokens,
    }
}

/// One gated metric that moved the wrong way against the baseline.
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct Regression {
    /// Which gated metric regressed (`reproduced` / `exact_match` / `mean_similarity`).
    pub metric: &'static str,
    /// Human-readable baseline→current movement for the failure message.
    pub detail: String,
}

/// Compare `current` against `baseline`, returning a [`Regression`] per gated
/// metric that degraded. `reproduced` and `exact_match` regress on a `true →
/// false` flip; `mean_similarity` regresses when it drops by more than
/// `similarity_tolerance`. Latency and token cost are informational and never
/// gated here (ADR-047).
pub fn check_regression(
    baseline: &EvalMetrics,
    current: &EvalMetrics,
    similarity_tolerance: f64,
) -> Vec<Regression> {
    let mut regressions = Vec::new();
    if baseline.reproduced && !current.reproduced {
        regressions.push(Regression {
            metric: "reproduced",
            detail: "baseline reproduced the run; current diverged".into(),
        });
    }
    if baseline.exact_match && !current.exact_match {
        regressions.push(Regression {
            metric: "exact_match",
            detail: "baseline matched the final output; current did not".into(),
        });
    }
    if baseline.mean_similarity - current.mean_similarity > similarity_tolerance {
        regressions.push(Regression {
            metric: "mean_similarity",
            detail: format!(
                "similarity {:.3} -> {:.3} exceeds tolerance {:.3}",
                baseline.mean_similarity, current.mean_similarity, similarity_tolerance
            ),
        });
    }
    regressions
}

/// In-process bus `ToolCtx` for replay; the tool doubles ignore it, but the
/// divergence engine requires a context, and the harness must not bind a
/// concrete bus into the eval crate's public API.
fn scripted_ctx() -> klieo_core::tool::ToolCtx {
    use klieo_bus_memory::{MemoryJobQueue, MemoryKv, MemoryPubsub};
    let pubsub = Arc::new(MemoryPubsub::default());
    let kv = Arc::new(MemoryKv::default());
    let jobs = Arc::new(MemoryJobQueue::new(pubsub.clone(), kv.clone()));
    klieo_core::tool::ToolCtx::new(pubsub, kv, jobs)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn metrics(reproduced: bool, mean_similarity: f64, exact_match: bool) -> EvalMetrics {
        EvalMetrics {
            reproduced,
            mean_similarity,
            exact_match,
            recorded_latency_ms: 10,
            recorded_total_tokens: 100,
        }
    }

    #[test]
    fn identical_baseline_and_current_is_clean() {
        let b = metrics(true, 1.0, true);
        assert!(check_regression(&b, &b.clone(), 0.05).is_empty());
    }

    #[test]
    fn reproduced_true_to_false_regresses() {
        let r = check_regression(&metrics(true, 1.0, true), &metrics(false, 1.0, true), 0.05);
        assert!(r.iter().any(|x| x.metric == "reproduced"));
    }

    #[test]
    fn exact_match_true_to_false_regresses() {
        let r = check_regression(&metrics(true, 1.0, true), &metrics(true, 1.0, false), 0.05);
        assert!(r.iter().any(|x| x.metric == "exact_match"));
    }

    #[test]
    fn similarity_drop_past_tolerance_regresses() {
        let r = check_regression(&metrics(true, 1.0, true), &metrics(true, 0.80, true), 0.05);
        assert!(r.iter().any(|x| x.metric == "mean_similarity"));
    }

    #[test]
    fn similarity_drop_within_tolerance_is_clean() {
        let r = check_regression(&metrics(true, 1.0, true), &metrics(true, 0.98, true), 0.05);
        assert!(r.is_empty());
    }

    #[test]
    fn latency_and_token_changes_are_not_gated() {
        let b = metrics(true, 1.0, true);
        let mut c = metrics(true, 1.0, true);
        c.recorded_latency_ms = 9_999;
        c.recorded_total_tokens = 9_999;
        assert!(
            check_regression(&b, &c, 0.05).is_empty(),
            "recorded latency/tokens are informational, never gated"
        );
    }

    #[test]
    fn improvements_do_not_regress() {
        // false -> true and a similarity rise must not flag.
        let r = check_regression(&metrics(false, 0.7, false), &metrics(true, 1.0, true), 0.05);
        assert!(r.is_empty());
    }
}