klieo-runlog 3.4.0

//! Soft-compare reproducibility reporting (ADR-046): replay a recorded run and
//! collect per-step textual divergences instead of failing on the first
//! mismatch, so non-deterministic runs yield an auditable report.

use crate::error::RunLogError;
use crate::replay::step_outcome;
use crate::types::{RunLog, StepKind};
use klieo_core::llm::LlmClient;
use klieo_core::tool::{ToolCtx, ToolInvoker};
use klieo_core::RunId;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;

/// Textual similarity in `0.0..=1.0` via the Sørensen–Dice coefficient over
/// character bigrams (`1.0` = identical, `0.0` = no shared bigrams). v1 is
/// deliberately textual, not semantic: it can over-report drift on paraphrases
/// (ADR-046 documents this limitation). Strings shorter than one bigram fall
/// back to exact match. Linear in the combined input length: bigram
/// multiplicities are counted in a map so matching avoids a quadratic scan.
pub(crate) fn text_similarity(a: &str, b: &str) -> f64 {
    if a == b {
        return 1.0;
    }
    let left: Vec<char> = a.chars().collect();
    let right: Vec<char> = b.chars().collect();
    if left.len() < 2 || right.len() < 2 {
        return 0.0;
    }
    let mut right_counts: HashMap<[char; 2], usize> = HashMap::with_capacity(right.len() - 1);
    for w in right.windows(2) {
        *right_counts.entry([w[0], w[1]]).or_insert(0) += 1;
    }
    let mut shared = 0usize;
    for w in left.windows(2) {
        if let Some(count) = right_counts.get_mut(&[w[0], w[1]]) {
            if *count > 0 {
                *count -= 1;
                shared += 1;
            }
        }
    }
    (2.0 * shared as f64) / (left.len() - 1 + right.len() - 1) as f64
}

/// Outcome of a reproducibility replay.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub enum ReproVerdict {
    /// Holds exactly when `DivergenceReport.divergences` is empty.
    Identical,
    /// Holds when `DivergenceReport.divergences` is non-empty; each entry's
    /// `similarity` quantifies that step's drift.
    Diverged,
}

/// Which replay path produced a [`DivergenceReport`]. This is the difference
/// between "the recording re-reads its own bytes" and "the agent's logic was
/// re-run", and it is the whole meaning of an `Identical` verdict — a renderer
/// MUST surface it (ADR-046 §4 / ADR-048 §Context).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[non_exhaustive]
pub enum ReplayMode {
    /// Doubles are reconstructed from the **same recording** the run produced
    /// (`scripted_*_from_runlog`), so `expected == actual` holds by
    /// construction. An `Identical` verdict here proves only internal
    /// self-consistency — it is **not** evidence the agent's logic reproduces
    /// the decision. This is the conservative default: a report deserialised
    /// without a mode (pre-`ReplayMode` data) is treated as self-consistency so
    /// it can never silently over-claim.
    #[default]
    ScriptedSelfConsistency,
    /// The current agent loop was re-driven against recorded LLM/tool doubles,
    /// so a divergence reflects a real change in agent logic since the recording
    /// and an `Identical` verdict is a genuine agent-logic reproduction. The
    /// live re-drive itself (`klieo_eval::eval_capture_live`) currently reports
    /// via `LiveEvalMetrics`; emitting a `DivergenceReport` carrying this mode is
    /// deferred (ADR-048). The variant exists so the verdict vocabulary is
    /// complete and the scripted path can name what it is *not*.
    LiveAgentReDrive,
}

impl ReplayMode {
    /// Human-readable qualifier appended to an `Identical` verdict so a reader
    /// never mistakes self-consistency for agent-logic reproduction.
    pub fn identical_qualifier(self) -> &'static str {
        match self {
            ReplayMode::ScriptedSelfConsistency => {
                "self-consistency only — NOT an agent-logic reproduction"
            }
            ReplayMode::LiveAgentReDrive => "agent-logic re-drive reproduced the recording",
        }
    }
}

/// One step where the replayed output differed from the recording. `similarity`
/// is `text_similarity(expected, actual)`; below `1.0` on a deterministic step
/// it is a true reproduction failure, while on a non-deterministic
/// (temperature>0) step it quantifies drift for an auditor.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct StepDivergence {
    /// Zero-based index into the source `RunLog.steps`.
    pub step_index: u32,
    /// Copied from the diverging step; only `LlmCall` and `ToolCall` can
    /// diverge, since other step kinds re-issue no I/O on replay.
    pub kind: StepKind,
    /// Recorded output, normalized through `flatten_output` — the exact form
    /// the comparison ran against.
    pub expected: String,
    /// Replayed output, normalized the same way as `expected`.
    pub actual: String,
    /// `text_similarity(expected, actual)`; read it per the type-level doc —
    /// a true failure on deterministic steps, drift on non-deterministic ones.
    pub similarity: f64,
}

/// Structured reproducibility result for one run: the per-step divergences and
/// the overall verdict. An empty `divergences` list means byte-identical
/// reproduction.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct DivergenceReport {
    /// Carried from the replayed `RunLog.run_id` so the report self-identifies
    /// once detached from its source.
    pub run_id: RunId,
    /// Ordered by step index, one entry per diverging step. Empty exactly when
    /// `verdict` is `Identical`.
    pub divergences: Vec<StepDivergence>,
    /// Redundant with `divergences.is_empty()`, stored so a serialized report
    /// states its outcome without the reader re-deriving it.
    pub verdict: ReproVerdict,
    /// Which replay path produced this report. An `Identical` verdict is only
    /// agent-logic reproduction when this is [`ReplayMode::LiveAgentReDrive`];
    /// otherwise it is self-consistency. `#[serde(default)]` keeps older
    /// serialized reports (no mode) deserialisable — they default to the
    /// conservative [`ReplayMode::ScriptedSelfConsistency`] so they never
    /// over-claim.
    #[serde(default)]
    pub mode: ReplayMode,
}

/// Replay `run_log` against the supplied doubles and report every step whose
/// output differs from the recording — the soft-compare counterpart to
/// [`crate::replay::replay`], which fast-fails on the first mismatch. A double
/// that errors (script exhausted, tool failure) is an infrastructure failure
/// and returns `Err`; a value difference is recorded as a [`StepDivergence`],
/// never an error.
pub async fn replay_with_divergence(
    run_log: &RunLog,
    llm: Arc<dyn LlmClient>,
    tools: Arc<dyn ToolInvoker>,
    ctx: ToolCtx,
) -> Result<DivergenceReport, RunLogError> {
    let mut divergences = Vec::new();
    for step in &run_log.steps {
        let Some(outcome) = step_outcome(step, &llm, &tools, &ctx).await? else {
            continue;
        };
        if outcome.actual != outcome.expected {
            divergences.push(StepDivergence {
                step_index: step.idx,
                kind: step.kind,
                similarity: text_similarity(&outcome.expected, &outcome.actual),
                expected: outcome.expected,
                actual: outcome.actual,
            });
        }
    }
    let verdict = if divergences.is_empty() {
        ReproVerdict::Identical
    } else {
        ReproVerdict::Diverged
    };
    Ok(DivergenceReport {
        run_id: run_log.run_id,
        divergences,
        verdict,
        // Self-consistency: the doubles come from the recording, so this never
        // claims re-drive. Agent-logic re-drive lives in
        // `klieo_eval::eval_capture_live` (a separate, final-output comparison);
        // routing that through a `LiveAgentReDrive`-labelled report is deferred
        // (ADR-048).
        mode: ReplayMode::ScriptedSelfConsistency,
    })
}

#[cfg(test)]
mod similarity_tests {
    use super::text_similarity;

    #[test]
    fn identical_strings_score_one() {
        assert_eq!(
            text_similarity("transfer approved", "transfer approved"),
            1.0
        );
    }

    #[test]
    fn both_empty_score_one() {
        assert_eq!(text_similarity("", ""), 1.0);
    }

    #[test]
    fn disjoint_strings_score_zero() {
        assert_eq!(text_similarity("abcd", "wxyz"), 0.0);
    }

    #[test]
    fn one_empty_one_nonempty_scores_zero() {
        assert_eq!(text_similarity("", "abcd"), 0.0);
    }

    #[test]
    fn partial_overlap_scores_between_zero_and_one() {
        let s = text_similarity("night", "nacht");
        assert!(s > 0.0 && s < 1.0, "expected partial overlap, got {s}");
    }

    #[test]
    fn single_char_strings_use_exact_match() {
        assert_eq!(text_similarity("a", "a"), 1.0);
        assert_eq!(text_similarity("a", "b"), 0.0);
    }

    #[test]
    fn repeated_bigrams_count_with_multiplicity() {
        assert_eq!(text_similarity("aaaa", "aaaa"), 1.0);
        // "aaab" shares only the bigrams it actually has, not unbounded "aa" hits.
        let s = text_similarity("aaaa", "aaab");
        assert!(s > 0.0 && s < 1.0, "expected partial overlap, got {s}");
    }
}

#[cfg(test)]
mod report_types_tests {
    use super::{DivergenceReport, ReplayMode, ReproVerdict, StepDivergence};
    use crate::types::StepKind;
    use klieo_core::RunId;

    #[test]
    fn identical_report_round_trips_with_empty_divergences() {
        let report = DivergenceReport {
            run_id: RunId::new(),
            divergences: vec![],
            verdict: ReproVerdict::Identical,
            mode: ReplayMode::ScriptedSelfConsistency,
        };
        let json = serde_json::to_string(&report).unwrap();
        let back: DivergenceReport = serde_json::from_str(&json).unwrap();
        assert!(back.divergences.is_empty());
        assert_eq!(back.verdict, ReproVerdict::Identical);
        assert_eq!(back.mode, ReplayMode::ScriptedSelfConsistency);
    }

    #[test]
    fn diverged_report_round_trips_with_step_detail() {
        let report = DivergenceReport {
            run_id: RunId::new(),
            divergences: vec![StepDivergence {
                step_index: 2,
                kind: StepKind::LlmCall,
                expected: "approve".into(),
                actual: "deny".into(),
                similarity: 0.0,
            }],
            verdict: ReproVerdict::Diverged,
            mode: ReplayMode::LiveAgentReDrive,
        };
        let json = serde_json::to_string(&report).unwrap();
        let back: DivergenceReport = serde_json::from_str(&json).unwrap();
        assert_eq!(back.divergences.len(), 1);
        assert_eq!(back.divergences[0].step_index, 2);
        assert_eq!(back.verdict, ReproVerdict::Diverged);
        assert_eq!(back.mode, ReplayMode::LiveAgentReDrive);
    }

    #[test]
    fn report_without_mode_field_deserializes_to_self_consistency() {
        // Pre-`ReplayMode` serialized report (no `mode` key) must stay readable
        // and default to the conservative mode — never silently over-claim.
        let legacy = format!(
            r#"{{"run_id":"{}","divergences":[],"verdict":"Identical"}}"#,
            RunId::new()
        );
        let back: DivergenceReport = serde_json::from_str(&legacy).unwrap();
        assert_eq!(back.verdict, ReproVerdict::Identical);
        assert_eq!(back.mode, ReplayMode::ScriptedSelfConsistency);
    }

    #[test]
    fn default_replay_mode_is_self_consistency() {
        assert_eq!(ReplayMode::default(), ReplayMode::ScriptedSelfConsistency);
    }

    #[test]
    fn each_mode_qualifier_states_its_kind_distinctly() {
        let scripted = ReplayMode::ScriptedSelfConsistency.identical_qualifier();
        let live = ReplayMode::LiveAgentReDrive.identical_qualifier();
        assert!(
            scripted.contains("NOT an agent-logic reproduction"),
            "self-consistency qualifier must disclaim reproduction: {scripted}"
        );
        assert!(
            live.contains("reproduced"),
            "re-drive qualifier must affirm reproduction: {live}"
        );
        assert_ne!(scripted, live, "the two modes must read differently");
    }
}

#[cfg(test)]
mod divergence_walk_tests {
    use super::*;
    use crate::replay::{NoopToolInvoker, ScriptedLlmClient};
    use crate::types::{RunLog, RunStatus, Step, Usage};
    use chrono::Utc;
    use std::sync::Arc;

    fn step(idx: u32, kind: StepKind, output: serde_json::Value) -> Step {
        Step {
            idx,
            kind,
            name: None,
            prompt_tokens: None,
            completion_tokens: None,
            cost_usd: None,
            input: serde_json::Value::Null,
            output,
            error: None,
            latency: std::time::Duration::ZERO,
            span_id: None,
        }
    }

    fn run_log(steps: Vec<Step>) -> RunLog {
        let now = Utc::now();
        RunLog {
            run_id: RunId::new(),
            agent: "t".into(),
            started_at: now,
            finished_at: Some(now),
            status: RunStatus::Completed,
            steps,
            tokens: Usage::default(),
            cost_estimate: None,
        }
    }

    fn ctx() -> ToolCtx {
        let bus = klieo_core::test_utils::noop_bus();
        ToolCtx::new(bus.0, bus.2, bus.3)
    }

    #[tokio::test]
    async fn identical_replay_yields_identical_verdict_no_divergences() {
        let log = run_log(vec![step(0, StepKind::LlmCall, serde_json::json!("hello"))]);
        let llm = Arc::new(ScriptedLlmClient::new("r", vec!["hello".into()]));
        let report = replay_with_divergence(&log, llm, Arc::new(NoopToolInvoker), ctx())
            .await
            .unwrap();
        assert!(report.divergences.is_empty());
        assert_eq!(report.verdict, ReproVerdict::Identical);
        // The scripted soft-compare path must label itself self-consistency so
        // an `Identical` here is never read as agent-logic reproduction.
        assert_eq!(report.mode, ReplayMode::ScriptedSelfConsistency);
    }

    #[tokio::test]
    async fn mismatched_llm_step_records_divergence_and_diverged_verdict() {
        let log = run_log(vec![step(
            0,
            StepKind::LlmCall,
            serde_json::json!("approve"),
        )]);
        let llm = Arc::new(ScriptedLlmClient::new("r", vec!["deny".into()]));
        let report = replay_with_divergence(&log, llm, Arc::new(NoopToolInvoker), ctx())
            .await
            .unwrap();
        assert_eq!(report.verdict, ReproVerdict::Diverged);
        assert_eq!(report.divergences.len(), 1);
        assert_eq!(report.divergences[0].step_index, 0);
        assert_eq!(report.divergences[0].expected, "approve");
        assert_eq!(report.divergences[0].actual, "deny");
        assert!(report.divergences[0].similarity < 1.0);
    }

    #[tokio::test]
    async fn walk_runs_to_completion_collecting_every_divergence() {
        let log = run_log(vec![
            step(0, StepKind::LlmCall, serde_json::json!("a")),
            step(1, StepKind::LlmCall, serde_json::json!("b")),
        ]);
        let llm = Arc::new(ScriptedLlmClient::new("r", vec!["x".into(), "y".into()]));
        let report = replay_with_divergence(&log, llm, Arc::new(NoopToolInvoker), ctx())
            .await
            .unwrap();
        assert_eq!(report.divergences.len(), 2);
        assert_eq!(report.divergences[0].step_index, 0);
        assert_eq!(report.divergences[0].expected, "a");
        assert_eq!(report.divergences[0].actual, "x");
        assert_eq!(report.divergences[1].step_index, 1);
        assert_eq!(report.divergences[1].expected, "b");
        assert_eq!(report.divergences[1].actual, "y");
    }

    #[tokio::test]
    async fn mismatched_tool_step_records_tool_divergence() {
        let tool_step = Step {
            idx: 0,
            kind: StepKind::ToolCall,
            name: Some("calc".into()),
            prompt_tokens: None,
            completion_tokens: None,
            cost_usd: None,
            input: serde_json::json!({}),
            output: serde_json::json!({"y": 4}),
            error: None,
            latency: std::time::Duration::ZERO,
            span_id: None,
        };
        let log = run_log(vec![tool_step]);
        let llm = Arc::new(ScriptedLlmClient::new("r", vec![]));
        let tools = Arc::new(klieo_core::test_utils::FakeToolInvoker::new().with_tool(
            "calc",
            "calculator",
            |_| Ok(serde_json::json!({"y": 99})),
        ));
        let report = replay_with_divergence(&log, llm, tools, ctx())
            .await
            .unwrap();
        assert_eq!(report.verdict, ReproVerdict::Diverged);
        assert_eq!(report.divergences.len(), 1);
        assert_eq!(report.divergences[0].kind, StepKind::ToolCall);
        assert_eq!(report.divergences[0].expected, "{\"y\":4}");
        assert_eq!(report.divergences[0].actual, "{\"y\":99}");
        assert!(report.divergences[0].similarity < 1.0);
    }

    #[tokio::test]
    async fn empty_runlog_yields_identical_with_no_divergences() {
        let log = run_log(vec![]);
        let llm = Arc::new(ScriptedLlmClient::new("r", vec![]));
        let report = replay_with_divergence(&log, llm, Arc::new(NoopToolInvoker), ctx())
            .await
            .unwrap();
        assert!(report.divergences.is_empty());
        assert_eq!(report.verdict, ReproVerdict::Identical);
    }

    #[tokio::test]
    async fn double_failure_is_err_with_preserved_source() {
        use std::error::Error;
        let log = run_log(vec![step(0, StepKind::LlmCall, serde_json::json!("a"))]);
        let llm = Arc::new(ScriptedLlmClient::new("r", vec![]));
        let err = replay_with_divergence(&log, llm, Arc::new(NoopToolInvoker), ctx())
            .await
            .unwrap_err();
        assert!(matches!(
            err,
            crate::error::RunLogError::ReplayStep { step: 0, .. }
        ));
        let source = err.source().expect("ReplayStep must carry a typed source");
        assert!(
            source.to_string().contains("exhausted"),
            "source should be the script-exhausted LlmError; got: {source}"
        );
    }

    #[tokio::test]
    async fn summary_and_ops_steps_are_skipped() {
        let log = run_log(vec![
            step(0, StepKind::SummaryCheckpoint, serde_json::Value::Null),
            step(1, StepKind::OpsEvent, serde_json::Value::Null),
            step(2, StepKind::LlmCall, serde_json::json!("ok")),
        ]);
        let llm = Arc::new(ScriptedLlmClient::new("r", vec!["ok".into()]));
        let report = replay_with_divergence(&log, llm, Arc::new(NoopToolInvoker), ctx())
            .await
            .unwrap();
        assert_eq!(report.verdict, ReproVerdict::Identical);
    }
}