klieo-eval 3.4.0

//! LLM-judge scorer over an existing replayed trajectory (plan item F,
//! klieo-vs-adk-improvement-plan-2026-07-21).
//!
//! [`eval_capture_determinism`](crate::eval_capture_determinism) (behind
//! `agent-eval`) answers "did the code make the same decision as the
//! recording?" — deterministic, CI-stable, audit-grade. This module answers a
//! different question: "is this trajectory *good*?" — inherently a judgment
//! call, so it calls an LLM and is **not** deterministic. Keeping it behind
//! its own `llm-judge` feature (rather than folding into `agent-eval`) keeps
//! the audit-grade replay surface free of that non-determinism.
//!
//! The judge takes an already-replayed [`RunLog`] — it does not re-run or
//! re-drive anything itself — and scores it against caller-supplied
//! [`JudgeCriterion`]s. Each per-criterion verdict is a
//! [`klieo_spec::Critique`], reusing `klieo-spec`'s pass/fail + free-form
//! rationale shape instead of inventing a parallel one; [`Judge`] itself is a
//! new trait because `klieo_spec::Critic<T>` returns a single `Critique` per
//! call and has no room for a named multi-criterion breakdown.

use std::sync::Arc;

use async_trait::async_trait;
use klieo_core::llm::ChatRequest;
use klieo_core::response::{parse_structured, KlieoResponse};
use klieo_core::{Error as CoreError, LlmClient, LlmError};
use klieo_runlog::types::RunLog;
use klieo_spec::Critique;
use serde::Deserialize;

/// One named quality bar a [`Judge`] scores a trajectory against.
///
/// `name` is the stable key callers use to look up the corresponding
/// [`CriterionVerdict`] via [`JudgeVerdict::verdict_for`]; `description` is
/// the free-form instruction sent to the judge LLM.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct JudgeCriterion {
    /// Stable key identifying this criterion in the resulting [`JudgeVerdict`].
    pub name: String,
    /// Free-form instruction describing what "pass" means for this criterion.
    pub description: String,
}

impl JudgeCriterion {
    /// Build a criterion from its name and scoring instruction.
    pub fn new(name: impl Into<String>, description: impl Into<String>) -> Self {
        Self {
            name: name.into(),
            description: description.into(),
        }
    }
}

/// One criterion's outcome — the criterion it scores, plus the
/// [`klieo_spec::Critique`] (pass/fail + rationale) the judge produced for it.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct CriterionVerdict {
    /// The [`JudgeCriterion::name`] this verdict answers.
    pub criterion: String,
    /// Pass/fail plus free-form rationale for this criterion.
    pub critique: Critique,
}

/// Full judge output for one replayed run — one [`CriterionVerdict`] per input
/// [`JudgeCriterion`].
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct JudgeVerdict {
    /// One verdict per requested criterion, in the order [`Judge::judge`] was
    /// called with.
    pub per_criterion: Vec<CriterionVerdict>,
}

impl JudgeVerdict {
    /// Whether every criterion passed. `true` on an empty verdict (vacuous).
    pub fn all_pass(&self) -> bool {
        self.per_criterion.iter().all(|v| v.critique.pass)
    }

    /// Look up the verdict for a given criterion name.
    pub fn verdict_for(&self, criterion: &str) -> Option<&CriterionVerdict> {
        self.per_criterion.iter().find(|v| v.criterion == criterion)
    }
}

/// Errors raised while judging a replayed trajectory.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum JudgeError {
    /// The underlying LLM call failed.
    #[error("llm-judge request failed: {0}")]
    Llm(#[source] LlmError),
    /// The LLM reply could not be parsed into the expected verdict shape.
    #[error("llm-judge response was not valid: {0}")]
    Parse(#[source] CoreError),
    /// Caller misuse — e.g. an empty criteria list.
    #[error("llm-judge config error: {0}")]
    Config(String),
    /// The parsed response omitted a verdict for a requested criterion.
    #[error("llm-judge response missing a verdict for criterion {0:?}")]
    MissingCriterion(String),
}

/// Scores an existing replayed [`RunLog`] against named criteria, producing a
/// per-criterion pass/fail + rationale [`JudgeVerdict`].
///
/// This is the exploratory-quality path: implementations typically call an
/// LLM and are therefore non-deterministic. Contrast with
/// [`eval_capture_determinism`](crate::eval_capture_determinism), which is
/// deterministic and audit-grade.
#[async_trait]
pub trait Judge: Send + Sync {
    /// Score `run_log` against `criteria`. Returns [`JudgeError::Config`] if
    /// `criteria` is empty — there is nothing to score.
    async fn judge(
        &self,
        run_log: &RunLog,
        criteria: &[JudgeCriterion],
    ) -> Result<JudgeVerdict, JudgeError>;
}

/// Default [`Judge`] impl: asks an injected `Arc<dyn LlmClient>` to score the
/// run and parses its reply into a [`JudgeVerdict`].
///
/// The LLM is injected rather than constructed internally so callers can
/// point it at any provider — and so tests can inject
/// `klieo_core::test_utils::FakeLlmClient` with a scripted verdict instead of
/// making a live call.
pub struct LlmJudge {
    llm: Arc<dyn LlmClient>,
}

impl LlmJudge {
    /// Build a judge backed by the given LLM client.
    pub fn new(llm: Arc<dyn LlmClient>) -> Self {
        Self { llm }
    }
}

#[async_trait]
impl Judge for LlmJudge {
    async fn judge(
        &self,
        run_log: &RunLog,
        criteria: &[JudgeCriterion],
    ) -> Result<JudgeVerdict, JudgeError> {
        if criteria.is_empty() {
            return Err(JudgeError::Config(
                "at least one criterion is required".into(),
            ));
        }
        let request = build_request(run_log, criteria);
        let response = self.llm.complete(request).await.map_err(JudgeError::Llm)?;
        let raw: RawJudgeResponse =
            parse_structured(&response.message.content).map_err(JudgeError::Parse)?;
        to_verdict(raw, criteria)
    }
}

const JUDGE_SYSTEM_PROMPT: &str = "You are an evaluation judge for an AI agent's \
    recorded run. You will be given the run's steps (tool calls and LLM calls, \
    in order) and a list of criteria. For EVERY criterion, decide pass or fail \
    and give a one- or two-sentence rationale grounded in the run's actual \
    steps. Respond with ONLY JSON matching this shape: \
    {\"verdicts\":[{\"criterion\":\"<name>\",\"pass\":<bool>,\"rationale\":\"<text>\"}]}";

fn build_request(run_log: &RunLog, criteria: &[JudgeCriterion]) -> ChatRequest {
    let steps_json =
        serde_json::to_string_pretty(&run_log.steps).unwrap_or_else(|_| "[]".to_string());
    let criteria_list = criteria
        .iter()
        .map(|c| format!("- {}: {}", c.name, c.description))
        .collect::<Vec<_>>()
        .join("\n");
    let user_prompt = format!(
        "Run steps (agent {:?}):\n{steps_json}\n\nCriteria:\n{criteria_list}",
        run_log.agent
    );
    ChatRequest::builder()
        .system(JUDGE_SYSTEM_PROMPT)
        .user(user_prompt)
        .build()
}

#[derive(Debug, Deserialize)]
struct RawCriterionVerdict {
    criterion: String,
    pass: bool,
    rationale: String,
}

#[derive(Debug, Deserialize)]
struct RawJudgeResponse {
    verdicts: Vec<RawCriterionVerdict>,
}

impl KlieoResponse for RawJudgeResponse {
    fn json_schema() -> serde_json::Value {
        serde_json::json!({
            "type": "object",
            "properties": {
                "verdicts": {
                    "type": "array",
                    "items": {
                        "type": "object",
                        "properties": {
                            "criterion": {"type": "string"},
                            "pass": {"type": "boolean"},
                            "rationale": {"type": "string"},
                        },
                        "required": ["criterion", "pass", "rationale"],
                    },
                },
            },
            "required": ["verdicts"],
        })
    }
}

/// Map the raw parsed response onto the requested criteria, matching by
/// name. Errors (rather than silently dropping) when the judge omitted a
/// requested criterion — a silent gap would let a criterion pass by
/// omission.
fn to_verdict(
    raw: RawJudgeResponse,
    criteria: &[JudgeCriterion],
) -> Result<JudgeVerdict, JudgeError> {
    let per_criterion = criteria
        .iter()
        .map(|c| {
            raw.verdicts
                .iter()
                .find(|v| v.criterion == c.name)
                .map(|v| CriterionVerdict {
                    criterion: c.name.clone(),
                    critique: if v.pass {
                        Critique::pass(v.rationale.clone())
                    } else {
                        Critique::fail(v.rationale.clone())
                    },
                })
                .ok_or_else(|| JudgeError::MissingCriterion(c.name.clone()))
        })
        .collect::<Result<Vec<_>, _>>()?;
    Ok(JudgeVerdict { per_criterion })
}

#[cfg(test)]
mod tests {
    use super::*;

    fn crit(name: &str) -> JudgeCriterion {
        JudgeCriterion::new(name, "desc")
    }

    #[test]
    fn all_pass_true_on_empty_verdict() {
        let v = JudgeVerdict {
            per_criterion: vec![],
        };
        assert!(v.all_pass());
    }

    #[test]
    fn all_pass_false_when_any_criterion_fails() {
        let v = JudgeVerdict {
            per_criterion: vec![
                CriterionVerdict {
                    criterion: "a".into(),
                    critique: Critique::pass("ok"),
                },
                CriterionVerdict {
                    criterion: "b".into(),
                    critique: Critique::fail("nope"),
                },
            ],
        };
        assert!(!v.all_pass());
    }

    #[test]
    fn to_verdict_maps_by_name_regardless_of_order() {
        let raw = RawJudgeResponse {
            verdicts: vec![
                RawCriterionVerdict {
                    criterion: "b".into(),
                    pass: false,
                    rationale: "bad".into(),
                },
                RawCriterionVerdict {
                    criterion: "a".into(),
                    pass: true,
                    rationale: "good".into(),
                },
            ],
        };
        let verdict = to_verdict(raw, &[crit("a"), crit("b")]).unwrap();
        assert_eq!(verdict.verdict_for("a").unwrap().critique.feedback, "good");
        assert_eq!(verdict.verdict_for("b").unwrap().critique.feedback, "bad");
    }

    #[test]
    fn to_verdict_errors_on_missing_criterion() {
        let raw = RawJudgeResponse { verdicts: vec![] };
        let err = to_verdict(raw, &[crit("a")]).unwrap_err();
        assert!(matches!(err, JudgeError::MissingCriterion(name) if name == "a"));
    }
}