wabot_testing/

llm_judge.rs

//! Grade a conversation with a real LLM. Port of
//! `wabot-ts/src/testing/LlmJudge.ts`.
//!
//! ```ignore
//! let judge = LlmJudge::new(adapter, vec![ModelRef::new("openai", "gpt-5")]);
//! judge
//!     .assert(harness.history(), "the bot gave the tracking number and stayed polite")
//!     .await?;
//! ```
//!
//! ## What this is for, and what it isn't
//!
//! Every other harness here asserts something exact — this reply,
//! that tool call. Some properties aren't exact: *did it stay on
//! topic*, *did it refuse without being rude*, *did it avoid
//! promising a refund*. Those are the ones that regress silently,
//! because nobody writes a brittle string match for them.
//!
//! It is **not** a unit test. It calls a paid API, it is slow, and
//! the same input can grade differently twice. Keep judged tests few,
//! behind an env var, and out of the loop developers run on every
//! save.
//!
//! ## The verdict comes back as a tool call, not as prose
//!
//! Asking a model for "PASS or FAIL" means parsing free-form text —
//! and a model that answers "PASS (with reservations)" quietly
//! becomes a failure, or worse, a pass. A forced tool call gives a
//! typed `pass: bool` the provider itself validated, and it works the
//! same across all six adapters because tool calling is the one thing
//! they all speak.
//!
//! A judge that answers with text anyway is an **error**, not a
//! failure: those are different things, and reporting "the criteria
//! were not met" when the judge never rendered a verdict would be a
//! lie about your code.

use std::sync::Arc;

use serde::Deserialize;
use thiserror::Error;
use wabot_core::validation::Validate;
use wabot_feature_chat_bot::{
    ChatAdapter, ChatAdapterRequest, ChatItem, ChatMessage, ModelRef, ToolDefinition, ToolParameter,
};
use wabot_feature_tool::schema_from_model_info;

/// The verdict's shape — the same `#[derive(Validate)]` path every
/// other tool schema comes from, so the judge is described to the
/// provider exactly as an application's tools are.
#[derive(Debug, Deserialize, wabot_macros::Validate)]
struct VerdictArgs {
    #[description("true if the transcript satisfies the criteria")]
    pass: bool,
    #[description("short explanation of the verdict")]
    reasoning: String,
}

const VERDICT_TOOL: &str = "submitVerdict";

const JUDGE_SYSTEM_PROMPT: &str = "\
You are a strict QA judge for chatbot conversations.
You will receive a chat transcript and evaluation criteria.
Evaluate whether the transcript satisfies ALL the criteria.
You MUST report your verdict by calling the submitVerdict tool exactly once.
Never reply with plain text.";

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Verdict {
    pub pass: bool,
    pub reasoning: String,
}

#[derive(Debug, Error)]
pub enum JudgeError {
    #[error("the judge model did not call {VERDICT_TOOL}. It said: {said}")]
    NoVerdict { said: String },
    #[error("the judge called {VERDICT_TOOL} with arguments that don't match: {detail}")]
    BadVerdict { detail: String },
    #[error("the judge's provider failed: {0}")]
    Adapter(String),
    /// The criteria were not met — the judge worked, your code
    /// didn't. Carries the reasoning so a failing test says *why*.
    #[error("criteria not satisfied: {criteria}\n{reasoning}")]
    Failed { criteria: String, reasoning: String },
}

/// What to grade: the items a harness recorded, or text you rendered
/// yourself.
pub enum Transcript {
    Items(Vec<ChatItem>),
    Text(String),
}

impl From<Vec<ChatItem>> for Transcript {
    fn from(items: Vec<ChatItem>) -> Self {
        Self::Items(items)
    }
}

impl From<&[ChatItem]> for Transcript {
    fn from(items: &[ChatItem]) -> Self {
        Self::Items(items.to_vec())
    }
}

impl From<String> for Transcript {
    fn from(text: String) -> Self {
        Self::Text(text)
    }
}

impl From<&str> for Transcript {
    fn from(text: &str) -> Self {
        Self::Text(text.to_string())
    }
}

impl Transcript {
    fn render(self) -> String {
        match self {
            Transcript::Text(text) => text,
            Transcript::Items(items) => render_transcript(&items),
        }
    }
}

/// One line per item, in a shape a model reads without instructions.
///
/// Tool calls are included with their arguments and result: half of
/// what is worth judging is whether the bot *looked something up*
/// before answering, and a transcript of prose alone can't show that.
pub fn render_transcript(items: &[ChatItem]) -> String {
    items
        .iter()
        .map(|item| match item {
            ChatItem::HumanMessage { human_message } => {
                format!("HUMAN: {}", describe_message(human_message))
            }
            ChatItem::BotMessage { bot_message } => {
                format!("BOT: {}", describe_message(bot_message))
            }
            ChatItem::FunctionCall { function_call } => format!(
                "TOOL CALL: {}({}) -> {}",
                function_call.name,
                function_call.arguments.as_deref().unwrap_or("{}"),
                function_call.result.as_deref().unwrap_or("(no result)")
            ),
        })
        .collect::<Vec<_>>()
        .join("\n")
}

fn describe_message(message: &ChatMessage) -> String {
    let mut parts = Vec::new();
    if let Some(text) = message.text.as_deref() {
        if !text.is_empty() {
            parts.push(text.to_string());
        }
    }
    // Attachments are named but not sent: the judge grades the
    // conversation, and shipping image bytes to it would cost tokens
    // for something it was not asked about.
    if let Some(images) = message.images.as_ref().filter(|i| !i.is_empty()) {
        parts.push(format!("[{} image(s)]", images.len()));
    }
    if let Some(documents) = message.documents.as_ref().filter(|d| !d.is_empty()) {
        parts.push(format!("[{} document(s)]", documents.len()));
    }
    parts.join(" ")
}

/// Grades a conversation with a real model.
pub struct LlmJudge {
    adapter: Arc<dyn ChatAdapter>,
    models: Vec<ModelRef>,
}

impl LlmJudge {
    pub fn new(adapter: Arc<dyn ChatAdapter>, models: Vec<ModelRef>) -> Self {
        Self { adapter, models }
    }

    /// The verdict, whatever it is.
    pub async fn evaluate(
        &self,
        transcript: impl Into<Transcript>,
        criteria: &str,
    ) -> Result<Verdict, JudgeError> {
        let transcript = transcript.into().render();

        let response = self
            .adapter
            .next_items(ChatAdapterRequest {
                models: self.models.clone(),
                system_prompt: JUDGE_SYSTEM_PROMPT.to_string(),
                tools: vec![verdict_tool()],
                prev_items: vec![ChatItem::HumanMessage {
                    human_message: ChatMessage::text(format!(
                        "## Criteria\n{criteria}\n\n## Transcript\n{transcript}\n\n\
                         Evaluate now and call {VERDICT_TOOL}."
                    )),
                }],
            })
            .await
            .map_err(|error| JudgeError::Adapter(error.to_string()))?;

        let call = response.next_items.iter().find_map(|item| match item {
            ChatItem::FunctionCall { function_call } if function_call.name == VERDICT_TOOL => {
                Some(function_call)
            }
            _ => None,
        });

        let Some(call) = call else {
            let said: Vec<String> = response
                .next_items
                .iter()
                .filter_map(|item| match item {
                    ChatItem::BotMessage { bot_message } => bot_message.text.clone(),
                    _ => None,
                })
                .collect();
            return Err(JudgeError::NoVerdict {
                said: if said.is_empty() {
                    "(nothing)".to_string()
                } else {
                    said.join(" | ")
                },
            });
        };

        let arguments = call.arguments.as_deref().unwrap_or("{}");
        let args: VerdictArgs =
            serde_json::from_str(arguments).map_err(|error| JudgeError::BadVerdict {
                detail: format!("{error} — got {arguments}"),
            })?;

        Ok(Verdict {
            pass: args.pass,
            reasoning: args.reasoning,
        })
    }

    /// Like [`evaluate`], but a failing verdict is an `Err` carrying
    /// the judge's reasoning — which is what makes a failing test
    /// readable.
    ///
    /// [`evaluate`]: Self::evaluate
    pub async fn assert(
        &self,
        transcript: impl Into<Transcript>,
        criteria: &str,
    ) -> Result<Verdict, JudgeError> {
        let verdict = self.evaluate(transcript, criteria).await?;
        if !verdict.pass {
            return Err(JudgeError::Failed {
                criteria: criteria.to_string(),
                reasoning: verdict.reasoning,
            });
        }
        Ok(verdict)
    }
}

/// The tool the judge must call.
///
/// Built from the same `schema_from_model_info` path an application's
/// tools go through, so the judge is described to the provider
/// exactly as they are — no second way of declaring a tool that could
/// behave differently.
///
/// There is no body: the *call* is the answer, and it is read out of
/// the response rather than dispatched.
pub fn verdict_tool() -> ToolDefinition {
    let schema = schema_from_model_info(
        VERDICT_TOOL,
        "Submit your evaluation verdict. You MUST always call this tool exactly once; \
         never answer with plain text.",
        "english",
        <VerdictArgs as Validate>::model_info(),
    );
    ToolDefinition {
        name: schema.name,
        description: schema.description,
        language: schema.language,
        parameters: schema
            .parameters
            .into_iter()
            .map(|parameter| ToolParameter {
                name: parameter.name,
                r#type: parameter.r#type,
                description: parameter.description,
                required: parameter.required,
            })
            .collect(),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use wabot_feature_chat_bot::FunctionCall;

    fn human(text: &str) -> ChatItem {
        ChatItem::HumanMessage {
            human_message: ChatMessage::text(text),
        }
    }

    fn bot(text: &str) -> ChatItem {
        ChatItem::BotMessage {
            bot_message: ChatMessage::text(text),
        }
    }

    #[test]
    fn a_transcript_shows_prose_and_tool_calls() {
        let rendered = render_transcript(&[
            human("where is my order?"),
            ChatItem::FunctionCall {
                function_call: FunctionCall {
                    id: "1".into(),
                    name: "read_order".into(),
                    arguments: Some("{\"id\":7}".into()),
                    result: Some("{\"status\":\"shipped\"}".into()),
                    signature: None,
                },
            },
            bot("It shipped yesterday."),
        ]);

        assert_eq!(
            rendered,
            "HUMAN: where is my order?\n\
             TOOL CALL: read_order({\"id\":7}) -> {\"status\":\"shipped\"}\n\
             BOT: It shipped yesterday."
        );
    }

    /// A call with no arguments recorded still renders, because "the
    /// bot called this and got nothing back" is exactly the kind of
    /// thing a judge is asked about.
    #[test]
    fn a_call_with_nothing_recorded_still_renders() {
        let rendered = render_transcript(&[ChatItem::FunctionCall {
            function_call: FunctionCall {
                id: "1".into(),
                name: "lookup".into(),
                arguments: None,
                result: None,
                signature: None,
            },
        }]);
        assert_eq!(rendered, "TOOL CALL: lookup({}) -> (no result)");
    }

    #[test]
    fn the_verdict_tool_asks_for_a_boolean_and_a_reason() {
        let schema = verdict_tool();
        assert_eq!(schema.name, "submitVerdict");

        let pass = schema
            .parameters
            .iter()
            .find(|parameter| parameter.name == "pass")
            .expect("pass");
        assert_eq!(pass.r#type, "boolean", "typed, not parsed out of prose");
        assert!(pass.required);

        assert!(schema
            .parameters
            .iter()
            .any(|parameter| parameter.name == "reasoning"));
    }
}