wabot_testing/

conformance.rs

//! A suite every chat adapter must pass. Port of
//! `wabot-ts/src/testing/conformance/chatAdapterConformanceCases.ts`.
//!
//! ```ignore
//! for case in chat_adapter_conformance(adapter, "gpt-4o-mini") {
//!     println!("{}: {:?}", case.name, case.run().await);
//! }
//! ```
//!
//! ## What a conformance suite is for
//!
//! Six adapters implement one trait, and each was tested against its
//! own mock server — which proves each speaks its own provider's
//! dialect, and nothing at all about whether they behave *the same*.
//! The differences that bite are not in the happy path: they are
//! whether a tool call comes back with its arguments intact, whether
//! usage is reported, whether a nulled optional argument survives the
//! round trip. This suite is the same questions asked of every
//! provider.
//!
//! ## Runner-agnostic on purpose
//!
//! Each case is a name plus a boxed future, not a `#[test]`. An
//! adapter crate wires them into its own test binary, and an
//! application can run them against a provider it added itself —
//! which is the point of publishing them rather than keeping them
//! internal.
//!
//! ## What is deliberately not here yet
//!
//! The TypeScript suite has cases this port cannot express:
//!
//! * **Nested, enum and typed-array tool parameters.** `ToolSchema`
//!   here is flat (`name` / `type` / `description` / `required`), a
//!   deliberate Phase-4e decision. An array's item type reaches the
//!   model as a bare `array`, so a case asserting the model filled a
//!   `string[]` correctly would be testing a schema the port doesn't
//!   emit.
//!
//! Listed rather than quietly dropped: this is the checklist for when
//! it lands.

use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;

use wabot_feature_chat_bot::{
    ChatAdapter, ChatAdapterRequest, ChatItem, ChatMessage, ChatMessageFile, FunctionCall,
    ModelRef, ToolDefinition, ToolParameter,
};

/// One case: a name and something to run.
pub struct ConformanceCase {
    pub name: &'static str,
    /// What it checks, in one line — printed by a runner so a failure
    /// reads as a property, not a number.
    pub asserts: &'static str,
    run: Box<dyn FnOnce() -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> + Send>,
}

impl ConformanceCase {
    pub async fn run(self) -> Result<(), String> {
        (self.run)().await
    }
}

fn case<F, Fut>(name: &'static str, asserts: &'static str, body: F) -> ConformanceCase
where
    F: FnOnce() -> Fut + Send + 'static,
    Fut: Future<Output = Result<(), String>> + Send + 'static,
{
    ConformanceCase {
        name,
        asserts,
        run: Box::new(move || Box::pin(body())),
    }
}

fn ensure(condition: bool, message: impl Into<String>) -> Result<(), String> {
    if condition {
        Ok(())
    } else {
        Err(message.into())
    }
}

fn human(text: &str) -> ChatItem {
    ChatItem::HumanMessage {
        human_message: ChatMessage::text(text),
    }
}

fn tool(name: &str, description: &str, parameters: Vec<ToolParameter>) -> ToolDefinition {
    ToolDefinition {
        name: name.into(),
        description: description.into(),
        language: "english".into(),
        parameters,
    }
}

fn parameter(name: &str, kind: &str, description: &str, required: bool) -> ToolParameter {
    ToolParameter {
        name: name.into(),
        r#type: kind.into(),
        description: description.into(),
        required,
    }
}

fn country_tools() -> Vec<ToolDefinition> {
    vec![
        tool(
            "getCountryTime",
            "return the current time of a country",
            vec![parameter("country", "string", "the country iso code", true)],
        ),
        tool(
            "getCountryMainLanguage",
            "return the main language of a country",
            vec![parameter("country", "string", "the country iso code", true)],
        ),
    ]
}

fn request(models: Vec<ModelRef>, prompt: &str) -> ChatAdapterRequest {
    ChatAdapterRequest {
        models,
        system_prompt: "You are a helpful assistant.".into(),
        tools: Vec::new(),
        prev_items: vec![human(prompt)],
    }
}

fn first_text(items: &[ChatItem]) -> Option<String> {
    items.iter().find_map(|item| match item {
        ChatItem::BotMessage { bot_message } => bot_message.text.clone(),
        _ => None,
    })
}

fn calls(items: &[ChatItem]) -> Vec<&FunctionCall> {
    items
        .iter()
        .filter_map(|item| match item {
            ChatItem::FunctionCall { function_call } => Some(function_call),
            _ => None,
        })
        .collect()
}

/// Every case an adapter must pass, for `model`.
pub fn chat_adapter_conformance(
    adapter: Arc<dyn ChatAdapter>,
    model: &str,
) -> Vec<ConformanceCase> {
    let models = vec![ModelRef::model(model)];

    vec![
        {
            let adapter = adapter.clone();
            let models = models.clone();
            case(
                "answers a human message",
                "a plain question comes back as one bot message with text",
                move || async move {
                    let response = adapter
                        .next_items(request(models, "Say the single word: pong"))
                        .await
                        .map_err(|error| error.to_string())?;
                    let text =
                        first_text(&response.next_items).ok_or("no bot message in the response")?;
                    ensure(!text.trim().is_empty(), "the bot message had no text")
                },
            )
        },
        {
            let adapter = adapter.clone();
            let models = models.clone();
            case(
                "reports usage",
                "input and output token counts come back non-zero",
                move || async move {
                    let response = adapter
                        .next_items(request(models, "Say the single word: pong"))
                        .await
                        .map_err(|error| error.to_string())?;
                    ensure(
                        response.usage.input_tokens > 0,
                        format!("input_tokens was {}", response.usage.input_tokens),
                    )?;
                    ensure(
                        response.usage.output_tokens > 0,
                        format!("output_tokens was {}", response.usage.output_tokens),
                    )
                },
            )
        },
        {
            let adapter = adapter.clone();
            case(
                "fails on an unknown model",
                "a bad request is an error, not an empty success",
                move || async move {
                    let outcome = adapter
                        .next_items(request(
                            vec![ModelRef::model("definitely-not-a-real-model-xyz")],
                            "hello",
                        ))
                        .await;
                    ensure(
                        outcome.is_err(),
                        "an unknown model was accepted — a caller cannot tell that from a real answer",
                    )
                },
            )
        },
        {
            let adapter = adapter.clone();
            let models = models.clone();
            case(
                "calls the right tool",
                "given two similar tools, the model picks the one the question needs, with its argument",
                move || async move {
                    let response = adapter
                        .next_items(ChatAdapterRequest {
                            models,
                            system_prompt: "Use the tools to answer.".into(),
                            tools: country_tools(),
                            prev_items: vec![human("What time is it in Japan?")],
                        })
                        .await
                        .map_err(|error| error.to_string())?;

                    let calls = calls(&response.next_items);
                    let call = calls.first().ok_or("the model called no tool")?;
                    ensure(
                        call.name == "getCountryTime",
                        format!("it called {} instead", call.name),
                    )?;

                    let arguments: serde_json::Value =
                        serde_json::from_str(call.arguments.as_deref().unwrap_or("{}"))
                            .map_err(|error| format!("arguments were not JSON: {error}"))?;
                    let country = arguments
                        .get("country")
                        .and_then(|value| value.as_str())
                        .ok_or("the call carried no country argument")?;
                    ensure(
                        country.to_lowercase().contains("jp")
                            || country.to_lowercase().contains("japan"),
                        format!("the country argument was {country:?}"),
                    )
                },
            )
        },
        {
            let adapter = adapter.clone();
            let models = models.clone();
            case(
                "consumes a tool result",
                "a function call with its result fed back produces an answer that uses it",
                move || async move {
                    let response = adapter
                        .next_items(ChatAdapterRequest {
                            models,
                            system_prompt: "Use the tools to answer.".into(),
                            tools: country_tools(),
                            prev_items: vec![
                                human(
                                    "What time is it in Japan? Include the station code \
                                     verbatim in your answer.",
                                ),
                                ChatItem::FunctionCall {
                                    function_call: FunctionCall {
                                        id: "call_1".into(),
                                        name: "getCountryTime".into(),
                                        arguments: Some(r#"{"country":"JP"}"#.into()),
                                        // An opaque token, not a time:
                                        // a model that answers "11:45
                                        // PM" for "23:45" has used the
                                        // result correctly, and an
                                        // assertion on the formatting
                                        // would call that a failure.
                                        result: Some(
                                            r#"{"time":"23:45","stationCode":"ZQX7"}"#.into(),
                                        ),
                                        signature: None,
                                    },
                                },
                            ],
                        })
                        .await
                        .map_err(|error| error.to_string())?;

                    let text = first_text(&response.next_items)
                        .ok_or("the model did not answer after the tool result")?;
                    ensure(
                        text.contains("ZQX7"),
                        format!("the answer ignored the tool's result: {text}"),
                    )
                },
            )
        },
        {
            let adapter = adapter.clone();
            let models = models.clone();
            case(
                "an optional argument may be nulled",
                "a parameter marked optional accepts null rather than forcing the model to invent a value",
                move || async move {
                    let response = adapter
                        .next_items(ChatAdapterRequest {
                            models,
                            system_prompt: "Use the tool. If you have no value for an optional \
                                            argument, pass null."
                                .into(),
                            tools: vec![tool(
                                "createNote",
                                "store a note",
                                vec![
                                    parameter("text", "string", "the note body", true),
                                    parameter(
                                        "folder",
                                        "string",
                                        "optional folder to file it under",
                                        false,
                                    ),
                                ],
                            )],
                            prev_items: vec![human("Save a note that says 'buy milk'.")],
                        })
                        .await
                        .map_err(|error| error.to_string())?;

                    let calls = calls(&response.next_items);
                    let call = calls.first().ok_or("the model called no tool")?;
                    let arguments: serde_json::Value =
                        serde_json::from_str(call.arguments.as_deref().unwrap_or("{}"))
                            .map_err(|error| format!("arguments were not JSON: {error}"))?;

                    ensure(
                        arguments.get("text").and_then(|v| v.as_str()).is_some(),
                        "the required argument is missing",
                    )?;
                    // The point is that the provider accepted the
                    // nullable spelling at all: absent or null both
                    // mean "no value", and both survive
                    // `normalize_optional_arguments`.
                    match arguments.get("folder") {
                        None | Some(serde_json::Value::Null) => Ok(()),
                        Some(other) => ensure(
                            other.is_string(),
                            format!("the optional argument came back as {other}"),
                        ),
                    }
                },
            )
        },
        {
            let adapter = adapter.clone();
            let models = models.clone();
            case(
                "keeps a multi-turn conversation",
                "earlier turns are sent, so the model can refer back to them",
                move || async move {
                    let response = adapter
                        .next_items(ChatAdapterRequest {
                            models,
                            system_prompt: "Answer briefly.".into(),
                            tools: Vec::new(),
                            prev_items: vec![
                                human("My favourite colour is chartreuse. Remember it."),
                                ChatItem::BotMessage {
                                    bot_message: ChatMessage::text("Noted."),
                                },
                                human("What is my favourite colour? Answer with one word."),
                            ],
                        })
                        .await
                        .map_err(|error| error.to_string())?;

                    let text = first_text(&response.next_items).ok_or("no answer")?;
                    ensure(
                        text.to_lowercase().contains("chartreuse"),
                        format!("the earlier turn did not reach the model: {text}"),
                    )
                },
            )
        },
        {
            let adapter = adapter.clone();
            let models = models.clone();
            case(
                "reads an attached image",
                "the image bytes reach the model, not just its filename",
                move || async move {
                    let response = adapter
                        .next_items(ChatAdapterRequest {
                            models,
                            system_prompt: "Answer in one word.".into(),
                            tools: Vec::new(),
                            prev_items: vec![ChatItem::HumanMessage {
                                human_message: ChatMessage {
                                    text: Some(
                                        "What colour fills this image? Answer with one word."
                                            .into(),
                                    ),
                                    images: Some(vec![public_image()]),
                                    ..ChatMessage::default()
                                },
                            }],
                        })
                        .await
                        .map_err(|error| error.to_string())?;

                    let text = first_text(&response.next_items).ok_or("no answer")?;
                    ensure(
                        text.to_lowercase().contains("red"),
                        format!("the model did not see the image: {text}"),
                    )
                },
            )
        },
        {
            let adapter = adapter.clone();
            let models = models.clone();
            case(
                "describes an attachment it cannot read",
                "an unsupported file is reported to the model rather than dropped",
                move || async move {
                    let response = adapter
                        .next_items(ChatAdapterRequest {
                            models,
                            system_prompt: "Answer briefly and truthfully.".into(),
                            tools: Vec::new(),
                            prev_items: vec![ChatItem::HumanMessage {
                                human_message: ChatMessage {
                                    text: Some(
                                        "Did I attach a file? Answer yes or no, then say its \
                                         format."
                                            .into(),
                                    ),
                                    images: Some(vec![ChatMessageFile {
                                        id: "weird-1".into(),
                                        mime_type: "image/vnd.adobe.photoshop".into(),
                                        name: Some("mockup.psd".into()),
                                        public_url: Some(
                                            "https://example.invalid/mockup.psd".into(),
                                        ),
                                        base64_url: None,
                                    }]),
                                    ..ChatMessage::default()
                                },
                            }],
                        })
                        .await
                        .map_err(|error| error.to_string())?;

                    let text = first_text(&response.next_items)
                        .ok_or("no answer")?
                        .to_lowercase();
                    // The property: the model *knows* something was
                    // attached. Dropping it silently would have it
                    // answer "no".
                    ensure(
                        text.contains("yes") || text.contains("psd") || text.contains("photoshop"),
                        format!("the model was not told a file was attached: {text}"),
                    )
                },
            )
        },
    ]
}

/// A 64×64 solid red PNG, inline.
///
/// Inline rather than a URL, and a colour rather than a photograph,
/// for the same reason: the case must fail only when *this port* is
/// wrong. A hosted image makes a third party's bot policy part of the
/// test (a first attempt used Wikipedia, and the provider's fetcher
/// was refused), and "is this a cat" makes the model's eyesight part
/// of it. What is being checked is that the bytes reached the model at
/// all.
fn public_image() -> ChatMessageFile {
    ChatMessageFile {
        id: "red-1".into(),
        mime_type: "image/png".into(),
        name: Some("red.png".into()),
        public_url: None,
        base64_url: Some(format!("data:image/png;base64,{RED_PNG}")),
    }
}

const RED_PNG: &str = "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PQQkAAAgAsetfWiP4FgYrsKZeS0BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEDgsqnc8OJg6Ln3AAAAAElFTkSuQmCC";

#[cfg(test)]
mod tests {
    use super::*;

    /// The suite has to be buildable without a provider — an adapter
    /// crate wires it up before it ever has a key.
    #[test]
    fn the_suite_lists_its_cases() {
        struct Never;
        #[async_trait::async_trait]
        impl ChatAdapter for Never {
            async fn next_items(
                &self,
                _request: ChatAdapterRequest,
            ) -> Result<
                wabot_feature_chat_bot::ChatAdapterResponse,
                wabot_feature_chat_bot::ChatAdapterError,
            > {
                unreachable!()
            }
        }

        let cases = chat_adapter_conformance(Arc::new(Never), "any");
        assert!(cases.len() >= 9);
        assert!(cases.iter().all(|case| !case.asserts.is_empty()));
        assert!(cases.iter().any(|case| case.name == "calls the right tool"));
    }
}