wabot-testing 0.1.0

//! The conformance suite against a real provider.
//!
//! Six adapters implement one trait, and each was tested against its
//! own mock server — which proves each speaks its provider's dialect
//! and nothing about whether they behave the same. This is the same
//! questions asked of every one of them.
//!
//! ```sh
//! WABOT_TEST_LLM_KEY=… WABOT_TEST_LLM_PROVIDER=openrouter \
//!   cargo test -p wabot-testing --test conformance_live -- --nocapture
//! ```
//!
//! Skips without a key: it calls a paid API and is slow.

use std::sync::Arc;

use wabot_addon_chat_bot_openai::{OpenaiChatAdapter, OpenaiConfig};
use wabot_addon_chat_bot_openrouter::{OpenRouterChatAdapter, OpenRouterConfig};
use wabot_feature_chat_bot::ChatAdapter;
use wabot_testing::conformance::chat_adapter_conformance;

fn adapter() -> Option<(Arc<dyn ChatAdapter>, String)> {
    let key = std::env::var("WABOT_TEST_LLM_KEY").ok()?;
    let provider = std::env::var("WABOT_TEST_LLM_PROVIDER").unwrap_or_else(|_| "openrouter".into());
    Some(match provider.as_str() {
        "openai" => (
            Arc::new(OpenaiChatAdapter::new(OpenaiConfig::new(key))),
            "gpt-4o-mini".to_string(),
        ),
        "openrouter" => (
            Arc::new(OpenRouterChatAdapter::new(OpenRouterConfig::new(key))),
            "openai/gpt-4o-mini".to_string(),
        ),
        other => panic!("unknown WABOT_TEST_LLM_PROVIDER {other:?}"),
    })
}

/// One test running every case, rather than a test per case: the
/// point is a **report** — an adapter that fails three cases should
/// say which three in one run, not stop at the first.
#[tokio::test]
async fn the_adapter_conforms() {
    let Some((adapter, model)) = adapter() else {
        eprintln!("skipping: WABOT_TEST_LLM_KEY not set");
        return;
    };

    let mut failures = Vec::new();
    for case in chat_adapter_conformance(adapter, &model) {
        let name = case.name;
        let asserts = case.asserts;
        match case.run().await {
            Ok(()) => eprintln!("  ok    {name}"),
            Err(error) => {
                eprintln!("  FAIL  {name} — {asserts}\n        {error}");
                failures.push(format!("{name}: {error}"));
            }
        }
    }

    assert!(
        failures.is_empty(),
        "{} of the conformance cases failed:\n{}",
        failures.len(),
        failures.join("\n")
    );
}