mod checks;
pub mod payloads;
mod report;
mod status;
use std::fmt;
use async_trait::async_trait;
use wiremock::MockServer;
pub use report::{Check, CheckResult, CheckStatus, ConformanceReport};
pub use status::{RowSupport, StatusRow, StatusSupport};
use crate::error::ProviderError;
use crate::provider::ModelProvider;
use crate::secret::ApiKey;
use checks::Context;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
#[non_exhaustive]
pub enum Scenario {
ValidStructured,
MalformedJson,
UnknownField,
MissingField,
MultipleActs,
ToolCallIds,
StreamingReconstruction,
CachedUsage,
EmptyOutput,
Refusal,
SlowResponse,
#[doc = concat!("(", stringify!(payloads::RETRY_AFTER_SECONDS), ").")]
RateLimited,
Authentication,
ContextOverflow,
SecretInBody,
Authorization,
ModelNotFound,
RequestTimeout,
InvalidRequest,
ServerError,
ServiceUnavailable,
ContentFilter,
ExpiredCredential,
QuotaExhausted,
}
impl Scenario {
pub const ALL: [Self; 14] = [
Self::ValidStructured,
Self::MalformedJson,
Self::UnknownField,
Self::MissingField,
Self::MultipleActs,
Self::ToolCallIds,
Self::StreamingReconstruction,
Self::CachedUsage,
Self::EmptyOutput,
Self::Refusal,
Self::SlowResponse,
Self::RateLimited,
Self::Authentication,
Self::ContextOverflow,
];
pub const STATUS: [Self; 9] = [
Self::Authorization,
Self::ModelNotFound,
Self::RequestTimeout,
Self::InvalidRequest,
Self::ServerError,
Self::ServiceUnavailable,
Self::ContentFilter,
Self::ExpiredCredential,
Self::QuotaExhausted,
];
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::ValidStructured => "valid_structured",
Self::MalformedJson => "malformed_json",
Self::UnknownField => "unknown_field",
Self::MissingField => "missing_field",
Self::MultipleActs => "multiple_acts",
Self::ToolCallIds => "tool_call_ids",
Self::StreamingReconstruction => "streaming_reconstruction",
Self::CachedUsage => "cached_usage",
Self::EmptyOutput => "empty_output",
Self::Refusal => "refusal",
Self::SlowResponse => "slow_response",
Self::RateLimited => "rate_limited",
Self::Authentication => "authentication",
Self::ContextOverflow => "context_overflow",
Self::SecretInBody => "secret_in_body",
Self::Authorization => "authorization",
Self::ModelNotFound => "model_not_found",
Self::RequestTimeout => "request_timeout",
Self::InvalidRequest => "invalid_request",
Self::ServerError => "server_error",
Self::ServiceUnavailable => "service_unavailable",
Self::ContentFilter => "content_filter",
Self::ExpiredCredential => "expired_credential",
Self::QuotaExhausted => "quota_exhausted",
}
}
}
impl fmt::Display for Scenario {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
#[async_trait]
pub trait WireFixtures: Send + Sync {
async fn mount(&self, server: &MockServer, scenario: Scenario);
fn status_support(&self, row: StatusRow) -> RowSupport {
let _ = row;
RowSupport::Mounted
}
fn feature_support(&self, check: Check) -> RowSupport {
let _ = check;
RowSupport::Mounted
}
}
fn declared(
support: &RowSupport,
wire_description: &str,
consequence: &str,
) -> Option<checks::Outcome> {
let reason = support.reason()?;
if reason.is_empty() {
return Some(checks::Outcome::Failed(format!(
"{wire_description} was declared not producible without a reason; \
name the deployment behaviour, because an unexplained skip and \
a dodged failure look the same in a report"
)));
}
Some(checks::Outcome::Skipped(format!(
"{wire_description} is not produced by this deployment ({reason}), {consequence}"
)))
}
fn feature_declaration<W: WireFixtures>(fixtures: &W, check: Check) -> Option<checks::Outcome> {
let support = fixtures.feature_support(check);
if support.reason().is_some() && !check.is_declarable() {
return Some(checks::Outcome::Failed(format!(
"{check} was declared not producible, but it is a property of the adapter \
rather than of the deployment around it and cannot be declared away"
)));
}
declared(
&support,
&format!("the behaviour {check} exercises"),
"so the row is unproven",
)
}
pub trait ProviderFactory: Send + Sync {
type Provider: ModelProvider + fmt::Debug;
fn build(&self, base_url: &str, api_key: ApiKey) -> Result<Self::Provider, ProviderError>;
}
pub async fn run_all<F: ProviderFactory, W: WireFixtures>(
factory: &F,
fixtures: &W,
) -> ConformanceReport {
let probe_server = MockServer::start().await;
let probe = match factory.build(&probe_server.uri(), ApiKey::new(payloads::DUMMY_API_KEY)) {
Ok(provider) => provider,
Err(error) => {
let mut report = ConformanceReport::new(
crate::ids::ProviderKey::from("unknown"),
crate::ids::ModelKey::from("unknown"),
);
for check in Check::run_order() {
report.push(checks::build_failure(check, &error));
}
return report;
}
};
let mut report = ConformanceReport::new(probe.provider_key(), probe.model_key());
let mut context = Context::new(probe.capabilities());
drop(probe);
drop(probe_server);
report.push(
checks::valid_structured_response(factory, fixtures)
.await
.into_result(Check::ValidStructuredResponse),
);
report.push(
checks::malformed_json(factory, fixtures)
.await
.into_result(Check::MalformedJson),
);
report.push(
checks::unknown_fields(factory, fixtures)
.await
.into_result(Check::UnknownFields),
);
report.push(
checks::missing_required_fields(factory, fixtures)
.await
.into_result(Check::MissingRequiredFields),
);
report.push(
checks::multiple_acts(factory, fixtures)
.await
.into_result(Check::MultipleActs),
);
report.push(
checks::tool_and_read_request_ids(factory, fixtures, &context)
.await
.into_result(Check::ToolAndReadRequestIds),
);
report.push(
match feature_declaration(fixtures, Check::StreamingReconstruction) {
Some(outcome) => outcome,
None => checks::streaming_reconstruction(factory, fixtures, &context).await,
}
.into_result(Check::StreamingReconstruction),
);
report.push(
match feature_declaration(fixtures, Check::StreamingIncremental) {
Some(outcome) => outcome,
None => checks::streaming_incremental(factory, fixtures, &context).await,
}
.into_result(Check::StreamingIncremental),
);
report.push(
match feature_declaration(fixtures, Check::StreamingUsageAgreement) {
Some(outcome) => outcome,
None => checks::streaming_usage_agreement(factory, fixtures, &context).await,
}
.into_result(Check::StreamingUsageAgreement),
);
report.push(
checks::token_usage_contract(factory, fixtures, &context)
.await
.into_result(Check::TokenUsageContract),
);
report.push(
checks::empty_output(factory, fixtures)
.await
.into_result(Check::EmptyOutput),
);
report.push(
match feature_declaration(fixtures, Check::Refusal) {
Some(outcome) => outcome,
None => checks::refusal(factory, fixtures, &mut context).await,
}
.into_result(Check::Refusal),
);
report.push(
checks::timeout(factory, fixtures, &mut context)
.await
.into_result(Check::Timeout),
);
report.push(
match feature_declaration(fixtures, Check::RateLimit) {
Some(outcome) => outcome,
None => checks::rate_limit(factory, fixtures, &mut context).await,
}
.into_result(Check::RateLimit),
);
report.push(
match feature_declaration(fixtures, Check::AuthenticationFailure) {
Some(outcome) => outcome,
None => checks::authentication_failure(factory, fixtures, &mut context).await,
}
.into_result(Check::AuthenticationFailure),
);
report.push(
checks::context_overflow(factory, fixtures, &mut context)
.await
.into_result(Check::ContextOverflow),
);
report.push(
checks::cancellation(factory, fixtures)
.await
.into_result(Check::Cancellation),
);
for row in StatusRow::ALL {
report.push(
status::status_mapping(factory, fixtures, row, &mut context)
.await
.into_result(Check::StatusMapping(row)),
);
}
report.push(checks::retry_classification(&context).into_result(Check::RetryClassification));
report.push(
checks::secret_redaction(factory, fixtures)
.await
.into_result(Check::SecretRedaction),
);
report.push(
checks::no_silent_capability_downgrade(factory, fixtures, &context)
.await
.into_result(Check::NoSilentCapabilityDowngrade),
);
refuse_undeclarable_declarations(&mut report, fixtures);
report
}
fn refuse_undeclarable_declarations<W: WireFixtures>(report: &mut ConformanceReport, fixtures: &W) {
for result in &mut report.results {
let check = result.check;
if check.is_declarable() || check.status_row().is_some() {
continue;
}
if fixtures.feature_support(check).reason().is_some() {
*result = CheckResult::failed(
check,
format!(
"{check} was declared not producible, but it is a property of the \
adapter rather than of the deployment around it and cannot be \
declared away"
),
);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn scenario_labels_are_unique() {
let mut labels: Vec<&str> = Scenario::ALL.iter().map(|s| s.as_str()).collect();
labels.push(Scenario::SecretInBody.as_str());
labels.extend(Scenario::STATUS.iter().map(|s| s.as_str()));
assert_eq!(labels.len(), 24);
labels.sort_unstable();
labels.dedup();
assert_eq!(labels.len(), 24);
assert_eq!(Scenario::ValidStructured.to_string(), "valid_structured");
}
#[test]
fn a_reason_less_declaration_fails_and_an_explained_one_skips() {
let unexplained = declared(
&RowSupport::not_producible(" "),
"HTTP 503",
"so it is unproven",
)
.expect("a declaration produces an outcome");
let checks::Outcome::Failed(detail) = unexplained else {
panic!("a declaration without a reason must fail its row");
};
assert!(detail.contains("without a reason"), "{detail}");
let explained = declared(
&RowSupport::not_producible("this daemon authenticates nothing"),
"HTTP 401",
"so authentication is unproven",
)
.expect("a declaration produces an outcome");
let checks::Outcome::Skipped(reason) = explained else {
panic!("an explained declaration must skip its row");
};
assert!(reason.contains("authenticates nothing"), "{reason}");
assert!(reason.contains("unproven"), "{reason}");
assert!(declared(&RowSupport::Mounted, "HTTP 401", "").is_none());
}
#[test]
fn a_declaration_on_a_row_that_describes_the_adapter_is_refused() {
struct DodgesTheSchemaRow;
#[async_trait]
impl WireFixtures for DodgesTheSchemaRow {
async fn mount(&self, _server: &MockServer, _scenario: Scenario) {}
fn feature_support(&self, check: Check) -> RowSupport {
match check {
Check::MalformedJson => {
RowSupport::not_producible("our vendor never sends bad JSON")
}
_ => RowSupport::Mounted,
}
}
}
let outcome = feature_declaration(&DodgesTheSchemaRow, Check::MalformedJson)
.expect("the declaration is answered");
let checks::Outcome::Failed(detail) = outcome else {
panic!("a row that describes the adapter cannot be declared away");
};
assert!(detail.contains("malformed_json"), "{detail}");
assert!(detail.contains("cannot be declared away"), "{detail}");
}
#[test]
fn every_status_row_names_a_scenario_a_fixture_can_mount() {
for row in StatusRow::ALL {
let Some(scenario) = row.scenario() else {
assert_eq!(row, StatusRow::ConnectionReset);
continue;
};
assert!(
Scenario::ALL.contains(&scenario) || Scenario::STATUS.contains(&scenario),
"{row} points at {scenario}, which no fixture is told to mount"
);
}
}
#[test]
fn a_fixture_that_says_nothing_is_taken_to_mount_every_row() {
struct Silent;
#[async_trait]
impl WireFixtures for Silent {
async fn mount(&self, _server: &MockServer, _scenario: Scenario) {}
}
for row in StatusRow::ALL {
assert_eq!(
Silent.status_support(row),
StatusSupport::Mounted,
"silence must never become a skip for {row}"
);
}
}
}