use bytes::Bytes;
use futures::StreamExt;
use futures::future::BoxFuture;
use crate::completion::{CompletionRequest, CompletionResponse};
use crate::error::ProviderError;
use crate::{
completion::FinishReason,
error::ErrorReport,
http_client,
message::AssistantContent,
streaming::{Item, StreamEvent},
};
#[derive(Debug, thiserror::Error)]
pub enum ConformanceError {
#[error(transparent)]
Completion(#[from] ProviderError),
#[error("{scenario} conformance failed for {provider}: {details}")]
Contract {
scenario: &'static str,
provider: &'static str,
details: String,
},
}
impl ConformanceError {
fn contract(
scenario: &'static str,
provider: &'static str,
details: impl Into<String>,
) -> Self {
Self::Contract {
scenario,
provider,
details: details.into(),
}
}
}
#[derive(Debug)]
pub struct ScenarioReport {
pub name: &'static str,
pub provider: &'static str,
pub observations: Vec<String>,
}
#[derive(Debug)]
pub enum ScenarioOutcome {
Ran(ScenarioReport),
Skipped {
name: &'static str,
provider: &'static str,
reason: &'static str,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct SuiteCapabilities {
pub partial_tool_args: bool,
pub zero_usage_terminal: bool,
pub bare_terminal: bool,
pub malformed_frame: bool,
pub unknown_event_frame: bool,
pub defective_known_frame: bool,
pub delta_less_prelude: bool,
pub refusal: bool,
pub interleaved_reasoning: bool,
}
impl SuiteCapabilities {
pub fn from_names(names: &[&str]) -> Result<Self, String> {
let mut caps = Self::default();
for name in names {
match *name {
"partial_tool_args" => caps.partial_tool_args = true,
"zero_usage_terminal" => caps.zero_usage_terminal = true,
"bare_terminal" => caps.bare_terminal = true,
"malformed_frame" => caps.malformed_frame = true,
"unknown_event_frame" => caps.unknown_event_frame = true,
"defective_known_frame" => caps.defective_known_frame = true,
"delta_less_prelude" => caps.delta_less_prelude = true,
"refusal" => caps.refusal = true,
"interleaved_reasoning" => caps.interleaved_reasoning = true,
other => {
return Err(format!(
"unknown capability name in suite manifest: {other}"
));
}
}
}
Ok(caps)
}
}
pub const CANONICAL_SCENARIOS: &[&str] = &[
"truncation_preserves_content_without_terminal",
"transport_error_after_tool_call_yields_err_then_end",
"malformed_frame_ends_the_reply",
"unknown_event_is_skipped",
"defective_known_event_ends_the_reply",
"delta_less_choice_prelude_is_a_noop",
"refusal_frames_deliver_text_without_error",
"bare_terminal_after_only_unparseable_frames_fabricates_nothing",
"usage_variants_are_reported_or_absent",
"interleaved_constant_id_reasoning_preserves_order",
];
pub const WIRE_FAMILIES: &[&str] = &[
"openai_chat",
"openai_responses",
"openai_responses_websocket",
"chatgpt",
"anthropic",
"gemini_rest",
"gemini_interactions",
"gemini_grpc",
"xai",
"copilot",
"bedrock",
"candle",
];
pub fn xfail_reason<'a>(xfail: &[&'a str], scenario: &str) -> Option<&'a str> {
xfail.iter().find_map(|entry| {
let (name, reason) = entry.split_once(':')?;
(name.trim() == scenario).then(|| reason.trim())
})
}
pub fn invalid_xfail_entries(xfail: &[&str]) -> Vec<String> {
xfail
.iter()
.filter(|entry| match entry.split_once(':') {
Some((name, reason)) => {
!CANONICAL_SCENARIOS.contains(&name.trim()) || reason.trim().is_empty()
}
None => true,
})
.map(std::string::ToString::to_string)
.collect()
}
pub fn check_gated_outcome(
scenario: &'static str,
capability: bool,
xfail: &[&str],
outcome: Result<ScenarioOutcome, ConformanceError>,
) -> Result<(), String> {
match (xfail_reason(xfail, scenario), outcome) {
(Some(reason), Err(error)) => {
eprintln!("xfail {scenario}: {reason} ({error})");
Ok(())
}
(Some(reason), Ok(_)) => Err(format!(
"{scenario} passed but is listed as xfail ({reason}); remove the xfail entry"
)),
(None, Err(error)) => Err(format!("{scenario} failed: {error}")),
(None, Ok(ScenarioOutcome::Ran(_))) => {
if capability {
Ok(())
} else {
Err(format!(
"{scenario} ran but the suite disclaims the capability; set the flag to true"
))
}
}
(None, Ok(ScenarioOutcome::Skipped { reason, .. })) => {
if capability {
Err(format!(
"{scenario} skipped ({reason}) but the suite declares the capability; \
a declared capability's scenario must run"
))
} else {
eprintln!("skipped {scenario}: {reason}");
Ok(())
}
}
}
}
pub fn check_ungated_outcome(
scenario: &'static str,
xfail: &[&str],
result: Result<ScenarioReport, ConformanceError>,
) -> Result<(), String> {
match (xfail_reason(xfail, scenario), result) {
(Some(reason), Err(error)) => {
eprintln!("xfail {scenario}: {reason} ({error})");
Ok(())
}
(Some(reason), Ok(_)) => Err(format!(
"{scenario} passed but is listed as xfail ({reason}); remove the xfail entry"
)),
(None, Err(error)) => Err(format!("{scenario} failed: {error}")),
(None, Ok(_)) => Ok(()),
}
}
#[derive(Clone)]
pub enum WireInput {
Bytes(Bytes),
Event(std::sync::Arc<dyn std::any::Any + Send + Sync>),
}
impl WireInput {
pub fn as_bytes(&self) -> Option<&Bytes> {
match self {
Self::Bytes(bytes) => Some(bytes),
Self::Event(_) => None,
}
}
pub fn downcast_event<T: 'static>(&self) -> Option<&T> {
match self {
Self::Bytes(_) => None,
Self::Event(event) => event.downcast_ref(),
}
}
}
impl From<Bytes> for WireInput {
fn from(bytes: Bytes) -> Self {
Self::Bytes(bytes)
}
}
impl std::fmt::Debug for WireInput {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Bytes(bytes) => formatter.debug_tuple("Bytes").field(bytes).finish(),
Self::Event(_) => formatter.write_str("Event(..)"),
}
}
}
pub fn event_frame<T: Send + Sync + 'static>(event: T) -> WireInput {
WireInput::Event(std::sync::Arc::new(event))
}
pub type WireChunks = Vec<http_client::Result<WireInput>>;
pub fn ok_chunks(frames: impl IntoIterator<Item = impl Into<WireInput>>) -> WireChunks {
frames.into_iter().map(|frame| Ok(frame.into())).collect()
}
pub fn transport_error_chunk() -> http_client::Result<WireInput> {
Err(http_client::Error::instance(std::io::Error::new(
std::io::ErrorKind::ConnectionReset,
"connection reset",
)))
}
pub fn assert_valid_event_stream(
items: &[Result<Item<StreamEvent>, ErrorReport>],
choice: &[AssistantContent],
) {
use crate::message::AssistantContent;
if let Some(error_index) = items.iter().position(Result::is_err) {
assert_eq!(
error_index + 1,
items.len(),
"law 1 (terminal error): an item followed the stream's error"
);
}
let events: Vec<&StreamEvent> = items
.iter()
.filter_map(|item| match item {
Ok(Item::Event(event)) => Some(event),
_ => None,
})
.collect();
let streamed_text: String = events
.iter()
.filter_map(|event| match event {
StreamEvent::Text { text, .. } => Some(text.as_str()),
_ => None,
})
.collect();
let aggregated_text: String = choice
.iter()
.filter_map(|content| match content {
AssistantContent::Text(text) => Some(text.text.as_str()),
_ => None,
})
.collect();
assert_eq!(
aggregated_text, streamed_text,
"law 2 (text conservation): aggregated text differs from the streamed fragments"
);
let yielded_calls = events
.iter()
.filter(|event| {
matches!(
event,
StreamEvent::End {
content: AssistantContent::ToolCall(_),
..
}
)
})
.count();
let aggregated_calls = choice
.iter()
.filter(|content| matches!(content, AssistantContent::ToolCall(_)))
.count();
assert_eq!(
aggregated_calls, yielded_calls,
"law 3 (completed-call conservation): {yielded_calls} calls yielded, \
{aggregated_calls} aggregated"
);
let serialized = serde_json::to_value(
items
.iter()
.filter_map(|item| item.as_ref().ok())
.collect::<Vec<_>>(),
)
.unwrap_or_default();
let read_back = crate::streaming::Transcript::parse_prefix(serialized);
assert!(
read_back.is_ok(),
"law 4 (sequence): the stream's events do not read back: {read_back:?}"
);
let yielded_reasoning = events.iter().any(|event| {
matches!(
event,
StreamEvent::End {
content: AssistantContent::Reasoning(_),
..
}
)
});
let aggregated_reasoning = choice
.iter()
.any(|content| matches!(content, AssistantContent::Reasoning(_)));
assert!(
yielded_reasoning || !aggregated_reasoning,
"law 5 (reasoning provenance): aggregated reasoning with no reasoning yielded"
);
}
#[derive(Debug)]
pub struct DrainedStream {
pub items: Vec<Result<Item<StreamEvent>, ErrorReport>>,
pub choice: Vec<AssistantContent>,
pub response: Option<CompletionResponse>,
}
impl DrainedStream {
fn events(&self) -> impl Iterator<Item = &StreamEvent> {
self.items.iter().filter_map(|item| match item {
Ok(Item::Event(event)) => Some(event),
_ => None,
})
}
pub fn texts(&self) -> Vec<&str> {
self.events()
.filter_map(|event| match event {
StreamEvent::Text { text, .. } => Some(text.as_str()),
_ => None,
})
.collect()
}
pub fn tool_call_names(&self) -> Vec<&str> {
self.events()
.filter_map(|event| match event {
StreamEvent::End {
content: AssistantContent::ToolCall(tool_call),
..
} => Some(tool_call.function.name.as_str()),
_ => None,
})
.collect()
}
pub fn unknown_values(&self) -> Vec<&serde_json::Value> {
self.items
.iter()
.filter_map(|item| match item {
Ok(Item::Unknown(value)) => Some(value.value()),
_ => None,
})
.collect()
}
pub fn error_count(&self) -> usize {
self.items.iter().filter(|item| item.is_err()).count()
}
fn has_terminal(&self) -> bool {
self.response.is_some()
}
fn completed_cleanly(&self) -> bool {
self.error_count() == 0 && self.response.is_some()
}
fn first_error_index(&self) -> Option<usize> {
self.items.iter().position(std::result::Result::is_err)
}
pub fn choice_texts(&self) -> Vec<&str> {
self.choice
.iter()
.filter_map(|content| match content {
AssistantContent::Text(text) => Some(text.text.as_str()),
_ => None,
})
.collect()
}
pub fn choice_reasoning(&self) -> Vec<&crate::message::Reasoning> {
self.choice
.iter()
.filter_map(|content| match content {
AssistantContent::Reasoning(reasoning) => Some(reasoning),
_ => None,
})
.collect()
}
}
type DriveFn = Box<
dyn Fn(WireChunks) -> BoxFuture<'static, Result<DrainedStream, ProviderError>> + Send + Sync,
>;
pub struct WireDriver {
pub provider: &'static str,
drive: DriveFn,
}
impl WireDriver {
pub fn new(
provider: &'static str,
drive: impl Fn(WireChunks) -> BoxFuture<'static, Result<DrainedStream, ProviderError>>
+ Send
+ Sync
+ 'static,
) -> Self {
Self {
provider,
drive: Box::new(drive),
}
}
pub async fn drive(&self, chunks: WireChunks) -> Result<DrainedStream, ProviderError> {
(self.drive)(chunks).await
}
}
pub struct RefusalFixture {
pub frames: Vec<WireInput>,
pub expected_text: &'static str,
}
pub struct InterleavedReasoningFixture {
pub frames: Vec<WireInput>,
pub first_reasoning: &'static str,
pub tool_name: &'static str,
pub second_reasoning: &'static str,
}
type BufferedDriveFn = Box<
dyn Fn(String) -> BoxFuture<'static, Result<Vec<AssistantContent>, ProviderError>>
+ Send
+ Sync,
>;
pub struct BufferedBodyDriver {
pub provider: &'static str,
drive: BufferedDriveFn,
}
impl BufferedBodyDriver {
pub fn new(
provider: &'static str,
drive: impl Fn(String) -> BoxFuture<'static, Result<Vec<AssistantContent>, ProviderError>>
+ Send
+ Sync
+ 'static,
) -> Self {
Self {
provider,
drive: Box::new(drive),
}
}
pub async fn drive(&self, body: String) -> Result<Vec<AssistantContent>, ProviderError> {
(self.drive)(body).await
}
}
pub struct ProviderWireFixture {
pub driver: WireDriver,
pub text_frames: Vec<WireInput>,
pub expected_texts: Vec<&'static str>,
pub tool_call_frames: Vec<WireInput>,
pub expected_tool_name: &'static str,
pub partial_tool_call_frames: Option<Vec<WireInput>>,
pub terminal_frames: Vec<WireInput>,
pub expected_usage_total: u64,
pub expected_finish_reason: Option<FinishReason>,
pub zero_usage_terminal_frames: Option<Vec<WireInput>>,
pub bare_terminal_frames: Option<Vec<WireInput>>,
pub malformed_frame: Option<WireInput>,
pub unknown_event_frame: Option<WireInput>,
pub defective_known_frame: Option<WireInput>,
pub delta_less_prelude_frame: Option<WireInput>,
pub refusal: Option<RefusalFixture>,
pub interleaved_reasoning: Option<InterleavedReasoningFixture>,
}
impl ProviderWireFixture {
pub fn capabilities(&self) -> SuiteCapabilities {
SuiteCapabilities {
partial_tool_args: self.partial_tool_call_frames.is_some(),
zero_usage_terminal: self.zero_usage_terminal_frames.is_some(),
bare_terminal: self.bare_terminal_frames.is_some(),
malformed_frame: self.malformed_frame.is_some(),
unknown_event_frame: self.unknown_event_frame.is_some(),
defective_known_frame: self.defective_known_frame.is_some(),
delta_less_prelude: self.delta_less_prelude_frame.is_some(),
refusal: self.refusal.is_some(),
interleaved_reasoning: self.interleaved_reasoning.is_some(),
}
}
}
fn concat_frames(parts: &[&[WireInput]]) -> Vec<WireInput> {
parts
.iter()
.flat_map(|frames| frames.iter().cloned())
.collect()
}
struct Checks {
name: &'static str,
provider: &'static str,
observations: Vec<String>,
}
impl Checks {
fn new(name: &'static str, provider: &'static str) -> Self {
Self {
name,
provider,
observations: Vec::new(),
}
}
fn fail(&self, details: impl Into<String>) -> ConformanceError {
ConformanceError::contract(self.name, self.provider, details)
}
fn require<D: Into<String>>(
&self,
held: bool,
details: impl FnOnce() -> D,
) -> Result<(), ConformanceError> {
if held {
return Ok(());
}
Err(self.fail(details()))
}
fn note(&mut self, observation: impl Into<String>) {
self.observations.push(observation.into());
}
fn skip(&self, reason: &'static str) -> ScenarioOutcome {
ScenarioOutcome::Skipped {
name: self.name,
provider: self.provider,
reason,
}
}
fn report(self) -> ScenarioReport {
ScenarioReport {
name: self.name,
provider: self.provider,
observations: self.observations,
}
}
fn ran(self) -> ScenarioOutcome {
ScenarioOutcome::Ran(self.report())
}
}
pub async fn truncation_preserves_content_without_terminal(
fixture: &ProviderWireFixture,
) -> Result<ScenarioReport, ConformanceError> {
let mut checks = Checks::new(
"truncation_preserves_content_without_terminal",
fixture.driver.provider,
);
let drained = fixture.driver.drive(Vec::new()).await?;
checks.require(
!drained.has_terminal(),
|| "an empty stream must not synthesize a terminal record",
)?;
checks.note("EOF before content: no terminal");
let drained = fixture
.driver
.drive(ok_chunks(fixture.text_frames.clone()))
.await?;
checks.require(drained.texts() == fixture.expected_texts, || {
format!(
"text delivered before truncation must be preserved: expected {:?}, observed {:?}",
fixture.expected_texts,
drained.texts()
)
})?;
checks.require(
!drained.has_terminal(),
|| "EOF after text deltas must not synthesize a terminal record",
)?;
checks.note("EOF mid-text: content preserved, no terminal");
if let Some(partial) = &fixture.partial_tool_call_frames {
let drained = fixture.driver.drive(ok_chunks(partial.clone())).await?;
checks.require(
!drained.has_terminal(),
|| "EOF mid-tool-arguments must not synthesize a terminal record",
)?;
checks.note("EOF mid-tool-args: no terminal");
}
let drained = fixture
.driver
.drive(ok_chunks(fixture.tool_call_frames.clone()))
.await?;
checks.require(
drained
.tool_call_names()
.iter()
.all(|name| *name == fixture.expected_tool_name),
|| {
format!(
"only the delivered call may surface: observed {:?}",
drained.tool_call_names()
)
},
)?;
checks.require(
!drained.has_terminal(),
|| "EOF after a tool call must not synthesize a terminal record",
)?;
checks.note("EOF after a tool call: no terminal");
Ok(checks.report())
}
pub async fn transport_error_after_tool_call_yields_err_then_end(
fixture: &ProviderWireFixture,
) -> Result<ScenarioReport, ConformanceError> {
let mut checks = Checks::new(
"transport_error_after_tool_call_yields_err_then_end",
fixture.driver.provider,
);
let mut chunks = ok_chunks(fixture.tool_call_frames.clone());
chunks.push(transport_error_chunk());
let drained = fixture.driver.drive(chunks).await?;
checks.require(
drained
.tool_call_names()
.iter()
.all(|name| *name == fixture.expected_tool_name),
|| {
format!(
"only the delivered call may precede the transport error: observed {:?}",
drained.tool_call_names()
)
},
)?;
let error_index = drained
.first_error_index()
.ok_or_else(|| checks.fail("the transport failure must reach the consumer"))?;
checks.require(
error_index + 1 == drained.items.len(),
|| "nothing may follow the terminal transport error",
)?;
checks.require(
!drained.has_terminal(),
|| "a transport failure must not be papered over with a terminal record",
)?;
checks.note("Err, then end; no terminal");
Ok(checks.report())
}
pub async fn malformed_frame_ends_the_reply(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
let mut checks = Checks::new("malformed_frame_ends_the_reply", fixture.driver.provider);
let Some(malformed) = &fixture.malformed_frame else {
return Ok(checks.skip("wire family cannot spell a frame-level decode failure"));
};
let frames = concat_frames(&[
&fixture.text_frames,
std::slice::from_ref(malformed),
&fixture.terminal_frames,
]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
checks.require(drained.error_count() == 1, || {
format!(
"the malformed frame must surface as exactly one Err item, observed {}",
drained.error_count()
)
})?;
checks.require(
drained.first_error_index() == Some(drained.items.len() - 1),
|| "the malformed frame's error must be the stream's last item",
)?;
checks.require(
drained.texts() == fixture.expected_texts,
|| "content before the malformed frame must be preserved",
)?;
checks.require(
!drained.has_terminal(),
|| "a reply cut by a corrupt frame has no response",
)?;
checks.note("Err surfaced last; content kept; no response");
Ok(checks.ran())
}
pub async fn unknown_event_is_skipped(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
let mut checks = Checks::new("unknown_event_is_skipped", fixture.driver.provider);
let Some(unknown) = &fixture.unknown_event_frame else {
return Ok(checks.skip("wire family cannot spell an unknown event type"));
};
let frames = concat_frames(&[
&fixture.text_frames,
std::slice::from_ref(unknown),
&fixture.terminal_frames,
]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
checks.require(
drained.error_count() == 0,
|| "an unknown event type must be skipped, not surfaced as an error",
)?;
checks.require(
drained.texts() == fixture.expected_texts && drained.response.is_some(),
|| "the stream must deliver its content and complete around the skipped event",
)?;
checks.require(drained.unknown_values().len() == 1, || {
format!(
"exactly one Unknown passthrough item must surface for the unknown frame, \
observed {}",
drained.unknown_values().len()
)
})?;
let control_frames = concat_frames(&[&fixture.text_frames, &fixture.terminal_frames]);
let control = fixture.driver.drive(ok_chunks(control_frames)).await?;
checks.require(
drained.choice == control.choice,
|| "the unknown frame must not perturb the aggregated assistant choice",
)?;
checks.note(
"unknown event skipped semantically, surfaced on the raw channel, \
choice unchanged, stream completed",
);
Ok(checks.ran())
}
pub async fn defective_known_event_ends_the_reply(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
let mut checks = Checks::new(
"defective_known_event_ends_the_reply",
fixture.driver.provider,
);
let Some(defective) = &fixture.defective_known_frame else {
return Ok(
checks.skip("wire family cannot spell a known event with a schema-defective payload")
);
};
let frames = concat_frames(&[
&fixture.text_frames,
std::slice::from_ref(defective),
&fixture.terminal_frames,
]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
checks.require(drained.error_count() == 1, || {
format!(
"a known event with a schema defect must surface exactly one Err item, observed {}",
drained.error_count()
)
})?;
checks.require(
!drained.has_terminal(),
|| "a reply cut by a defective event has no response",
)?;
checks.note("defective known event ended the reply with its Err");
Ok(checks.ran())
}
pub async fn delta_less_choice_prelude_is_a_noop(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
let mut checks = Checks::new(
"delta_less_choice_prelude_is_a_noop",
fixture.driver.provider,
);
let Some(prelude) = &fixture.delta_less_prelude_frame else {
return Ok(checks.skip("wire family has no delta-less prelude shape"));
};
let frames = concat_frames(&[
std::slice::from_ref(prelude),
&fixture.text_frames,
&fixture.terminal_frames,
]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
checks.require(
drained.error_count() == 0,
|| "the delta-less prelude must not surface an error",
)?;
checks.require(
drained.texts() == fixture.expected_texts && drained.response.is_some(),
|| "the prelude must not perturb content delivery or the terminal",
)?;
checks.note("delta-less prelude ignored; stream unaffected");
Ok(checks.ran())
}
pub async fn refusal_frames_deliver_text_without_error(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
let mut checks = Checks::new(
"refusal_frames_deliver_text_without_error",
fixture.driver.provider,
);
let Some(refusal) = &fixture.refusal else {
return Ok(checks.skip("wire family has no refusal channel"));
};
let frames = concat_frames(&[&refusal.frames, &fixture.terminal_frames]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
checks.require(
drained.error_count() == 0,
|| "refusal content must not surface as an error",
)?;
let delivered = drained.texts().concat();
checks.require(delivered == refusal.expected_text, || {
format!(
"refusal text must be delivered: expected {:?}, observed {delivered:?}",
refusal.expected_text
)
})?;
checks.require(
drained.response.is_some(),
|| "a refused turn still ends with the provider's genuine terminal",
)?;
checks.note("refusal text delivered without error");
Ok(checks.ran())
}
pub async fn terminal_body_content_merges_per_kind(
driver: &BufferedBodyDriver,
cases: Vec<(&'static str, String)>,
expected_text: &str,
) -> Result<ScenarioReport, ConformanceError> {
let mut checks = Checks::new("terminal_body_content_merges_per_kind", driver.provider);
for (label, body) in cases {
let choice = driver.drive(body).await?;
let choice_text: String = choice
.iter()
.filter_map(|content| match content {
AssistantContent::Text(text) => Some(text.text.as_str()),
_ => None,
})
.collect();
let occurrences = choice_text.matches(expected_text).count();
checks.require(occurrences == 1, || {
format!(
"{label}: terminal-body text must appear exactly once in the choice, observed {occurrences} in {choice_text:?}"
)
})?;
checks.note(format!("{label}: text merged exactly once"));
}
Ok(checks.report())
}
pub async fn bare_terminal_after_only_unparseable_frames_fabricates_nothing(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
let mut checks = Checks::new(
"bare_terminal_after_only_unparseable_frames_fabricates_nothing",
fixture.driver.provider,
);
let Some(bare_terminal) = &fixture.bare_terminal_frames else {
return Ok(checks.skip("wire family has no data-less terminal signal"));
};
let Some(malformed) = &fixture.malformed_frame else {
return Ok(checks.skip("wire family cannot spell a frame-level decode failure"));
};
let frames = concat_frames(&[std::slice::from_ref(malformed), bare_terminal]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
checks.require(
drained.error_count() != 0,
|| "the unparseable frame must surface as an Err item",
)?;
checks.require(
!drained.has_terminal(),
|| "a bare terminal with no decoded frame must not fabricate a terminal record",
)?;
checks.note("no fabricated terminal after only-unparseable frames");
Ok(checks.ran())
}
pub async fn usage_variants_are_reported_or_absent(
fixture: &ProviderWireFixture,
) -> Result<ScenarioReport, ConformanceError> {
let mut checks = Checks::new(
"usage_variants_are_reported_or_absent",
fixture.driver.provider,
);
let frames = concat_frames(&[&fixture.text_frames, &fixture.terminal_frames]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
let response = drained
.response
.as_ref()
.ok_or_else(|| checks.fail("the genuine terminal must produce a record"))?;
checks.require(
response.usage.total_tokens == Some(fixture.expected_usage_total),
|| {
format!(
"terminal usage must be preserved: expected total {}, observed {:?}",
fixture.expected_usage_total, response.usage.total_tokens
)
},
)?;
checks.require(
response.finish_reason() == fixture.expected_finish_reason,
|| {
format!(
"terminal finish reason must be normalized: expected {:?}, observed {:?}",
fixture.expected_finish_reason,
response.finish_reason()
)
},
)?;
checks.note(format!(
"usage total {} and finish reason {:?} preserved",
fixture.expected_usage_total, fixture.expected_finish_reason
));
if let Some(zero_usage) = &fixture.zero_usage_terminal_frames {
let frames = concat_frames(&[&fixture.text_frames, zero_usage]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
let response = drained.response.as_ref().ok_or_else(|| {
checks.fail("a usage-less genuine terminal must still complete the stream")
})?;
checks.require(!response.usage.is_reported(), || {
format!(
"missing usage metrics must leave every counter unreported, not invented: {:?}",
response.usage
)
})?;
checks.note("usage-less terminal completed with no counter reported");
}
Ok(checks.report())
}
pub async fn reasoning_summary_deltas_are_superseded_without_duplication(
driver: &WireDriver,
frames: Vec<WireInput>,
summary_text: &str,
) -> Result<ScenarioReport, ConformanceError> {
let mut checks = Checks::new(
"reasoning_summary_deltas_are_superseded_without_duplication",
driver.provider,
);
let drained = driver.drive(ok_chunks(frames)).await?;
checks.require(
drained.completed_cleanly(),
|| "the reasoning stream must complete without errors",
)?;
let reasoning = drained.choice_reasoning();
let occurrences: usize = reasoning
.iter()
.map(|item| item.text.matches(summary_text).count())
.sum();
checks.require(occurrences == 1, || {
format!(
"the summary must appear exactly once in the aggregated choice, observed {occurrences} across {reasoning:?}"
)
})?;
checks.require(reasoning.len() == 1, || {
format!(
"deltas and their full block must collapse to one reasoning item, observed {}",
reasoning.len()
)
})?;
checks.note("summary aggregated exactly once");
Ok(checks.report())
}
pub async fn multi_part_same_id_reasoning_keeps_every_part(
driver: &WireDriver,
frames: Vec<WireInput>,
expected_parts: &[&str],
) -> Result<ScenarioReport, ConformanceError> {
let mut checks = Checks::new(
"multi_part_same_id_reasoning_keeps_every_part",
driver.provider,
);
let drained = driver.drive(ok_chunks(frames)).await?;
checks.require(
drained.completed_cleanly(),
|| "the reasoning stream must complete without errors",
)?;
let items: Vec<String> = drained
.choice_reasoning()
.iter()
.filter_map(|item| item.native.as_ref())
.map(|native| native.item.to_string())
.collect();
let in_order = |item: &String| {
let mut rest = item.as_str();
expected_parts.iter().all(|part| match rest.find(part) {
Some(at) => {
rest = &rest[at + part.len()..];
true
}
None => false,
})
};
let observed = items;
checks.require(observed.len() == 1 && observed.iter().all(in_order), || {
format!(
"every same-id reasoning part must survive in order: expected {expected_parts:?}, observed {observed:?}"
)
})?;
checks.note(format!(
"all {} reasoning parts survived",
expected_parts.len()
));
Ok(checks.report())
}
pub async fn interleaved_reasoning_aggregates_to_one_item(
driver: &WireDriver,
frames: Vec<WireInput>,
expected_text: &str,
) -> Result<ScenarioReport, ConformanceError> {
let mut checks = Checks::new(
"interleaved_reasoning_aggregates_to_one_item",
driver.provider,
);
let drained = driver.drive(ok_chunks(frames)).await?;
checks.require(
drained.completed_cleanly(),
|| "the interleaved stream must complete without errors",
)?;
let reasoning = drained.choice_reasoning();
checks.require(reasoning.len() == 1, || {
format!(
"interleaved deltas and their completed block must collapse to one reasoning item, observed {}",
reasoning.len()
)
})?;
let carries_text = reasoning.iter().any(|item| item.text == expected_text);
checks.require(carries_text, || {
format!("the reasoning item must carry the completed block's text {expected_text:?}")
})?;
checks.note("exactly one reasoning item with the completed content");
Ok(checks.report())
}
pub async fn interleaved_constant_id_reasoning_preserves_order(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
let mut checks = Checks::new(
"interleaved_constant_id_reasoning_preserves_order",
fixture.driver.provider,
);
let Some(interleaved) = &fixture.interleaved_reasoning else {
return Ok(checks.skip("wire fixture supplies no interleaved reasoning frames"));
};
let drained = fixture
.driver
.drive(ok_chunks(interleaved.frames.clone()))
.await?;
checks.require(
drained.completed_cleanly(),
|| "the interleaved stream must complete without errors",
)?;
assert_reasoning_tool_reasoning(
&checks,
&drained,
interleaved.first_reasoning,
interleaved.tool_name,
interleaved.second_reasoning,
)?;
checks.note("boundary kept: reasoning, tool call, reasoning in order");
Ok(checks.ran())
}
pub async fn interleaved_signed_full_reasoning_does_not_erase_prior_thought(
driver: &WireDriver,
frames: Vec<WireInput>,
first: &str,
tool_name: &str,
second: &str,
) -> Result<ScenarioReport, ConformanceError> {
let mut checks = Checks::new(
"interleaved_signed_full_reasoning_does_not_erase_prior_thought",
driver.provider,
);
let drained = driver.drive(ok_chunks(frames)).await?;
checks.require(
drained.completed_cleanly(),
|| "the interleaved stream must complete without errors",
)?;
assert_reasoning_tool_reasoning(&checks, &drained, first, tool_name, second)?;
let signed = drained
.choice_reasoning()
.last()
.is_some_and(|reasoning| reasoning.native.is_some());
checks.require(signed, || "the post-boundary block must keep its signature")?;
checks.note("pre-boundary thought survived; signed block completed the post-boundary part");
Ok(checks.report())
}
fn assert_reasoning_tool_reasoning(
checks: &Checks,
drained: &DrainedStream,
first: &str,
tool_name: &str,
second: &str,
) -> Result<(), ConformanceError> {
let shape: Vec<String> = drained
.choice
.iter()
.map(|content| match content {
AssistantContent::Reasoning(reasoning) => format!("reasoning:{}", reasoning.text),
AssistantContent::ToolCall(tool_call) => {
format!("tool:{}", tool_call.function.name)
}
AssistantContent::Text(text) => format!("text:{}", text.text),
AssistantContent::Image(_) => "image".to_string(),
AssistantContent::Opaque(opaque) => format!("opaque:{}", opaque.kind().unwrap_or("")),
})
.collect();
let expected = vec![
format!("reasoning:{first}"),
format!("tool:{tool_name}"),
format!("reasoning:{second}"),
];
checks.require(shape == expected, || {
format!("the boundary must survive aggregation: expected {expected:?}, observed {shape:?}")
})
}
pub mod fixtures {
use super::*;
use crate::driver::{Model, Transport};
use crate::operation::Completion;
use crate::test_utils::SequencedStreamingHttpClient;
use crate::wire::Wire;
use serde_json::json;
pub async fn drain(mut stream: crate::streaming::CompletionStream) -> DrainedStream {
let mut items = Vec::new();
while let Some(item) = stream.next().await {
items.push(item.map_err(|error| ErrorReport::from(&error)));
}
let partial = stream.partial().choice;
let response = stream.finish().await.ok();
let drained = DrainedStream {
items,
choice: response
.as_ref()
.map_or(partial, |response| response.choice.clone()),
response,
};
super::assert_valid_event_stream(&drained.items, &drained.choice);
drained
}
pub async fn drain_observed<W, T>(
model: &Model<W, T>,
request: crate::completion::CompletionRequest,
) -> Result<DrainedStream, ProviderError>
where
W: Wire<Op = Completion>,
T: Transport<W>,
{
let log = std::sync::Arc::new(crate::observe::ObservationLog::default());
let context = crate::observe::AdapterContext::new(
log.clone(),
crate::observe::Subject::default(),
"conformance",
);
let drained = match model.stream_observed(request, context) {
Ok(stream) => Ok(drain(stream).await),
Err(error) => Err(error),
};
let events: Vec<_> = log
.trace()
.observations
.iter()
.filter_map(|o| match &o.action {
crate::observe::Action::Adapter { observation } => Some(observation.event.clone()),
_ => None,
})
.collect();
assert!(
matches!(
events.first(),
Some(crate::observe::AdapterEvent::Started { .. })
),
"the wire must attach the observation context it was handed: {events:?}"
);
assert!(
matches!(
events.last(),
Some(crate::observe::AdapterEvent::Finished { .. })
),
"the attempt must close: {events:?}"
);
drained
}
fn byte_chunks(chunks: WireChunks) -> Result<Vec<http_client::Result<Bytes>>, ProviderError> {
chunks
.into_iter()
.map(|chunk| match chunk {
Ok(WireInput::Bytes(bytes)) => Ok(Ok(bytes)),
Ok(WireInput::Event(_)) => Err(ProviderError::Provider(
"typed-event frame fed to a byte-transport driver".to_string(),
)),
Err(error) => Ok(Err(error)),
})
.collect()
}
fn byte_driver<W>(
provider: &'static str,
bind: fn(SequencedStreamingHttpClient) -> Model<W, SequencedStreamingHttpClient>,
) -> WireDriver
where
W: Wire<Op = Completion, Payload = crate::wire::Encoded, Frame = crate::wire::WireFrame>,
{
WireDriver::new(provider, move |chunks| {
Box::pin(async move {
let model = bind(SequencedStreamingHttpClient::new(byte_chunks(chunks)?));
let request = CompletionRequest::new("hello");
drain_observed(&model, request).await
})
})
}
fn sse(frame: &serde_json::Value) -> WireInput {
WireInput::Bytes(Bytes::from(format!("data: {frame}\n\n")))
}
fn sse_raw(data: &str) -> WireInput {
WireInput::Bytes(Bytes::from(format!("data: {data}\n\n")))
}
fn frame_text(frame: &WireInput) -> String {
frame
.as_bytes()
.map(|bytes| String::from_utf8_lossy(bytes).into_owned())
.unwrap_or_default()
}
pub mod openai_chat {
use super::*;
fn driver() -> WireDriver {
byte_driver("openai", |transport| {
crate::driver::Model::new(
crate::providers::openai::wire::OpenAIConfig::with_key(
&crate::providers::openai::wire::OPENAI,
"test-key",
)
.chat("gpt-4o"),
transport,
)
})
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![sse(&json!({
"id": "chatcmpl-1",
"model": "gpt-4o-2024-08-06",
"choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": null}],
"usage": null,
}))],
expected_texts: vec!["hi"],
tool_call_frames: vec![
sse(&json!({
"choices": [{"index": 0, "delta": {"tool_calls": [{
"index": 0,
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": ""},
}]}, "finish_reason": null}],
})),
sse(&json!({
"choices": [{"index": 0, "delta": {"tool_calls": [{
"index": 0,
"function": {"arguments": "{\"city\":\"Tokyo\"}"},
}]}, "finish_reason": null}],
})),
],
expected_tool_name: "get_weather",
partial_tool_call_frames: Some(vec![sse(&json!({
"choices": [{"index": 0, "delta": {"tool_calls": [{
"index": 0,
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": "{\"cit"},
}]}, "finish_reason": null}],
}))]),
terminal_frames: vec![
sse(&json!({
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
"usage": null,
})),
sse(&json!({
"choices": [],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
})),
sse_raw("[DONE]"),
],
expected_usage_total: 15,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: Some(vec![
sse(&json!({
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
"usage": null,
})),
sse_raw("[DONE]"),
]),
bare_terminal_frames: Some(vec![sse_raw("[DONE]")]),
malformed_frame: Some(sse_raw("{not json")),
unknown_event_frame: None,
defective_known_frame: None,
delta_less_prelude_frame: Some(sse_raw(
r#"{"id":"","object":"","choices":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"}}}]}"#,
)),
refusal: None,
interleaved_reasoning: None,
}
}
}
pub mod openai_responses {
use super::*;
pub fn driver() -> WireDriver {
byte_driver("openai", |transport| {
crate::driver::Model::new(
crate::providers::openai::OpenAIConfig::new("test-key").responses("gpt-5.4"),
transport,
)
})
}
fn completed_response(
usage: Option<&serde_json::Value>,
output: &serde_json::Value,
) -> serde_json::Value {
json!({
"id": "resp_1",
"object": "response",
"created_at": 0,
"status": "completed",
"model": "gpt-5.4",
"output": output,
"tools": [],
"usage": usage,
})
}
fn terminal(usage: Option<&serde_json::Value>, output: &serde_json::Value) -> WireInput {
sse(&json!({
"type": "response.completed",
"sequence_number": 99,
"response": completed_response(usage, output),
}))
}
fn usage_json() -> serde_json::Value {
json!({
"input_tokens": 10,
"output_tokens": 5,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 15,
})
}
fn text_delta(text: &str) -> WireInput {
sse(&json!({
"type": "response.output_text.delta",
"content_index": 0,
"delta": text,
"item_id": "msg_1",
"output_index": 0,
"sequence_number": 1,
}))
}
fn tool_call_done() -> WireInput {
sse(&json!({
"type": "response.output_item.done",
"output_index": 0,
"sequence_number": 2,
"item": {
"type": "function_call",
"id": "fc_1",
"arguments": "{\"city\":\"Tokyo\"}",
"call_id": "call_1",
"name": "get_weather",
"status": "completed",
},
}))
}
pub fn incomplete_mid_tool_call_frames() -> Vec<WireInput> {
vec![
sse(&json!({
"type": "response.output_item.added",
"output_index": 0,
"sequence_number": 1,
"item": {
"type": "function_call",
"id": "fc_1",
"arguments": "",
"call_id": "call_1",
"name": "add",
"status": "in_progress",
},
})),
sse(&json!({
"type": "response.function_call_arguments.delta",
"item_id": "fc_1",
"output_index": 0,
"sequence_number": 2,
"delta": "{\"x",
})),
sse(&json!({
"type": "response.function_call_arguments.delta",
"item_id": "fc_1",
"output_index": 0,
"sequence_number": 3,
"delta": "\":48151",
})),
sse(&json!({
"type": "response.function_call_arguments.done",
"item_id": "fc_1",
"output_index": 0,
"sequence_number": 4,
"arguments": "{\"x\":48151",
})),
sse(&json!({
"type": "response.output_item.done",
"output_index": 0,
"sequence_number": 5,
"item": {
"type": "function_call",
"id": "fc_1",
"arguments": "{\"x\":48151",
"call_id": "call_1",
"name": "add",
"status": "incomplete",
},
})),
sse(&json!({
"type": "response.incomplete",
"sequence_number": 6,
"response": {
"id": "resp_1",
"object": "response",
"created_at": 0,
"status": "incomplete",
"incomplete_details": {"reason": "max_output_tokens"},
"model": "gpt-5.4",
"output": [{
"type": "function_call",
"id": "fc_1",
"arguments": "{\"x\":48151",
"call_id": "call_1",
"name": "add",
"status": "incomplete",
}],
"tools": [],
"usage": usage_json(),
},
})),
]
}
fn reasoning_done_item(
id: &str,
summary: &serde_json::Value,
content: &serde_json::Value,
encrypted: Option<&str>,
) -> WireInput {
let mut item = json!({
"type": "reasoning",
"id": id,
"summary": summary,
"content": content,
"status": "completed",
});
if let (Some(encrypted), Some(object)) = (encrypted, item.as_object_mut()) {
object.insert("encrypted_content".to_string(), json!(encrypted));
}
sse(&json!({
"type": "response.output_item.done",
"output_index": 0,
"sequence_number": 3,
"item": item,
}))
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![text_delta("hi")],
expected_texts: vec!["hi"],
tool_call_frames: vec![tool_call_done()],
expected_tool_name: "get_weather",
partial_tool_call_frames: Some(vec![
sse(&json!({
"type": "response.output_item.added",
"output_index": 0,
"sequence_number": 1,
"item": {
"type": "function_call",
"id": "fc_1",
"arguments": "",
"call_id": "call_1",
"name": "get_weather",
"status": "in_progress",
},
})),
sse(&json!({
"type": "response.function_call_arguments.delta",
"item_id": "fc_1",
"output_index": 0,
"sequence_number": 2,
"delta": "{\"cit",
})),
]),
terminal_frames: vec![terminal(Some(&usage_json()), &json!([]))],
expected_usage_total: 15,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: Some(vec![terminal(None, &json!([]))]),
bare_terminal_frames: None,
malformed_frame: Some(sse_raw("{not json")),
unknown_event_frame: Some(sse(&json!({
"type": "response.web_search_call.searching",
"output_index": 0,
"sequence_number": 4,
"item_id": "ws_1",
}))),
defective_known_frame: None,
delta_less_prelude_frame: None,
refusal: Some(RefusalFixture {
frames: vec![sse(&json!({
"type": "response.refusal.delta",
"content_index": 0,
"delta": "I cannot help with that.",
"item_id": "msg_1",
"output_index": 0,
"sequence_number": 1,
}))],
expected_text: "I cannot help with that.",
}),
interleaved_reasoning: None,
}
}
pub fn buffered_driver() -> BufferedBodyDriver {
BufferedBodyDriver::new("chatgpt", |body| {
Box::pin(async move {
let model = crate::driver::Model::new(
crate::providers::openai::OpenAIConfig::with_key(
&crate::providers::chatgpt::DIALECT,
"test-token",
)
.with_account_id("account-id")
.responses("gpt-5.4"),
crate::test_utils::RecordingHttpClient::new(body),
);
let request = CompletionRequest::new("hello");
let response = model.call(request).await?;
Ok(response.choice)
})
})
}
fn message_output(text: &str) -> serde_json::Value {
json!([{
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": text, "annotations": []}],
}])
}
pub fn terminal_body_only_sse_body(text: &str) -> String {
frame_text(&terminal(Some(&usage_json()), &message_output(text)))
}
pub fn terminal_body_and_delta_sse_body(text: &str) -> String {
let frames = [
text_delta(text),
terminal(Some(&usage_json()), &message_output(text)),
];
frames.iter().map(frame_text).collect()
}
pub fn delta_only_sse_body(text: &str) -> String {
let frames = [text_delta(text), terminal(Some(&usage_json()), &json!([]))];
frames.iter().map(frame_text).collect()
}
pub fn envelope_less_reasoning_supersede_sse_body() -> (String, &'static str) {
let delta = json!({
"type": "response.reasoning_summary_text.delta",
"delta": "step 1",
});
let frames = [
sse(&delta),
reasoning_done_item(
"rs_1",
&json!([{"type": "summary_text", "text": "step 1"}]),
&json!([]),
None,
),
terminal(Some(&usage_json()), &json!([])),
];
(frames.iter().map(frame_text).collect(), "step 1")
}
pub fn reasoning_summary_supersede_frames() -> (Vec<WireInput>, &'static str) {
let frames = vec![
sse(&json!({
"type": "response.reasoning_summary_text.delta",
"item_id": "rs_1",
"output_index": 0,
"summary_index": 0,
"sequence_number": 1,
"delta": "step 1",
})),
reasoning_done_item(
"rs_1",
&json!([{"type": "summary_text", "text": "step 1"}]),
&json!([]),
None,
),
terminal(Some(&usage_json()), &json!([])),
];
(frames, "step 1")
}
pub fn multi_part_reasoning_frames() -> (Vec<WireInput>, Vec<&'static str>) {
let frames = vec![
reasoning_done_item(
"rs_1",
&json!([
{"type": "summary_text", "text": "s1"},
{"type": "summary_text", "text": "s2"},
]),
&json!([{"type": "reasoning_text", "text": "visible"}]),
Some("enc_blob"),
),
terminal(Some(&usage_json()), &json!([])),
];
(frames, vec!["s1", "s2", "visible", "enc_blob"])
}
pub fn interleaved_reasoning_frames() -> (Vec<WireInput>, &'static str) {
let frames = vec![
sse(&json!({
"type": "response.reasoning_text.delta",
"item_id": "rs_2",
"output_index": 0,
"content_index": 0,
"sequence_number": 1,
"delta": "full ",
})),
sse(&json!({
"type": "response.output_item.done",
"output_index": 1,
"sequence_number": 2,
"item": {
"type": "function_call",
"id": "fc_1",
"arguments": "{\"city\":\"Tokyo\"}",
"call_id": "call_1",
"name": "get_weather",
"status": "completed",
},
})),
reasoning_done_item(
"rs_2",
&json!([]),
&json!([{"type": "reasoning_text", "text": "full reasoning"}]),
None,
),
terminal(Some(&usage_json()), &json!([])),
];
(frames, "full reasoning")
}
}
pub mod gemini_rest {
use super::*;
fn driver() -> WireDriver {
byte_driver("gemini", |transport| {
crate::driver::Model::new(
crate::providers::gemini::GeminiConfig::new("test-key").completion(
crate::providers::gemini::completion::GEMINI_2_5_PRO_PREVIEW_06_05,
),
transport,
)
})
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![sse(&json!({
"candidates": [{"content": {"parts": [{"text": "hi"}], "role": "model"}}],
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))],
expected_texts: vec!["hi"],
tool_call_frames: vec![sse(&json!({
"candidates": [{"content": {"parts": [{
"functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
}], "role": "model"}}],
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))],
expected_tool_name: "get_weather",
partial_tool_call_frames: None,
terminal_frames: vec![sse(&json!({
"candidates": [{
"content": {"parts": [], "role": "model"},
"finishReason": "STOP",
}],
"usageMetadata": {
"promptTokenCount": 5,
"candidatesTokenCount": 2,
"totalTokenCount": 7,
},
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))],
expected_usage_total: 7,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: Some(vec![sse(&json!({
"candidates": [{
"content": {"parts": [], "role": "model"},
"finishReason": "STOP",
}],
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))]),
bare_terminal_frames: None,
malformed_frame: Some(sse_raw("{not json")),
unknown_event_frame: Some(sse_raw(r#"{"noise":true}"#)),
defective_known_frame: Some(sse_raw(r#"{"candidates": 42}"#)),
delta_less_prelude_frame: None,
refusal: None,
interleaved_reasoning: Some(interleaved_thought_fixture()),
}
}
fn chunk(parts: &serde_json::Value) -> WireInput {
sse(&json!({
"candidates": [{"content": {"parts": parts, "role": "model"}}],
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))
}
fn terminal_frame() -> WireInput {
sse(&json!({
"candidates": [{
"content": {"parts": [], "role": "model"},
"finishReason": "STOP",
}],
"usageMetadata": {
"promptTokenCount": 5,
"candidatesTokenCount": 2,
"totalTokenCount": 7,
},
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))
}
fn interleaved_thought_fixture() -> InterleavedReasoningFixture {
InterleavedReasoningFixture {
frames: vec![
chunk(&json!([{"text": "before tool", "thought": true}])),
chunk(&json!([{
"functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
}])),
chunk(&json!([{"text": "after tool", "thought": true}])),
terminal_frame(),
],
first_reasoning: "before tool",
tool_name: "get_weather",
second_reasoning: "after tool",
}
}
pub fn interleaved_signed_thought_frames()
-> (Vec<WireInput>, &'static str, &'static str, &'static str) {
let frames = vec![
chunk(&json!([{"text": "before tool", "thought": true}])),
chunk(&json!([{
"functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
}])),
chunk(&json!([{
"text": "signed conclusion",
"thought": true,
"thoughtSignature": "sig-1",
}])),
terminal_frame(),
];
(frames, "before tool", "get_weather", "signed conclusion")
}
}
pub mod interactions {
use super::*;
fn driver() -> WireDriver {
byte_driver("gemini", |transport| {
crate::driver::Model::new(
crate::providers::gemini::GeminiConfig::new("test-key")
.interactions("gemini-2.5-pro"),
transport,
)
})
}
fn completed(usage: Option<serde_json::Value>) -> WireInput {
let mut interaction = json!({
"id": "int-1",
"model": "gemini-2.5-pro",
"status": "completed",
});
if let (Some(usage), Some(object)) = (usage, interaction.as_object_mut()) {
object.insert("usage".to_string(), usage);
}
sse(&json!({
"event_type": "interaction.completed",
"interaction": interaction,
}))
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![sse(&json!({
"event_type": "step.delta",
"index": 0,
"delta": {"type": "text", "text": "hi"},
}))],
expected_texts: vec!["hi"],
tool_call_frames: vec![sse(&json!({
"event_type": "step.delta",
"index": 0,
"delta": {
"type": "function_call",
"name": "get_weather",
"arguments": {"city": "Tokyo"},
"id": "call-1",
},
}))],
expected_tool_name: "get_weather",
partial_tool_call_frames: None,
terminal_frames: vec![completed(Some(json!({
"total_input_tokens": 5,
"total_output_tokens": 2,
"total_tokens": 7,
})))],
expected_usage_total: 7,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: Some(vec![completed(None)]),
bare_terminal_frames: None,
malformed_frame: Some(sse_raw("{not json")),
unknown_event_frame: Some(sse(&json!({
"event_type": "future.event",
"index": 0,
}))),
defective_known_frame: Some(sse_raw(
r#"{"event_type":"step.delta","index":0,"delta":42}"#,
)),
delta_less_prelude_frame: None,
refusal: None,
interleaved_reasoning: Some(interleaved_thought_fixture()),
}
}
fn interleaved_thought_fixture() -> InterleavedReasoningFixture {
let frames = vec![
sse(&json!({
"event_type": "step.delta",
"index": 0,
"delta": {
"type": "thought_summary",
"content": {"type": "text", "text": "before tool"},
},
})),
sse(&json!({
"event_type": "step.delta",
"index": 1,
"delta": {
"type": "function_call",
"name": "get_weather",
"arguments": {"city": "Tokyo"},
"id": "call-1",
},
})),
sse(&json!({
"event_type": "step.delta",
"index": 2,
"delta": {
"type": "thought_summary",
"content": {"type": "text", "text": "after tool"},
},
})),
completed(Some(json!({
"total_input_tokens": 5,
"total_output_tokens": 2,
"total_tokens": 7,
}))),
];
InterleavedReasoningFixture {
frames,
first_reasoning: "before tool",
tool_name: "get_weather",
second_reasoning: "after tool",
}
}
}
pub mod anthropic {
use super::*;
fn driver() -> WireDriver {
byte_driver("anthropic", |transport| {
crate::driver::Model::new(
crate::providers::anthropic::wire::AnthropicConfig::new("test-key")
.completion(crate::providers::anthropic::completion::CLAUDE_SONNET_4_6),
transport,
)
})
}
fn message_start() -> WireInput {
sse(&json!({
"type": "message_start",
"message": {
"id": "msg_1",
"role": "assistant",
"content": [],
"model": "claude-sonnet-4-6",
"stop_reason": null,
"stop_sequence": null,
"usage": {"input_tokens": 5, "output_tokens": 0},
},
}))
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![
message_start(),
sse(&json!({
"type": "content_block_start",
"index": 0,
"content_block": {"type": "text", "text": ""},
})),
sse(&json!({
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": "hi"},
})),
],
expected_texts: vec!["hi"],
tool_call_frames: vec![
sse(&json!({
"type": "content_block_start",
"index": 0,
"content_block": {
"type": "tool_use",
"id": "toolu_1",
"name": "get_weather",
"input": {},
},
})),
sse(&json!({
"type": "content_block_delta",
"index": 0,
"delta": {"type": "input_json_delta", "partial_json": "{\"city\":\"Tokyo\"}"},
})),
sse(&json!({"type": "content_block_stop", "index": 0})),
],
expected_tool_name: "get_weather",
partial_tool_call_frames: Some(vec![
sse(&json!({
"type": "content_block_start",
"index": 0,
"content_block": {
"type": "tool_use",
"id": "toolu_1",
"name": "get_weather",
"input": {},
},
})),
sse(&json!({
"type": "content_block_delta",
"index": 0,
"delta": {"type": "input_json_delta", "partial_json": "{\"cit"},
})),
]),
terminal_frames: vec![sse(&json!({
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": null},
"usage": {"output_tokens": 4},
}))],
expected_usage_total: 9,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: None,
bare_terminal_frames: Some(vec![sse(&json!({"type": "message_stop"}))]),
malformed_frame: Some(sse_raw("{not json")),
unknown_event_frame: Some(sse(&json!({
"type": "content_block_heartbeat",
"index": 0,
}))),
defective_known_frame: Some(sse_raw(
r#"{"type":"content_block_delta","index":0,"delta":42}"#,
)),
delta_less_prelude_frame: None,
refusal: None,
interleaved_reasoning: None,
}
}
}
}