use bytes::Bytes;
use futures::StreamExt;
use futures::future::BoxFuture;
use crate::{
completion::{CompletionError, FinishReason},
http_client,
message::AssistantContent,
streaming::{StreamFinal, StreamedAssistantContent},
};
#[derive(Debug, thiserror::Error)]
pub enum ConformanceError {
#[error(transparent)]
Completion(#[from] CompletionError),
#[error("{scenario} conformance failed for {provider}: {details}")]
Contract {
scenario: &'static str,
provider: &'static str,
details: String,
},
}
impl ConformanceError {
fn contract(
scenario: &'static str,
provider: &'static str,
details: impl Into<String>,
) -> Self {
Self::Contract {
scenario,
provider,
details: details.into(),
}
}
}
#[derive(Debug)]
pub struct ScenarioReport {
pub name: &'static str,
pub provider: &'static str,
pub observations: Vec<String>,
}
#[derive(Debug)]
pub enum ScenarioOutcome {
Ran(ScenarioReport),
Skipped {
name: &'static str,
provider: &'static str,
reason: &'static str,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct SuiteCapabilities {
pub partial_tool_args: bool,
pub zero_usage_terminal: bool,
pub bare_terminal: bool,
pub malformed_frame: bool,
pub unknown_event_frame: bool,
pub defective_known_frame: bool,
pub delta_less_prelude: bool,
pub refusal: bool,
pub interleaved_reasoning: bool,
}
impl SuiteCapabilities {
pub fn from_names(names: &[&str]) -> Result<Self, String> {
let mut caps = Self::default();
for name in names {
match *name {
"partial_tool_args" => caps.partial_tool_args = true,
"zero_usage_terminal" => caps.zero_usage_terminal = true,
"bare_terminal" => caps.bare_terminal = true,
"malformed_frame" => caps.malformed_frame = true,
"unknown_event_frame" => caps.unknown_event_frame = true,
"defective_known_frame" => caps.defective_known_frame = true,
"delta_less_prelude" => caps.delta_less_prelude = true,
"refusal" => caps.refusal = true,
"interleaved_reasoning" => caps.interleaved_reasoning = true,
other => {
return Err(format!(
"unknown capability name in suite manifest: {other}"
));
}
}
}
Ok(caps)
}
}
pub const CANONICAL_SCENARIOS: &[&str] = &[
"truncation_preserves_content_without_terminal",
"transport_error_after_tool_call_yields_err_then_end",
"malformed_frame_surfaces_err_and_terminal_still_completes",
"unknown_event_is_skipped",
"defective_known_event_surfaces_err",
"delta_less_choice_prelude_is_a_noop",
"refusal_frames_deliver_text_without_error",
"bare_terminal_after_only_unparseable_frames_fabricates_nothing",
"usage_variants_are_reported_or_zero_sentinel",
"interleaved_constant_id_reasoning_preserves_order",
];
pub const WIRE_FAMILIES: &[&str] = &[
"openai_chat",
"openai_responses",
"openai_responses_websocket",
"chatgpt",
"anthropic",
"gemini_rest",
"gemini_interactions",
"gemini_grpc",
"cohere",
"ollama",
"xai",
"copilot",
"bedrock",
"candle",
];
pub fn xfail_reason<'a>(xfail: &[&'a str], scenario: &str) -> Option<&'a str> {
xfail.iter().find_map(|entry| {
let (name, reason) = entry.split_once(':')?;
(name.trim() == scenario).then(|| reason.trim())
})
}
pub fn invalid_xfail_entries(xfail: &[&str]) -> Vec<String> {
xfail
.iter()
.filter(|entry| match entry.split_once(':') {
Some((name, reason)) => {
!CANONICAL_SCENARIOS.contains(&name.trim()) || reason.trim().is_empty()
}
None => true,
})
.map(|entry| entry.to_string())
.collect()
}
pub fn check_gated_outcome(
scenario: &'static str,
capability: bool,
xfail: &[&str],
outcome: Result<ScenarioOutcome, ConformanceError>,
) -> Result<(), String> {
match (xfail_reason(xfail, scenario), outcome) {
(Some(reason), Err(error)) => {
eprintln!("xfail {scenario}: {reason} ({error})");
Ok(())
}
(Some(reason), Ok(_)) => Err(format!(
"{scenario} passed but is listed as xfail ({reason}); remove the xfail entry"
)),
(None, Err(error)) => Err(format!("{scenario} failed: {error}")),
(None, Ok(ScenarioOutcome::Ran(_))) => {
if capability {
Ok(())
} else {
Err(format!(
"{scenario} ran but the suite disclaims the capability; set the flag to true"
))
}
}
(None, Ok(ScenarioOutcome::Skipped { reason, .. })) => {
if capability {
Err(format!(
"{scenario} skipped ({reason}) but the suite declares the capability; \
a declared capability's scenario must run"
))
} else {
eprintln!("skipped {scenario}: {reason}");
Ok(())
}
}
}
}
pub fn check_ungated_outcome(
scenario: &'static str,
xfail: &[&str],
result: Result<ScenarioReport, ConformanceError>,
) -> Result<(), String> {
match (xfail_reason(xfail, scenario), result) {
(Some(reason), Err(error)) => {
eprintln!("xfail {scenario}: {reason} ({error})");
Ok(())
}
(Some(reason), Ok(_)) => Err(format!(
"{scenario} passed but is listed as xfail ({reason}); remove the xfail entry"
)),
(None, Err(error)) => Err(format!("{scenario} failed: {error}")),
(None, Ok(_)) => Ok(()),
}
}
#[derive(Clone)]
pub enum WireInput {
Bytes(Bytes),
Event(std::sync::Arc<dyn std::any::Any + Send + Sync>),
}
impl WireInput {
pub fn as_bytes(&self) -> Option<&Bytes> {
match self {
Self::Bytes(bytes) => Some(bytes),
Self::Event(_) => None,
}
}
pub fn downcast_event<T: 'static>(&self) -> Option<&T> {
match self {
Self::Bytes(_) => None,
Self::Event(event) => event.downcast_ref(),
}
}
}
impl From<Bytes> for WireInput {
fn from(bytes: Bytes) -> Self {
Self::Bytes(bytes)
}
}
impl std::fmt::Debug for WireInput {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Bytes(bytes) => formatter.debug_tuple("Bytes").field(bytes).finish(),
Self::Event(_) => formatter.write_str("Event(..)"),
}
}
}
pub fn event_frame<T: Send + Sync + 'static>(event: T) -> WireInput {
WireInput::Event(std::sync::Arc::new(event))
}
pub type WireChunks = Vec<http_client::Result<WireInput>>;
pub fn ok_chunks(frames: impl IntoIterator<Item = impl Into<WireInput>>) -> WireChunks {
frames.into_iter().map(|frame| Ok(frame.into())).collect()
}
pub fn transport_error_chunk() -> http_client::Result<WireInput> {
Err(http_client::Error::InvalidStatusCodeWithMessage(
http::StatusCode::BAD_GATEWAY,
"connection reset".to_string(),
))
}
pub fn assert_valid_event_stream(
items: &[Result<crate::streaming::StreamedAssistantContent, CompletionError>],
choice: &[AssistantContent],
) {
use crate::message::AssistantContent;
use crate::streaming::StreamedAssistantContent as Item;
let ok_items: Vec<&Item> = items.iter().filter_map(|item| item.as_ref().ok()).collect();
let final_count = ok_items
.iter()
.filter(|item| matches!(item, Item::Final(_)))
.count();
assert!(
final_count <= 1,
"law 1 (terminal latch): {final_count} terminal records yielded"
);
if let Some(final_index) = ok_items
.iter()
.position(|item| matches!(item, Item::Final(_)))
{
for item in ok_items.get(final_index + 1..).unwrap_or_default() {
assert!(
matches!(item, Item::Unknown(_)),
"law 1 (terminal latch): content item after the terminal record: {item:?}"
);
}
}
let streamed_text: String = ok_items
.iter()
.filter_map(|item| match item {
Item::Text(text) => Some(text.text.as_str()),
_ => None,
})
.collect();
let aggregated_text: String = choice
.iter()
.filter_map(|content| match content {
AssistantContent::Text(text) => Some(text.text.as_str()),
_ => None,
})
.collect();
assert_eq!(
aggregated_text, streamed_text,
"law 2 (text conservation): aggregated text differs from the streamed deltas"
);
let yielded_calls = ok_items
.iter()
.filter(|item| matches!(item, Item::ToolCall { .. }))
.count();
let aggregated_calls = choice
.iter()
.filter(|content| matches!(content, AssistantContent::ToolCall(_)))
.count();
assert_eq!(
aggregated_calls, yielded_calls,
"law 3 (completed-call conservation): {yielded_calls} calls yielded, \
{aggregated_calls} aggregated"
);
let mut seen_delta_ids: Vec<&str> = Vec::new();
let mut completed_ids: Vec<&str> = Vec::new();
for item in &ok_items {
match item {
Item::ToolCallDelta {
internal_call_id, ..
} => {
assert!(
!completed_ids.contains(&internal_call_id.as_str()),
"law 4: a delta for internal id {internal_call_id} arrived after its \
completed call"
);
seen_delta_ids.push(internal_call_id);
}
Item::ToolCall {
internal_call_id, ..
} => completed_ids.push(internal_call_id),
_ => {}
}
}
let mut completed_reasoning_ids: Vec<&str> = Vec::new();
for item in &ok_items {
if let Item::Reasoning { id, .. } = item {
assert!(
!id.is_empty(),
"law 4b (reasoning correlation): a completed block carries an empty correlator"
);
assert!(
!completed_reasoning_ids.contains(&id.as_str()),
"law 4b (reasoning correlation): two completed blocks share correlator {id}"
);
completed_reasoning_ids.push(id);
}
}
let yielded_reasoning = ok_items
.iter()
.any(|item| matches!(item, Item::Reasoning { .. } | Item::ReasoningDelta { .. }));
let aggregated_reasoning = choice
.iter()
.any(|content| matches!(content, AssistantContent::Reasoning(_)));
assert!(
yielded_reasoning || !aggregated_reasoning,
"law 5 (reasoning provenance): aggregated reasoning with no reasoning yielded"
);
let yielded_full_block = ok_items
.iter()
.any(|item| matches!(item, Item::Reasoning { .. }));
if yielded_reasoning && !yielded_full_block {
let streamed_reasoning: String = ok_items
.iter()
.filter_map(|item| match item {
Item::ReasoningDelta { reasoning, .. } => Some(reasoning.as_str()),
_ => None,
})
.collect();
let aggregated_reasoning_text: String = choice
.iter()
.filter_map(|content| match content {
AssistantContent::Reasoning(reasoning) => Some(reasoning.content.iter()),
_ => None,
})
.flatten()
.filter_map(|part| match part {
crate::message::ReasoningContent::Text { text, .. } => Some(text.as_str()),
_ => None,
})
.collect();
assert_eq!(
aggregated_reasoning_text, streamed_reasoning,
"law 5 (reasoning conservation): with no full block, the aggregated reasoning \
must be exactly the concatenated deltas"
);
}
}
#[derive(Debug)]
pub struct DrainedStream {
pub items: Vec<Result<StreamedAssistantContent, CompletionError>>,
pub choice: Vec<AssistantContent>,
pub response: Option<StreamFinal>,
}
impl DrainedStream {
pub fn texts(&self) -> Vec<&str> {
self.items
.iter()
.filter_map(|item| match item {
Ok(StreamedAssistantContent::Text(text)) => Some(text.text.as_str()),
_ => None,
})
.collect()
}
pub fn tool_call_names(&self) -> Vec<&str> {
self.items
.iter()
.filter_map(|item| match item {
Ok(StreamedAssistantContent::ToolCall { tool_call, .. }) => {
Some(tool_call.function.name.as_str())
}
_ => None,
})
.collect()
}
pub fn unknown_values(&self) -> Vec<&serde_json::Value> {
self.items
.iter()
.filter_map(|item| match item {
Ok(StreamedAssistantContent::Unknown(value)) => Some(value.value()),
_ => None,
})
.collect()
}
pub fn error_count(&self) -> usize {
self.items.iter().filter(|item| item.is_err()).count()
}
pub fn final_count(&self) -> usize {
self.items
.iter()
.filter(|item| matches!(item, Ok(StreamedAssistantContent::Final(_))))
.count()
}
fn first_error_index(&self) -> Option<usize> {
self.items.iter().position(|item| item.is_err())
}
pub fn choice_texts(&self) -> Vec<&str> {
self.choice
.iter()
.filter_map(|content| match content {
AssistantContent::Text(text) => Some(text.text.as_str()),
_ => None,
})
.collect()
}
pub fn choice_reasoning(&self) -> Vec<&crate::message::Reasoning> {
self.choice
.iter()
.filter_map(|content| match content {
AssistantContent::Reasoning(reasoning) => Some(reasoning),
_ => None,
})
.collect()
}
pub fn choice_tool_call_names(&self) -> Vec<&str> {
self.choice
.iter()
.filter_map(|content| match content {
AssistantContent::ToolCall(tool_call) => Some(tool_call.function.name.as_str()),
_ => None,
})
.collect()
}
}
type DriveFn = Box<
dyn Fn(WireChunks) -> BoxFuture<'static, Result<DrainedStream, CompletionError>> + Send + Sync,
>;
pub struct WireDriver {
pub provider: &'static str,
drive: DriveFn,
}
impl WireDriver {
pub fn new(
provider: &'static str,
drive: impl Fn(WireChunks) -> BoxFuture<'static, Result<DrainedStream, CompletionError>>
+ Send
+ Sync
+ 'static,
) -> Self {
Self {
provider,
drive: Box::new(drive),
}
}
pub async fn drive(&self, chunks: WireChunks) -> Result<DrainedStream, CompletionError> {
(self.drive)(chunks).await
}
}
pub struct RefusalFixture {
pub frames: Vec<WireInput>,
pub expected_text: &'static str,
}
pub struct InterleavedReasoningFixture {
pub frames: Vec<WireInput>,
pub first_reasoning: &'static str,
pub tool_name: &'static str,
pub second_reasoning: &'static str,
}
type BufferedDriveFn = Box<
dyn Fn(String) -> BoxFuture<'static, Result<Vec<AssistantContent>, CompletionError>>
+ Send
+ Sync,
>;
pub struct BufferedBodyDriver {
pub provider: &'static str,
drive: BufferedDriveFn,
}
impl BufferedBodyDriver {
pub fn new(
provider: &'static str,
drive: impl Fn(String) -> BoxFuture<'static, Result<Vec<AssistantContent>, CompletionError>>
+ Send
+ Sync
+ 'static,
) -> Self {
Self {
provider,
drive: Box::new(drive),
}
}
pub async fn drive(&self, body: String) -> Result<Vec<AssistantContent>, CompletionError> {
(self.drive)(body).await
}
}
pub struct ProviderWireFixture {
pub driver: WireDriver,
pub text_frames: Vec<WireInput>,
pub expected_texts: Vec<&'static str>,
pub tool_call_frames: Vec<WireInput>,
pub expected_tool_name: &'static str,
pub partial_tool_call_frames: Option<Vec<WireInput>>,
pub terminal_frames: Vec<WireInput>,
pub expected_usage_total: u64,
pub expected_finish_reason: Option<FinishReason>,
pub zero_usage_terminal_frames: Option<Vec<WireInput>>,
pub bare_terminal_frames: Option<Vec<WireInput>>,
pub malformed_frame: Option<WireInput>,
pub unknown_event_frame: Option<WireInput>,
pub defective_known_frame: Option<WireInput>,
pub delta_less_prelude_frame: Option<WireInput>,
pub refusal: Option<RefusalFixture>,
pub interleaved_reasoning: Option<InterleavedReasoningFixture>,
}
impl ProviderWireFixture {
pub fn capabilities(&self) -> SuiteCapabilities {
SuiteCapabilities {
partial_tool_args: self.partial_tool_call_frames.is_some(),
zero_usage_terminal: self.zero_usage_terminal_frames.is_some(),
bare_terminal: self.bare_terminal_frames.is_some(),
malformed_frame: self.malformed_frame.is_some(),
unknown_event_frame: self.unknown_event_frame.is_some(),
defective_known_frame: self.defective_known_frame.is_some(),
delta_less_prelude: self.delta_less_prelude_frame.is_some(),
refusal: self.refusal.is_some(),
interleaved_reasoning: self.interleaved_reasoning.is_some(),
}
}
}
fn concat_frames(parts: &[&[WireInput]]) -> Vec<WireInput> {
parts
.iter()
.flat_map(|frames| frames.iter().cloned())
.collect()
}
pub async fn truncation_preserves_content_without_terminal(
fixture: &ProviderWireFixture,
) -> Result<ScenarioReport, ConformanceError> {
const SCENARIO: &str = "truncation_preserves_content_without_terminal";
let provider = fixture.driver.provider;
let mut observations = Vec::new();
let drained = fixture.driver.drive(Vec::new()).await?;
if drained.response.is_some() || drained.final_count() != 0 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"an empty stream must not synthesize a terminal record",
));
}
observations.push("EOF before content: no terminal".to_string());
let drained = fixture
.driver
.drive(ok_chunks(fixture.text_frames.clone()))
.await?;
if drained.texts() != fixture.expected_texts {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"text delivered before truncation must be preserved: expected {:?}, observed {:?}",
fixture.expected_texts,
drained.texts()
),
));
}
if drained.response.is_some() || drained.final_count() != 0 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"EOF after text deltas must not synthesize a terminal record",
));
}
observations.push("EOF mid-text: content preserved, no terminal".to_string());
if let Some(partial) = &fixture.partial_tool_call_frames {
let drained = fixture.driver.drive(ok_chunks(partial.clone())).await?;
if drained.response.is_some() || drained.final_count() != 0 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"EOF mid-tool-arguments must not synthesize a terminal record",
));
}
observations.push("EOF mid-tool-args: no terminal".to_string());
}
let drained = fixture
.driver
.drive(ok_chunks(fixture.tool_call_frames.clone()))
.await?;
if drained.tool_call_names() != vec![fixture.expected_tool_name] {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"a fully-delivered tool call must survive truncation: observed {:?}",
drained.tool_call_names()
),
));
}
if drained.response.is_some() || drained.final_count() != 0 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"EOF after a delivered tool call must not synthesize a terminal record",
));
}
observations.push("EOF after tool-complete: tool call preserved, no terminal".to_string());
Ok(ScenarioReport {
name: SCENARIO,
provider,
observations,
})
}
pub async fn transport_error_after_tool_call_yields_err_then_end(
fixture: &ProviderWireFixture,
) -> Result<ScenarioReport, ConformanceError> {
const SCENARIO: &str = "transport_error_after_tool_call_yields_err_then_end";
let provider = fixture.driver.provider;
let mut chunks = ok_chunks(fixture.tool_call_frames.clone());
chunks.push(transport_error_chunk());
let drained = fixture.driver.drive(chunks).await?;
if drained.tool_call_names() != vec![fixture.expected_tool_name] {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"the delivered tool call must precede the transport error: observed {:?}",
drained.tool_call_names()
),
));
}
let error_index = drained.first_error_index().ok_or_else(|| {
ConformanceError::contract(
SCENARIO,
provider,
"the transport failure must reach the consumer",
)
})?;
if error_index + 1 != drained.items.len() {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"nothing may follow the terminal transport error",
));
}
if drained.response.is_some() || drained.final_count() != 0 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"a transport failure must not be papered over with a terminal record",
));
}
Ok(ScenarioReport {
name: SCENARIO,
provider,
observations: vec!["tool call, then Err, then end; no terminal".to_string()],
})
}
pub async fn malformed_frame_surfaces_err_and_terminal_still_completes(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
const SCENARIO: &str = "malformed_frame_surfaces_err_and_terminal_still_completes";
let provider = fixture.driver.provider;
let Some(malformed) = &fixture.malformed_frame else {
return Ok(ScenarioOutcome::Skipped {
name: SCENARIO,
provider,
reason: "wire family cannot spell a frame-level decode failure",
});
};
let frames = concat_frames(&[
&fixture.text_frames,
std::slice::from_ref(malformed),
&fixture.terminal_frames,
]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
if drained.error_count() != 1 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"the malformed frame must surface as exactly one Err item, observed {}",
drained.error_count()
),
));
}
if drained.texts() != fixture.expected_texts {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"content around the malformed frame must be preserved",
));
}
if drained.response.is_none() {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the genuine terminal after a recoverable parse error must still complete the stream",
));
}
Ok(ScenarioOutcome::Ran(ScenarioReport {
name: SCENARIO,
provider,
observations: vec!["Err surfaced, terminal still completed".to_string()],
}))
}
pub async fn unknown_event_is_skipped(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
const SCENARIO: &str = "unknown_event_is_skipped";
let provider = fixture.driver.provider;
let Some(unknown) = &fixture.unknown_event_frame else {
return Ok(ScenarioOutcome::Skipped {
name: SCENARIO,
provider,
reason: "wire family cannot spell an unknown event type",
});
};
let frames = concat_frames(&[
&fixture.text_frames,
std::slice::from_ref(unknown),
&fixture.terminal_frames,
]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
if drained.error_count() != 0 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"an unknown event type must be skipped, not surfaced as an error",
));
}
if drained.texts() != fixture.expected_texts || drained.response.is_none() {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the stream must deliver its content and complete around the skipped event",
));
}
if drained.unknown_values().len() != 1 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"exactly one Unknown passthrough item must surface for the unknown frame, \
observed {}",
drained.unknown_values().len()
),
));
}
let control_frames = concat_frames(&[&fixture.text_frames, &fixture.terminal_frames]);
let control = fixture.driver.drive(ok_chunks(control_frames)).await?;
if drained.choice != control.choice {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the unknown frame must not perturb the aggregated assistant choice",
));
}
Ok(ScenarioOutcome::Ran(ScenarioReport {
name: SCENARIO,
provider,
observations: vec![
"unknown event skipped semantically, surfaced on the raw channel, \
choice unchanged, stream completed"
.to_string(),
],
}))
}
pub async fn defective_known_event_surfaces_err(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
const SCENARIO: &str = "defective_known_event_surfaces_err";
let provider = fixture.driver.provider;
let Some(defective) = &fixture.defective_known_frame else {
return Ok(ScenarioOutcome::Skipped {
name: SCENARIO,
provider,
reason: "wire family cannot spell a known event with a schema-defective payload",
});
};
let frames = concat_frames(&[
&fixture.text_frames,
std::slice::from_ref(defective),
&fixture.terminal_frames,
]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
if drained.error_count() != 1 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"a known event with a schema defect must surface exactly one Err item, observed {}",
drained.error_count()
),
));
}
if drained.response.is_none() {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the genuine terminal must still complete the stream after the defective frame",
));
}
Ok(ScenarioOutcome::Ran(ScenarioReport {
name: SCENARIO,
provider,
observations: vec!["defective known event surfaced as Err; stream completed".to_string()],
}))
}
pub async fn delta_less_choice_prelude_is_a_noop(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
const SCENARIO: &str = "delta_less_choice_prelude_is_a_noop";
let provider = fixture.driver.provider;
let Some(prelude) = &fixture.delta_less_prelude_frame else {
return Ok(ScenarioOutcome::Skipped {
name: SCENARIO,
provider,
reason: "wire family has no delta-less prelude shape",
});
};
let frames = concat_frames(&[
std::slice::from_ref(prelude),
&fixture.text_frames,
&fixture.terminal_frames,
]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
if drained.error_count() != 0 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the delta-less prelude must not surface an error",
));
}
if drained.texts() != fixture.expected_texts || drained.response.is_none() {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the prelude must not perturb content delivery or the terminal",
));
}
Ok(ScenarioOutcome::Ran(ScenarioReport {
name: SCENARIO,
provider,
observations: vec!["delta-less prelude ignored; stream unaffected".to_string()],
}))
}
pub async fn refusal_frames_deliver_text_without_error(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
const SCENARIO: &str = "refusal_frames_deliver_text_without_error";
let provider = fixture.driver.provider;
let Some(refusal) = &fixture.refusal else {
return Ok(ScenarioOutcome::Skipped {
name: SCENARIO,
provider,
reason: "wire family has no refusal channel",
});
};
let frames = concat_frames(&[&refusal.frames, &fixture.terminal_frames]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
if drained.error_count() != 0 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"refusal content must not surface as an error",
));
}
let delivered = drained.texts().concat();
if delivered != refusal.expected_text {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"refusal text must be delivered: expected {:?}, observed {delivered:?}",
refusal.expected_text
),
));
}
if drained.response.is_none() {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"a refused turn still ends with the provider's genuine terminal",
));
}
Ok(ScenarioOutcome::Ran(ScenarioReport {
name: SCENARIO,
provider,
observations: vec!["refusal text delivered without error".to_string()],
}))
}
pub async fn terminal_body_content_merges_per_kind(
driver: &BufferedBodyDriver,
cases: Vec<(&'static str, String)>,
expected_text: &str,
) -> Result<ScenarioReport, ConformanceError> {
const SCENARIO: &str = "terminal_body_content_merges_per_kind";
let provider = driver.provider;
let mut observations = Vec::new();
for (label, body) in cases {
let choice = driver.drive(body).await?;
let choice_text: String = choice
.iter()
.filter_map(|content| match content {
AssistantContent::Text(text) => Some(text.text.as_str()),
_ => None,
})
.collect();
let occurrences = choice_text.matches(expected_text).count();
if occurrences != 1 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"{label}: terminal-body text must appear exactly once in the choice, observed {occurrences} in {choice_text:?}"
),
));
}
observations.push(format!("{label}: text merged exactly once"));
}
Ok(ScenarioReport {
name: SCENARIO,
provider,
observations,
})
}
pub async fn bare_terminal_after_only_unparseable_frames_fabricates_nothing(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
const SCENARIO: &str = "bare_terminal_after_only_unparseable_frames_fabricates_nothing";
let provider = fixture.driver.provider;
let Some(bare_terminal) = &fixture.bare_terminal_frames else {
return Ok(ScenarioOutcome::Skipped {
name: SCENARIO,
provider,
reason: "wire family has no data-less terminal signal",
});
};
let Some(malformed) = &fixture.malformed_frame else {
return Ok(ScenarioOutcome::Skipped {
name: SCENARIO,
provider,
reason: "wire family cannot spell a frame-level decode failure",
});
};
let frames = concat_frames(&[std::slice::from_ref(malformed), bare_terminal]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
if drained.error_count() == 0 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the unparseable frame must surface as an Err item",
));
}
if drained.response.is_some() || drained.final_count() != 0 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"a bare terminal with no decoded frame must not fabricate a terminal record",
));
}
Ok(ScenarioOutcome::Ran(ScenarioReport {
name: SCENARIO,
provider,
observations: vec!["no fabricated terminal after only-unparseable frames".to_string()],
}))
}
pub async fn usage_variants_are_reported_or_zero_sentinel(
fixture: &ProviderWireFixture,
) -> Result<ScenarioReport, ConformanceError> {
const SCENARIO: &str = "usage_variants_are_reported_or_zero_sentinel";
let provider = fixture.driver.provider;
let mut observations = Vec::new();
let frames = concat_frames(&[&fixture.text_frames, &fixture.terminal_frames]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
let response = drained.response.as_ref().ok_or_else(|| {
ConformanceError::contract(
SCENARIO,
provider,
"the genuine terminal must produce a record",
)
})?;
if response.usage.total_tokens != fixture.expected_usage_total {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"terminal usage must be preserved: expected total {}, observed {}",
fixture.expected_usage_total, response.usage.total_tokens
),
));
}
if response.finish_reason != fixture.expected_finish_reason {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"terminal finish reason must be normalized: expected {:?}, observed {:?}",
fixture.expected_finish_reason, response.finish_reason
),
));
}
observations.push(format!(
"usage total {} and finish reason {:?} preserved",
fixture.expected_usage_total, fixture.expected_finish_reason
));
if let Some(zero_usage) = &fixture.zero_usage_terminal_frames {
let frames = concat_frames(&[&fixture.text_frames, zero_usage]);
let drained = fixture.driver.drive(ok_chunks(frames)).await?;
let response = drained.response.as_ref().ok_or_else(|| {
ConformanceError::contract(
SCENARIO,
provider,
"a usage-less genuine terminal must still complete the stream",
)
})?;
if response.usage.total_tokens != 0 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"missing usage metrics must be the zero-usage sentinel, not invented values",
));
}
observations.push("usage-less terminal completed with the zero sentinel".to_string());
}
Ok(ScenarioReport {
name: SCENARIO,
provider,
observations,
})
}
pub async fn reasoning_summary_deltas_are_superseded_without_duplication(
driver: &WireDriver,
frames: Vec<WireInput>,
summary_text: &str,
) -> Result<ScenarioReport, ConformanceError> {
const SCENARIO: &str = "reasoning_summary_deltas_are_superseded_without_duplication";
let provider = driver.provider;
let drained = driver.drive(ok_chunks(frames)).await?;
if drained.error_count() != 0 || drained.response.is_none() {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the reasoning stream must complete without errors",
));
}
let reasoning = drained.choice_reasoning();
let occurrences: usize = reasoning
.iter()
.flat_map(|item| item.content.iter())
.filter(|content| match content {
crate::message::ReasoningContent::Summary(text)
| crate::message::ReasoningContent::Text { text, .. } => text.contains(summary_text),
_ => false,
})
.count();
if occurrences != 1 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"the summary must appear exactly once in the aggregated choice, observed {occurrences} across {reasoning:?}"
),
));
}
if reasoning.len() != 1 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"deltas and their full block must collapse to one reasoning item, observed {}",
reasoning.len()
),
));
}
Ok(ScenarioReport {
name: SCENARIO,
provider,
observations: vec!["summary aggregated exactly once".to_string()],
})
}
pub async fn multi_part_same_id_reasoning_keeps_every_part(
driver: &WireDriver,
frames: Vec<WireInput>,
expected_parts: &[&str],
) -> Result<ScenarioReport, ConformanceError> {
const SCENARIO: &str = "multi_part_same_id_reasoning_keeps_every_part";
let provider = driver.provider;
let drained = driver.drive(ok_chunks(frames)).await?;
if drained.error_count() != 0 || drained.response.is_none() {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the reasoning stream must complete without errors",
));
}
let observed: Vec<String> = drained
.choice_reasoning()
.iter()
.flat_map(|item| item.content.iter())
.map(|content| match content {
crate::message::ReasoningContent::Summary(text) => text.clone(),
crate::message::ReasoningContent::Text { text, .. } => text.clone(),
crate::message::ReasoningContent::Encrypted(data) => data.clone(),
crate::message::ReasoningContent::Redacted { data } => data.clone(),
})
.collect();
if observed != expected_parts {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"every same-id reasoning part must survive in order: expected {expected_parts:?}, observed {observed:?}"
),
));
}
Ok(ScenarioReport {
name: SCENARIO,
provider,
observations: vec![format!(
"all {} reasoning parts survived",
expected_parts.len()
)],
})
}
pub async fn interleaved_reasoning_aggregates_to_one_item(
driver: &WireDriver,
frames: Vec<WireInput>,
expected_text: &str,
) -> Result<ScenarioReport, ConformanceError> {
const SCENARIO: &str = "interleaved_reasoning_aggregates_to_one_item";
let provider = driver.provider;
let drained = driver.drive(ok_chunks(frames)).await?;
if drained.error_count() != 0 || drained.response.is_none() {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the interleaved stream must complete without errors",
));
}
let reasoning = drained.choice_reasoning();
if reasoning.len() != 1 {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!(
"interleaved deltas and their completed block must collapse to one reasoning item, observed {}",
reasoning.len()
),
));
}
let carries_text = reasoning
.iter()
.flat_map(|item| item.content.iter())
.any(|content| match content {
crate::message::ReasoningContent::Summary(text)
| crate::message::ReasoningContent::Text { text, .. } => text == expected_text,
_ => false,
});
if !carries_text {
return Err(ConformanceError::contract(
SCENARIO,
provider,
format!("the reasoning item must carry the completed block's text {expected_text:?}"),
));
}
Ok(ScenarioReport {
name: SCENARIO,
provider,
observations: vec!["exactly one reasoning item with the completed content".to_string()],
})
}
pub async fn interleaved_constant_id_reasoning_preserves_order(
fixture: &ProviderWireFixture,
) -> Result<ScenarioOutcome, ConformanceError> {
const SCENARIO: &str = "interleaved_constant_id_reasoning_preserves_order";
let provider = fixture.driver.provider;
let Some(interleaved) = &fixture.interleaved_reasoning else {
return Ok(ScenarioOutcome::Skipped {
name: SCENARIO,
provider,
reason: "wire fixture supplies no interleaved reasoning frames",
});
};
let drained = fixture
.driver
.drive(ok_chunks(interleaved.frames.clone()))
.await?;
if drained.error_count() != 0 || drained.response.is_none() {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the interleaved stream must complete without errors",
));
}
assert_reasoning_tool_reasoning(
SCENARIO,
provider,
&drained,
interleaved.first_reasoning,
interleaved.tool_name,
interleaved.second_reasoning,
)?;
Ok(ScenarioOutcome::Ran(ScenarioReport {
name: SCENARIO,
provider,
observations: vec!["boundary kept: reasoning, tool call, reasoning in order".to_string()],
}))
}
pub async fn interleaved_signed_full_reasoning_does_not_erase_prior_thought(
driver: &WireDriver,
frames: Vec<WireInput>,
first: &str,
tool_name: &str,
second: &str,
) -> Result<ScenarioReport, ConformanceError> {
const SCENARIO: &str = "interleaved_signed_full_reasoning_does_not_erase_prior_thought";
let provider = driver.provider;
let drained = driver.drive(ok_chunks(frames)).await?;
if drained.error_count() != 0 || drained.response.is_none() {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the interleaved stream must complete without errors",
));
}
assert_reasoning_tool_reasoning(SCENARIO, provider, &drained, first, tool_name, second)?;
let signed = drained.choice_reasoning().last().is_some_and(|reasoning| {
reasoning.content.iter().any(|content| {
matches!(
content,
crate::message::ReasoningContent::Text {
signature: Some(_),
..
}
)
})
});
if !signed {
return Err(ConformanceError::contract(
SCENARIO,
provider,
"the post-boundary block must keep its signature",
));
}
Ok(ScenarioReport {
name: SCENARIO,
provider,
observations: vec![
"pre-boundary thought survived; signed block completed the post-boundary part"
.to_string(),
],
})
}
fn assert_reasoning_tool_reasoning(
scenario: &'static str,
provider: &'static str,
drained: &DrainedStream,
first: &str,
tool_name: &str,
second: &str,
) -> Result<(), ConformanceError> {
let shape: Vec<String> = drained
.choice
.iter()
.map(|content| match content {
AssistantContent::Reasoning(reasoning) => {
let text: String = reasoning
.content
.iter()
.filter_map(|content| match content {
crate::message::ReasoningContent::Summary(text)
| crate::message::ReasoningContent::Text { text, .. } => {
Some(text.as_str())
}
_ => None,
})
.collect();
format!("reasoning:{text}")
}
AssistantContent::ToolCall(tool_call) => {
format!("tool:{}", tool_call.function.name)
}
AssistantContent::Text(text) => format!("text:{}", text.text),
AssistantContent::Image(_) => "image".to_string(),
})
.collect();
let expected = vec![
format!("reasoning:{first}"),
format!("tool:{tool_name}"),
format!("reasoning:{second}"),
];
if shape != expected {
return Err(ConformanceError::contract(
scenario,
provider,
format!(
"the boundary must survive aggregation: expected {expected:?}, observed {shape:?}"
),
));
}
Ok(())
}
#[cfg(all(not(target_family = "wasm"), feature = "websocket"))]
pub async fn drain_openai_responses_websocket_events(
provider: &'static str,
events: Vec<
Result<
crate::providers::openai::responses_api::websocket::ResponsesWebSocketEvent,
CompletionError,
>,
>,
) -> DrainedStream {
use crate::providers::openai::responses_api::ResponsesUsage;
use crate::providers::openai::responses_api::streaming::{
RawChoiceAccumulator, ResponseChunkKind, ResponsesStreamOptions, normalize_responses_stream,
};
use crate::providers::openai::responses_api::websocket::ResponsesWebSocketEvent;
let mut accumulator = RawChoiceAccumulator::new(ResponsesUsage::new());
let mut raw = Vec::new();
let mut errored = false;
for event in events {
match event {
Ok(ResponsesWebSocketEvent::Item(chunk)) => raw.extend(
accumulator
.decode_item_chunk(chunk, ResponsesStreamOptions::strict())
.into_iter()
.map(Ok),
),
Ok(ResponsesWebSocketEvent::Response(chunk)) => {
let terminal = matches!(
chunk.kind,
ResponseChunkKind::ResponseCompleted
| ResponseChunkKind::ResponseFailed
| ResponseChunkKind::ResponseIncomplete
);
if let Err(error) =
accumulator.record_response_chunk(chunk.kind, chunk.response, "")
{
raw.extend(accumulator.take_tool_calls().into_iter().map(Ok));
raw.push(Err(error));
errored = true;
break;
}
if terminal {
break;
}
}
Ok(ResponsesWebSocketEvent::Unknown(value)) => {
raw.push(Ok(crate::streaming::RawStreamingChoice::Unknown(value)));
}
Ok(ResponsesWebSocketEvent::Done(_)) => {}
Ok(ResponsesWebSocketEvent::Error(error)) => {
raw.extend(accumulator.take_tool_calls().into_iter().map(Ok));
raw.push(Err(CompletionError::ProviderError(error.to_string())));
errored = true;
break;
}
Err(error) => {
raw.extend(accumulator.take_tool_calls().into_iter().map(Ok));
raw.push(Err(error));
errored = true;
break;
}
}
}
if !errored {
raw.extend(accumulator.finish().into_iter().map(Ok));
}
let stream = normalize_responses_stream(provider, Box::pin(futures::stream::iter(raw)));
fixtures::drain(stream).await
}
pub mod fixtures {
use super::*;
use crate::client::CompletionClient;
use crate::completion::CompletionModel;
use crate::test_utils::SequencedStreamingHttpClient;
use serde_json::json;
pub async fn drain(mut stream: crate::streaming::StreamingCompletionResponse) -> DrainedStream {
let mut items = Vec::new();
while let Some(item) = stream.next().await {
items.push(item);
}
let drained = DrainedStream {
items,
choice: stream.choice.clone(),
response: stream.response.clone(),
};
super::assert_valid_event_stream(&drained.items, &drained.choice);
drained
}
fn byte_chunks(chunks: WireChunks) -> Result<Vec<http_client::Result<Bytes>>, CompletionError> {
chunks
.into_iter()
.map(|chunk| match chunk {
Ok(WireInput::Bytes(bytes)) => Ok(Ok(bytes)),
Ok(WireInput::Event(_)) => Err(CompletionError::ProviderError(
"typed-event frame fed to a byte-transport driver".to_string(),
)),
Err(error) => Ok(Err(error)),
})
.collect()
}
fn sse(frame: &serde_json::Value) -> WireInput {
WireInput::Bytes(Bytes::from(format!("data: {frame}\n\n")))
}
fn sse_raw(data: &str) -> WireInput {
WireInput::Bytes(Bytes::from(format!("data: {data}\n\n")))
}
fn ndjson(frame: &serde_json::Value) -> WireInput {
WireInput::Bytes(Bytes::from(format!("{frame}\n")))
}
fn frame_text(frame: &WireInput) -> String {
frame
.as_bytes()
.map(|bytes| String::from_utf8_lossy(bytes).into_owned())
.unwrap_or_default()
}
pub mod openai_chat {
use super::*;
fn driver() -> WireDriver {
WireDriver::new("openai", |chunks| {
Box::pin(async move {
let client = crate::providers::openai::Client::builder()
.http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
.api_key("test-key")
.build()?
.completions_api();
let model = client.completion_model("gpt-4o");
let request = model.completion_request("hello").build();
let stream = model.stream(request).await?;
Ok(drain(stream).await)
})
})
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![sse(&json!({
"id": "chatcmpl-1",
"model": "gpt-4o-2024-08-06",
"choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": null}],
"usage": null,
}))],
expected_texts: vec!["hi"],
tool_call_frames: vec![
sse(&json!({
"choices": [{"index": 0, "delta": {"tool_calls": [{
"index": 0,
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": ""},
}]}, "finish_reason": null}],
})),
sse(&json!({
"choices": [{"index": 0, "delta": {"tool_calls": [{
"index": 0,
"function": {"arguments": "{\"city\":\"Tokyo\"}"},
}]}, "finish_reason": null}],
})),
],
expected_tool_name: "get_weather",
partial_tool_call_frames: Some(vec![sse(&json!({
"choices": [{"index": 0, "delta": {"tool_calls": [{
"index": 0,
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": "{\"cit"},
}]}, "finish_reason": null}],
}))]),
terminal_frames: vec![
sse(&json!({
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
"usage": null,
})),
sse(&json!({
"choices": [],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
})),
sse_raw("[DONE]"),
],
expected_usage_total: 15,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: Some(vec![
sse(&json!({
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
"usage": null,
})),
sse_raw("[DONE]"),
]),
bare_terminal_frames: Some(vec![sse_raw("[DONE]")]),
malformed_frame: Some(sse_raw("{not json")),
unknown_event_frame: None,
defective_known_frame: Some(sse_raw(r#"{"choices": 42}"#)),
delta_less_prelude_frame: Some(sse_raw(
r#"{"id":"","object":"","choices":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"}}}]}"#,
)),
refusal: None,
interleaved_reasoning: None,
}
}
}
pub mod openai_responses {
use super::*;
pub fn driver() -> WireDriver {
WireDriver::new("openai", |chunks| {
Box::pin(async move {
let client = crate::providers::openai::Client::builder()
.http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
.api_key("test-key")
.build()?;
let model = client.completion_model("gpt-5.4");
let request = model.completion_request("hello").build();
let stream = model.stream(request).await?;
Ok(drain(stream).await)
})
})
}
fn completed_response(
usage: Option<serde_json::Value>,
output: serde_json::Value,
) -> serde_json::Value {
json!({
"id": "resp_1",
"object": "response",
"created_at": 0,
"status": "completed",
"model": "gpt-5.4",
"output": output,
"tools": [],
"usage": usage,
})
}
fn terminal(usage: Option<serde_json::Value>, output: serde_json::Value) -> WireInput {
sse(&json!({
"type": "response.completed",
"sequence_number": 99,
"response": completed_response(usage, output),
}))
}
fn usage_json() -> serde_json::Value {
json!({
"input_tokens": 10,
"output_tokens": 5,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 15,
})
}
fn text_delta(text: &str) -> WireInput {
sse(&json!({
"type": "response.output_text.delta",
"content_index": 0,
"delta": text,
"item_id": "msg_1",
"output_index": 0,
"sequence_number": 1,
}))
}
fn tool_call_done() -> WireInput {
sse(&json!({
"type": "response.output_item.done",
"output_index": 0,
"sequence_number": 2,
"item": {
"type": "function_call",
"id": "fc_1",
"arguments": "{\"city\":\"Tokyo\"}",
"call_id": "call_1",
"name": "get_weather",
"status": "completed",
},
}))
}
pub fn incomplete_mid_tool_call_frames() -> Vec<WireInput> {
vec![
sse(&json!({
"type": "response.output_item.added",
"output_index": 0,
"sequence_number": 1,
"item": {
"type": "function_call",
"id": "fc_1",
"arguments": "",
"call_id": "call_1",
"name": "add",
"status": "in_progress",
},
})),
sse(&json!({
"type": "response.function_call_arguments.delta",
"item_id": "fc_1",
"output_index": 0,
"sequence_number": 2,
"delta": "{\"x",
})),
sse(&json!({
"type": "response.function_call_arguments.delta",
"item_id": "fc_1",
"output_index": 0,
"sequence_number": 3,
"delta": "\":48151",
})),
sse(&json!({
"type": "response.function_call_arguments.done",
"item_id": "fc_1",
"output_index": 0,
"sequence_number": 4,
"arguments": "{\"x\":48151",
})),
sse(&json!({
"type": "response.output_item.done",
"output_index": 0,
"sequence_number": 5,
"item": {
"type": "function_call",
"id": "fc_1",
"arguments": "{\"x\":48151",
"call_id": "call_1",
"name": "add",
"status": "incomplete",
},
})),
sse(&json!({
"type": "response.incomplete",
"sequence_number": 6,
"response": {
"id": "resp_1",
"object": "response",
"created_at": 0,
"status": "incomplete",
"incomplete_details": {"reason": "max_output_tokens"},
"model": "gpt-5.4",
"output": [{
"type": "function_call",
"id": "fc_1",
"arguments": "{\"x\":48151",
"call_id": "call_1",
"name": "add",
"status": "incomplete",
}],
"tools": [],
"usage": usage_json(),
},
})),
]
}
fn reasoning_done_item(
id: &str,
summary: serde_json::Value,
content: serde_json::Value,
encrypted: Option<&str>,
) -> WireInput {
let mut item = json!({
"type": "reasoning",
"id": id,
"summary": summary,
"content": content,
"status": "completed",
});
if let (Some(encrypted), Some(object)) = (encrypted, item.as_object_mut()) {
object.insert("encrypted_content".to_string(), json!(encrypted));
}
sse(&json!({
"type": "response.output_item.done",
"output_index": 0,
"sequence_number": 3,
"item": item,
}))
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![text_delta("hi")],
expected_texts: vec!["hi"],
tool_call_frames: vec![tool_call_done()],
expected_tool_name: "get_weather",
partial_tool_call_frames: Some(vec![
sse(&json!({
"type": "response.output_item.added",
"output_index": 0,
"sequence_number": 1,
"item": {
"type": "function_call",
"id": "fc_1",
"arguments": "",
"call_id": "call_1",
"name": "get_weather",
"status": "in_progress",
},
})),
sse(&json!({
"type": "response.function_call_arguments.delta",
"item_id": "fc_1",
"output_index": 0,
"sequence_number": 2,
"delta": "{\"cit",
})),
]),
terminal_frames: vec![terminal(Some(usage_json()), json!([]))],
expected_usage_total: 15,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: Some(vec![terminal(None, json!([]))]),
bare_terminal_frames: None,
malformed_frame: Some(sse_raw("{not json")),
unknown_event_frame: Some(sse(&json!({
"type": "response.web_search_call.searching",
"output_index": 0,
"sequence_number": 4,
"item_id": "ws_1",
}))),
defective_known_frame: Some(sse(&json!({
"type": "response.content_part.added",
"item_id": "msg_1",
"output_index": 0,
"content_index": 0,
"sequence_number": 5,
"part": {"type": "output_text", "text": 42},
}))),
delta_less_prelude_frame: None,
refusal: Some(RefusalFixture {
frames: vec![sse(&json!({
"type": "response.refusal.delta",
"content_index": 0,
"delta": "I cannot help with that.",
"item_id": "msg_1",
"output_index": 0,
"sequence_number": 1,
}))],
expected_text: "I cannot help with that.",
}),
interleaved_reasoning: None,
}
}
pub fn buffered_driver() -> BufferedBodyDriver {
BufferedBodyDriver::new("chatgpt", |body| {
Box::pin(async move {
let client = crate::providers::chatgpt::Client::builder()
.api_key(crate::providers::chatgpt::ChatGPTAuth::AccessToken {
access_token: "test-token".to_string(),
account_id: Some("account-id".to_string()),
})
.http_client(crate::test_utils::RecordingHttpClient::new(body))
.build()?;
let model = client.completion_model("gpt-5.4");
let request = model.completion_request("hello").build();
let response = model.completion(request).await?;
Ok(response.choice)
})
})
}
fn message_output(text: &str) -> serde_json::Value {
json!([{
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": text, "annotations": []}],
}])
}
pub fn terminal_body_only_sse_body(text: &str) -> String {
frame_text(&terminal(Some(usage_json()), message_output(text)))
}
pub fn terminal_body_and_delta_sse_body(text: &str) -> String {
let frames = [
text_delta(text),
terminal(Some(usage_json()), message_output(text)),
];
frames.iter().map(frame_text).collect()
}
pub fn delta_only_sse_body(text: &str) -> String {
let frames = [text_delta(text), terminal(Some(usage_json()), json!([]))];
frames.iter().map(frame_text).collect()
}
pub fn envelope_less_reasoning_supersede_sse_body() -> (String, &'static str) {
let delta = json!({
"type": "response.reasoning_summary_text.delta",
"delta": "step 1",
});
let frames = [
sse(&delta),
reasoning_done_item(
"rs_1",
json!([{"type": "summary_text", "text": "step 1"}]),
json!([]),
None,
),
terminal(Some(usage_json()), json!([])),
];
(frames.iter().map(frame_text).collect(), "step 1")
}
pub fn reasoning_summary_supersede_frames() -> (Vec<WireInput>, &'static str) {
let frames = vec![
sse(&json!({
"type": "response.reasoning_summary_text.delta",
"item_id": "rs_1",
"output_index": 0,
"summary_index": 0,
"sequence_number": 1,
"delta": "step 1",
})),
reasoning_done_item(
"rs_1",
json!([{"type": "summary_text", "text": "step 1"}]),
json!([]),
None,
),
terminal(Some(usage_json()), json!([])),
];
(frames, "step 1")
}
pub fn multi_part_reasoning_frames() -> (Vec<WireInput>, Vec<&'static str>) {
let frames = vec![
reasoning_done_item(
"rs_1",
json!([
{"type": "summary_text", "text": "s1"},
{"type": "summary_text", "text": "s2"},
]),
json!([{"type": "reasoning_text", "text": "visible"}]),
Some("enc_blob"),
),
terminal(Some(usage_json()), json!([])),
];
(frames, vec!["s1", "s2", "visible", "enc_blob"])
}
pub fn interleaved_reasoning_frames() -> (Vec<WireInput>, &'static str) {
let frames = vec![
sse(&json!({
"type": "response.reasoning_text.delta",
"item_id": "rs_2",
"output_index": 0,
"content_index": 0,
"sequence_number": 1,
"delta": "thinking",
})),
tool_call_done(),
reasoning_done_item(
"rs_2",
json!([]),
json!([{"type": "reasoning_text", "text": "full reasoning"}]),
None,
),
terminal(Some(usage_json()), json!([])),
];
(frames, "full reasoning")
}
}
pub mod gemini_rest {
use super::*;
fn driver() -> WireDriver {
WireDriver::new("gemini", |chunks| {
Box::pin(async move {
let client = crate::providers::gemini::Client::builder()
.api_key("test-key")
.http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
.build()?;
let model = client.completion_model(
crate::providers::gemini::completion::GEMINI_2_5_PRO_PREVIEW_06_05,
);
let request = model.completion_request("hello").build();
let stream = model.stream(request).await?;
Ok(drain(stream).await)
})
})
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![sse(&json!({
"candidates": [{"content": {"parts": [{"text": "hi"}], "role": "model"}}],
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))],
expected_texts: vec!["hi"],
tool_call_frames: vec![sse(&json!({
"candidates": [{"content": {"parts": [{
"functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
}], "role": "model"}}],
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))],
expected_tool_name: "get_weather",
partial_tool_call_frames: None,
terminal_frames: vec![sse(&json!({
"candidates": [{
"content": {"parts": [], "role": "model"},
"finishReason": "STOP",
}],
"usageMetadata": {
"promptTokenCount": 5,
"candidatesTokenCount": 2,
"totalTokenCount": 7,
},
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))],
expected_usage_total: 7,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: Some(vec![sse(&json!({
"candidates": [{
"content": {"parts": [], "role": "model"},
"finishReason": "STOP",
}],
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))]),
bare_terminal_frames: None,
malformed_frame: Some(sse_raw("{not json")),
unknown_event_frame: Some(sse_raw(r#"{"noise":true}"#)),
defective_known_frame: Some(sse_raw(r#"{"candidates": 42}"#)),
delta_less_prelude_frame: None,
refusal: None,
interleaved_reasoning: Some(interleaved_thought_fixture()),
}
}
fn chunk(parts: serde_json::Value) -> WireInput {
sse(&json!({
"candidates": [{"content": {"parts": parts, "role": "model"}}],
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))
}
fn terminal_frame() -> WireInput {
sse(&json!({
"candidates": [{
"content": {"parts": [], "role": "model"},
"finishReason": "STOP",
}],
"usageMetadata": {
"promptTokenCount": 5,
"candidatesTokenCount": 2,
"totalTokenCount": 7,
},
"responseId": "resp-1",
"modelVersion": "gemini-2.5-pro",
}))
}
fn interleaved_thought_fixture() -> InterleavedReasoningFixture {
InterleavedReasoningFixture {
frames: vec![
chunk(json!([{"text": "before tool", "thought": true}])),
chunk(json!([{
"functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
}])),
chunk(json!([{"text": "after tool", "thought": true}])),
terminal_frame(),
],
first_reasoning: "before tool",
tool_name: "get_weather",
second_reasoning: "after tool",
}
}
pub fn interleaved_signed_thought_frames()
-> (Vec<WireInput>, &'static str, &'static str, &'static str) {
let frames = vec![
chunk(json!([{"text": "before tool", "thought": true}])),
chunk(json!([{
"functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
}])),
chunk(json!([{
"text": "signed conclusion",
"thought": true,
"thoughtSignature": "sig-1",
}])),
terminal_frame(),
];
(frames, "before tool", "get_weather", "signed conclusion")
}
}
pub mod interactions {
use super::*;
fn driver() -> WireDriver {
WireDriver::new("gemini", |chunks| {
Box::pin(async move {
let client = crate::providers::gemini::Client::builder()
.api_key("test-key")
.http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
.build()?
.interactions_api();
let model = client.completion_model("gemini-2.5-pro");
let request = model.completion_request("hello").build();
let stream = model.stream(request).await?;
Ok(drain(stream).await)
})
})
}
fn completed(usage: Option<serde_json::Value>) -> WireInput {
let mut interaction = json!({
"id": "int-1",
"model": "gemini-2.5-pro",
"status": "completed",
});
if let (Some(usage), Some(object)) = (usage, interaction.as_object_mut()) {
object.insert("usage".to_string(), usage);
}
sse(&json!({
"event_type": "interaction.completed",
"interaction": interaction,
}))
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![sse(&json!({
"event_type": "step.delta",
"index": 0,
"delta": {"type": "text", "text": "hi"},
}))],
expected_texts: vec!["hi"],
tool_call_frames: vec![sse(&json!({
"event_type": "step.delta",
"index": 0,
"delta": {
"type": "function_call",
"name": "get_weather",
"arguments": {"city": "Tokyo"},
"id": "call-1",
},
}))],
expected_tool_name: "get_weather",
partial_tool_call_frames: None,
terminal_frames: vec![completed(Some(json!({
"total_input_tokens": 5,
"total_output_tokens": 2,
"total_tokens": 7,
})))],
expected_usage_total: 7,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: Some(vec![completed(None)]),
bare_terminal_frames: None,
malformed_frame: Some(sse_raw("{not json")),
unknown_event_frame: Some(sse(&json!({
"event_type": "future.event",
"index": 0,
}))),
defective_known_frame: Some(sse_raw(
r#"{"event_type":"step.delta","index":0,"delta":42}"#,
)),
delta_less_prelude_frame: None,
refusal: None,
interleaved_reasoning: Some(interleaved_thought_fixture()),
}
}
fn interleaved_thought_fixture() -> InterleavedReasoningFixture {
let frames = vec![
sse(&json!({
"event_type": "step.delta",
"index": 0,
"delta": {
"type": "thought_summary",
"content": {"text": "before tool"},
},
})),
sse(&json!({
"event_type": "step.delta",
"index": 0,
"delta": {
"type": "function_call",
"name": "get_weather",
"arguments": {"city": "Tokyo"},
"id": "call-1",
},
})),
sse(&json!({
"event_type": "step.delta",
"index": 0,
"delta": {
"type": "thought_summary",
"content": {"text": "after tool"},
},
})),
completed(Some(json!({
"total_input_tokens": 5,
"total_output_tokens": 2,
"total_tokens": 7,
}))),
];
InterleavedReasoningFixture {
frames,
first_reasoning: "before tool",
tool_name: "get_weather",
second_reasoning: "after tool",
}
}
}
pub mod anthropic {
use super::*;
fn driver() -> WireDriver {
WireDriver::new("anthropic", |chunks| {
Box::pin(async move {
let client = crate::providers::anthropic::Client::builder()
.api_key("test-key")
.http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
.build()?;
let model = client.completion_model(
crate::providers::anthropic::completion::CLAUDE_SONNET_4_6,
);
let request = model.completion_request("hello").build();
let stream = model.stream(request).await?;
Ok(drain(stream).await)
})
})
}
fn message_start() -> WireInput {
sse(&json!({
"type": "message_start",
"message": {
"id": "msg_1",
"role": "assistant",
"content": [],
"model": "claude-sonnet-4-6",
"stop_reason": null,
"stop_sequence": null,
"usage": {"input_tokens": 5, "output_tokens": 0},
},
}))
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![
message_start(),
sse(&json!({
"type": "content_block_start",
"index": 0,
"content_block": {"type": "text", "text": ""},
})),
sse(&json!({
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": "hi"},
})),
],
expected_texts: vec!["hi"],
tool_call_frames: vec![
sse(&json!({
"type": "content_block_start",
"index": 0,
"content_block": {
"type": "tool_use",
"id": "toolu_1",
"name": "get_weather",
"input": {},
},
})),
sse(&json!({
"type": "content_block_delta",
"index": 0,
"delta": {"type": "input_json_delta", "partial_json": "{\"city\":\"Tokyo\"}"},
})),
sse(&json!({"type": "content_block_stop", "index": 0})),
],
expected_tool_name: "get_weather",
partial_tool_call_frames: Some(vec![
sse(&json!({
"type": "content_block_start",
"index": 0,
"content_block": {
"type": "tool_use",
"id": "toolu_1",
"name": "get_weather",
"input": {},
},
})),
sse(&json!({
"type": "content_block_delta",
"index": 0,
"delta": {"type": "input_json_delta", "partial_json": "{\"cit"},
})),
]),
terminal_frames: vec![sse(&json!({
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": null},
"usage": {"output_tokens": 4},
}))],
expected_usage_total: 9,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: None,
bare_terminal_frames: Some(vec![sse(&json!({"type": "message_stop"}))]),
malformed_frame: Some(sse_raw("{not json")),
unknown_event_frame: Some(sse(&json!({
"type": "content_block_heartbeat",
"index": 0,
}))),
defective_known_frame: Some(sse_raw(
r#"{"type":"content_block_delta","index":0,"delta":42}"#,
)),
delta_less_prelude_frame: None,
refusal: None,
interleaved_reasoning: None,
}
}
}
pub mod cohere {
use super::*;
fn driver() -> WireDriver {
WireDriver::new("cohere", |chunks| {
Box::pin(async move {
let client = crate::providers::cohere::Client::builder()
.api_key("test-key")
.http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
.build()?;
let model =
client.completion_model(crate::providers::cohere::COMMAND_R_08_2024);
let request = model.completion_request("hello").build();
let stream = model.stream(request).await?;
Ok(drain(stream).await)
})
})
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![
sse(&json!({"type": "message-start", "id": "msg_1"})),
sse(&json!({
"type": "content-delta",
"delta": {"message": {"content": {"text": "hi"}}},
})),
],
expected_texts: vec!["hi"],
tool_call_frames: vec![
sse(&json!({
"type": "tool-call-start",
"delta": {"message": {"tool_calls": {
"id": "call_1",
"function": {"name": "get_weather", "arguments": ""},
}}},
})),
sse(&json!({
"type": "tool-call-delta",
"delta": {"message": {"tool_calls": {
"function": {"arguments": "{\"city\":\"Tokyo\"}"},
}}},
})),
sse(&json!({"type": "tool-call-end"})),
],
expected_tool_name: "get_weather",
partial_tool_call_frames: Some(vec![sse(&json!({
"type": "tool-call-start",
"delta": {"message": {"tool_calls": {
"id": "call_1",
"function": {"name": "get_weather", "arguments": "{\"cit"},
}}},
}))]),
terminal_frames: vec![sse(&json!({
"type": "message-end",
"delta": {
"finish_reason": "COMPLETE",
"usage": {"tokens": {"input_tokens": 10, "output_tokens": 4}},
},
}))],
expected_usage_total: 14,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: Some(vec![sse(&json!({"type": "message-end"}))]),
bare_terminal_frames: None,
malformed_frame: Some(sse_raw("{not json")),
unknown_event_frame: Some(sse(&json!({
"type": "citation-start",
"delta": {"message": {"citations": {}}},
}))),
defective_known_frame: Some(sse_raw(r#"{"type":"content-delta","delta":42}"#)),
delta_less_prelude_frame: None,
refusal: None,
interleaved_reasoning: Some(interleaved_thinking_fixture()),
}
}
fn interleaved_thinking_fixture() -> InterleavedReasoningFixture {
let frames = vec![
sse(&json!({"type": "message-start", "id": "msg_1"})),
sse(&json!({
"type": "content-delta",
"delta": {"message": {"content": {"thinking": "before tool"}}},
})),
sse(&json!({
"type": "tool-call-start",
"delta": {"message": {"tool_calls": {
"id": "call_1",
"function": {"name": "get_weather", "arguments": "{\"city\":\"Tokyo\"}"},
}}},
})),
sse(&json!({"type": "tool-call-end"})),
sse(&json!({
"type": "content-delta",
"delta": {"message": {"content": {"thinking": "after tool"}}},
})),
sse(&json!({
"type": "message-end",
"delta": {
"finish_reason": "COMPLETE",
"usage": {"tokens": {"input_tokens": 10, "output_tokens": 4}},
},
})),
];
InterleavedReasoningFixture {
frames,
first_reasoning: "before tool",
tool_name: "get_weather",
second_reasoning: "after tool",
}
}
}
pub mod ollama {
use super::*;
fn driver() -> WireDriver {
WireDriver::new("ollama", |chunks| {
Box::pin(async move {
let client = crate::providers::ollama::Client::builder()
.api_key("test-key")
.http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
.build()?;
let model = client.completion_model("llama3.2");
let request = model.completion_request("hello").build();
let stream = model.stream(request).await?;
Ok(drain(stream).await)
})
})
}
pub fn fixture() -> ProviderWireFixture {
ProviderWireFixture {
driver: driver(),
text_frames: vec![ndjson(&json!({
"model": "llama3.2",
"created_at": "2023-08-04T19:22:45.499127Z",
"message": {"role": "assistant", "content": "hi"},
"done": false,
}))],
expected_texts: vec!["hi"],
tool_call_frames: vec![ndjson(&json!({
"model": "llama3.2",
"created_at": "2023-08-04T19:22:45.499127Z",
"message": {"role": "assistant", "content": "", "tool_calls": [{
"function": {"name": "get_weather", "arguments": {"city": "Tokyo"}},
}]},
"done": false,
}))],
expected_tool_name: "get_weather",
partial_tool_call_frames: None,
terminal_frames: vec![ndjson(&json!({
"model": "llama3.2",
"created_at": "2023-08-04T19:22:47.499127Z",
"message": {"role": "assistant", "content": ""},
"done": true,
"done_reason": "stop",
"prompt_eval_count": 10,
"eval_count": 4,
}))],
expected_usage_total: 14,
expected_finish_reason: Some(FinishReason::Stop),
zero_usage_terminal_frames: Some(vec![ndjson(&json!({
"model": "llama3.2",
"created_at": "2023-08-04T19:22:47.499127Z",
"message": {"role": "assistant", "content": ""},
"done": true,
"done_reason": "stop",
}))]),
bare_terminal_frames: None,
malformed_frame: Some(WireInput::Bytes(Bytes::from_static(b"{not json\n"))),
unknown_event_frame: None,
defective_known_frame: Some(ndjson(&json!({
"model": "llama3.2",
"created_at": "2023-08-04T19:22:46.499127Z",
"message": {"role": "assistant", "content": 42},
"done": false,
}))),
delta_less_prelude_frame: None,
refusal: None,
interleaved_reasoning: Some(interleaved_thinking_fixture()),
}
}
fn interleaved_thinking_fixture() -> InterleavedReasoningFixture {
let frames = vec![
ndjson(&json!({
"model": "llama3.2",
"created_at": "2023-08-04T19:22:45.499127Z",
"message": {"role": "assistant", "content": "", "thinking": "before tool"},
"done": false,
})),
ndjson(&json!({
"model": "llama3.2",
"created_at": "2023-08-04T19:22:45.599127Z",
"message": {"role": "assistant", "content": "", "tool_calls": [{
"function": {"name": "get_weather", "arguments": {"city": "Tokyo"}},
}]},
"done": false,
})),
ndjson(&json!({
"model": "llama3.2",
"created_at": "2023-08-04T19:22:45.699127Z",
"message": {"role": "assistant", "content": "", "thinking": "after tool"},
"done": false,
})),
ndjson(&json!({
"model": "llama3.2",
"created_at": "2023-08-04T19:22:47.499127Z",
"message": {"role": "assistant", "content": ""},
"done": true,
"done_reason": "stop",
"prompt_eval_count": 10,
"eval_count": 4,
})),
];
InterleavedReasoningFixture {
frames,
first_reasoning: "before tool",
tool_name: "get_weather",
second_reasoning: "after tool",
}
}
}
}