1use bytes::Bytes;
25use futures::StreamExt;
26use futures::future::BoxFuture;
27
28use crate::{
29 completion::{CompletionError, FinishReason},
30 http_client,
31 message::AssistantContent,
32 streaming::{StreamFinal, StreamedAssistantContent},
33};
34
35#[derive(Debug, thiserror::Error)]
37pub enum ConformanceError {
38 #[error(transparent)]
40 Completion(#[from] CompletionError),
41 #[error("{scenario} conformance failed for {provider}: {details}")]
43 Contract {
44 scenario: &'static str,
46 provider: &'static str,
48 details: String,
50 },
51}
52
53impl ConformanceError {
54 fn contract(
55 scenario: &'static str,
56 provider: &'static str,
57 details: impl Into<String>,
58 ) -> Self {
59 Self::Contract {
60 scenario,
61 provider,
62 details: details.into(),
63 }
64 }
65}
66
67#[derive(Debug)]
69pub struct ScenarioReport {
70 pub name: &'static str,
72 pub provider: &'static str,
74 pub observations: Vec<String>,
76}
77
78#[derive(Debug)]
87pub enum ScenarioOutcome {
88 Ran(ScenarioReport),
90 Skipped {
92 name: &'static str,
94 provider: &'static str,
96 reason: &'static str,
98 },
99}
100
101#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
109pub struct SuiteCapabilities {
110 pub partial_tool_args: bool,
113 pub zero_usage_terminal: bool,
116 pub bare_terminal: bool,
118 pub malformed_frame: bool,
120 pub unknown_event_frame: bool,
122 pub defective_known_frame: bool,
125 pub delta_less_prelude: bool,
128 pub refusal: bool,
130 pub interleaved_reasoning: bool,
134}
135
136impl SuiteCapabilities {
137 pub fn from_names(names: &[&str]) -> Result<Self, String> {
142 let mut caps = Self::default();
143 for name in names {
144 match *name {
145 "partial_tool_args" => caps.partial_tool_args = true,
146 "zero_usage_terminal" => caps.zero_usage_terminal = true,
147 "bare_terminal" => caps.bare_terminal = true,
148 "malformed_frame" => caps.malformed_frame = true,
149 "unknown_event_frame" => caps.unknown_event_frame = true,
150 "defective_known_frame" => caps.defective_known_frame = true,
151 "delta_less_prelude" => caps.delta_less_prelude = true,
152 "refusal" => caps.refusal = true,
153 "interleaved_reasoning" => caps.interleaved_reasoning = true,
154 other => {
155 return Err(format!(
156 "unknown capability name in suite manifest: {other}"
157 ));
158 }
159 }
160 }
161 Ok(caps)
162 }
163}
164
165pub const CANONICAL_SCENARIOS: &[&str] = &[
169 "truncation_preserves_content_without_terminal",
170 "transport_error_after_tool_call_yields_err_then_end",
171 "malformed_frame_surfaces_err_and_terminal_still_completes",
172 "unknown_event_is_skipped",
173 "defective_known_event_surfaces_err",
174 "delta_less_choice_prelude_is_a_noop",
175 "refusal_frames_deliver_text_without_error",
176 "bare_terminal_after_only_unparseable_frames_fabricates_nothing",
177 "usage_variants_are_reported_or_zero_sentinel",
178 "interleaved_constant_id_reasoning_preserves_order",
179];
180
181pub const WIRE_FAMILIES: &[&str] = &[
186 "openai_chat",
187 "openai_responses",
188 "openai_responses_websocket",
189 "chatgpt",
190 "anthropic",
191 "gemini_rest",
192 "gemini_interactions",
193 "gemini_grpc",
194 "cohere",
195 "ollama",
196 "xai",
197 "copilot",
198 "bedrock",
199 "candle",
200];
201
202pub fn xfail_reason<'a>(xfail: &[&'a str], scenario: &str) -> Option<&'a str> {
205 xfail.iter().find_map(|entry| {
206 let (name, reason) = entry.split_once(':')?;
207 (name.trim() == scenario).then(|| reason.trim())
208 })
209}
210
211pub fn invalid_xfail_entries(xfail: &[&str]) -> Vec<String> {
213 xfail
214 .iter()
215 .filter(|entry| match entry.split_once(':') {
216 Some((name, reason)) => {
217 !CANONICAL_SCENARIOS.contains(&name.trim()) || reason.trim().is_empty()
218 }
219 None => true,
220 })
221 .map(|entry| entry.to_string())
222 .collect()
223}
224
225pub fn check_gated_outcome(
233 scenario: &'static str,
234 capability: bool,
235 xfail: &[&str],
236 outcome: Result<ScenarioOutcome, ConformanceError>,
237) -> Result<(), String> {
238 match (xfail_reason(xfail, scenario), outcome) {
239 (Some(reason), Err(error)) => {
240 eprintln!("xfail {scenario}: {reason} ({error})");
241 Ok(())
242 }
243 (Some(reason), Ok(_)) => Err(format!(
244 "{scenario} passed but is listed as xfail ({reason}); remove the xfail entry"
245 )),
246 (None, Err(error)) => Err(format!("{scenario} failed: {error}")),
247 (None, Ok(ScenarioOutcome::Ran(_))) => {
248 if capability {
249 Ok(())
250 } else {
251 Err(format!(
252 "{scenario} ran but the suite disclaims the capability; set the flag to true"
253 ))
254 }
255 }
256 (None, Ok(ScenarioOutcome::Skipped { reason, .. })) => {
257 if capability {
258 Err(format!(
259 "{scenario} skipped ({reason}) but the suite declares the capability; \
260 a declared capability's scenario must run"
261 ))
262 } else {
263 eprintln!("skipped {scenario}: {reason}");
264 Ok(())
265 }
266 }
267 }
268}
269
270pub fn check_ungated_outcome(
272 scenario: &'static str,
273 xfail: &[&str],
274 result: Result<ScenarioReport, ConformanceError>,
275) -> Result<(), String> {
276 match (xfail_reason(xfail, scenario), result) {
277 (Some(reason), Err(error)) => {
278 eprintln!("xfail {scenario}: {reason} ({error})");
279 Ok(())
280 }
281 (Some(reason), Ok(_)) => Err(format!(
282 "{scenario} passed but is listed as xfail ({reason}); remove the xfail entry"
283 )),
284 (None, Err(error)) => Err(format!("{scenario} failed: {error}")),
285 (None, Ok(_)) => Ok(()),
286 }
287}
288
289#[derive(Clone)]
296pub enum WireInput {
297 Bytes(Bytes),
299 Event(std::sync::Arc<dyn std::any::Any + Send + Sync>),
301}
302
303impl WireInput {
304 pub fn as_bytes(&self) -> Option<&Bytes> {
306 match self {
307 Self::Bytes(bytes) => Some(bytes),
308 Self::Event(_) => None,
309 }
310 }
311
312 pub fn downcast_event<T: 'static>(&self) -> Option<&T> {
314 match self {
315 Self::Bytes(_) => None,
316 Self::Event(event) => event.downcast_ref(),
317 }
318 }
319}
320
321impl From<Bytes> for WireInput {
322 fn from(bytes: Bytes) -> Self {
323 Self::Bytes(bytes)
324 }
325}
326
327impl std::fmt::Debug for WireInput {
328 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
329 match self {
330 Self::Bytes(bytes) => formatter.debug_tuple("Bytes").field(bytes).finish(),
331 Self::Event(_) => formatter.write_str("Event(..)"),
332 }
333 }
334}
335
336pub fn event_frame<T: Send + Sync + 'static>(event: T) -> WireInput {
338 WireInput::Event(std::sync::Arc::new(event))
339}
340
341pub type WireChunks = Vec<http_client::Result<WireInput>>;
344
345pub fn ok_chunks(frames: impl IntoIterator<Item = impl Into<WireInput>>) -> WireChunks {
347 frames.into_iter().map(|frame| Ok(frame.into())).collect()
348}
349
350pub fn transport_error_chunk() -> http_client::Result<WireInput> {
352 Err(http_client::Error::InvalidStatusCodeWithMessage(
353 http::StatusCode::BAD_GATEWAY,
354 "connection reset".to_string(),
355 ))
356}
357
358pub fn assert_valid_event_stream(
384 items: &[Result<crate::streaming::StreamedAssistantContent, CompletionError>],
385 choice: &[AssistantContent],
386) {
387 use crate::message::AssistantContent;
388 use crate::streaming::StreamedAssistantContent as Item;
389
390 let ok_items: Vec<&Item> = items.iter().filter_map(|item| item.as_ref().ok()).collect();
391
392 let final_count = ok_items
394 .iter()
395 .filter(|item| matches!(item, Item::Final(_)))
396 .count();
397 assert!(
398 final_count <= 1,
399 "law 1 (terminal latch): {final_count} terminal records yielded"
400 );
401 if let Some(final_index) = ok_items
402 .iter()
403 .position(|item| matches!(item, Item::Final(_)))
404 {
405 for item in ok_items.get(final_index + 1..).unwrap_or_default() {
406 assert!(
407 matches!(item, Item::Unknown(_)),
408 "law 1 (terminal latch): content item after the terminal record: {item:?}"
409 );
410 }
411 }
412
413 let streamed_text: String = ok_items
415 .iter()
416 .filter_map(|item| match item {
417 Item::Text(text) => Some(text.text.as_str()),
418 _ => None,
419 })
420 .collect();
421 let aggregated_text: String = choice
422 .iter()
423 .filter_map(|content| match content {
424 AssistantContent::Text(text) => Some(text.text.as_str()),
425 _ => None,
426 })
427 .collect();
428 assert_eq!(
429 aggregated_text, streamed_text,
430 "law 2 (text conservation): aggregated text differs from the streamed deltas"
431 );
432
433 let yielded_calls = ok_items
435 .iter()
436 .filter(|item| matches!(item, Item::ToolCall { .. }))
437 .count();
438 let aggregated_calls = choice
439 .iter()
440 .filter(|content| matches!(content, AssistantContent::ToolCall(_)))
441 .count();
442 assert_eq!(
443 aggregated_calls, yielded_calls,
444 "law 3 (completed-call conservation): {yielded_calls} calls yielded, \
445 {aggregated_calls} aggregated"
446 );
447
448 let mut seen_delta_ids: Vec<&str> = Vec::new();
450 let mut completed_ids: Vec<&str> = Vec::new();
451 for item in &ok_items {
452 match item {
453 Item::ToolCallDelta {
454 internal_call_id, ..
455 } => {
456 assert!(
457 !completed_ids.contains(&internal_call_id.as_str()),
458 "law 4: a delta for internal id {internal_call_id} arrived after its \
459 completed call"
460 );
461 seen_delta_ids.push(internal_call_id);
462 }
463 Item::ToolCall {
464 internal_call_id, ..
465 } => completed_ids.push(internal_call_id),
466 _ => {}
467 }
468 }
469
470 let mut completed_reasoning_ids: Vec<&str> = Vec::new();
476 for item in &ok_items {
477 if let Item::Reasoning { id, .. } = item {
478 assert!(
479 !id.is_empty(),
480 "law 4b (reasoning correlation): a completed block carries an empty correlator"
481 );
482 assert!(
483 !completed_reasoning_ids.contains(&id.as_str()),
484 "law 4b (reasoning correlation): two completed blocks share correlator {id}"
485 );
486 completed_reasoning_ids.push(id);
487 }
488 }
489
490 let yielded_reasoning = ok_items
492 .iter()
493 .any(|item| matches!(item, Item::Reasoning { .. } | Item::ReasoningDelta { .. }));
494 let aggregated_reasoning = choice
495 .iter()
496 .any(|content| matches!(content, AssistantContent::Reasoning(_)));
497 assert!(
498 yielded_reasoning || !aggregated_reasoning,
499 "law 5 (reasoning provenance): aggregated reasoning with no reasoning yielded"
500 );
501 let yielded_full_block = ok_items
502 .iter()
503 .any(|item| matches!(item, Item::Reasoning { .. }));
504 if yielded_reasoning && !yielded_full_block {
505 let streamed_reasoning: String = ok_items
506 .iter()
507 .filter_map(|item| match item {
508 Item::ReasoningDelta { reasoning, .. } => Some(reasoning.as_str()),
509 _ => None,
510 })
511 .collect();
512 let aggregated_reasoning_text: String = choice
513 .iter()
514 .filter_map(|content| match content {
515 AssistantContent::Reasoning(reasoning) => Some(reasoning.content.iter()),
516 _ => None,
517 })
518 .flatten()
519 .filter_map(|part| match part {
520 crate::message::ReasoningContent::Text { text, .. } => Some(text.as_str()),
521 _ => None,
522 })
523 .collect();
524 assert_eq!(
525 aggregated_reasoning_text, streamed_reasoning,
526 "law 5 (reasoning conservation): with no full block, the aggregated reasoning \
527 must be exactly the concatenated deltas"
528 );
529 }
530}
531
532#[derive(Debug)]
535pub struct DrainedStream {
536 pub items: Vec<Result<StreamedAssistantContent, CompletionError>>,
538 pub choice: Vec<AssistantContent>,
540 pub response: Option<StreamFinal>,
542}
543
544impl DrainedStream {
545 pub fn texts(&self) -> Vec<&str> {
547 self.items
548 .iter()
549 .filter_map(|item| match item {
550 Ok(StreamedAssistantContent::Text(text)) => Some(text.text.as_str()),
551 _ => None,
552 })
553 .collect()
554 }
555
556 pub fn tool_call_names(&self) -> Vec<&str> {
558 self.items
559 .iter()
560 .filter_map(|item| match item {
561 Ok(StreamedAssistantContent::ToolCall { tool_call, .. }) => {
562 Some(tool_call.function.name.as_str())
563 }
564 _ => None,
565 })
566 .collect()
567 }
568
569 pub fn unknown_values(&self) -> Vec<&serde_json::Value> {
572 self.items
573 .iter()
574 .filter_map(|item| match item {
575 Ok(StreamedAssistantContent::Unknown(value)) => Some(value.value()),
576 _ => None,
577 })
578 .collect()
579 }
580
581 pub fn error_count(&self) -> usize {
583 self.items.iter().filter(|item| item.is_err()).count()
584 }
585
586 pub fn final_count(&self) -> usize {
588 self.items
589 .iter()
590 .filter(|item| matches!(item, Ok(StreamedAssistantContent::Final(_))))
591 .count()
592 }
593
594 fn first_error_index(&self) -> Option<usize> {
596 self.items.iter().position(|item| item.is_err())
597 }
598
599 pub fn choice_texts(&self) -> Vec<&str> {
601 self.choice
602 .iter()
603 .filter_map(|content| match content {
604 AssistantContent::Text(text) => Some(text.text.as_str()),
605 _ => None,
606 })
607 .collect()
608 }
609
610 pub fn choice_reasoning(&self) -> Vec<&crate::message::Reasoning> {
612 self.choice
613 .iter()
614 .filter_map(|content| match content {
615 AssistantContent::Reasoning(reasoning) => Some(reasoning),
616 _ => None,
617 })
618 .collect()
619 }
620
621 pub fn choice_tool_call_names(&self) -> Vec<&str> {
623 self.choice
624 .iter()
625 .filter_map(|content| match content {
626 AssistantContent::ToolCall(tool_call) => Some(tool_call.function.name.as_str()),
627 _ => None,
628 })
629 .collect()
630 }
631}
632
633type DriveFn = Box<
634 dyn Fn(WireChunks) -> BoxFuture<'static, Result<DrainedStream, CompletionError>> + Send + Sync,
635>;
636
637pub struct WireDriver {
643 pub provider: &'static str,
645 drive: DriveFn,
646}
647
648impl WireDriver {
649 pub fn new(
651 provider: &'static str,
652 drive: impl Fn(WireChunks) -> BoxFuture<'static, Result<DrainedStream, CompletionError>>
653 + Send
654 + Sync
655 + 'static,
656 ) -> Self {
657 Self {
658 provider,
659 drive: Box::new(drive),
660 }
661 }
662
663 pub async fn drive(&self, chunks: WireChunks) -> Result<DrainedStream, CompletionError> {
665 (self.drive)(chunks).await
666 }
667}
668
669pub struct RefusalFixture {
671 pub frames: Vec<WireInput>,
673 pub expected_text: &'static str,
675}
676
677pub struct InterleavedReasoningFixture {
682 pub frames: Vec<WireInput>,
684 pub first_reasoning: &'static str,
686 pub tool_name: &'static str,
688 pub second_reasoning: &'static str,
690}
691
692type BufferedDriveFn = Box<
693 dyn Fn(String) -> BoxFuture<'static, Result<Vec<AssistantContent>, CompletionError>>
694 + Send
695 + Sync,
696>;
697
698pub struct BufferedBodyDriver {
701 pub provider: &'static str,
703 drive: BufferedDriveFn,
704}
705
706impl BufferedBodyDriver {
707 pub fn new(
709 provider: &'static str,
710 drive: impl Fn(String) -> BoxFuture<'static, Result<Vec<AssistantContent>, CompletionError>>
711 + Send
712 + Sync
713 + 'static,
714 ) -> Self {
715 Self {
716 provider,
717 drive: Box::new(drive),
718 }
719 }
720
721 pub async fn drive(&self, body: String) -> Result<Vec<AssistantContent>, CompletionError> {
723 (self.drive)(body).await
724 }
725}
726
727pub struct ProviderWireFixture {
732 pub driver: WireDriver,
734 pub text_frames: Vec<WireInput>,
736 pub expected_texts: Vec<&'static str>,
738 pub tool_call_frames: Vec<WireInput>,
741 pub expected_tool_name: &'static str,
743 pub partial_tool_call_frames: Option<Vec<WireInput>>,
746 pub terminal_frames: Vec<WireInput>,
748 pub expected_usage_total: u64,
750 pub expected_finish_reason: Option<FinishReason>,
752 pub zero_usage_terminal_frames: Option<Vec<WireInput>>,
754 pub bare_terminal_frames: Option<Vec<WireInput>>,
757 pub malformed_frame: Option<WireInput>,
761 pub unknown_event_frame: Option<WireInput>,
763 pub defective_known_frame: Option<WireInput>,
765 pub delta_less_prelude_frame: Option<WireInput>,
767 pub refusal: Option<RefusalFixture>,
769 pub interleaved_reasoning: Option<InterleavedReasoningFixture>,
773}
774
775impl ProviderWireFixture {
776 pub fn capabilities(&self) -> SuiteCapabilities {
784 SuiteCapabilities {
785 partial_tool_args: self.partial_tool_call_frames.is_some(),
786 zero_usage_terminal: self.zero_usage_terminal_frames.is_some(),
787 bare_terminal: self.bare_terminal_frames.is_some(),
788 malformed_frame: self.malformed_frame.is_some(),
789 unknown_event_frame: self.unknown_event_frame.is_some(),
790 defective_known_frame: self.defective_known_frame.is_some(),
791 delta_less_prelude: self.delta_less_prelude_frame.is_some(),
792 refusal: self.refusal.is_some(),
793 interleaved_reasoning: self.interleaved_reasoning.is_some(),
794 }
795 }
796}
797
798fn concat_frames(parts: &[&[WireInput]]) -> Vec<WireInput> {
799 parts
800 .iter()
801 .flat_map(|frames| frames.iter().cloned())
802 .collect()
803}
804
805pub async fn truncation_preserves_content_without_terminal(
813 fixture: &ProviderWireFixture,
814) -> Result<ScenarioReport, ConformanceError> {
815 const SCENARIO: &str = "truncation_preserves_content_without_terminal";
816 let provider = fixture.driver.provider;
817 let mut observations = Vec::new();
818
819 let drained = fixture.driver.drive(Vec::new()).await?;
821 if drained.response.is_some() || drained.final_count() != 0 {
822 return Err(ConformanceError::contract(
823 SCENARIO,
824 provider,
825 "an empty stream must not synthesize a terminal record",
826 ));
827 }
828 observations.push("EOF before content: no terminal".to_string());
829
830 let drained = fixture
832 .driver
833 .drive(ok_chunks(fixture.text_frames.clone()))
834 .await?;
835 if drained.texts() != fixture.expected_texts {
836 return Err(ConformanceError::contract(
837 SCENARIO,
838 provider,
839 format!(
840 "text delivered before truncation must be preserved: expected {:?}, observed {:?}",
841 fixture.expected_texts,
842 drained.texts()
843 ),
844 ));
845 }
846 if drained.response.is_some() || drained.final_count() != 0 {
847 return Err(ConformanceError::contract(
848 SCENARIO,
849 provider,
850 "EOF after text deltas must not synthesize a terminal record",
851 ));
852 }
853 observations.push("EOF mid-text: content preserved, no terminal".to_string());
854
855 if let Some(partial) = &fixture.partial_tool_call_frames {
857 let drained = fixture.driver.drive(ok_chunks(partial.clone())).await?;
858 if drained.response.is_some() || drained.final_count() != 0 {
859 return Err(ConformanceError::contract(
860 SCENARIO,
861 provider,
862 "EOF mid-tool-arguments must not synthesize a terminal record",
863 ));
864 }
865 observations.push("EOF mid-tool-args: no terminal".to_string());
866 }
867
868 let drained = fixture
870 .driver
871 .drive(ok_chunks(fixture.tool_call_frames.clone()))
872 .await?;
873 if drained.tool_call_names() != vec![fixture.expected_tool_name] {
874 return Err(ConformanceError::contract(
875 SCENARIO,
876 provider,
877 format!(
878 "a fully-delivered tool call must survive truncation: observed {:?}",
879 drained.tool_call_names()
880 ),
881 ));
882 }
883 if drained.response.is_some() || drained.final_count() != 0 {
884 return Err(ConformanceError::contract(
885 SCENARIO,
886 provider,
887 "EOF after a delivered tool call must not synthesize a terminal record",
888 ));
889 }
890 observations.push("EOF after tool-complete: tool call preserved, no terminal".to_string());
891
892 Ok(ScenarioReport {
893 name: SCENARIO,
894 provider,
895 observations,
896 })
897}
898
899pub async fn transport_error_after_tool_call_yields_err_then_end(
906 fixture: &ProviderWireFixture,
907) -> Result<ScenarioReport, ConformanceError> {
908 const SCENARIO: &str = "transport_error_after_tool_call_yields_err_then_end";
909 let provider = fixture.driver.provider;
910
911 let mut chunks = ok_chunks(fixture.tool_call_frames.clone());
912 chunks.push(transport_error_chunk());
913 let drained = fixture.driver.drive(chunks).await?;
914
915 if drained.tool_call_names() != vec![fixture.expected_tool_name] {
916 return Err(ConformanceError::contract(
917 SCENARIO,
918 provider,
919 format!(
920 "the delivered tool call must precede the transport error: observed {:?}",
921 drained.tool_call_names()
922 ),
923 ));
924 }
925 let error_index = drained.first_error_index().ok_or_else(|| {
926 ConformanceError::contract(
927 SCENARIO,
928 provider,
929 "the transport failure must reach the consumer",
930 )
931 })?;
932 if error_index + 1 != drained.items.len() {
933 return Err(ConformanceError::contract(
934 SCENARIO,
935 provider,
936 "nothing may follow the terminal transport error",
937 ));
938 }
939 if drained.response.is_some() || drained.final_count() != 0 {
940 return Err(ConformanceError::contract(
941 SCENARIO,
942 provider,
943 "a transport failure must not be papered over with a terminal record",
944 ));
945 }
946
947 Ok(ScenarioReport {
948 name: SCENARIO,
949 provider,
950 observations: vec!["tool call, then Err, then end; no terminal".to_string()],
951 })
952}
953
954pub async fn malformed_frame_surfaces_err_and_terminal_still_completes(
961 fixture: &ProviderWireFixture,
962) -> Result<ScenarioOutcome, ConformanceError> {
963 const SCENARIO: &str = "malformed_frame_surfaces_err_and_terminal_still_completes";
964 let provider = fixture.driver.provider;
965 let Some(malformed) = &fixture.malformed_frame else {
966 return Ok(ScenarioOutcome::Skipped {
967 name: SCENARIO,
968 provider,
969 reason: "wire family cannot spell a frame-level decode failure",
970 });
971 };
972
973 let frames = concat_frames(&[
974 &fixture.text_frames,
975 std::slice::from_ref(malformed),
976 &fixture.terminal_frames,
977 ]);
978 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
979
980 if drained.error_count() != 1 {
981 return Err(ConformanceError::contract(
982 SCENARIO,
983 provider,
984 format!(
985 "the malformed frame must surface as exactly one Err item, observed {}",
986 drained.error_count()
987 ),
988 ));
989 }
990 if drained.texts() != fixture.expected_texts {
991 return Err(ConformanceError::contract(
992 SCENARIO,
993 provider,
994 "content around the malformed frame must be preserved",
995 ));
996 }
997 if drained.response.is_none() {
998 return Err(ConformanceError::contract(
999 SCENARIO,
1000 provider,
1001 "the genuine terminal after a recoverable parse error must still complete the stream",
1002 ));
1003 }
1004
1005 Ok(ScenarioOutcome::Ran(ScenarioReport {
1006 name: SCENARIO,
1007 provider,
1008 observations: vec!["Err surfaced, terminal still completed".to_string()],
1009 }))
1010}
1011
1012pub async fn unknown_event_is_skipped(
1018 fixture: &ProviderWireFixture,
1019) -> Result<ScenarioOutcome, ConformanceError> {
1020 const SCENARIO: &str = "unknown_event_is_skipped";
1021 let provider = fixture.driver.provider;
1022 let Some(unknown) = &fixture.unknown_event_frame else {
1023 return Ok(ScenarioOutcome::Skipped {
1024 name: SCENARIO,
1025 provider,
1026 reason: "wire family cannot spell an unknown event type",
1027 });
1028 };
1029
1030 let frames = concat_frames(&[
1031 &fixture.text_frames,
1032 std::slice::from_ref(unknown),
1033 &fixture.terminal_frames,
1034 ]);
1035 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
1036
1037 if drained.error_count() != 0 {
1038 return Err(ConformanceError::contract(
1039 SCENARIO,
1040 provider,
1041 "an unknown event type must be skipped, not surfaced as an error",
1042 ));
1043 }
1044 if drained.texts() != fixture.expected_texts || drained.response.is_none() {
1045 return Err(ConformanceError::contract(
1046 SCENARIO,
1047 provider,
1048 "the stream must deliver its content and complete around the skipped event",
1049 ));
1050 }
1051 if drained.unknown_values().len() != 1 {
1054 return Err(ConformanceError::contract(
1055 SCENARIO,
1056 provider,
1057 format!(
1058 "exactly one Unknown passthrough item must surface for the unknown frame, \
1059 observed {}",
1060 drained.unknown_values().len()
1061 ),
1062 ));
1063 }
1064
1065 let control_frames = concat_frames(&[&fixture.text_frames, &fixture.terminal_frames]);
1068 let control = fixture.driver.drive(ok_chunks(control_frames)).await?;
1069 if drained.choice != control.choice {
1070 return Err(ConformanceError::contract(
1071 SCENARIO,
1072 provider,
1073 "the unknown frame must not perturb the aggregated assistant choice",
1074 ));
1075 }
1076
1077 Ok(ScenarioOutcome::Ran(ScenarioReport {
1078 name: SCENARIO,
1079 provider,
1080 observations: vec![
1081 "unknown event skipped semantically, surfaced on the raw channel, \
1082 choice unchanged, stream completed"
1083 .to_string(),
1084 ],
1085 }))
1086}
1087
1088pub async fn defective_known_event_surfaces_err(
1096 fixture: &ProviderWireFixture,
1097) -> Result<ScenarioOutcome, ConformanceError> {
1098 const SCENARIO: &str = "defective_known_event_surfaces_err";
1099 let provider = fixture.driver.provider;
1100 let Some(defective) = &fixture.defective_known_frame else {
1101 return Ok(ScenarioOutcome::Skipped {
1102 name: SCENARIO,
1103 provider,
1104 reason: "wire family cannot spell a known event with a schema-defective payload",
1105 });
1106 };
1107
1108 let frames = concat_frames(&[
1109 &fixture.text_frames,
1110 std::slice::from_ref(defective),
1111 &fixture.terminal_frames,
1112 ]);
1113 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
1114
1115 if drained.error_count() != 1 {
1116 return Err(ConformanceError::contract(
1117 SCENARIO,
1118 provider,
1119 format!(
1120 "a known event with a schema defect must surface exactly one Err item, observed {}",
1121 drained.error_count()
1122 ),
1123 ));
1124 }
1125 if drained.response.is_none() {
1126 return Err(ConformanceError::contract(
1127 SCENARIO,
1128 provider,
1129 "the genuine terminal must still complete the stream after the defective frame",
1130 ));
1131 }
1132
1133 Ok(ScenarioOutcome::Ran(ScenarioReport {
1134 name: SCENARIO,
1135 provider,
1136 observations: vec!["defective known event surfaced as Err; stream completed".to_string()],
1137 }))
1138}
1139
1140pub async fn delta_less_choice_prelude_is_a_noop(
1146 fixture: &ProviderWireFixture,
1147) -> Result<ScenarioOutcome, ConformanceError> {
1148 const SCENARIO: &str = "delta_less_choice_prelude_is_a_noop";
1149 let provider = fixture.driver.provider;
1150 let Some(prelude) = &fixture.delta_less_prelude_frame else {
1151 return Ok(ScenarioOutcome::Skipped {
1152 name: SCENARIO,
1153 provider,
1154 reason: "wire family has no delta-less prelude shape",
1155 });
1156 };
1157
1158 let frames = concat_frames(&[
1159 std::slice::from_ref(prelude),
1160 &fixture.text_frames,
1161 &fixture.terminal_frames,
1162 ]);
1163 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
1164
1165 if drained.error_count() != 0 {
1166 return Err(ConformanceError::contract(
1167 SCENARIO,
1168 provider,
1169 "the delta-less prelude must not surface an error",
1170 ));
1171 }
1172 if drained.texts() != fixture.expected_texts || drained.response.is_none() {
1173 return Err(ConformanceError::contract(
1174 SCENARIO,
1175 provider,
1176 "the prelude must not perturb content delivery or the terminal",
1177 ));
1178 }
1179
1180 Ok(ScenarioOutcome::Ran(ScenarioReport {
1181 name: SCENARIO,
1182 provider,
1183 observations: vec!["delta-less prelude ignored; stream unaffected".to_string()],
1184 }))
1185}
1186
1187pub async fn refusal_frames_deliver_text_without_error(
1192 fixture: &ProviderWireFixture,
1193) -> Result<ScenarioOutcome, ConformanceError> {
1194 const SCENARIO: &str = "refusal_frames_deliver_text_without_error";
1195 let provider = fixture.driver.provider;
1196 let Some(refusal) = &fixture.refusal else {
1197 return Ok(ScenarioOutcome::Skipped {
1198 name: SCENARIO,
1199 provider,
1200 reason: "wire family has no refusal channel",
1201 });
1202 };
1203
1204 let frames = concat_frames(&[&refusal.frames, &fixture.terminal_frames]);
1205 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
1206
1207 if drained.error_count() != 0 {
1208 return Err(ConformanceError::contract(
1209 SCENARIO,
1210 provider,
1211 "refusal content must not surface as an error",
1212 ));
1213 }
1214 let delivered = drained.texts().concat();
1215 if delivered != refusal.expected_text {
1216 return Err(ConformanceError::contract(
1217 SCENARIO,
1218 provider,
1219 format!(
1220 "refusal text must be delivered: expected {:?}, observed {delivered:?}",
1221 refusal.expected_text
1222 ),
1223 ));
1224 }
1225 if drained.response.is_none() {
1226 return Err(ConformanceError::contract(
1227 SCENARIO,
1228 provider,
1229 "a refused turn still ends with the provider's genuine terminal",
1230 ));
1231 }
1232
1233 Ok(ScenarioOutcome::Ran(ScenarioReport {
1234 name: SCENARIO,
1235 provider,
1236 observations: vec!["refusal text delivered without error".to_string()],
1237 }))
1238}
1239
1240pub async fn terminal_body_content_merges_per_kind(
1249 driver: &BufferedBodyDriver,
1250 cases: Vec<(&'static str, String)>,
1251 expected_text: &str,
1252) -> Result<ScenarioReport, ConformanceError> {
1253 const SCENARIO: &str = "terminal_body_content_merges_per_kind";
1254 let provider = driver.provider;
1255 let mut observations = Vec::new();
1256
1257 for (label, body) in cases {
1258 let choice = driver.drive(body).await?;
1259 let choice_text: String = choice
1260 .iter()
1261 .filter_map(|content| match content {
1262 AssistantContent::Text(text) => Some(text.text.as_str()),
1263 _ => None,
1264 })
1265 .collect();
1266 let occurrences = choice_text.matches(expected_text).count();
1267 if occurrences != 1 {
1268 return Err(ConformanceError::contract(
1269 SCENARIO,
1270 provider,
1271 format!(
1272 "{label}: terminal-body text must appear exactly once in the choice, observed {occurrences} in {choice_text:?}"
1273 ),
1274 ));
1275 }
1276 observations.push(format!("{label}: text merged exactly once"));
1277 }
1278
1279 Ok(ScenarioReport {
1280 name: SCENARIO,
1281 provider,
1282 observations,
1283 })
1284}
1285
1286pub async fn bare_terminal_after_only_unparseable_frames_fabricates_nothing(
1293 fixture: &ProviderWireFixture,
1294) -> Result<ScenarioOutcome, ConformanceError> {
1295 const SCENARIO: &str = "bare_terminal_after_only_unparseable_frames_fabricates_nothing";
1296 let provider = fixture.driver.provider;
1297 let Some(bare_terminal) = &fixture.bare_terminal_frames else {
1298 return Ok(ScenarioOutcome::Skipped {
1299 name: SCENARIO,
1300 provider,
1301 reason: "wire family has no data-less terminal signal",
1302 });
1303 };
1304 let Some(malformed) = &fixture.malformed_frame else {
1305 return Ok(ScenarioOutcome::Skipped {
1306 name: SCENARIO,
1307 provider,
1308 reason: "wire family cannot spell a frame-level decode failure",
1309 });
1310 };
1311
1312 let frames = concat_frames(&[std::slice::from_ref(malformed), bare_terminal]);
1313 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
1314
1315 if drained.error_count() == 0 {
1316 return Err(ConformanceError::contract(
1317 SCENARIO,
1318 provider,
1319 "the unparseable frame must surface as an Err item",
1320 ));
1321 }
1322 if drained.response.is_some() || drained.final_count() != 0 {
1323 return Err(ConformanceError::contract(
1324 SCENARIO,
1325 provider,
1326 "a bare terminal with no decoded frame must not fabricate a terminal record",
1327 ));
1328 }
1329
1330 Ok(ScenarioOutcome::Ran(ScenarioReport {
1331 name: SCENARIO,
1332 provider,
1333 observations: vec!["no fabricated terminal after only-unparseable frames".to_string()],
1334 }))
1335}
1336
1337pub async fn usage_variants_are_reported_or_zero_sentinel(
1344 fixture: &ProviderWireFixture,
1345) -> Result<ScenarioReport, ConformanceError> {
1346 const SCENARIO: &str = "usage_variants_are_reported_or_zero_sentinel";
1347 let provider = fixture.driver.provider;
1348 let mut observations = Vec::new();
1349
1350 let frames = concat_frames(&[&fixture.text_frames, &fixture.terminal_frames]);
1351 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
1352 let response = drained.response.as_ref().ok_or_else(|| {
1353 ConformanceError::contract(
1354 SCENARIO,
1355 provider,
1356 "the genuine terminal must produce a record",
1357 )
1358 })?;
1359 if response.usage.total_tokens != fixture.expected_usage_total {
1360 return Err(ConformanceError::contract(
1361 SCENARIO,
1362 provider,
1363 format!(
1364 "terminal usage must be preserved: expected total {}, observed {}",
1365 fixture.expected_usage_total, response.usage.total_tokens
1366 ),
1367 ));
1368 }
1369 if response.finish_reason != fixture.expected_finish_reason {
1370 return Err(ConformanceError::contract(
1371 SCENARIO,
1372 provider,
1373 format!(
1374 "terminal finish reason must be normalized: expected {:?}, observed {:?}",
1375 fixture.expected_finish_reason, response.finish_reason
1376 ),
1377 ));
1378 }
1379 observations.push(format!(
1380 "usage total {} and finish reason {:?} preserved",
1381 fixture.expected_usage_total, fixture.expected_finish_reason
1382 ));
1383
1384 if let Some(zero_usage) = &fixture.zero_usage_terminal_frames {
1385 let frames = concat_frames(&[&fixture.text_frames, zero_usage]);
1386 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
1387 let response = drained.response.as_ref().ok_or_else(|| {
1388 ConformanceError::contract(
1389 SCENARIO,
1390 provider,
1391 "a usage-less genuine terminal must still complete the stream",
1392 )
1393 })?;
1394 if response.usage.total_tokens != 0 {
1395 return Err(ConformanceError::contract(
1396 SCENARIO,
1397 provider,
1398 "missing usage metrics must be the zero-usage sentinel, not invented values",
1399 ));
1400 }
1401 observations.push("usage-less terminal completed with the zero sentinel".to_string());
1402 }
1403
1404 Ok(ScenarioReport {
1405 name: SCENARIO,
1406 provider,
1407 observations,
1408 })
1409}
1410
1411pub async fn reasoning_summary_deltas_are_superseded_without_duplication(
1420 driver: &WireDriver,
1421 frames: Vec<WireInput>,
1422 summary_text: &str,
1423) -> Result<ScenarioReport, ConformanceError> {
1424 const SCENARIO: &str = "reasoning_summary_deltas_are_superseded_without_duplication";
1425 let provider = driver.provider;
1426
1427 let drained = driver.drive(ok_chunks(frames)).await?;
1428 if drained.error_count() != 0 || drained.response.is_none() {
1429 return Err(ConformanceError::contract(
1430 SCENARIO,
1431 provider,
1432 "the reasoning stream must complete without errors",
1433 ));
1434 }
1435 let reasoning = drained.choice_reasoning();
1436 let occurrences: usize = reasoning
1437 .iter()
1438 .flat_map(|item| item.content.iter())
1439 .filter(|content| match content {
1440 crate::message::ReasoningContent::Summary(text)
1441 | crate::message::ReasoningContent::Text { text, .. } => text.contains(summary_text),
1442 _ => false,
1443 })
1444 .count();
1445 if occurrences != 1 {
1446 return Err(ConformanceError::contract(
1447 SCENARIO,
1448 provider,
1449 format!(
1450 "the summary must appear exactly once in the aggregated choice, observed {occurrences} across {reasoning:?}"
1451 ),
1452 ));
1453 }
1454 if reasoning.len() != 1 {
1455 return Err(ConformanceError::contract(
1456 SCENARIO,
1457 provider,
1458 format!(
1459 "deltas and their full block must collapse to one reasoning item, observed {}",
1460 reasoning.len()
1461 ),
1462 ));
1463 }
1464
1465 Ok(ScenarioReport {
1466 name: SCENARIO,
1467 provider,
1468 observations: vec!["summary aggregated exactly once".to_string()],
1469 })
1470}
1471
1472pub async fn multi_part_same_id_reasoning_keeps_every_part(
1480 driver: &WireDriver,
1481 frames: Vec<WireInput>,
1482 expected_parts: &[&str],
1483) -> Result<ScenarioReport, ConformanceError> {
1484 const SCENARIO: &str = "multi_part_same_id_reasoning_keeps_every_part";
1485 let provider = driver.provider;
1486
1487 let drained = driver.drive(ok_chunks(frames)).await?;
1488 if drained.error_count() != 0 || drained.response.is_none() {
1489 return Err(ConformanceError::contract(
1490 SCENARIO,
1491 provider,
1492 "the reasoning stream must complete without errors",
1493 ));
1494 }
1495 let observed: Vec<String> = drained
1496 .choice_reasoning()
1497 .iter()
1498 .flat_map(|item| item.content.iter())
1499 .map(|content| match content {
1500 crate::message::ReasoningContent::Summary(text) => text.clone(),
1501 crate::message::ReasoningContent::Text { text, .. } => text.clone(),
1502 crate::message::ReasoningContent::Encrypted(data) => data.clone(),
1503 crate::message::ReasoningContent::Redacted { data } => data.clone(),
1504 })
1505 .collect();
1506 if observed != expected_parts {
1507 return Err(ConformanceError::contract(
1508 SCENARIO,
1509 provider,
1510 format!(
1511 "every same-id reasoning part must survive in order: expected {expected_parts:?}, observed {observed:?}"
1512 ),
1513 ));
1514 }
1515
1516 Ok(ScenarioReport {
1517 name: SCENARIO,
1518 provider,
1519 observations: vec![format!(
1520 "all {} reasoning parts survived",
1521 expected_parts.len()
1522 )],
1523 })
1524}
1525
1526pub async fn interleaved_reasoning_aggregates_to_one_item(
1534 driver: &WireDriver,
1535 frames: Vec<WireInput>,
1536 expected_text: &str,
1537) -> Result<ScenarioReport, ConformanceError> {
1538 const SCENARIO: &str = "interleaved_reasoning_aggregates_to_one_item";
1539 let provider = driver.provider;
1540
1541 let drained = driver.drive(ok_chunks(frames)).await?;
1542 if drained.error_count() != 0 || drained.response.is_none() {
1543 return Err(ConformanceError::contract(
1544 SCENARIO,
1545 provider,
1546 "the interleaved stream must complete without errors",
1547 ));
1548 }
1549 let reasoning = drained.choice_reasoning();
1550 if reasoning.len() != 1 {
1551 return Err(ConformanceError::contract(
1552 SCENARIO,
1553 provider,
1554 format!(
1555 "interleaved deltas and their completed block must collapse to one reasoning item, observed {}",
1556 reasoning.len()
1557 ),
1558 ));
1559 }
1560 let carries_text = reasoning
1561 .iter()
1562 .flat_map(|item| item.content.iter())
1563 .any(|content| match content {
1564 crate::message::ReasoningContent::Summary(text)
1565 | crate::message::ReasoningContent::Text { text, .. } => text == expected_text,
1566 _ => false,
1567 });
1568 if !carries_text {
1569 return Err(ConformanceError::contract(
1570 SCENARIO,
1571 provider,
1572 format!("the reasoning item must carry the completed block's text {expected_text:?}"),
1573 ));
1574 }
1575
1576 Ok(ScenarioReport {
1577 name: SCENARIO,
1578 provider,
1579 observations: vec!["exactly one reasoning item with the completed content".to_string()],
1580 })
1581}
1582
1583pub async fn interleaved_constant_id_reasoning_preserves_order(
1592 fixture: &ProviderWireFixture,
1593) -> Result<ScenarioOutcome, ConformanceError> {
1594 const SCENARIO: &str = "interleaved_constant_id_reasoning_preserves_order";
1595 let provider = fixture.driver.provider;
1596 let Some(interleaved) = &fixture.interleaved_reasoning else {
1597 return Ok(ScenarioOutcome::Skipped {
1598 name: SCENARIO,
1599 provider,
1600 reason: "wire fixture supplies no interleaved reasoning frames",
1601 });
1602 };
1603
1604 let drained = fixture
1605 .driver
1606 .drive(ok_chunks(interleaved.frames.clone()))
1607 .await?;
1608 if drained.error_count() != 0 || drained.response.is_none() {
1609 return Err(ConformanceError::contract(
1610 SCENARIO,
1611 provider,
1612 "the interleaved stream must complete without errors",
1613 ));
1614 }
1615 assert_reasoning_tool_reasoning(
1616 SCENARIO,
1617 provider,
1618 &drained,
1619 interleaved.first_reasoning,
1620 interleaved.tool_name,
1621 interleaved.second_reasoning,
1622 )?;
1623
1624 Ok(ScenarioOutcome::Ran(ScenarioReport {
1625 name: SCENARIO,
1626 provider,
1627 observations: vec!["boundary kept: reasoning, tool call, reasoning in order".to_string()],
1628 }))
1629}
1630
1631pub async fn interleaved_signed_full_reasoning_does_not_erase_prior_thought(
1640 driver: &WireDriver,
1641 frames: Vec<WireInput>,
1642 first: &str,
1643 tool_name: &str,
1644 second: &str,
1645) -> Result<ScenarioReport, ConformanceError> {
1646 const SCENARIO: &str = "interleaved_signed_full_reasoning_does_not_erase_prior_thought";
1647 let provider = driver.provider;
1648
1649 let drained = driver.drive(ok_chunks(frames)).await?;
1650 if drained.error_count() != 0 || drained.response.is_none() {
1651 return Err(ConformanceError::contract(
1652 SCENARIO,
1653 provider,
1654 "the interleaved stream must complete without errors",
1655 ));
1656 }
1657 assert_reasoning_tool_reasoning(SCENARIO, provider, &drained, first, tool_name, second)?;
1658 let signed = drained.choice_reasoning().last().is_some_and(|reasoning| {
1659 reasoning.content.iter().any(|content| {
1660 matches!(
1661 content,
1662 crate::message::ReasoningContent::Text {
1663 signature: Some(_),
1664 ..
1665 }
1666 )
1667 })
1668 });
1669 if !signed {
1670 return Err(ConformanceError::contract(
1671 SCENARIO,
1672 provider,
1673 "the post-boundary block must keep its signature",
1674 ));
1675 }
1676
1677 Ok(ScenarioReport {
1678 name: SCENARIO,
1679 provider,
1680 observations: vec![
1681 "pre-boundary thought survived; signed block completed the post-boundary part"
1682 .to_string(),
1683 ],
1684 })
1685}
1686
1687fn assert_reasoning_tool_reasoning(
1690 scenario: &'static str,
1691 provider: &'static str,
1692 drained: &DrainedStream,
1693 first: &str,
1694 tool_name: &str,
1695 second: &str,
1696) -> Result<(), ConformanceError> {
1697 let shape: Vec<String> = drained
1698 .choice
1699 .iter()
1700 .map(|content| match content {
1701 AssistantContent::Reasoning(reasoning) => {
1702 let text: String = reasoning
1703 .content
1704 .iter()
1705 .filter_map(|content| match content {
1706 crate::message::ReasoningContent::Summary(text)
1707 | crate::message::ReasoningContent::Text { text, .. } => {
1708 Some(text.as_str())
1709 }
1710 _ => None,
1711 })
1712 .collect();
1713 format!("reasoning:{text}")
1714 }
1715 AssistantContent::ToolCall(tool_call) => {
1716 format!("tool:{}", tool_call.function.name)
1717 }
1718 AssistantContent::Text(text) => format!("text:{}", text.text),
1719 AssistantContent::Image(_) => "image".to_string(),
1720 })
1721 .collect();
1722 let expected = vec![
1723 format!("reasoning:{first}"),
1724 format!("tool:{tool_name}"),
1725 format!("reasoning:{second}"),
1726 ];
1727 if shape != expected {
1728 return Err(ConformanceError::contract(
1729 scenario,
1730 provider,
1731 format!(
1732 "the boundary must survive aggregation: expected {expected:?}, observed {shape:?}"
1733 ),
1734 ));
1735 }
1736 Ok(())
1737}
1738
1739#[cfg(all(not(target_family = "wasm"), feature = "websocket"))]
1751pub async fn drain_openai_responses_websocket_events(
1752 provider: &'static str,
1753 events: Vec<
1754 Result<
1755 crate::providers::openai::responses_api::websocket::ResponsesWebSocketEvent,
1756 CompletionError,
1757 >,
1758 >,
1759) -> DrainedStream {
1760 use crate::providers::openai::responses_api::ResponsesUsage;
1761 use crate::providers::openai::responses_api::streaming::{
1762 RawChoiceAccumulator, ResponseChunkKind, ResponsesStreamOptions, normalize_responses_stream,
1763 };
1764 use crate::providers::openai::responses_api::websocket::ResponsesWebSocketEvent;
1765
1766 let mut accumulator = RawChoiceAccumulator::new(ResponsesUsage::new());
1767 let mut raw = Vec::new();
1768 let mut errored = false;
1769 for event in events {
1770 match event {
1771 Ok(ResponsesWebSocketEvent::Item(chunk)) => raw.extend(
1772 accumulator
1773 .decode_item_chunk(chunk, ResponsesStreamOptions::strict())
1774 .into_iter()
1775 .map(Ok),
1776 ),
1777 Ok(ResponsesWebSocketEvent::Response(chunk)) => {
1778 let terminal = matches!(
1779 chunk.kind,
1780 ResponseChunkKind::ResponseCompleted
1781 | ResponseChunkKind::ResponseFailed
1782 | ResponseChunkKind::ResponseIncomplete
1783 );
1784 if let Err(error) =
1785 accumulator.record_response_chunk(chunk.kind, chunk.response, "")
1786 {
1787 raw.extend(accumulator.take_tool_calls().into_iter().map(Ok));
1788 raw.push(Err(error));
1789 errored = true;
1790 break;
1791 }
1792 if terminal {
1793 break;
1794 }
1795 }
1796 Ok(ResponsesWebSocketEvent::Unknown(value)) => {
1799 raw.push(Ok(crate::streaming::RawStreamingChoice::Unknown(value)));
1800 }
1801 Ok(ResponsesWebSocketEvent::Done(_)) => {}
1805 Ok(ResponsesWebSocketEvent::Error(error)) => {
1806 raw.extend(accumulator.take_tool_calls().into_iter().map(Ok));
1807 raw.push(Err(CompletionError::ProviderError(error.to_string())));
1808 errored = true;
1809 break;
1810 }
1811 Err(error) => {
1812 raw.extend(accumulator.take_tool_calls().into_iter().map(Ok));
1813 raw.push(Err(error));
1814 errored = true;
1815 break;
1816 }
1817 }
1818 }
1819 if !errored {
1820 raw.extend(accumulator.finish().into_iter().map(Ok));
1821 }
1822
1823 let stream = normalize_responses_stream(provider, Box::pin(futures::stream::iter(raw)));
1824 fixtures::drain(stream).await
1825}
1826
1827pub mod fixtures {
1829 use super::*;
1830 use crate::client::CompletionClient;
1831 use crate::completion::CompletionModel;
1832 use crate::test_utils::SequencedStreamingHttpClient;
1833 use serde_json::json;
1834
1835 pub async fn drain(mut stream: crate::streaming::StreamingCompletionResponse) -> DrainedStream {
1839 let mut items = Vec::new();
1840 while let Some(item) = stream.next().await {
1841 items.push(item);
1842 }
1843 let drained = DrainedStream {
1844 items,
1845 choice: stream.choice.clone(),
1846 response: stream.response.clone(),
1847 };
1848 super::assert_valid_event_stream(&drained.items, &drained.choice);
1852 drained
1853 }
1854
1855 fn byte_chunks(chunks: WireChunks) -> Result<Vec<http_client::Result<Bytes>>, CompletionError> {
1859 chunks
1860 .into_iter()
1861 .map(|chunk| match chunk {
1862 Ok(WireInput::Bytes(bytes)) => Ok(Ok(bytes)),
1863 Ok(WireInput::Event(_)) => Err(CompletionError::ProviderError(
1864 "typed-event frame fed to a byte-transport driver".to_string(),
1865 )),
1866 Err(error) => Ok(Err(error)),
1867 })
1868 .collect()
1869 }
1870
1871 fn sse(frame: &serde_json::Value) -> WireInput {
1872 WireInput::Bytes(Bytes::from(format!("data: {frame}\n\n")))
1873 }
1874
1875 fn sse_raw(data: &str) -> WireInput {
1876 WireInput::Bytes(Bytes::from(format!("data: {data}\n\n")))
1877 }
1878
1879 fn ndjson(frame: &serde_json::Value) -> WireInput {
1880 WireInput::Bytes(Bytes::from(format!("{frame}\n")))
1881 }
1882
1883 fn frame_text(frame: &WireInput) -> String {
1886 frame
1887 .as_bytes()
1888 .map(|bytes| String::from_utf8_lossy(bytes).into_owned())
1889 .unwrap_or_default()
1890 }
1891
1892 pub mod openai_chat {
1894 use super::*;
1895
1896 fn driver() -> WireDriver {
1897 WireDriver::new("openai", |chunks| {
1898 Box::pin(async move {
1899 let client = crate::providers::openai::Client::builder()
1900 .http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
1901 .api_key("test-key")
1902 .build()?
1903 .completions_api();
1904 let model = client.completion_model("gpt-4o");
1905 let request = model.completion_request("hello").build();
1906 let stream = model.stream(request).await?;
1907 Ok(drain(stream).await)
1908 })
1909 })
1910 }
1911
1912 pub fn fixture() -> ProviderWireFixture {
1914 ProviderWireFixture {
1915 driver: driver(),
1916 text_frames: vec![sse(&json!({
1917 "id": "chatcmpl-1",
1918 "model": "gpt-4o-2024-08-06",
1919 "choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": null}],
1920 "usage": null,
1921 }))],
1922 expected_texts: vec!["hi"],
1923 tool_call_frames: vec![
1924 sse(&json!({
1925 "choices": [{"index": 0, "delta": {"tool_calls": [{
1926 "index": 0,
1927 "id": "call_1",
1928 "type": "function",
1929 "function": {"name": "get_weather", "arguments": ""},
1930 }]}, "finish_reason": null}],
1931 })),
1932 sse(&json!({
1933 "choices": [{"index": 0, "delta": {"tool_calls": [{
1934 "index": 0,
1935 "function": {"arguments": "{\"city\":\"Tokyo\"}"},
1936 }]}, "finish_reason": null}],
1937 })),
1938 ],
1942 expected_tool_name: "get_weather",
1943 partial_tool_call_frames: Some(vec![sse(&json!({
1944 "choices": [{"index": 0, "delta": {"tool_calls": [{
1945 "index": 0,
1946 "id": "call_1",
1947 "type": "function",
1948 "function": {"name": "get_weather", "arguments": "{\"cit"},
1949 }]}, "finish_reason": null}],
1950 }))]),
1951 terminal_frames: vec![
1952 sse(&json!({
1953 "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
1954 "usage": null,
1955 })),
1956 sse(&json!({
1957 "choices": [],
1958 "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
1959 })),
1960 sse_raw("[DONE]"),
1961 ],
1962 expected_usage_total: 15,
1963 expected_finish_reason: Some(FinishReason::Stop),
1964 zero_usage_terminal_frames: Some(vec![
1965 sse(&json!({
1966 "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
1967 "usage": null,
1968 })),
1969 sse_raw("[DONE]"),
1970 ]),
1971 bare_terminal_frames: Some(vec![sse_raw("[DONE]")]),
1972 malformed_frame: Some(sse_raw("{not json")),
1973 unknown_event_frame: None,
1974 defective_known_frame: Some(sse_raw(r#"{"choices": 42}"#)),
1978 delta_less_prelude_frame: Some(sse_raw(
1981 r#"{"id":"","object":"","choices":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"}}}]}"#,
1982 )),
1983 refusal: None,
1984 interleaved_reasoning: None,
1996 }
1997 }
1998 }
1999
2000 pub mod openai_responses {
2002 use super::*;
2003
2004 pub fn driver() -> WireDriver {
2006 WireDriver::new("openai", |chunks| {
2007 Box::pin(async move {
2008 let client = crate::providers::openai::Client::builder()
2009 .http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
2010 .api_key("test-key")
2011 .build()?;
2012 let model = client.completion_model("gpt-5.4");
2013 let request = model.completion_request("hello").build();
2014 let stream = model.stream(request).await?;
2015 Ok(drain(stream).await)
2016 })
2017 })
2018 }
2019
2020 fn completed_response(
2021 usage: Option<serde_json::Value>,
2022 output: serde_json::Value,
2023 ) -> serde_json::Value {
2024 json!({
2025 "id": "resp_1",
2026 "object": "response",
2027 "created_at": 0,
2028 "status": "completed",
2029 "model": "gpt-5.4",
2030 "output": output,
2031 "tools": [],
2032 "usage": usage,
2033 })
2034 }
2035
2036 fn terminal(usage: Option<serde_json::Value>, output: serde_json::Value) -> WireInput {
2037 sse(&json!({
2038 "type": "response.completed",
2039 "sequence_number": 99,
2040 "response": completed_response(usage, output),
2041 }))
2042 }
2043
2044 fn usage_json() -> serde_json::Value {
2045 json!({
2046 "input_tokens": 10,
2047 "output_tokens": 5,
2048 "output_tokens_details": {"reasoning_tokens": 0},
2049 "total_tokens": 15,
2050 })
2051 }
2052
2053 fn text_delta(text: &str) -> WireInput {
2054 sse(&json!({
2055 "type": "response.output_text.delta",
2056 "content_index": 0,
2057 "delta": text,
2058 "item_id": "msg_1",
2059 "output_index": 0,
2060 "sequence_number": 1,
2061 }))
2062 }
2063
2064 fn tool_call_done() -> WireInput {
2065 sse(&json!({
2066 "type": "response.output_item.done",
2067 "output_index": 0,
2068 "sequence_number": 2,
2069 "item": {
2070 "type": "function_call",
2071 "id": "fc_1",
2072 "arguments": "{\"city\":\"Tokyo\"}",
2073 "call_id": "call_1",
2074 "name": "get_weather",
2075 "status": "completed",
2076 },
2077 }))
2078 }
2079
2080 pub fn incomplete_mid_tool_call_frames() -> Vec<WireInput> {
2087 vec![
2088 sse(&json!({
2089 "type": "response.output_item.added",
2090 "output_index": 0,
2091 "sequence_number": 1,
2092 "item": {
2093 "type": "function_call",
2094 "id": "fc_1",
2095 "arguments": "",
2096 "call_id": "call_1",
2097 "name": "add",
2098 "status": "in_progress",
2099 },
2100 })),
2101 sse(&json!({
2102 "type": "response.function_call_arguments.delta",
2103 "item_id": "fc_1",
2104 "output_index": 0,
2105 "sequence_number": 2,
2106 "delta": "{\"x",
2107 })),
2108 sse(&json!({
2109 "type": "response.function_call_arguments.delta",
2110 "item_id": "fc_1",
2111 "output_index": 0,
2112 "sequence_number": 3,
2113 "delta": "\":48151",
2114 })),
2115 sse(&json!({
2116 "type": "response.function_call_arguments.done",
2117 "item_id": "fc_1",
2118 "output_index": 0,
2119 "sequence_number": 4,
2120 "arguments": "{\"x\":48151",
2121 })),
2122 sse(&json!({
2123 "type": "response.output_item.done",
2124 "output_index": 0,
2125 "sequence_number": 5,
2126 "item": {
2127 "type": "function_call",
2128 "id": "fc_1",
2129 "arguments": "{\"x\":48151",
2130 "call_id": "call_1",
2131 "name": "add",
2132 "status": "incomplete",
2133 },
2134 })),
2135 sse(&json!({
2136 "type": "response.incomplete",
2137 "sequence_number": 6,
2138 "response": {
2139 "id": "resp_1",
2140 "object": "response",
2141 "created_at": 0,
2142 "status": "incomplete",
2143 "incomplete_details": {"reason": "max_output_tokens"},
2144 "model": "gpt-5.4",
2145 "output": [{
2146 "type": "function_call",
2147 "id": "fc_1",
2148 "arguments": "{\"x\":48151",
2149 "call_id": "call_1",
2150 "name": "add",
2151 "status": "incomplete",
2152 }],
2153 "tools": [],
2154 "usage": usage_json(),
2155 },
2156 })),
2157 ]
2158 }
2159
2160 fn reasoning_done_item(
2161 id: &str,
2162 summary: serde_json::Value,
2163 content: serde_json::Value,
2164 encrypted: Option<&str>,
2165 ) -> WireInput {
2166 let mut item = json!({
2167 "type": "reasoning",
2168 "id": id,
2169 "summary": summary,
2170 "content": content,
2171 "status": "completed",
2172 });
2173 if let (Some(encrypted), Some(object)) = (encrypted, item.as_object_mut()) {
2174 object.insert("encrypted_content".to_string(), json!(encrypted));
2175 }
2176 sse(&json!({
2177 "type": "response.output_item.done",
2178 "output_index": 0,
2179 "sequence_number": 3,
2180 "item": item,
2181 }))
2182 }
2183
2184 pub fn fixture() -> ProviderWireFixture {
2186 ProviderWireFixture {
2187 driver: driver(),
2188 text_frames: vec![text_delta("hi")],
2189 expected_texts: vec!["hi"],
2190 tool_call_frames: vec![tool_call_done()],
2191 expected_tool_name: "get_weather",
2192 partial_tool_call_frames: Some(vec![
2193 sse(&json!({
2194 "type": "response.output_item.added",
2195 "output_index": 0,
2196 "sequence_number": 1,
2197 "item": {
2198 "type": "function_call",
2199 "id": "fc_1",
2200 "arguments": "",
2201 "call_id": "call_1",
2202 "name": "get_weather",
2203 "status": "in_progress",
2204 },
2205 })),
2206 sse(&json!({
2207 "type": "response.function_call_arguments.delta",
2208 "item_id": "fc_1",
2209 "output_index": 0,
2210 "sequence_number": 2,
2211 "delta": "{\"cit",
2212 })),
2213 ]),
2214 terminal_frames: vec![terminal(Some(usage_json()), json!([]))],
2215 expected_usage_total: 15,
2216 expected_finish_reason: Some(FinishReason::Stop),
2217 zero_usage_terminal_frames: Some(vec![terminal(None, json!([]))]),
2218 bare_terminal_frames: None,
2219 malformed_frame: Some(sse_raw("{not json")),
2220 unknown_event_frame: Some(sse(&json!({
2221 "type": "response.web_search_call.searching",
2222 "output_index": 0,
2223 "sequence_number": 4,
2224 "item_id": "ws_1",
2225 }))),
2226 defective_known_frame: Some(sse(&json!({
2229 "type": "response.content_part.added",
2230 "item_id": "msg_1",
2231 "output_index": 0,
2232 "content_index": 0,
2233 "sequence_number": 5,
2234 "part": {"type": "output_text", "text": 42},
2235 }))),
2236 delta_less_prelude_frame: None,
2237 refusal: Some(RefusalFixture {
2238 frames: vec![sse(&json!({
2239 "type": "response.refusal.delta",
2240 "content_index": 0,
2241 "delta": "I cannot help with that.",
2242 "item_id": "msg_1",
2243 "output_index": 0,
2244 "sequence_number": 1,
2245 }))],
2246 expected_text: "I cannot help with that.",
2247 }),
2248 interleaved_reasoning: None,
2249 }
2250 }
2251
2252 pub fn buffered_driver() -> BufferedBodyDriver {
2262 BufferedBodyDriver::new("chatgpt", |body| {
2263 Box::pin(async move {
2264 let client = crate::providers::chatgpt::Client::builder()
2265 .api_key(crate::providers::chatgpt::ChatGPTAuth::AccessToken {
2266 access_token: "test-token".to_string(),
2267 account_id: Some("account-id".to_string()),
2268 })
2269 .http_client(crate::test_utils::RecordingHttpClient::new(body))
2270 .build()?;
2271 let model = client.completion_model("gpt-5.4");
2272 let request = model.completion_request("hello").build();
2273 let response = model.completion(request).await?;
2274 Ok(response.choice)
2275 })
2276 })
2277 }
2278
2279 fn message_output(text: &str) -> serde_json::Value {
2280 json!([{
2281 "type": "message",
2282 "id": "msg_1",
2283 "role": "assistant",
2284 "status": "completed",
2285 "content": [{"type": "output_text", "text": text, "annotations": []}],
2286 }])
2287 }
2288
2289 pub fn terminal_body_only_sse_body(text: &str) -> String {
2291 frame_text(&terminal(Some(usage_json()), message_output(text)))
2292 }
2293
2294 pub fn terminal_body_and_delta_sse_body(text: &str) -> String {
2296 let frames = [
2297 text_delta(text),
2298 terminal(Some(usage_json()), message_output(text)),
2299 ];
2300 frames.iter().map(frame_text).collect()
2301 }
2302
2303 pub fn delta_only_sse_body(text: &str) -> String {
2306 let frames = [text_delta(text), terminal(Some(usage_json()), json!([]))];
2307 frames.iter().map(frame_text).collect()
2308 }
2309
2310 pub fn envelope_less_reasoning_supersede_sse_body() -> (String, &'static str) {
2317 let delta = json!({
2318 "type": "response.reasoning_summary_text.delta",
2319 "delta": "step 1",
2320 });
2321 let frames = [
2322 sse(&delta),
2323 reasoning_done_item(
2324 "rs_1",
2325 json!([{"type": "summary_text", "text": "step 1"}]),
2326 json!([]),
2327 None,
2328 ),
2329 terminal(Some(usage_json()), json!([])),
2330 ];
2331 (frames.iter().map(frame_text).collect(), "step 1")
2332 }
2333
2334 pub fn reasoning_summary_supersede_frames() -> (Vec<WireInput>, &'static str) {
2338 let frames = vec![
2339 sse(&json!({
2340 "type": "response.reasoning_summary_text.delta",
2341 "item_id": "rs_1",
2342 "output_index": 0,
2343 "summary_index": 0,
2344 "sequence_number": 1,
2345 "delta": "step 1",
2346 })),
2347 reasoning_done_item(
2348 "rs_1",
2349 json!([{"type": "summary_text", "text": "step 1"}]),
2350 json!([]),
2351 None,
2352 ),
2353 terminal(Some(usage_json()), json!([])),
2354 ];
2355 (frames, "step 1")
2356 }
2357
2358 pub fn multi_part_reasoning_frames() -> (Vec<WireInput>, Vec<&'static str>) {
2361 let frames = vec![
2362 reasoning_done_item(
2363 "rs_1",
2364 json!([
2365 {"type": "summary_text", "text": "s1"},
2366 {"type": "summary_text", "text": "s2"},
2367 ]),
2368 json!([{"type": "reasoning_text", "text": "visible"}]),
2369 Some("enc_blob"),
2370 ),
2371 terminal(Some(usage_json()), json!([])),
2372 ];
2373 (frames, vec!["s1", "s2", "visible", "enc_blob"])
2374 }
2375
2376 pub fn interleaved_reasoning_frames() -> (Vec<WireInput>, &'static str) {
2379 let frames = vec![
2380 sse(&json!({
2381 "type": "response.reasoning_text.delta",
2382 "item_id": "rs_2",
2383 "output_index": 0,
2384 "content_index": 0,
2385 "sequence_number": 1,
2386 "delta": "thinking",
2387 })),
2388 tool_call_done(),
2389 reasoning_done_item(
2390 "rs_2",
2391 json!([]),
2392 json!([{"type": "reasoning_text", "text": "full reasoning"}]),
2393 None,
2394 ),
2395 terminal(Some(usage_json()), json!([])),
2396 ];
2397 (frames, "full reasoning")
2398 }
2399 }
2400
2401 pub mod gemini_rest {
2403 use super::*;
2404
2405 fn driver() -> WireDriver {
2406 WireDriver::new("gemini", |chunks| {
2407 Box::pin(async move {
2408 let client = crate::providers::gemini::Client::builder()
2409 .api_key("test-key")
2410 .http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
2411 .build()?;
2412 let model = client.completion_model(
2413 crate::providers::gemini::completion::GEMINI_2_5_PRO_PREVIEW_06_05,
2414 );
2415 let request = model.completion_request("hello").build();
2416 let stream = model.stream(request).await?;
2417 Ok(drain(stream).await)
2418 })
2419 })
2420 }
2421
2422 pub fn fixture() -> ProviderWireFixture {
2424 ProviderWireFixture {
2425 driver: driver(),
2426 text_frames: vec![sse(&json!({
2427 "candidates": [{"content": {"parts": [{"text": "hi"}], "role": "model"}}],
2428 "responseId": "resp-1",
2429 "modelVersion": "gemini-2.5-pro",
2430 }))],
2431 expected_texts: vec!["hi"],
2432 tool_call_frames: vec![sse(&json!({
2433 "candidates": [{"content": {"parts": [{
2434 "functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
2435 }], "role": "model"}}],
2436 "responseId": "resp-1",
2437 "modelVersion": "gemini-2.5-pro",
2438 }))],
2439 expected_tool_name: "get_weather",
2440 partial_tool_call_frames: None,
2442 terminal_frames: vec![sse(&json!({
2443 "candidates": [{
2444 "content": {"parts": [], "role": "model"},
2445 "finishReason": "STOP",
2446 }],
2447 "usageMetadata": {
2448 "promptTokenCount": 5,
2449 "candidatesTokenCount": 2,
2450 "totalTokenCount": 7,
2451 },
2452 "responseId": "resp-1",
2453 "modelVersion": "gemini-2.5-pro",
2454 }))],
2455 expected_usage_total: 7,
2456 expected_finish_reason: Some(FinishReason::Stop),
2457 zero_usage_terminal_frames: Some(vec![sse(&json!({
2458 "candidates": [{
2459 "content": {"parts": [], "role": "model"},
2460 "finishReason": "STOP",
2461 }],
2462 "responseId": "resp-1",
2463 "modelVersion": "gemini-2.5-pro",
2464 }))]),
2465 bare_terminal_frames: None,
2466 malformed_frame: Some(sse_raw("{not json")),
2467 unknown_event_frame: Some(sse_raw(r#"{"noise":true}"#)),
2471 defective_known_frame: Some(sse_raw(r#"{"candidates": 42}"#)),
2472 delta_less_prelude_frame: None,
2473 refusal: None,
2474 interleaved_reasoning: Some(interleaved_thought_fixture()),
2475 }
2476 }
2477
2478 fn chunk(parts: serde_json::Value) -> WireInput {
2479 sse(&json!({
2480 "candidates": [{"content": {"parts": parts, "role": "model"}}],
2481 "responseId": "resp-1",
2482 "modelVersion": "gemini-2.5-pro",
2483 }))
2484 }
2485
2486 fn terminal_frame() -> WireInput {
2487 sse(&json!({
2488 "candidates": [{
2489 "content": {"parts": [], "role": "model"},
2490 "finishReason": "STOP",
2491 }],
2492 "usageMetadata": {
2493 "promptTokenCount": 5,
2494 "candidatesTokenCount": 2,
2495 "totalTokenCount": 7,
2496 },
2497 "responseId": "resp-1",
2498 "modelVersion": "gemini-2.5-pro",
2499 }))
2500 }
2501
2502 fn interleaved_thought_fixture() -> InterleavedReasoningFixture {
2505 InterleavedReasoningFixture {
2506 frames: vec![
2507 chunk(json!([{"text": "before tool", "thought": true}])),
2508 chunk(json!([{
2509 "functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
2510 }])),
2511 chunk(json!([{"text": "after tool", "thought": true}])),
2512 terminal_frame(),
2513 ],
2514 first_reasoning: "before tool",
2515 tool_name: "get_weather",
2516 second_reasoning: "after tool",
2517 }
2518 }
2519
2520 pub fn interleaved_signed_thought_frames()
2523 -> (Vec<WireInput>, &'static str, &'static str, &'static str) {
2524 let frames = vec![
2525 chunk(json!([{"text": "before tool", "thought": true}])),
2526 chunk(json!([{
2527 "functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
2528 }])),
2529 chunk(json!([{
2530 "text": "signed conclusion",
2531 "thought": true,
2532 "thoughtSignature": "sig-1",
2533 }])),
2534 terminal_frame(),
2535 ];
2536 (frames, "before tool", "get_weather", "signed conclusion")
2537 }
2538 }
2539
2540 pub mod interactions {
2542 use super::*;
2543
2544 fn driver() -> WireDriver {
2545 WireDriver::new("gemini", |chunks| {
2546 Box::pin(async move {
2547 let client = crate::providers::gemini::Client::builder()
2548 .api_key("test-key")
2549 .http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
2550 .build()?
2551 .interactions_api();
2552 let model = client.completion_model("gemini-2.5-pro");
2553 let request = model.completion_request("hello").build();
2554 let stream = model.stream(request).await?;
2555 Ok(drain(stream).await)
2556 })
2557 })
2558 }
2559
2560 fn completed(usage: Option<serde_json::Value>) -> WireInput {
2561 let mut interaction = json!({
2562 "id": "int-1",
2563 "model": "gemini-2.5-pro",
2564 "status": "completed",
2565 });
2566 if let (Some(usage), Some(object)) = (usage, interaction.as_object_mut()) {
2567 object.insert("usage".to_string(), usage);
2568 }
2569 sse(&json!({
2570 "event_type": "interaction.completed",
2571 "interaction": interaction,
2572 }))
2573 }
2574
2575 pub fn fixture() -> ProviderWireFixture {
2577 ProviderWireFixture {
2578 driver: driver(),
2579 text_frames: vec![sse(&json!({
2580 "event_type": "step.delta",
2581 "index": 0,
2582 "delta": {"type": "text", "text": "hi"},
2583 }))],
2584 expected_texts: vec!["hi"],
2585 tool_call_frames: vec![sse(&json!({
2586 "event_type": "step.delta",
2587 "index": 0,
2588 "delta": {
2589 "type": "function_call",
2590 "name": "get_weather",
2591 "arguments": {"city": "Tokyo"},
2592 "id": "call-1",
2593 },
2594 }))],
2595 expected_tool_name: "get_weather",
2596 partial_tool_call_frames: None,
2599 terminal_frames: vec![completed(Some(json!({
2600 "total_input_tokens": 5,
2601 "total_output_tokens": 2,
2602 "total_tokens": 7,
2603 })))],
2604 expected_usage_total: 7,
2605 expected_finish_reason: Some(FinishReason::Stop),
2606 zero_usage_terminal_frames: Some(vec![completed(None)]),
2607 bare_terminal_frames: None,
2608 malformed_frame: Some(sse_raw("{not json")),
2609 unknown_event_frame: Some(sse(&json!({
2610 "event_type": "future.event",
2611 "index": 0,
2612 }))),
2613 defective_known_frame: Some(sse_raw(
2616 r#"{"event_type":"step.delta","index":0,"delta":42}"#,
2617 )),
2618 delta_less_prelude_frame: None,
2619 refusal: None,
2620 interleaved_reasoning: Some(interleaved_thought_fixture()),
2621 }
2622 }
2623
2624 fn interleaved_thought_fixture() -> InterleavedReasoningFixture {
2628 let frames = vec![
2629 sse(&json!({
2630 "event_type": "step.delta",
2631 "index": 0,
2632 "delta": {
2633 "type": "thought_summary",
2634 "content": {"text": "before tool"},
2635 },
2636 })),
2637 sse(&json!({
2638 "event_type": "step.delta",
2639 "index": 0,
2640 "delta": {
2641 "type": "function_call",
2642 "name": "get_weather",
2643 "arguments": {"city": "Tokyo"},
2644 "id": "call-1",
2645 },
2646 })),
2647 sse(&json!({
2648 "event_type": "step.delta",
2649 "index": 0,
2650 "delta": {
2651 "type": "thought_summary",
2652 "content": {"text": "after tool"},
2653 },
2654 })),
2655 completed(Some(json!({
2656 "total_input_tokens": 5,
2657 "total_output_tokens": 2,
2658 "total_tokens": 7,
2659 }))),
2660 ];
2661 InterleavedReasoningFixture {
2662 frames,
2663 first_reasoning: "before tool",
2664 tool_name: "get_weather",
2665 second_reasoning: "after tool",
2666 }
2667 }
2668 }
2669
2670 pub mod anthropic {
2672 use super::*;
2673
2674 fn driver() -> WireDriver {
2675 WireDriver::new("anthropic", |chunks| {
2676 Box::pin(async move {
2677 let client = crate::providers::anthropic::Client::builder()
2678 .api_key("test-key")
2679 .http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
2680 .build()?;
2681 let model = client.completion_model(
2682 crate::providers::anthropic::completion::CLAUDE_SONNET_4_6,
2683 );
2684 let request = model.completion_request("hello").build();
2685 let stream = model.stream(request).await?;
2686 Ok(drain(stream).await)
2687 })
2688 })
2689 }
2690
2691 fn message_start() -> WireInput {
2692 sse(&json!({
2693 "type": "message_start",
2694 "message": {
2695 "id": "msg_1",
2696 "role": "assistant",
2697 "content": [],
2698 "model": "claude-sonnet-4-6",
2699 "stop_reason": null,
2700 "stop_sequence": null,
2701 "usage": {"input_tokens": 5, "output_tokens": 0},
2702 },
2703 }))
2704 }
2705
2706 pub fn fixture() -> ProviderWireFixture {
2708 ProviderWireFixture {
2709 driver: driver(),
2710 text_frames: vec![
2711 message_start(),
2712 sse(&json!({
2713 "type": "content_block_start",
2714 "index": 0,
2715 "content_block": {"type": "text", "text": ""},
2716 })),
2717 sse(&json!({
2718 "type": "content_block_delta",
2719 "index": 0,
2720 "delta": {"type": "text_delta", "text": "hi"},
2721 })),
2722 ],
2723 expected_texts: vec!["hi"],
2724 tool_call_frames: vec![
2725 sse(&json!({
2726 "type": "content_block_start",
2727 "index": 0,
2728 "content_block": {
2729 "type": "tool_use",
2730 "id": "toolu_1",
2731 "name": "get_weather",
2732 "input": {},
2733 },
2734 })),
2735 sse(&json!({
2736 "type": "content_block_delta",
2737 "index": 0,
2738 "delta": {"type": "input_json_delta", "partial_json": "{\"city\":\"Tokyo\"}"},
2739 })),
2740 sse(&json!({"type": "content_block_stop", "index": 0})),
2743 ],
2744 expected_tool_name: "get_weather",
2745 partial_tool_call_frames: Some(vec![
2746 sse(&json!({
2747 "type": "content_block_start",
2748 "index": 0,
2749 "content_block": {
2750 "type": "tool_use",
2751 "id": "toolu_1",
2752 "name": "get_weather",
2753 "input": {},
2754 },
2755 })),
2756 sse(&json!({
2757 "type": "content_block_delta",
2758 "index": 0,
2759 "delta": {"type": "input_json_delta", "partial_json": "{\"cit"},
2760 })),
2761 ]),
2762 terminal_frames: vec![sse(&json!({
2763 "type": "message_delta",
2764 "delta": {"stop_reason": "end_turn", "stop_sequence": null},
2765 "usage": {"output_tokens": 4},
2766 }))],
2767 expected_usage_total: 9,
2769 expected_finish_reason: Some(FinishReason::Stop),
2770 zero_usage_terminal_frames: None,
2773 bare_terminal_frames: Some(vec![sse(&json!({"type": "message_stop"}))]),
2776 malformed_frame: Some(sse_raw("{not json")),
2777 unknown_event_frame: Some(sse(&json!({
2778 "type": "content_block_heartbeat",
2779 "index": 0,
2780 }))),
2781 defective_known_frame: Some(sse_raw(
2784 r#"{"type":"content_block_delta","index":0,"delta":42}"#,
2785 )),
2786 delta_less_prelude_frame: None,
2787 refusal: None,
2788 interleaved_reasoning: None,
2789 }
2790 }
2791 }
2792
2793 pub mod cohere {
2795 use super::*;
2796
2797 fn driver() -> WireDriver {
2798 WireDriver::new("cohere", |chunks| {
2799 Box::pin(async move {
2800 let client = crate::providers::cohere::Client::builder()
2801 .api_key("test-key")
2802 .http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
2803 .build()?;
2804 let model =
2805 client.completion_model(crate::providers::cohere::COMMAND_R_08_2024);
2806 let request = model.completion_request("hello").build();
2807 let stream = model.stream(request).await?;
2808 Ok(drain(stream).await)
2809 })
2810 })
2811 }
2812
2813 pub fn fixture() -> ProviderWireFixture {
2815 ProviderWireFixture {
2816 driver: driver(),
2817 text_frames: vec![
2818 sse(&json!({"type": "message-start", "id": "msg_1"})),
2819 sse(&json!({
2820 "type": "content-delta",
2821 "delta": {"message": {"content": {"text": "hi"}}},
2822 })),
2823 ],
2824 expected_texts: vec!["hi"],
2825 tool_call_frames: vec![
2826 sse(&json!({
2827 "type": "tool-call-start",
2828 "delta": {"message": {"tool_calls": {
2829 "id": "call_1",
2830 "function": {"name": "get_weather", "arguments": ""},
2831 }}},
2832 })),
2833 sse(&json!({
2834 "type": "tool-call-delta",
2835 "delta": {"message": {"tool_calls": {
2836 "function": {"arguments": "{\"city\":\"Tokyo\"}"},
2837 }}},
2838 })),
2839 sse(&json!({"type": "tool-call-end"})),
2840 ],
2841 expected_tool_name: "get_weather",
2842 partial_tool_call_frames: Some(vec![sse(&json!({
2843 "type": "tool-call-start",
2844 "delta": {"message": {"tool_calls": {
2845 "id": "call_1",
2846 "function": {"name": "get_weather", "arguments": "{\"cit"},
2847 }}},
2848 }))]),
2849 terminal_frames: vec![sse(&json!({
2850 "type": "message-end",
2851 "delta": {
2852 "finish_reason": "COMPLETE",
2853 "usage": {"tokens": {"input_tokens": 10, "output_tokens": 4}},
2854 },
2855 }))],
2856 expected_usage_total: 14,
2857 expected_finish_reason: Some(FinishReason::Stop),
2858 zero_usage_terminal_frames: Some(vec![sse(&json!({"type": "message-end"}))]),
2859 bare_terminal_frames: None,
2860 malformed_frame: Some(sse_raw("{not json")),
2861 unknown_event_frame: Some(sse(&json!({
2862 "type": "citation-start",
2863 "delta": {"message": {"citations": {}}},
2864 }))),
2865 defective_known_frame: Some(sse_raw(r#"{"type":"content-delta","delta":42}"#)),
2866 delta_less_prelude_frame: None,
2867 refusal: None,
2868 interleaved_reasoning: Some(interleaved_thinking_fixture()),
2869 }
2870 }
2871
2872 fn interleaved_thinking_fixture() -> InterleavedReasoningFixture {
2876 let frames = vec![
2877 sse(&json!({"type": "message-start", "id": "msg_1"})),
2878 sse(&json!({
2879 "type": "content-delta",
2880 "delta": {"message": {"content": {"thinking": "before tool"}}},
2881 })),
2882 sse(&json!({
2883 "type": "tool-call-start",
2884 "delta": {"message": {"tool_calls": {
2885 "id": "call_1",
2886 "function": {"name": "get_weather", "arguments": "{\"city\":\"Tokyo\"}"},
2887 }}},
2888 })),
2889 sse(&json!({"type": "tool-call-end"})),
2890 sse(&json!({
2891 "type": "content-delta",
2892 "delta": {"message": {"content": {"thinking": "after tool"}}},
2893 })),
2894 sse(&json!({
2895 "type": "message-end",
2896 "delta": {
2897 "finish_reason": "COMPLETE",
2898 "usage": {"tokens": {"input_tokens": 10, "output_tokens": 4}},
2899 },
2900 })),
2901 ];
2902 InterleavedReasoningFixture {
2903 frames,
2904 first_reasoning: "before tool",
2905 tool_name: "get_weather",
2906 second_reasoning: "after tool",
2907 }
2908 }
2909 }
2910
2911 pub mod ollama {
2913 use super::*;
2914
2915 fn driver() -> WireDriver {
2916 WireDriver::new("ollama", |chunks| {
2917 Box::pin(async move {
2918 let client = crate::providers::ollama::Client::builder()
2919 .api_key("test-key")
2920 .http_client(SequencedStreamingHttpClient::new(byte_chunks(chunks)?))
2921 .build()?;
2922 let model = client.completion_model("llama3.2");
2923 let request = model.completion_request("hello").build();
2924 let stream = model.stream(request).await?;
2925 Ok(drain(stream).await)
2926 })
2927 })
2928 }
2929
2930 pub fn fixture() -> ProviderWireFixture {
2932 ProviderWireFixture {
2933 driver: driver(),
2934 text_frames: vec![ndjson(&json!({
2935 "model": "llama3.2",
2936 "created_at": "2023-08-04T19:22:45.499127Z",
2937 "message": {"role": "assistant", "content": "hi"},
2938 "done": false,
2939 }))],
2940 expected_texts: vec!["hi"],
2941 tool_call_frames: vec![ndjson(&json!({
2942 "model": "llama3.2",
2943 "created_at": "2023-08-04T19:22:45.499127Z",
2944 "message": {"role": "assistant", "content": "", "tool_calls": [{
2945 "function": {"name": "get_weather", "arguments": {"city": "Tokyo"}},
2946 }]},
2947 "done": false,
2948 }))],
2949 expected_tool_name: "get_weather",
2950 partial_tool_call_frames: None,
2952 terminal_frames: vec![ndjson(&json!({
2953 "model": "llama3.2",
2954 "created_at": "2023-08-04T19:22:47.499127Z",
2955 "message": {"role": "assistant", "content": ""},
2956 "done": true,
2957 "done_reason": "stop",
2958 "prompt_eval_count": 10,
2959 "eval_count": 4,
2960 }))],
2961 expected_usage_total: 14,
2962 expected_finish_reason: Some(FinishReason::Stop),
2963 zero_usage_terminal_frames: Some(vec![ndjson(&json!({
2964 "model": "llama3.2",
2965 "created_at": "2023-08-04T19:22:47.499127Z",
2966 "message": {"role": "assistant", "content": ""},
2967 "done": true,
2968 "done_reason": "stop",
2969 }))]),
2970 bare_terminal_frames: None,
2971 malformed_frame: Some(WireInput::Bytes(Bytes::from_static(b"{not json\n"))),
2972 unknown_event_frame: None,
2973 defective_known_frame: Some(ndjson(&json!({
2974 "model": "llama3.2",
2975 "created_at": "2023-08-04T19:22:46.499127Z",
2976 "message": {"role": "assistant", "content": 42},
2977 "done": false,
2978 }))),
2979 delta_less_prelude_frame: None,
2980 refusal: None,
2981 interleaved_reasoning: Some(interleaved_thinking_fixture()),
2982 }
2983 }
2984
2985 fn interleaved_thinking_fixture() -> InterleavedReasoningFixture {
2988 let frames = vec![
2989 ndjson(&json!({
2990 "model": "llama3.2",
2991 "created_at": "2023-08-04T19:22:45.499127Z",
2992 "message": {"role": "assistant", "content": "", "thinking": "before tool"},
2993 "done": false,
2994 })),
2995 ndjson(&json!({
2996 "model": "llama3.2",
2997 "created_at": "2023-08-04T19:22:45.599127Z",
2998 "message": {"role": "assistant", "content": "", "tool_calls": [{
2999 "function": {"name": "get_weather", "arguments": {"city": "Tokyo"}},
3000 }]},
3001 "done": false,
3002 })),
3003 ndjson(&json!({
3004 "model": "llama3.2",
3005 "created_at": "2023-08-04T19:22:45.699127Z",
3006 "message": {"role": "assistant", "content": "", "thinking": "after tool"},
3007 "done": false,
3008 })),
3009 ndjson(&json!({
3010 "model": "llama3.2",
3011 "created_at": "2023-08-04T19:22:47.499127Z",
3012 "message": {"role": "assistant", "content": ""},
3013 "done": true,
3014 "done_reason": "stop",
3015 "prompt_eval_count": 10,
3016 "eval_count": 4,
3017 })),
3018 ];
3019 InterleavedReasoningFixture {
3020 frames,
3021 first_reasoning: "before tool",
3022 tool_name: "get_weather",
3023 second_reasoning: "after tool",
3024 }
3025 }
3026 }
3027}