1use bytes::Bytes;
19use futures::StreamExt;
20use futures::future::BoxFuture;
21
22use crate::completion::{CompletionRequest, CompletionResponse};
23use crate::error::ProviderError;
24use crate::{
25 completion::FinishReason,
26 error::ErrorReport,
27 http_client,
28 message::AssistantContent,
29 streaming::{Item, StreamEvent},
30};
31
32#[derive(Debug, thiserror::Error)]
34pub enum ConformanceError {
35 #[error(transparent)]
37 Completion(#[from] ProviderError),
38 #[error("{scenario} conformance failed for {provider}: {details}")]
40 Contract {
41 scenario: &'static str,
43 provider: &'static str,
45 details: String,
47 },
48}
49
50impl ConformanceError {
51 fn contract(
52 scenario: &'static str,
53 provider: &'static str,
54 details: impl Into<String>,
55 ) -> Self {
56 Self::Contract {
57 scenario,
58 provider,
59 details: details.into(),
60 }
61 }
62}
63
64#[derive(Debug)]
73pub enum ScenarioOutcome {
74 Ran,
76 Skipped {
78 name: &'static str,
80 provider: &'static str,
82 reason: &'static str,
84 },
85}
86
87#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
95pub struct SuiteCapabilities {
96 pub partial_tool_args: bool,
99 pub zero_usage_terminal: bool,
102 pub bare_terminal: bool,
104 pub malformed_frame: bool,
106 pub unknown_event_frame: bool,
108 pub defective_known_frame: bool,
111 pub delta_less_prelude: bool,
114 pub refusal: bool,
116 pub interleaved_reasoning: bool,
120}
121
122impl SuiteCapabilities {
123 pub fn from_names(names: &[&str]) -> Result<Self, String> {
128 let mut caps = Self::default();
129 for name in names {
130 match *name {
131 "partial_tool_args" => caps.partial_tool_args = true,
132 "zero_usage_terminal" => caps.zero_usage_terminal = true,
133 "bare_terminal" => caps.bare_terminal = true,
134 "malformed_frame" => caps.malformed_frame = true,
135 "unknown_event_frame" => caps.unknown_event_frame = true,
136 "defective_known_frame" => caps.defective_known_frame = true,
137 "delta_less_prelude" => caps.delta_less_prelude = true,
138 "refusal" => caps.refusal = true,
139 "interleaved_reasoning" => caps.interleaved_reasoning = true,
140 other => {
141 return Err(format!(
142 "unknown capability name in suite manifest: {other}"
143 ));
144 }
145 }
146 }
147 Ok(caps)
148 }
149}
150
151pub const CANONICAL_SCENARIOS: &[&str] = &[
155 "truncation_preserves_content_without_terminal",
156 "transport_error_after_tool_call_yields_err_then_end",
157 "malformed_frame_ends_the_reply",
158 "unknown_event_is_skipped",
159 "defective_known_event_ends_the_reply",
160 "delta_less_choice_prelude_is_a_noop",
161 "refusal_frames_deliver_text_without_error",
162 "bare_terminal_after_only_unparseable_frames_fabricates_nothing",
163 "usage_variants_are_reported_or_absent",
164 "interleaved_constant_id_reasoning_preserves_order",
165];
166
167pub const WIRE_FAMILIES: &[&str] = &[
172 "openai_chat",
173 "openai_responses",
174 "openai_responses_websocket",
175 "chatgpt",
176 "anthropic",
177 "gemini_rest",
178 "gemini_interactions",
179 "gemini_grpc",
180 "xai",
181 "copilot",
182 "bedrock",
183 "candle",
184];
185
186pub fn check_gated_outcome(
194 scenario: &'static str,
195 capability: bool,
196 outcome: Result<ScenarioOutcome, ConformanceError>,
197) -> Result<(), String> {
198 match outcome {
199 Err(error) => Err(format!("{scenario} failed: {error}")),
200 Ok(ScenarioOutcome::Ran) => {
201 if capability {
202 Ok(())
203 } else {
204 Err(format!(
205 "{scenario} ran but the suite disclaims the capability; set the flag to true"
206 ))
207 }
208 }
209 Ok(ScenarioOutcome::Skipped { reason, .. }) => {
210 if capability {
211 Err(format!(
212 "{scenario} skipped ({reason}) but the suite declares the capability; \
213 a declared capability's scenario must run"
214 ))
215 } else {
216 eprintln!("skipped {scenario}: {reason}");
217 Ok(())
218 }
219 }
220 }
221}
222
223pub fn check_ungated_outcome(
225 scenario: &'static str,
226 result: Result<(), ConformanceError>,
227) -> Result<(), String> {
228 check_gated_outcome(scenario, true, result.map(|()| ScenarioOutcome::Ran))
230}
231
232#[derive(Clone)]
239pub enum WireInput {
240 Bytes(Bytes),
242 Event(std::sync::Arc<dyn std::any::Any + Send + Sync>),
244}
245
246impl WireInput {
247 pub fn as_bytes(&self) -> Option<&Bytes> {
249 match self {
250 Self::Bytes(bytes) => Some(bytes),
251 Self::Event(_) => None,
252 }
253 }
254
255 pub fn downcast_event<T: 'static>(&self) -> Option<&T> {
257 match self {
258 Self::Bytes(_) => None,
259 Self::Event(event) => event.downcast_ref(),
260 }
261 }
262}
263
264impl From<Bytes> for WireInput {
265 fn from(bytes: Bytes) -> Self {
266 Self::Bytes(bytes)
267 }
268}
269
270impl std::fmt::Debug for WireInput {
271 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
272 match self {
273 Self::Bytes(bytes) => formatter.debug_tuple("Bytes").field(bytes).finish(),
274 Self::Event(_) => formatter.write_str("Event(..)"),
275 }
276 }
277}
278
279pub fn event_frame<T: Send + Sync + 'static>(event: T) -> WireInput {
281 WireInput::Event(std::sync::Arc::new(event))
282}
283
284pub type WireChunks = Vec<http_client::Result<WireInput>>;
287
288pub fn ok_chunks(frames: impl IntoIterator<Item = impl Into<WireInput>>) -> WireChunks {
290 frames.into_iter().map(|frame| Ok(frame.into())).collect()
291}
292
293pub fn transport_error_chunk() -> http_client::Result<WireInput> {
296 Err(http_client::Error::transient(std::io::Error::new(
297 std::io::ErrorKind::ConnectionReset,
298 "connection reset",
299 )))
300}
301
302pub fn assert_valid_event_stream(
327 items: &[Result<Item<StreamEvent>, ErrorReport>],
328 choice: &[AssistantContent],
329) {
330 use crate::message::AssistantContent;
331
332 if let Some(error_index) = items.iter().position(Result::is_err) {
334 assert_eq!(
335 error_index + 1,
336 items.len(),
337 "law 1 (terminal error): an item followed the stream's error"
338 );
339 }
340 let events: Vec<&StreamEvent> = items
341 .iter()
342 .filter_map(|item| match item {
343 Ok(Item::Event(event)) => Some(event),
344 _ => None,
345 })
346 .collect();
347
348 let streamed_text: String = events
350 .iter()
351 .filter_map(|event| match event {
352 StreamEvent::Text { text, .. } => Some(text.as_str()),
353 _ => None,
354 })
355 .collect();
356 let aggregated_text: String = choice
357 .iter()
358 .filter_map(|content| match content {
359 AssistantContent::Text(text) => Some(text.text.as_str()),
360 _ => None,
361 })
362 .collect();
363 assert_eq!(
364 aggregated_text, streamed_text,
365 "law 2 (text conservation): aggregated text differs from the streamed fragments"
366 );
367
368 let yielded_calls = events
370 .iter()
371 .filter(|event| {
372 matches!(
373 event,
374 StreamEvent::End {
375 content: AssistantContent::ToolCall(_),
376 ..
377 }
378 )
379 })
380 .count();
381 let aggregated_calls = choice
382 .iter()
383 .filter(|content| matches!(content, AssistantContent::ToolCall(_)))
384 .count();
385 assert_eq!(
386 aggregated_calls, yielded_calls,
387 "law 3 (completed-call conservation): {yielded_calls} calls yielded, \
388 {aggregated_calls} aggregated"
389 );
390
391 let serialized = serde_json::to_value(
393 items
394 .iter()
395 .filter_map(|item| item.as_ref().ok())
396 .collect::<Vec<_>>(),
397 )
398 .unwrap_or_default();
399 let read_back = crate::streaming::StreamRecord::parse_prefix(serialized);
400 assert!(
401 read_back.is_ok(),
402 "law 4 (sequence): the stream's events do not read back: {read_back:?}"
403 );
404
405 let yielded_reasoning = events.iter().any(|event| {
407 matches!(
408 event,
409 StreamEvent::End {
410 content: AssistantContent::Reasoning(_),
411 ..
412 }
413 )
414 });
415 let aggregated_reasoning = choice
416 .iter()
417 .any(|content| matches!(content, AssistantContent::Reasoning(_)));
418 assert!(
419 yielded_reasoning || !aggregated_reasoning,
420 "law 5 (reasoning provenance): aggregated reasoning with no reasoning yielded"
421 );
422}
423
424#[derive(Debug)]
427pub struct DrainedStream {
428 pub items: Vec<Result<Item<StreamEvent>, ErrorReport>>,
430 pub choice: Vec<AssistantContent>,
433 pub response: Option<CompletionResponse>,
435}
436
437impl DrainedStream {
438 fn events(&self) -> impl Iterator<Item = &StreamEvent> {
439 self.items.iter().filter_map(|item| match item {
440 Ok(Item::Event(event)) => Some(event),
441 _ => None,
442 })
443 }
444
445 pub fn texts(&self) -> Vec<&str> {
447 self.events()
448 .filter_map(|event| match event {
449 StreamEvent::Text { text, .. } => Some(text.as_str()),
450 _ => None,
451 })
452 .collect()
453 }
454
455 pub fn tool_call_names(&self) -> Vec<&str> {
457 self.events()
458 .filter_map(|event| match event {
459 StreamEvent::End {
460 content: AssistantContent::ToolCall(tool_call),
461 ..
462 } => Some(tool_call.function.name.as_str()),
463 _ => None,
464 })
465 .collect()
466 }
467
468 pub fn unknown_values(&self) -> Vec<&serde_json::Value> {
470 self.items
471 .iter()
472 .filter_map(|item| match item {
473 Ok(Item::Unknown(value)) => Some(value.value()),
474 _ => None,
475 })
476 .collect()
477 }
478
479 pub fn error_count(&self) -> usize {
481 self.items.iter().filter(|item| item.is_err()).count()
482 }
483
484 fn has_terminal(&self) -> bool {
486 self.response.is_some()
487 }
488
489 fn completed_cleanly(&self) -> bool {
492 self.error_count() == 0 && self.response.is_some()
493 }
494
495 fn first_error_index(&self) -> Option<usize> {
497 self.items.iter().position(std::result::Result::is_err)
498 }
499
500 pub fn choice_texts(&self) -> Vec<&str> {
502 self.choice
503 .iter()
504 .filter_map(|content| match content {
505 AssistantContent::Text(text) => Some(text.text.as_str()),
506 _ => None,
507 })
508 .collect()
509 }
510
511 pub fn choice_reasoning(&self) -> Vec<&crate::message::Reasoning> {
513 self.choice
514 .iter()
515 .filter_map(|content| match content {
516 AssistantContent::Reasoning(reasoning) => Some(reasoning),
517 _ => None,
518 })
519 .collect()
520 }
521}
522
523type DriveFn = Box<
524 dyn Fn(WireChunks) -> BoxFuture<'static, Result<DrainedStream, ProviderError>> + Send + Sync,
525>;
526
527pub struct WireDriver {
533 pub provider: &'static str,
535 drive: DriveFn,
536}
537
538impl WireDriver {
539 pub fn new(
541 provider: &'static str,
542 drive: impl Fn(WireChunks) -> BoxFuture<'static, Result<DrainedStream, ProviderError>>
543 + Send
544 + Sync
545 + 'static,
546 ) -> Self {
547 Self {
548 provider,
549 drive: Box::new(drive),
550 }
551 }
552
553 pub async fn drive(&self, chunks: WireChunks) -> Result<DrainedStream, ProviderError> {
555 (self.drive)(chunks).await
556 }
557}
558
559pub struct RefusalFixture {
561 pub frames: Vec<WireInput>,
563 pub expected_text: &'static str,
565}
566
567pub struct InterleavedReasoningFixture {
572 pub frames: Vec<WireInput>,
574 pub first_reasoning: &'static str,
576 pub tool_name: &'static str,
578 pub second_reasoning: &'static str,
580}
581
582type BufferedDriveFn = Box<
583 dyn Fn(String) -> BoxFuture<'static, Result<Vec<AssistantContent>, ProviderError>>
584 + Send
585 + Sync,
586>;
587
588pub struct BufferedBodyDriver {
591 pub provider: &'static str,
593 drive: BufferedDriveFn,
594}
595
596impl BufferedBodyDriver {
597 pub fn new(
599 provider: &'static str,
600 drive: impl Fn(String) -> BoxFuture<'static, Result<Vec<AssistantContent>, ProviderError>>
601 + Send
602 + Sync
603 + 'static,
604 ) -> Self {
605 Self {
606 provider,
607 drive: Box::new(drive),
608 }
609 }
610
611 pub async fn drive(&self, body: String) -> Result<Vec<AssistantContent>, ProviderError> {
613 (self.drive)(body).await
614 }
615}
616
617pub struct ProviderWireFixture {
622 pub driver: WireDriver,
624 pub text_frames: Vec<WireInput>,
626 pub expected_texts: Vec<&'static str>,
628 pub tool_call_frames: Vec<WireInput>,
631 pub expected_tool_name: &'static str,
633 pub partial_tool_call_frames: Option<Vec<WireInput>>,
636 pub terminal_frames: Vec<WireInput>,
638 pub expected_usage_total: u64,
640 pub expected_finish_reason: Option<FinishReason>,
642 pub zero_usage_terminal_frames: Option<Vec<WireInput>>,
644 pub bare_terminal_frames: Option<Vec<WireInput>>,
647 pub malformed_frame: Option<WireInput>,
651 pub unknown_event_frame: Option<WireInput>,
653 pub defective_known_frame: Option<WireInput>,
655 pub delta_less_prelude_frame: Option<WireInput>,
657 pub refusal: Option<RefusalFixture>,
659 pub interleaved_reasoning: Option<InterleavedReasoningFixture>,
663}
664
665impl ProviderWireFixture {
666 pub fn capabilities(&self) -> SuiteCapabilities {
674 SuiteCapabilities {
675 partial_tool_args: self.partial_tool_call_frames.is_some(),
676 zero_usage_terminal: self.zero_usage_terminal_frames.is_some(),
677 bare_terminal: self.bare_terminal_frames.is_some(),
678 malformed_frame: self.malformed_frame.is_some(),
679 unknown_event_frame: self.unknown_event_frame.is_some(),
680 defective_known_frame: self.defective_known_frame.is_some(),
681 delta_less_prelude: self.delta_less_prelude_frame.is_some(),
682 refusal: self.refusal.is_some(),
683 interleaved_reasoning: self.interleaved_reasoning.is_some(),
684 }
685 }
686}
687
688fn concat_frames(parts: &[&[WireInput]]) -> Vec<WireInput> {
689 parts
690 .iter()
691 .flat_map(|frames| frames.iter().cloned())
692 .collect()
693}
694
695struct Checks {
699 name: &'static str,
700 provider: &'static str,
701}
702
703impl Checks {
704 fn new(name: &'static str, provider: &'static str) -> Self {
705 Self { name, provider }
706 }
707
708 fn fail(&self, details: impl Into<String>) -> ConformanceError {
710 ConformanceError::contract(self.name, self.provider, details)
711 }
712
713 fn require<D: Into<String>>(
716 &self,
717 held: bool,
718 details: impl FnOnce() -> D,
719 ) -> Result<(), ConformanceError> {
720 if held {
721 return Ok(());
722 }
723 Err(self.fail(details()))
724 }
725
726 fn skip(&self, reason: &'static str) -> ScenarioOutcome {
730 ScenarioOutcome::Skipped {
731 name: self.name,
732 provider: self.provider,
733 reason,
734 }
735 }
736}
737
738pub async fn truncation_preserves_content_without_terminal(
744 fixture: &ProviderWireFixture,
745) -> Result<(), ConformanceError> {
746 let checks = Checks::new(
747 "truncation_preserves_content_without_terminal",
748 fixture.driver.provider,
749 );
750
751 let drained = fixture.driver.drive(Vec::new()).await?;
753 checks.require(
754 !drained.has_terminal(),
755 || "an empty stream must not synthesize a terminal record",
756 )?;
757
758 let drained = fixture
760 .driver
761 .drive(ok_chunks(fixture.text_frames.clone()))
762 .await?;
763 checks.require(drained.texts() == fixture.expected_texts, || {
764 format!(
765 "text delivered before truncation must be preserved: expected {:?}, observed {:?}",
766 fixture.expected_texts,
767 drained.texts()
768 )
769 })?;
770 checks.require(
771 !drained.has_terminal(),
772 || "EOF after text deltas must not synthesize a terminal record",
773 )?;
774
775 if let Some(partial) = &fixture.partial_tool_call_frames {
777 let drained = fixture.driver.drive(ok_chunks(partial.clone())).await?;
778 checks.require(
779 !drained.has_terminal(),
780 || "EOF mid-tool-arguments must not synthesize a terminal record",
781 )?;
782 }
783
784 let drained = fixture
788 .driver
789 .drive(ok_chunks(fixture.tool_call_frames.clone()))
790 .await?;
791 checks.require(
792 drained
793 .tool_call_names()
794 .iter()
795 .all(|name| *name == fixture.expected_tool_name),
796 || {
797 format!(
798 "only the delivered call may surface: observed {:?}",
799 drained.tool_call_names()
800 )
801 },
802 )?;
803 checks.require(
804 !drained.has_terminal(),
805 || "EOF after a tool call must not synthesize a terminal record",
806 )?;
807
808 Ok(())
809}
810
811pub async fn transport_error_after_tool_call_yields_err_then_end(
815 fixture: &ProviderWireFixture,
816) -> Result<(), ConformanceError> {
817 let checks = Checks::new(
818 "transport_error_after_tool_call_yields_err_then_end",
819 fixture.driver.provider,
820 );
821
822 let mut chunks = ok_chunks(fixture.tool_call_frames.clone());
823 chunks.push(transport_error_chunk());
824 let drained = fixture.driver.drive(chunks).await?;
825
826 checks.require(
827 drained
828 .tool_call_names()
829 .iter()
830 .all(|name| *name == fixture.expected_tool_name),
831 || {
832 format!(
833 "only the delivered call may precede the transport error: observed {:?}",
834 drained.tool_call_names()
835 )
836 },
837 )?;
838 let error_index = drained
839 .first_error_index()
840 .ok_or_else(|| checks.fail("the transport failure must reach the consumer"))?;
841 checks.require(
842 error_index + 1 == drained.items.len(),
843 || "nothing may follow the terminal transport error",
844 )?;
845 checks.require(
846 !drained.has_terminal(),
847 || "a transport failure must not be papered over with a terminal record",
848 )?;
849
850 Ok(())
851}
852
853pub async fn malformed_frame_ends_the_reply(
857 fixture: &ProviderWireFixture,
858) -> Result<ScenarioOutcome, ConformanceError> {
859 let checks = Checks::new("malformed_frame_ends_the_reply", fixture.driver.provider);
860 let Some(malformed) = &fixture.malformed_frame else {
861 return Ok(checks.skip("wire family cannot spell a frame-level decode failure"));
862 };
863
864 let frames = concat_frames(&[
865 &fixture.text_frames,
866 std::slice::from_ref(malformed),
867 &fixture.terminal_frames,
868 ]);
869 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
870
871 checks.require(drained.error_count() == 1, || {
872 format!(
873 "the malformed frame must surface as exactly one Err item, observed {}",
874 drained.error_count()
875 )
876 })?;
877 checks.require(
878 drained.first_error_index() == Some(drained.items.len() - 1),
879 || "the malformed frame's error must be the stream's last item",
880 )?;
881 checks.require(
882 drained.texts() == fixture.expected_texts,
883 || "content before the malformed frame must be preserved",
884 )?;
885 checks.require(
886 !drained.has_terminal(),
887 || "a reply cut by a corrupt frame has no response",
888 )?;
889
890 Ok(ScenarioOutcome::Ran)
891}
892
893pub async fn unknown_event_is_skipped(
896 fixture: &ProviderWireFixture,
897) -> Result<ScenarioOutcome, ConformanceError> {
898 let checks = Checks::new("unknown_event_is_skipped", fixture.driver.provider);
899 let Some(unknown) = &fixture.unknown_event_frame else {
900 return Ok(checks.skip("wire family cannot spell an unknown event type"));
901 };
902
903 let frames = concat_frames(&[
904 &fixture.text_frames,
905 std::slice::from_ref(unknown),
906 &fixture.terminal_frames,
907 ]);
908 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
909
910 checks.require(
911 drained.error_count() == 0,
912 || "an unknown event type must be skipped, not surfaced as an error",
913 )?;
914 checks.require(
915 drained.texts() == fixture.expected_texts && drained.response.is_some(),
916 || "the stream must deliver its content and complete around the skipped event",
917 )?;
918 checks.require(drained.unknown_values().len() == 1, || {
921 format!(
922 "exactly one Unknown passthrough item must surface for the unknown frame, \
923 observed {}",
924 drained.unknown_values().len()
925 )
926 })?;
927
928 let control_frames = concat_frames(&[&fixture.text_frames, &fixture.terminal_frames]);
931 let control = fixture.driver.drive(ok_chunks(control_frames)).await?;
932 checks.require(
933 drained.choice == control.choice,
934 || "the unknown frame must not perturb the aggregated assistant choice",
935 )?;
936
937 Ok(ScenarioOutcome::Ran)
938}
939
940pub async fn defective_known_event_ends_the_reply(
946 fixture: &ProviderWireFixture,
947) -> Result<ScenarioOutcome, ConformanceError> {
948 let checks = Checks::new(
949 "defective_known_event_ends_the_reply",
950 fixture.driver.provider,
951 );
952 let Some(defective) = &fixture.defective_known_frame else {
953 return Ok(
954 checks.skip("wire family cannot spell a known event with a schema-defective payload")
955 );
956 };
957
958 let frames = concat_frames(&[
959 &fixture.text_frames,
960 std::slice::from_ref(defective),
961 &fixture.terminal_frames,
962 ]);
963 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
964
965 checks.require(drained.error_count() == 1, || {
966 format!(
967 "a known event with a schema defect must surface exactly one Err item, observed {}",
968 drained.error_count()
969 )
970 })?;
971 checks.require(
972 !drained.has_terminal(),
973 || "a reply cut by a defective event has no response",
974 )?;
975
976 Ok(ScenarioOutcome::Ran)
977}
978
979pub async fn delta_less_choice_prelude_is_a_noop(
982 fixture: &ProviderWireFixture,
983) -> Result<ScenarioOutcome, ConformanceError> {
984 let checks = Checks::new(
985 "delta_less_choice_prelude_is_a_noop",
986 fixture.driver.provider,
987 );
988 let Some(prelude) = &fixture.delta_less_prelude_frame else {
989 return Ok(checks.skip("wire family has no delta-less prelude shape"));
990 };
991
992 let frames = concat_frames(&[
993 std::slice::from_ref(prelude),
994 &fixture.text_frames,
995 &fixture.terminal_frames,
996 ]);
997 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
998
999 checks.require(
1000 drained.error_count() == 0,
1001 || "the delta-less prelude must not surface an error",
1002 )?;
1003 checks.require(
1004 drained.texts() == fixture.expected_texts && drained.response.is_some(),
1005 || "the prelude must not perturb content delivery or the terminal",
1006 )?;
1007
1008 Ok(ScenarioOutcome::Ran)
1009}
1010
1011pub async fn refusal_frames_deliver_text_without_error(
1013 fixture: &ProviderWireFixture,
1014) -> Result<ScenarioOutcome, ConformanceError> {
1015 let checks = Checks::new(
1016 "refusal_frames_deliver_text_without_error",
1017 fixture.driver.provider,
1018 );
1019 let Some(refusal) = &fixture.refusal else {
1020 return Ok(checks.skip("wire family has no refusal channel"));
1021 };
1022
1023 let frames = concat_frames(&[&refusal.frames, &fixture.terminal_frames]);
1024 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
1025
1026 checks.require(
1027 drained.error_count() == 0,
1028 || "refusal content must not surface as an error",
1029 )?;
1030 let delivered = drained.texts().concat();
1031 checks.require(delivered == refusal.expected_text, || {
1032 format!(
1033 "refusal text must be delivered: expected {:?}, observed {delivered:?}",
1034 refusal.expected_text
1035 )
1036 })?;
1037 checks.require(
1038 drained.response.is_some(),
1039 || "a refused turn still ends with the provider's genuine terminal",
1040 )?;
1041
1042 Ok(ScenarioOutcome::Ran)
1043}
1044
1045pub async fn terminal_body_content_merges_per_kind(
1049 driver: &BufferedBodyDriver,
1050 cases: Vec<(&'static str, String)>,
1051 expected_text: &str,
1052) -> Result<(), ConformanceError> {
1053 let checks = Checks::new("terminal_body_content_merges_per_kind", driver.provider);
1054
1055 for (label, body) in cases {
1056 let choice = driver.drive(body).await?;
1057 let choice_text: String = choice
1058 .iter()
1059 .filter_map(|content| match content {
1060 AssistantContent::Text(text) => Some(text.text.as_str()),
1061 _ => None,
1062 })
1063 .collect();
1064 let occurrences = choice_text.matches(expected_text).count();
1065 checks.require(occurrences == 1, || {
1066 format!(
1067 "{label}: terminal-body text must appear exactly once in the choice, observed {occurrences} in {choice_text:?}"
1068 )
1069 })?;
1070 }
1071
1072 Ok(())
1073}
1074
1075pub async fn bare_terminal_after_only_unparseable_frames_fabricates_nothing(
1079 fixture: &ProviderWireFixture,
1080) -> Result<ScenarioOutcome, ConformanceError> {
1081 let checks = Checks::new(
1082 "bare_terminal_after_only_unparseable_frames_fabricates_nothing",
1083 fixture.driver.provider,
1084 );
1085 let Some(bare_terminal) = &fixture.bare_terminal_frames else {
1086 return Ok(checks.skip("wire family has no data-less terminal signal"));
1087 };
1088 let Some(malformed) = &fixture.malformed_frame else {
1089 return Ok(checks.skip("wire family cannot spell a frame-level decode failure"));
1090 };
1091
1092 let frames = concat_frames(&[std::slice::from_ref(malformed), bare_terminal]);
1093 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
1094
1095 checks.require(
1096 drained.error_count() != 0,
1097 || "the unparseable frame must surface as an Err item",
1098 )?;
1099 checks.require(
1100 !drained.has_terminal(),
1101 || "a bare terminal with no decoded frame must not fabricate a terminal record",
1102 )?;
1103
1104 Ok(ScenarioOutcome::Ran)
1105}
1106
1107pub async fn usage_variants_are_reported_or_absent(
1113 fixture: &ProviderWireFixture,
1114) -> Result<(), ConformanceError> {
1115 let checks = Checks::new(
1116 "usage_variants_are_reported_or_absent",
1117 fixture.driver.provider,
1118 );
1119
1120 let frames = concat_frames(&[&fixture.text_frames, &fixture.terminal_frames]);
1121 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
1122 let response = drained
1123 .response
1124 .as_ref()
1125 .ok_or_else(|| checks.fail("the genuine terminal must produce a record"))?;
1126 checks.require(
1127 response.usage.total_tokens == Some(fixture.expected_usage_total),
1128 || {
1129 format!(
1130 "terminal usage must be preserved: expected total {}, observed {:?}",
1131 fixture.expected_usage_total, response.usage.total_tokens
1132 )
1133 },
1134 )?;
1135 checks.require(
1136 response.finish_reason() == fixture.expected_finish_reason,
1137 || {
1138 format!(
1139 "terminal finish reason must be normalized: expected {:?}, observed {:?}",
1140 fixture.expected_finish_reason,
1141 response.finish_reason()
1142 )
1143 },
1144 )?;
1145
1146 if let Some(zero_usage) = &fixture.zero_usage_terminal_frames {
1147 let frames = concat_frames(&[&fixture.text_frames, zero_usage]);
1148 let drained = fixture.driver.drive(ok_chunks(frames)).await?;
1149 let response = drained.response.as_ref().ok_or_else(|| {
1150 checks.fail("a usage-less genuine terminal must still complete the stream")
1151 })?;
1152 checks.require(!response.usage.is_reported(), || {
1153 format!(
1154 "missing usage metrics must leave every counter unreported, not invented: {:?}",
1155 response.usage
1156 )
1157 })?;
1158 }
1159
1160 Ok(())
1161}
1162
1163pub async fn reasoning_summary_deltas_are_superseded_without_duplication(
1170 driver: &WireDriver,
1171 frames: Vec<WireInput>,
1172 summary_text: &str,
1173) -> Result<(), ConformanceError> {
1174 let checks = Checks::new(
1175 "reasoning_summary_deltas_are_superseded_without_duplication",
1176 driver.provider,
1177 );
1178
1179 let drained = driver.drive(ok_chunks(frames)).await?;
1180 checks.require(
1181 drained.completed_cleanly(),
1182 || "the reasoning stream must complete without errors",
1183 )?;
1184 let reasoning = drained.choice_reasoning();
1185 let occurrences: usize = reasoning
1186 .iter()
1187 .map(|item| item.text.matches(summary_text).count())
1188 .sum();
1189 checks.require(occurrences == 1, || {
1190 format!(
1191 "the summary must appear exactly once in the aggregated choice, observed {occurrences} across {reasoning:?}"
1192 )
1193 })?;
1194 checks.require(reasoning.len() == 1, || {
1195 format!(
1196 "deltas and their full block must collapse to one reasoning item, observed {}",
1197 reasoning.len()
1198 )
1199 })?;
1200
1201 Ok(())
1202}
1203
1204pub async fn multi_part_same_id_reasoning_keeps_every_part(
1208 driver: &WireDriver,
1209 frames: Vec<WireInput>,
1210 expected_parts: &[&str],
1211) -> Result<(), ConformanceError> {
1212 let checks = Checks::new(
1213 "multi_part_same_id_reasoning_keeps_every_part",
1214 driver.provider,
1215 );
1216
1217 let drained = driver.drive(ok_chunks(frames)).await?;
1218 checks.require(
1219 drained.completed_cleanly(),
1220 || "the reasoning stream must complete without errors",
1221 )?;
1222 let items: Vec<String> = drained
1223 .choice_reasoning()
1224 .iter()
1225 .filter_map(|item| item.native.as_ref())
1226 .map(|native| native.item.to_string())
1227 .collect();
1228 let in_order = |item: &String| {
1229 let mut rest = item.as_str();
1230 expected_parts.iter().all(|part| match rest.find(part) {
1231 Some(at) => {
1232 rest = &rest[at + part.len()..];
1233 true
1234 }
1235 None => false,
1236 })
1237 };
1238 let observed = items;
1239 checks.require(observed.len() == 1 && observed.iter().all(in_order), || {
1240 format!(
1241 "every same-id reasoning part must survive in order: expected {expected_parts:?}, observed {observed:?}"
1242 )
1243 })?;
1244
1245 Ok(())
1246}
1247
1248pub async fn interleaved_reasoning_aggregates_to_one_item(
1255 driver: &WireDriver,
1256 frames: Vec<WireInput>,
1257 expected_text: &str,
1258) -> Result<(), ConformanceError> {
1259 let checks = Checks::new(
1260 "interleaved_reasoning_aggregates_to_one_item",
1261 driver.provider,
1262 );
1263
1264 let drained = driver.drive(ok_chunks(frames)).await?;
1265 checks.require(
1266 drained.completed_cleanly(),
1267 || "the interleaved stream must complete without errors",
1268 )?;
1269 let reasoning = drained.choice_reasoning();
1270 checks.require(reasoning.len() == 1, || {
1271 format!(
1272 "interleaved deltas and their completed block must collapse to one reasoning item, observed {}",
1273 reasoning.len()
1274 )
1275 })?;
1276 let carries_text = reasoning.iter().any(|item| item.text == expected_text);
1277 checks.require(carries_text, || {
1278 format!("the reasoning item must carry the completed block's text {expected_text:?}")
1279 })?;
1280
1281 Ok(())
1282}
1283
1284pub async fn interleaved_constant_id_reasoning_preserves_order(
1289 fixture: &ProviderWireFixture,
1290) -> Result<ScenarioOutcome, ConformanceError> {
1291 let checks = Checks::new(
1292 "interleaved_constant_id_reasoning_preserves_order",
1293 fixture.driver.provider,
1294 );
1295 let Some(interleaved) = &fixture.interleaved_reasoning else {
1296 return Ok(checks.skip("wire fixture supplies no interleaved reasoning frames"));
1297 };
1298
1299 let drained = fixture
1300 .driver
1301 .drive(ok_chunks(interleaved.frames.clone()))
1302 .await?;
1303 checks.require(
1304 drained.completed_cleanly(),
1305 || "the interleaved stream must complete without errors",
1306 )?;
1307 assert_reasoning_tool_reasoning(
1308 &checks,
1309 &drained,
1310 interleaved.first_reasoning,
1311 interleaved.tool_name,
1312 interleaved.second_reasoning,
1313 )?;
1314
1315 Ok(ScenarioOutcome::Ran)
1316}
1317
1318pub async fn interleaved_signed_full_reasoning_does_not_erase_prior_thought(
1325 driver: &WireDriver,
1326 frames: Vec<WireInput>,
1327 first: &str,
1328 tool_name: &str,
1329 second: &str,
1330) -> Result<(), ConformanceError> {
1331 let checks = Checks::new(
1332 "interleaved_signed_full_reasoning_does_not_erase_prior_thought",
1333 driver.provider,
1334 );
1335
1336 let drained = driver.drive(ok_chunks(frames)).await?;
1337 checks.require(
1338 drained.completed_cleanly(),
1339 || "the interleaved stream must complete without errors",
1340 )?;
1341 assert_reasoning_tool_reasoning(&checks, &drained, first, tool_name, second)?;
1342 let signed = drained
1343 .choice_reasoning()
1344 .last()
1345 .is_some_and(|reasoning| reasoning.native.is_some());
1346 checks.require(signed, || "the post-boundary block must keep its signature")?;
1347
1348 Ok(())
1349}
1350
1351fn assert_reasoning_tool_reasoning(
1354 checks: &Checks,
1355 drained: &DrainedStream,
1356 first: &str,
1357 tool_name: &str,
1358 second: &str,
1359) -> Result<(), ConformanceError> {
1360 let shape: Vec<String> = drained
1361 .choice
1362 .iter()
1363 .map(|content| match content {
1364 AssistantContent::Reasoning(reasoning) => format!("reasoning:{}", reasoning.text),
1365 AssistantContent::ToolCall(tool_call) => {
1366 format!("tool:{}", tool_call.function.name)
1367 }
1368 AssistantContent::Text(text) => format!("text:{}", text.text),
1369 AssistantContent::Image(_) => "image".to_string(),
1370 AssistantContent::Opaque(opaque) => format!("opaque:{}", opaque.kind().unwrap_or("")),
1371 })
1372 .collect();
1373 let expected = vec![
1374 format!("reasoning:{first}"),
1375 format!("tool:{tool_name}"),
1376 format!("reasoning:{second}"),
1377 ];
1378 checks.require(shape == expected, || {
1379 format!("the boundary must survive aggregation: expected {expected:?}, observed {shape:?}")
1380 })
1381}
1382
1383pub mod fixtures {
1385 use super::*;
1386 use crate::driver::Model;
1387 use crate::operation::Completion;
1388 use crate::test_utils::SequencedStreamingHttpClient;
1389 use crate::wire::Wire;
1390 use serde_json::json;
1391
1392 pub async fn drain(mut stream: crate::streaming::CompletionStream) -> DrainedStream {
1396 let mut items = Vec::new();
1397 while let Some(item) = stream.next().await {
1398 items.push(item.map_err(|error| ErrorReport::from(&error)));
1399 }
1400 let partial = stream.partial().choice;
1401 let response = stream.finish().await.ok();
1402 let drained = DrainedStream {
1403 items,
1404 choice: response
1405 .as_ref()
1406 .map_or(partial, |response| response.choice.clone()),
1407 response,
1408 };
1409 super::assert_valid_event_stream(&drained.items, &drained.choice);
1412 drained
1413 }
1414
1415 fn byte_chunks(chunks: WireChunks) -> Result<Vec<http_client::Result<Bytes>>, ProviderError> {
1419 chunks
1420 .into_iter()
1421 .map(|chunk| match chunk {
1422 Ok(WireInput::Bytes(bytes)) => Ok(Ok(bytes)),
1423 Ok(WireInput::Event(_)) => Err(ProviderError::Provider(
1424 "typed-event frame fed to a byte-transport driver".to_string(),
1425 )),
1426 Err(error) => Ok(Err(error)),
1427 })
1428 .collect()
1429 }
1430
1431 fn byte_driver<W>(
1439 provider: &'static str,
1440 bind: fn(SequencedStreamingHttpClient) -> Model<W, SequencedStreamingHttpClient>,
1441 ) -> WireDriver
1442 where
1443 W: Wire<Op = Completion, Payload = crate::wire::Encoded, Frame = crate::wire::WireFrame>,
1444 {
1445 WireDriver::new(provider, move |chunks| {
1446 Box::pin(async move {
1447 let model = bind(SequencedStreamingHttpClient::new(byte_chunks(chunks)?));
1448 Ok(drain(model.stream(CompletionRequest::new("hello"))?).await)
1449 })
1450 })
1451 }
1452
1453 fn sse(frame: &serde_json::Value) -> WireInput {
1454 WireInput::Bytes(Bytes::from(format!("data: {frame}\n\n")))
1455 }
1456
1457 fn sse_raw(data: &str) -> WireInput {
1458 WireInput::Bytes(Bytes::from(format!("data: {data}\n\n")))
1459 }
1460
1461 fn frame_text(frame: &WireInput) -> String {
1464 frame
1465 .as_bytes()
1466 .map(|bytes| String::from_utf8_lossy(bytes).into_owned())
1467 .unwrap_or_default()
1468 }
1469
1470 pub mod openai_chat {
1472 use super::*;
1473
1474 fn driver() -> WireDriver {
1475 byte_driver("openai", |transport| {
1476 crate::driver::Model::new(
1477 crate::providers::openai::wire::OpenAIConfig::with_key(
1478 &crate::providers::openai::wire::OPENAI,
1479 "test-key",
1480 )
1481 .chat("gpt-4o"),
1482 transport,
1483 )
1484 })
1485 }
1486
1487 pub fn fixture() -> ProviderWireFixture {
1489 ProviderWireFixture {
1490 driver: driver(),
1491 text_frames: vec![sse(&json!({
1492 "id": "chatcmpl-1",
1493 "model": "gpt-4o-2024-08-06",
1494 "choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": null}],
1495 "usage": null,
1496 }))],
1497 expected_texts: vec!["hi"],
1498 tool_call_frames: vec![
1499 sse(&json!({
1500 "choices": [{"index": 0, "delta": {"tool_calls": [{
1501 "index": 0,
1502 "id": "call_1",
1503 "type": "function",
1504 "function": {"name": "get_weather", "arguments": ""},
1505 }]}, "finish_reason": null}],
1506 })),
1507 sse(&json!({
1508 "choices": [{"index": 0, "delta": {"tool_calls": [{
1509 "index": 0,
1510 "function": {"arguments": "{\"city\":\"Tokyo\"}"},
1511 }]}, "finish_reason": null}],
1512 })),
1513 ],
1517 expected_tool_name: "get_weather",
1518 partial_tool_call_frames: Some(vec![sse(&json!({
1519 "choices": [{"index": 0, "delta": {"tool_calls": [{
1520 "index": 0,
1521 "id": "call_1",
1522 "type": "function",
1523 "function": {"name": "get_weather", "arguments": "{\"cit"},
1524 }]}, "finish_reason": null}],
1525 }))]),
1526 terminal_frames: vec![
1527 sse(&json!({
1528 "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
1529 "usage": null,
1530 })),
1531 sse(&json!({
1532 "choices": [],
1533 "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
1534 })),
1535 sse_raw("[DONE]"),
1536 ],
1537 expected_usage_total: 15,
1538 expected_finish_reason: Some(FinishReason::Stop),
1539 zero_usage_terminal_frames: Some(vec![
1540 sse(&json!({
1541 "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
1542 "usage": null,
1543 })),
1544 sse_raw("[DONE]"),
1545 ]),
1546 bare_terminal_frames: Some(vec![sse_raw("[DONE]")]),
1547 malformed_frame: Some(sse_raw("{not json")),
1548 unknown_event_frame: None,
1549 defective_known_frame: None,
1555 delta_less_prelude_frame: Some(sse_raw(
1558 r#"{"id":"","object":"","choices":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"}}}]}"#,
1559 )),
1560 refusal: None,
1561 interleaved_reasoning: None,
1573 }
1574 }
1575 }
1576
1577 pub mod openai_responses {
1579 use super::*;
1580
1581 pub fn driver() -> WireDriver {
1583 byte_driver("openai", |transport| {
1584 crate::driver::Model::new(
1585 crate::providers::openai::OpenAIConfig::new("test-key").responses("gpt-5.4"),
1586 transport,
1587 )
1588 })
1589 }
1590
1591 fn completed_response(
1592 usage: Option<&serde_json::Value>,
1593 output: &serde_json::Value,
1594 ) -> serde_json::Value {
1595 json!({
1596 "id": "resp_1",
1597 "object": "response",
1598 "created_at": 0,
1599 "status": "completed",
1600 "model": "gpt-5.4",
1601 "output": output,
1602 "tools": [],
1603 "usage": usage,
1604 })
1605 }
1606
1607 fn terminal(usage: Option<&serde_json::Value>, output: &serde_json::Value) -> WireInput {
1608 sse(&json!({
1609 "type": "response.completed",
1610 "sequence_number": 99,
1611 "response": completed_response(usage, output),
1612 }))
1613 }
1614
1615 fn usage_json() -> serde_json::Value {
1616 json!({
1617 "input_tokens": 10,
1618 "output_tokens": 5,
1619 "output_tokens_details": {"reasoning_tokens": 0},
1620 "total_tokens": 15,
1621 })
1622 }
1623
1624 fn text_delta(text: &str) -> WireInput {
1625 sse(&json!({
1626 "type": "response.output_text.delta",
1627 "content_index": 0,
1628 "delta": text,
1629 "item_id": "msg_1",
1630 "output_index": 0,
1631 "sequence_number": 1,
1632 }))
1633 }
1634
1635 fn tool_call_done() -> WireInput {
1636 sse(&json!({
1637 "type": "response.output_item.done",
1638 "output_index": 0,
1639 "sequence_number": 2,
1640 "item": {
1641 "type": "function_call",
1642 "id": "fc_1",
1643 "arguments": "{\"city\":\"Tokyo\"}",
1644 "call_id": "call_1",
1645 "name": "get_weather",
1646 "status": "completed",
1647 },
1648 }))
1649 }
1650
1651 pub fn incomplete_mid_tool_call_frames() -> Vec<WireInput> {
1658 vec![
1659 sse(&json!({
1660 "type": "response.output_item.added",
1661 "output_index": 0,
1662 "sequence_number": 1,
1663 "item": {
1664 "type": "function_call",
1665 "id": "fc_1",
1666 "arguments": "",
1667 "call_id": "call_1",
1668 "name": "add",
1669 "status": "in_progress",
1670 },
1671 })),
1672 sse(&json!({
1673 "type": "response.function_call_arguments.delta",
1674 "item_id": "fc_1",
1675 "output_index": 0,
1676 "sequence_number": 2,
1677 "delta": "{\"x",
1678 })),
1679 sse(&json!({
1680 "type": "response.function_call_arguments.delta",
1681 "item_id": "fc_1",
1682 "output_index": 0,
1683 "sequence_number": 3,
1684 "delta": "\":48151",
1685 })),
1686 sse(&json!({
1687 "type": "response.function_call_arguments.done",
1688 "item_id": "fc_1",
1689 "output_index": 0,
1690 "sequence_number": 4,
1691 "arguments": "{\"x\":48151",
1692 })),
1693 sse(&json!({
1694 "type": "response.output_item.done",
1695 "output_index": 0,
1696 "sequence_number": 5,
1697 "item": {
1698 "type": "function_call",
1699 "id": "fc_1",
1700 "arguments": "{\"x\":48151",
1701 "call_id": "call_1",
1702 "name": "add",
1703 "status": "incomplete",
1704 },
1705 })),
1706 sse(&json!({
1707 "type": "response.incomplete",
1708 "sequence_number": 6,
1709 "response": {
1710 "id": "resp_1",
1711 "object": "response",
1712 "created_at": 0,
1713 "status": "incomplete",
1714 "incomplete_details": {"reason": "max_output_tokens"},
1715 "model": "gpt-5.4",
1716 "output": [{
1717 "type": "function_call",
1718 "id": "fc_1",
1719 "arguments": "{\"x\":48151",
1720 "call_id": "call_1",
1721 "name": "add",
1722 "status": "incomplete",
1723 }],
1724 "tools": [],
1725 "usage": usage_json(),
1726 },
1727 })),
1728 ]
1729 }
1730
1731 fn reasoning_done_item(
1732 id: &str,
1733 summary: &serde_json::Value,
1734 content: &serde_json::Value,
1735 encrypted: Option<&str>,
1736 ) -> WireInput {
1737 let mut item = json!({
1738 "type": "reasoning",
1739 "id": id,
1740 "summary": summary,
1741 "content": content,
1742 "status": "completed",
1743 });
1744 if let (Some(encrypted), Some(object)) = (encrypted, item.as_object_mut()) {
1745 object.insert("encrypted_content".to_string(), json!(encrypted));
1746 }
1747 sse(&json!({
1748 "type": "response.output_item.done",
1749 "output_index": 0,
1750 "sequence_number": 3,
1751 "item": item,
1752 }))
1753 }
1754
1755 pub fn fixture() -> ProviderWireFixture {
1757 ProviderWireFixture {
1758 driver: driver(),
1759 text_frames: vec![text_delta("hi")],
1760 expected_texts: vec!["hi"],
1761 tool_call_frames: vec![tool_call_done()],
1762 expected_tool_name: "get_weather",
1763 partial_tool_call_frames: Some(vec![
1764 sse(&json!({
1765 "type": "response.output_item.added",
1766 "output_index": 0,
1767 "sequence_number": 1,
1768 "item": {
1769 "type": "function_call",
1770 "id": "fc_1",
1771 "arguments": "",
1772 "call_id": "call_1",
1773 "name": "get_weather",
1774 "status": "in_progress",
1775 },
1776 })),
1777 sse(&json!({
1778 "type": "response.function_call_arguments.delta",
1779 "item_id": "fc_1",
1780 "output_index": 0,
1781 "sequence_number": 2,
1782 "delta": "{\"cit",
1783 })),
1784 ]),
1785 terminal_frames: vec![terminal(Some(&usage_json()), &json!([]))],
1786 expected_usage_total: 15,
1787 expected_finish_reason: Some(FinishReason::Stop),
1788 zero_usage_terminal_frames: Some(vec![terminal(None, &json!([]))]),
1789 bare_terminal_frames: None,
1790 malformed_frame: Some(sse_raw("{not json")),
1791 unknown_event_frame: Some(sse(&json!({
1792 "type": "response.web_search_call.searching",
1793 "output_index": 0,
1794 "sequence_number": 4,
1795 "item_id": "ws_1",
1796 }))),
1797 defective_known_frame: None,
1800 delta_less_prelude_frame: None,
1801 refusal: Some(RefusalFixture {
1802 frames: vec![sse(&json!({
1803 "type": "response.refusal.delta",
1804 "content_index": 0,
1805 "delta": "I cannot help with that.",
1806 "item_id": "msg_1",
1807 "output_index": 0,
1808 "sequence_number": 1,
1809 }))],
1810 expected_text: "I cannot help with that.",
1811 }),
1812 interleaved_reasoning: None,
1813 }
1814 }
1815
1816 pub fn buffered_driver() -> BufferedBodyDriver {
1825 BufferedBodyDriver::new("chatgpt", |body| {
1826 Box::pin(async move {
1827 let model = crate::driver::Model::new(
1828 crate::providers::openai::OpenAIConfig::with_key(
1829 &crate::providers::chatgpt::DIALECT,
1830 "test-token",
1831 )
1832 .with_account_id("account-id")
1833 .responses("gpt-5.4"),
1834 crate::test_utils::RecordingHttpClient::new(body),
1835 );
1836 let request = CompletionRequest::new("hello");
1837 let response = model.call(request).await?;
1838 Ok(response.choice)
1839 })
1840 })
1841 }
1842
1843 fn message_output(text: &str) -> serde_json::Value {
1844 json!([{
1845 "type": "message",
1846 "id": "msg_1",
1847 "role": "assistant",
1848 "status": "completed",
1849 "content": [{"type": "output_text", "text": text, "annotations": []}],
1850 }])
1851 }
1852
1853 pub fn terminal_body_only_sse_body(text: &str) -> String {
1855 frame_text(&terminal(Some(&usage_json()), &message_output(text)))
1856 }
1857
1858 pub fn terminal_body_and_delta_sse_body(text: &str) -> String {
1860 let frames = [
1861 text_delta(text),
1862 terminal(Some(&usage_json()), &message_output(text)),
1863 ];
1864 frames.iter().map(frame_text).collect()
1865 }
1866
1867 pub fn delta_only_sse_body(text: &str) -> String {
1870 let frames = [text_delta(text), terminal(Some(&usage_json()), &json!([]))];
1871 frames.iter().map(frame_text).collect()
1872 }
1873
1874 pub fn envelope_less_reasoning_supersede_sse_body() -> (String, &'static str) {
1881 let delta = json!({
1882 "type": "response.reasoning_summary_text.delta",
1883 "delta": "step 1",
1884 });
1885 let frames = [
1886 sse(&delta),
1887 reasoning_done_item(
1888 "rs_1",
1889 &json!([{"type": "summary_text", "text": "step 1"}]),
1890 &json!([]),
1891 None,
1892 ),
1893 terminal(Some(&usage_json()), &json!([])),
1894 ];
1895 (frames.iter().map(frame_text).collect(), "step 1")
1896 }
1897
1898 pub fn reasoning_summary_supersede_frames() -> (Vec<WireInput>, &'static str) {
1902 let frames = vec![
1903 sse(&json!({
1904 "type": "response.reasoning_summary_text.delta",
1905 "item_id": "rs_1",
1906 "output_index": 0,
1907 "summary_index": 0,
1908 "sequence_number": 1,
1909 "delta": "step 1",
1910 })),
1911 reasoning_done_item(
1912 "rs_1",
1913 &json!([{"type": "summary_text", "text": "step 1"}]),
1914 &json!([]),
1915 None,
1916 ),
1917 terminal(Some(&usage_json()), &json!([])),
1918 ];
1919 (frames, "step 1")
1920 }
1921
1922 pub fn multi_part_reasoning_frames() -> (Vec<WireInput>, Vec<&'static str>) {
1925 let frames = vec![
1926 reasoning_done_item(
1927 "rs_1",
1928 &json!([
1929 {"type": "summary_text", "text": "s1"},
1930 {"type": "summary_text", "text": "s2"},
1931 ]),
1932 &json!([{"type": "reasoning_text", "text": "visible"}]),
1933 Some("enc_blob"),
1934 ),
1935 terminal(Some(&usage_json()), &json!([])),
1936 ];
1937 (frames, vec!["s1", "s2", "visible", "enc_blob"])
1938 }
1939
1940 pub fn interleaved_reasoning_frames() -> (Vec<WireInput>, &'static str) {
1943 let frames = vec![
1944 sse(&json!({
1945 "type": "response.reasoning_text.delta",
1946 "item_id": "rs_2",
1947 "output_index": 0,
1948 "content_index": 0,
1949 "sequence_number": 1,
1950 "delta": "full ",
1951 })),
1952 sse(&json!({
1954 "type": "response.output_item.done",
1955 "output_index": 1,
1956 "sequence_number": 2,
1957 "item": {
1958 "type": "function_call",
1959 "id": "fc_1",
1960 "arguments": "{\"city\":\"Tokyo\"}",
1961 "call_id": "call_1",
1962 "name": "get_weather",
1963 "status": "completed",
1964 },
1965 })),
1966 reasoning_done_item(
1967 "rs_2",
1968 &json!([]),
1969 &json!([{"type": "reasoning_text", "text": "full reasoning"}]),
1970 None,
1971 ),
1972 terminal(Some(&usage_json()), &json!([])),
1973 ];
1974 (frames, "full reasoning")
1975 }
1976 }
1977
1978 pub mod gemini_rest {
1980 use super::*;
1981
1982 fn driver() -> WireDriver {
1983 byte_driver("gemini", |transport| {
1984 crate::driver::Model::new(
1985 crate::providers::gemini::GeminiConfig::new("test-key").completion(
1986 crate::providers::gemini::completion::GEMINI_2_5_PRO_PREVIEW_06_05,
1987 ),
1988 transport,
1989 )
1990 })
1991 }
1992
1993 pub fn fixture() -> ProviderWireFixture {
1995 ProviderWireFixture {
1996 driver: driver(),
1997 text_frames: vec![sse(&json!({
1998 "candidates": [{"content": {"parts": [{"text": "hi"}], "role": "model"}}],
1999 "responseId": "resp-1",
2000 "modelVersion": "gemini-2.5-pro",
2001 }))],
2002 expected_texts: vec!["hi"],
2003 tool_call_frames: vec![sse(&json!({
2004 "candidates": [{"content": {"parts": [{
2005 "functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
2006 }], "role": "model"}}],
2007 "responseId": "resp-1",
2008 "modelVersion": "gemini-2.5-pro",
2009 }))],
2010 expected_tool_name: "get_weather",
2011 partial_tool_call_frames: None,
2013 terminal_frames: vec![sse(&json!({
2014 "candidates": [{
2015 "content": {"parts": [], "role": "model"},
2016 "finishReason": "STOP",
2017 }],
2018 "usageMetadata": {
2019 "promptTokenCount": 5,
2020 "candidatesTokenCount": 2,
2021 "totalTokenCount": 7,
2022 },
2023 "responseId": "resp-1",
2024 "modelVersion": "gemini-2.5-pro",
2025 }))],
2026 expected_usage_total: 7,
2027 expected_finish_reason: Some(FinishReason::Stop),
2028 zero_usage_terminal_frames: Some(vec![sse(&json!({
2029 "candidates": [{
2030 "content": {"parts": [], "role": "model"},
2031 "finishReason": "STOP",
2032 }],
2033 "responseId": "resp-1",
2034 "modelVersion": "gemini-2.5-pro",
2035 }))]),
2036 bare_terminal_frames: None,
2037 malformed_frame: Some(sse_raw("{not json")),
2038 unknown_event_frame: Some(sse_raw(r#"{"noise":true}"#)),
2042 defective_known_frame: Some(sse_raw(r#"{"candidates": 42}"#)),
2043 delta_less_prelude_frame: None,
2044 refusal: None,
2045 interleaved_reasoning: Some(interleaved_thought_fixture()),
2046 }
2047 }
2048
2049 fn chunk(parts: &serde_json::Value) -> WireInput {
2050 sse(&json!({
2051 "candidates": [{"content": {"parts": parts, "role": "model"}}],
2052 "responseId": "resp-1",
2053 "modelVersion": "gemini-2.5-pro",
2054 }))
2055 }
2056
2057 fn terminal_frame() -> WireInput {
2058 sse(&json!({
2059 "candidates": [{
2060 "content": {"parts": [], "role": "model"},
2061 "finishReason": "STOP",
2062 }],
2063 "usageMetadata": {
2064 "promptTokenCount": 5,
2065 "candidatesTokenCount": 2,
2066 "totalTokenCount": 7,
2067 },
2068 "responseId": "resp-1",
2069 "modelVersion": "gemini-2.5-pro",
2070 }))
2071 }
2072
2073 fn interleaved_thought_fixture() -> InterleavedReasoningFixture {
2076 InterleavedReasoningFixture {
2077 frames: vec![
2078 chunk(&json!([{"text": "before tool", "thought": true}])),
2079 chunk(&json!([{
2080 "functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
2081 }])),
2082 chunk(&json!([{"text": "after tool", "thought": true}])),
2083 terminal_frame(),
2084 ],
2085 first_reasoning: "before tool",
2086 tool_name: "get_weather",
2087 second_reasoning: "after tool",
2088 }
2089 }
2090
2091 pub fn interleaved_signed_thought_frames()
2094 -> (Vec<WireInput>, &'static str, &'static str, &'static str) {
2095 let frames = vec![
2096 chunk(&json!([{"text": "before tool", "thought": true}])),
2097 chunk(&json!([{
2098 "functionCall": {"name": "get_weather", "args": {"city": "Tokyo"}},
2099 }])),
2100 chunk(&json!([{
2101 "text": "signed conclusion",
2102 "thought": true,
2103 "thoughtSignature": "sig-1",
2104 }])),
2105 terminal_frame(),
2106 ];
2107 (frames, "before tool", "get_weather", "signed conclusion")
2108 }
2109 }
2110
2111 pub mod interactions {
2113 use super::*;
2114
2115 fn driver() -> WireDriver {
2116 byte_driver("gemini", |transport| {
2117 crate::driver::Model::new(
2118 crate::providers::gemini::GeminiConfig::new("test-key")
2119 .interactions("gemini-2.5-pro"),
2120 transport,
2121 )
2122 })
2123 }
2124
2125 fn completed(usage: Option<serde_json::Value>) -> WireInput {
2126 let mut interaction = json!({
2127 "id": "int-1",
2128 "model": "gemini-2.5-pro",
2129 "status": "completed",
2130 });
2131 if let (Some(usage), Some(object)) = (usage, interaction.as_object_mut()) {
2132 object.insert("usage".to_string(), usage);
2133 }
2134 sse(&json!({
2135 "event_type": "interaction.completed",
2136 "interaction": interaction,
2137 }))
2138 }
2139
2140 pub fn fixture() -> ProviderWireFixture {
2142 ProviderWireFixture {
2143 driver: driver(),
2144 text_frames: vec![sse(&json!({
2145 "event_type": "step.delta",
2146 "index": 0,
2147 "delta": {"type": "text", "text": "hi"},
2148 }))],
2149 expected_texts: vec!["hi"],
2150 tool_call_frames: vec![sse(&json!({
2151 "event_type": "step.delta",
2152 "index": 0,
2153 "delta": {
2154 "type": "function_call",
2155 "name": "get_weather",
2156 "arguments": {"city": "Tokyo"},
2157 "id": "call-1",
2158 },
2159 }))],
2160 expected_tool_name: "get_weather",
2161 partial_tool_call_frames: None,
2164 terminal_frames: vec![completed(Some(json!({
2165 "total_input_tokens": 5,
2166 "total_output_tokens": 2,
2167 "total_tokens": 7,
2168 })))],
2169 expected_usage_total: 7,
2170 expected_finish_reason: Some(FinishReason::Stop),
2171 zero_usage_terminal_frames: Some(vec![completed(None)]),
2172 bare_terminal_frames: None,
2173 malformed_frame: Some(sse_raw("{not json")),
2174 unknown_event_frame: Some(sse(&json!({
2175 "event_type": "future.event",
2176 "index": 0,
2177 }))),
2178 defective_known_frame: Some(sse_raw(
2181 r#"{"event_type":"step.delta","index":0,"delta":42}"#,
2182 )),
2183 delta_less_prelude_frame: None,
2184 refusal: None,
2185 interleaved_reasoning: Some(interleaved_thought_fixture()),
2186 }
2187 }
2188
2189 fn interleaved_thought_fixture() -> InterleavedReasoningFixture {
2192 let frames = vec![
2193 sse(&json!({
2194 "event_type": "step.delta",
2195 "index": 0,
2196 "delta": {
2197 "type": "thought_summary",
2198 "content": {"type": "text", "text": "before tool"},
2199 },
2200 })),
2201 sse(&json!({
2202 "event_type": "step.delta",
2203 "index": 1,
2204 "delta": {
2205 "type": "function_call",
2206 "name": "get_weather",
2207 "arguments": {"city": "Tokyo"},
2208 "id": "call-1",
2209 },
2210 })),
2211 sse(&json!({
2212 "event_type": "step.delta",
2213 "index": 2,
2214 "delta": {
2215 "type": "thought_summary",
2216 "content": {"type": "text", "text": "after tool"},
2217 },
2218 })),
2219 completed(Some(json!({
2220 "total_input_tokens": 5,
2221 "total_output_tokens": 2,
2222 "total_tokens": 7,
2223 }))),
2224 ];
2225 InterleavedReasoningFixture {
2226 frames,
2227 first_reasoning: "before tool",
2228 tool_name: "get_weather",
2229 second_reasoning: "after tool",
2230 }
2231 }
2232 }
2233
2234 pub mod anthropic {
2236 use super::*;
2237
2238 fn driver() -> WireDriver {
2239 byte_driver("anthropic", |transport| {
2240 crate::driver::Model::new(
2241 crate::providers::anthropic::wire::AnthropicConfig::new("test-key")
2242 .completion(crate::providers::anthropic::completion::CLAUDE_SONNET_4_6),
2243 transport,
2244 )
2245 })
2246 }
2247
2248 fn message_start() -> WireInput {
2249 sse(&json!({
2250 "type": "message_start",
2251 "message": {
2252 "id": "msg_1",
2253 "role": "assistant",
2254 "content": [],
2255 "model": "claude-sonnet-4-6",
2256 "stop_reason": null,
2257 "stop_sequence": null,
2258 "usage": {"input_tokens": 5, "output_tokens": 0},
2259 },
2260 }))
2261 }
2262
2263 pub fn fixture() -> ProviderWireFixture {
2265 ProviderWireFixture {
2266 driver: driver(),
2267 text_frames: vec![
2268 message_start(),
2269 sse(&json!({
2270 "type": "content_block_start",
2271 "index": 0,
2272 "content_block": {"type": "text", "text": ""},
2273 })),
2274 sse(&json!({
2275 "type": "content_block_delta",
2276 "index": 0,
2277 "delta": {"type": "text_delta", "text": "hi"},
2278 })),
2279 ],
2280 expected_texts: vec!["hi"],
2281 tool_call_frames: vec![
2282 sse(&json!({
2283 "type": "content_block_start",
2284 "index": 0,
2285 "content_block": {
2286 "type": "tool_use",
2287 "id": "toolu_1",
2288 "name": "get_weather",
2289 "input": {},
2290 },
2291 })),
2292 sse(&json!({
2293 "type": "content_block_delta",
2294 "index": 0,
2295 "delta": {"type": "input_json_delta", "partial_json": "{\"city\":\"Tokyo\"}"},
2296 })),
2297 sse(&json!({"type": "content_block_stop", "index": 0})),
2300 ],
2301 expected_tool_name: "get_weather",
2302 partial_tool_call_frames: Some(vec![
2303 sse(&json!({
2304 "type": "content_block_start",
2305 "index": 0,
2306 "content_block": {
2307 "type": "tool_use",
2308 "id": "toolu_1",
2309 "name": "get_weather",
2310 "input": {},
2311 },
2312 })),
2313 sse(&json!({
2314 "type": "content_block_delta",
2315 "index": 0,
2316 "delta": {"type": "input_json_delta", "partial_json": "{\"cit"},
2317 })),
2318 ]),
2319 terminal_frames: vec![sse(&json!({
2320 "type": "message_delta",
2321 "delta": {"stop_reason": "end_turn", "stop_sequence": null},
2322 "usage": {"output_tokens": 4},
2323 }))],
2324 expected_usage_total: 9,
2326 expected_finish_reason: Some(FinishReason::Stop),
2327 zero_usage_terminal_frames: None,
2330 bare_terminal_frames: Some(vec![sse(&json!({"type": "message_stop"}))]),
2333 malformed_frame: Some(sse_raw("{not json")),
2334 unknown_event_frame: Some(sse(&json!({
2335 "type": "content_block_heartbeat",
2336 "index": 0,
2337 }))),
2338 defective_known_frame: Some(sse_raw(
2341 r#"{"type":"content_block_delta","index":0,"delta":42}"#,
2342 )),
2343 delta_less_prelude_frame: None,
2344 refusal: None,
2345 interleaved_reasoning: None,
2346 }
2347 }
2348 }
2349}