use bytes::Bytes;
use futures::StreamExt;
use serde_json::json;
use super::{Model, Transport};
use crate::completion::CompletionRequest;
use crate::error::{EncodeError, ProviderError};
use crate::http_client::framing::Framing;
use crate::message::{CallId, ToolName};
use crate::operation::{Block, Completion, Finish};
use crate::streaming::Streamed;
use crate::test_utils::{
MockHttpResponse, MockStreamingClient, NonSuccessStreamingClient, PacedStreamingClient,
RecordingHttpClient, SequencedHttpClient, SilentStreamingClient, TraceCapture,
};
use crate::wire::{
Body, Decoder, Descriptor, Encoded, Flow, Mode, Out, Wire, WireEvent, WireFrame,
};
pub(crate) async fn call<W, H>(
wire: &W,
http: &H,
request: crate::wire::Request<W>,
) -> Result<crate::wire::Response<W>, ProviderError>
where
W: Wire,
H: Transport<W>,
{
Model::new(wire.clone(), http.clone()).call(request).await
}
pub(crate) fn stream<W, H>(
wire: &W,
http: &H,
request: crate::wire::Request<W>,
) -> Result<Streamed<W::Op>, ProviderError>
where
W: Wire,
H: Transport<W>,
{
Model::new(wire.clone(), http.clone()).stream(request)
}
pub(crate) async fn reply_usage<W, T>(
wire: W,
http: T,
mode: crate::wire::Mode,
) -> crate::completion::Usage
where
W: crate::wire::Wire<Op = crate::operation::Completion>,
T: Transport<W>,
{
let model = Model::new(wire, http);
let request = crate::completion::CompletionRequest::new("hi");
let response = match mode {
crate::wire::Mode::Unary => model.call(request).await,
crate::wire::Mode::Streaming => match model.stream(request) {
Ok(stream) => stream.finish().await,
Err(error) => Err(error),
},
}
.unwrap_or_else(|error| panic!("the reply folds: {error}"));
response.usage
}
#[derive(Clone, Debug, PartialEq)]
struct Echo {
framing: Framing,
request_id_header: Option<&'static str>,
relaxed_content_type: bool,
}
impl Echo {
fn unary() -> Self {
Self {
framing: Framing::Whole,
request_id_header: Some("request-id"),
relaxed_content_type: false,
}
}
fn streaming() -> Self {
Self {
framing: Framing::Sse,
request_id_header: Some("request-id"),
relaxed_content_type: false,
}
}
fn sse_unary() -> Self {
Self::streaming()
}
fn relaxed(mut self) -> Self {
self.relaxed_content_type = true;
self
}
}
#[derive(serde::Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
enum Frame {
Message { text: String, usage: Usage },
Delta { text: String },
Stop { usage: Usage },
Tool {
name: String,
arguments: String,
usage: Option<Usage>,
},
Error { message: String },
}
#[derive(Clone, Copy, serde::Deserialize)]
struct Usage {
output_tokens: u64,
}
#[derive(Default)]
struct EchoDecoder<'id> {
brand: std::marker::PhantomData<fn(&'id ()) -> &'id ()>,
}
impl<'id> Decoder<'id, Completion> for EchoDecoder<'id> {
type Event = Frame;
fn classify(&self, frame: WireFrame) -> WireEvent<Self::Event> {
crate::providers::internal::wire::classify_tagged_frame(&frame.as_str(), "type", |kind| {
matches!(kind, "message" | "delta" | "stop" | "tool" | "error")
})
}
fn decode(
&mut self,
event: Frame,
mut out: Out<'id, Completion>,
) -> Result<Flow, ProviderError> {
match event {
Frame::Message { text, usage } => {
out.run(Block::Text, &text)?;
return self.end(out, usage);
}
Frame::Delta { text } => {
out.run(Block::Text, &text)?;
}
Frame::Stop { usage } => return self.end(out, usage),
Frame::Error { message } => return Err(ProviderError::Provider(message)),
Frame::Tool {
name,
arguments,
usage,
} => {
out.end_run()?;
let name = ToolName::new(name)
.map_err(|error| ProviderError::Response(error.to_string()))?;
let index = out.fresh_index();
out.whole(
index,
Block::Call {
id: CallId::from_wire("call_1"),
name,
},
serde_json::Value::Null,
&arguments,
)?;
if let Some(usage) = usage {
return self.end(out, usage);
}
}
}
Ok(Flow::More)
}
}
impl<'id> EchoDecoder<'id> {
fn end(&mut self, mut out: Out<'id, Completion>, usage: Usage) -> Result<Flow, ProviderError> {
out.end_run()?;
Ok(out.end(Finish {
usage: crate::completion::Usage {
output_tokens: Some(usage.output_tokens),
..crate::completion::Usage::default()
},
model: Some("echo-1".to_owned()),
..Finish::default()
}))
}
}
impl crate::completion::ReplayTarget for Echo {
fn map_options(
&self,
_request: &crate::completion::CompletionRequest,
fields: crate::completion::options::OptionFields<'_>,
) -> crate::completion::options::OptionMap {
crate::test_utils::refuse_options(fields)
}
fn api(&self) -> crate::message::Api {
crate::message::Api::from_static("echo.chat")
}
fn provider(&self) -> &str {
"echo"
}
fn model(&self) -> &str {
"echo-1"
}
fn accepts(&self, _model: &str) -> crate::completion::Accepts {
crate::completion::Accepts::ALL
}
}
#[derive(Default)]
struct EchoDocument(Option<serde_json::Map<String, serde_json::Value>>);
impl crate::wire::document::Serves<Completion> for EchoDocument {}
impl crate::wire::document::Reassemble<Frame> for EchoDocument {
fn absorb(&mut self, frame: &Frame) {
let document = self.0.get_or_insert_with(|| {
serde_json::Map::from_iter([
("type".to_owned(), json!("message")),
("text".to_owned(), json!("")),
])
});
let usage = |usage: &Usage| json!({"output_tokens": usage.output_tokens});
match frame {
Frame::Message { text, usage: used } => {
*document = serde_json::Map::from_iter([
("type".to_owned(), json!("message")),
("text".to_owned(), json!(text)),
("usage".to_owned(), usage(used)),
]);
}
Frame::Delta { text } => {
if let Some(serde_json::Value::String(held)) = document.get_mut("text") {
held.push_str(text);
}
}
Frame::Stop { usage: used } => {
document.insert("usage".to_owned(), usage(used));
}
Frame::Tool {
name,
arguments,
usage: used,
} => {
document.insert("name".to_owned(), json!(name));
document.insert("arguments".to_owned(), json!(arguments));
if let Some(used) = used {
document.insert("usage".to_owned(), usage(used));
}
}
Frame::Error { .. } => {}
}
}
fn finish(self) -> serde_json::Value {
self.0
.map_or(serde_json::Value::Null, serde_json::Value::Object)
}
}
impl Wire for Echo {
type Op = Completion;
type Payload = Encoded;
type Frame = WireFrame;
type Event = Frame;
type Decoder<'id> = EchoDecoder<'id>;
type Reassembler = EchoDocument;
fn describe(&self) -> Descriptor<'_> {
Descriptor::new("echo").model("echo-1").replay(self)
}
fn encode(&self, request: CompletionRequest, _mode: Mode) -> Result<Encoded, EncodeError> {
let body = serde_json::to_vec(&serde_json::json!({
"messages": request.chat_history.len(),
}))?;
let request =
http::Request::post("https://echo.invalid/v1/messages").body(Body::Bytes(body))?;
let encoded =
Encoded::new(request, self.framing).with_request_id_header(self.request_id_header);
Ok(if self.relaxed_content_type {
encoded.with_relaxed_content_type()
} else {
encoded
})
}
fn decoder<'id>(&self) -> Self::Decoder<'id> {
EchoDecoder::default()
}
}
fn prompt() -> CompletionRequest {
CompletionRequest::new("say hi")
}
const UNARY_BODY: &str = r#"{"type":"message","text":"hi there","usage":{"output_tokens":3}}"#;
#[tokio::test]
async fn a_unary_reply_that_is_not_the_event_stream_it_asked_for_fails_the_call() {
let http = SequencedHttpClient::new([MockHttpResponse::success_typed(
r#"{"error":"this endpoint speaks JSON"}"#,
"application/json",
)]);
let error = call(&Echo::sse_unary(), &http, prompt())
.await
.expect_err("a JSON reply to an SSE wire is the wrong endpoint");
assert!(
error.to_string().contains("content type"),
"the error names what was wrong: {error}"
);
}
#[tokio::test]
async fn a_relaxed_wire_still_reads_a_unary_reply_that_names_no_content_type() {
let http =
SequencedHttpClient::new([MockHttpResponse::success(format!("data: {UNARY_BODY}\n\n"))]);
let response = call(&Echo::sse_unary().relaxed(), &http, prompt())
.await
.expect("the reply decodes");
assert_eq!(text_of(&response), Some("hi there"));
}
#[tokio::test]
async fn a_non_success_streaming_response_is_rejected_as_the_streams_only_item() {
let mut headers = http::HeaderMap::new();
headers.insert("retry-after", http::HeaderValue::from_static("13"));
headers.insert("request-id", http::HeaderValue::from_static("req_3"));
let http = NonSuccessStreamingClient {
status: http::StatusCode::SERVICE_UNAVAILABLE,
headers,
body: Bytes::from_static(b"{\"error\":\"down\"}"),
};
let frames = stream(&Echo::streaming(), &http, prompt()).expect("the stream opens");
let items: Vec<_> = frames.collect().await;
assert_eq!(items.len(), 1, "nothing follows the rejection: {items:?}");
let error = items
.into_iter()
.next()
.expect("one item")
.expect_err("the item is the rejected reply");
assert_eq!(
error.provider_response_status(),
Some(http::StatusCode::SERVICE_UNAVAILABLE)
);
assert_eq!(error.provider_request_id(), Some("req_3"));
assert_eq!(
error
.provider_response_headers()
.and_then(|headers| headers.get("retry-after"))
.and_then(|value| value.to_str().ok()),
Some("13")
);
assert!(
error
.provider_response_body()
.is_some_and(|body| body.contains("down")),
"the reply body is the error: {error:?}"
);
}
#[derive(Clone)]
struct Broken(Vec<&'static str>);
impl Transport<Echo> for Broken {
fn send(&self, _payload: Encoded, _exchange: super::Exchange) -> super::Opening<WireFrame> {
let frames = self
.0
.iter()
.map(|frame| Ok(WireFrame::Text((*frame).to_owned())))
.chain([Err(ProviderError::Response(
"the connection dropped".to_owned(),
))])
.collect::<Vec<_>>();
super::Opening::ready(super::Opened::new(futures::stream::iter(frames)))
}
}
#[derive(Clone)]
struct LateOpening {
opens_after: std::time::Duration,
gap: std::time::Duration,
frame: &'static str,
}
impl Transport<Echo> for LateOpening {
fn send(&self, _payload: Encoded, _exchange: super::Exchange) -> super::Opening<WireFrame> {
let (opens_after, gap, frame) = (self.opens_after, self.gap, self.frame);
super::Opening::new(async move {
tokio::time::sleep(opens_after).await;
let frames = futures::stream::once(async move {
tokio::time::sleep(gap).await;
Ok(WireFrame::Text(frame.to_owned()))
})
.chain(futures::stream::pending());
Ok(super::Opened::new(frames))
})
}
}
#[derive(Clone)]
struct Paced {
frames: Vec<&'static str>,
gap: std::time::Duration,
}
impl Transport<Echo> for Paced {
fn send(&self, _payload: Encoded, _exchange: super::Exchange) -> super::Opening<WireFrame> {
let gap = self.gap;
let frames = futures::stream::iter(self.frames.clone())
.then(move |frame| async move {
tokio::time::sleep(gap).await;
Ok(WireFrame::Text(frame.to_owned()))
})
.chain(futures::stream::pending());
super::Opening::ready(super::Opened::new(frames))
}
}
#[tokio::test]
async fn a_streamed_reply_that_falls_silent_stalls() {
let idle = std::time::Duration::from_millis(200);
let delta = r#"{"type":"delta","text":"hi"}"#;
let stop = r#"{"type":"stop","usage":{"output_tokens":1}}"#;
let steady = Paced {
frames: vec![delta, delta, delta, delta, stop],
gap: idle / 4,
};
let response = Model::new(Echo::streaming(), steady)
.with_idle_timeout(Some(idle))
.stream(prompt())
.expect("the stream opens")
.finish()
.await;
assert!(response.is_ok(), "a steady reply ends: {response:?}");
let silent = Paced {
frames: vec![delta],
gap: idle / 4,
};
let stalled = Model::new(Echo::streaming(), silent)
.with_idle_timeout(Some(idle))
.stream(prompt())
.expect("the stream opens")
.finish()
.await;
match stalled {
Err(error @ ProviderError::Stalled(after)) => {
assert_eq!(after, idle);
assert!(error.is_retryable(), "a stall is retried");
}
other => panic!("a silent reply stalls: {other:?}"),
}
}
#[tokio::test]
async fn comments_alone_stall_a_streamed_reply() {
let idle = std::time::Duration::from_millis(400);
let pinging = PacedStreamingClient {
chunks: vec![": keep-alive\n\n"; 64],
gap: idle / 8,
};
let started = std::time::Instant::now();
let stalled = Model::new(Echo::streaming(), pinging)
.with_idle_timeout(Some(idle))
.stream(prompt())
.expect("the stream opens")
.finish()
.await;
let took = started.elapsed();
assert!(
matches!(stalled, Err(ProviderError::Stalled(after)) if after == idle),
"a reply of comments stalls: {stalled:?}"
);
assert!(took < idle * 2, "stalled {took:?} after it opened");
}
#[tokio::test]
async fn frames_within_the_idle_timeout_never_stall() {
let idle = std::time::Duration::from_millis(200);
let mut chunks = Vec::new();
for _ in 0..6 {
chunks.push(": keep-alive\n\n");
chunks.push("data: {\"type\":\"delta\",\"text\":\"hi\"}\n\n");
}
chunks.push("data: {\"type\":\"stop\",\"usage\":{\"output_tokens\":1}}\n\n");
let trickle = PacedStreamingClient {
chunks,
gap: idle / 6,
};
let response = Model::new(Echo::streaming(), trickle)
.with_idle_timeout(Some(idle))
.stream(prompt())
.expect("the stream opens")
.finish()
.await;
assert!(response.is_ok(), "a trickling reply ends: {response:?}");
}
#[tokio::test]
async fn opening_and_the_first_frame_share_the_idle_timeout() {
let idle = std::time::Duration::from_millis(400);
let late = LateOpening {
opens_after: idle * 3 / 4,
gap: idle * 3 / 4,
frame: r#"{"type":"delta","text":"hi"}"#,
};
let mut stream = Model::new(Echo::streaming(), late)
.with_idle_timeout(Some(idle))
.stream(prompt())
.expect("the stream opens");
let first = stream.next().await;
assert!(
matches!(first, Some(Err(ProviderError::Stalled(after))) if after == idle),
"a late first frame stalls: {first:?}"
);
}
#[tokio::test]
async fn a_slow_reader_stalls_an_idle_timeout_after_the_last_frame() {
let idle = std::time::Duration::from_millis(400);
let once = PacedStreamingClient {
chunks: vec!["data: {\"type\":\"delta\",\"text\":\"hi\"}\n\n"],
gap: std::time::Duration::ZERO,
};
let mut stream = Model::new(Echo::streaming(), once)
.with_idle_timeout(Some(idle))
.stream(prompt())
.expect("the stream opens");
let started = std::time::Instant::now();
assert!(
matches!(stream.next().await, Some(Ok(_))),
"the delta comes"
);
tokio::time::sleep(idle * 3 / 4).await;
let mut last = None;
while let Some(item) = stream.next().await {
last = Some(item);
}
let took = started.elapsed();
assert!(
matches!(last, Some(Err(ProviderError::Stalled(_)))),
"a silent reply stalls: {last:?}"
);
assert!(took < idle + idle / 2, "stalled {took:?} after the frame");
}
#[tokio::test]
async fn an_idle_timeout_past_the_timer_waits_indefinitely() {
let delta = r#"{"type":"delta","text":"hi"}"#;
let stop = r#"{"type":"stop","usage":{"output_tokens":1}}"#;
let steady = Paced {
frames: vec![delta, stop],
gap: std::time::Duration::from_millis(10),
};
let response = Model::new(Echo::streaming(), steady)
.with_idle_timeout(Some(std::time::Duration::MAX))
.stream(prompt())
.expect("the stream opens")
.finish()
.await;
assert!(response.is_ok(), "the reply ends: {response:?}");
}
#[tokio::test]
async fn a_reply_that_never_opens_stalls() {
let idle = std::time::Duration::from_millis(100);
let stalled = match Model::new(Echo::streaming(), SilentStreamingClient)
.with_idle_timeout(Some(idle))
.stream(prompt())
{
Ok(stream) => stream.finish().await,
Err(error) => Err(error),
};
assert!(
matches!(stalled, Err(ProviderError::Stalled(after)) if after == idle),
"a reply that never opens stalls: {stalled:?}"
);
}
#[tokio::test]
async fn a_streamed_reply_records_its_document_whole_cut_or_failed() {
let delta = r#"{"type":"delta","text":"hi "}"#;
let whole = MockStreamingClient {
sse_bytes: Bytes::from(format!(
"data: {delta}\n\ndata: {{\"type\":\"delta\",\"text\":\"there\"}}\n\n\
data: {{\"type\":\"stop\",\"usage\":{{\"output_tokens\":3}}}}\n\n"
)),
};
let response = Model::new(Echo::streaming(), whole)
.stream(prompt())
.expect("the stream opens")
.finish()
.await
.expect("the reply ends");
assert_eq!(
response.raw,
serde_json::from_str::<serde_json::Value>(UNARY_BODY).expect("the unary body is JSON"),
"the stream rebuilds the unary document"
);
let so_far = json!({"type": "message", "text": "hi "});
let cut = MockStreamingClient {
sse_bytes: Bytes::from(format!("data: {delta}\n\n")),
};
let mut stream = Model::new(Echo::streaming(), cut)
.stream(prompt())
.expect("the stream opens");
while stream.next().await.is_some() {}
assert_eq!(
stream.partial().raw,
so_far,
"a cut stream keeps the document so far"
);
let mut stream = Model::new(Echo::streaming(), Broken(vec![delta]))
.stream(prompt())
.expect("the stream opens");
let items: Vec<_> = (&mut stream).collect().await;
assert!(
items.last().is_some_and(Result::is_err),
"the transport failure ends the stream: {items:?}"
);
assert_eq!(
stream.partial().raw,
so_far,
"a failed stream keeps the document so far"
);
}
#[tokio::test]
async fn a_stream_the_caller_stopped_polling_records_no_raw() {
let http = MockStreamingClient {
sse_bytes: Bytes::from_static(
b"data: {\"type\":\"delta\",\"text\":\"hi \"}\n\n\
data: {\"type\":\"delta\",\"text\":\"there\"}\n\n",
),
};
let mut stream = Model::new(Echo::streaming(), http)
.stream(prompt())
.expect("the stream opens");
let _first = stream.next().await;
assert!(stream.partial().raw.is_null());
}
#[tokio::test]
async fn a_malformed_tool_input_is_kept_with_its_raw_text() {
let http = RecordingHttpClient::new(
r#"{"type":"tool","name":"add","arguments":"{not json","usage":{"output_tokens":1}}"#,
);
let response = call(&Echo::unary(), &http, prompt())
.await
.expect("a malformed call does not fail the reply");
let call_of = |response: &crate::completion::CompletionResponse| {
response
.tool_calls()
.map(|call| {
(
call.function.arguments_value(),
call.function.invalid_arguments.clone(),
)
})
.collect::<Vec<_>>()
};
assert_eq!(
call_of(&response),
[(serde_json::json!({}), Some("{not json".to_owned()))]
);
let http = MockStreamingClient {
sse_bytes: Bytes::from_static(
b"data: {\"type\":\"tool\",\"name\":\"add\",\"arguments\":\"{not json\"}\n\n\
data: {\"type\":\"stop\",\"usage\":{\"output_tokens\":1}}\n\n",
),
};
let streamed = Model::new(Echo::streaming(), http)
.stream(prompt())
.expect("the stream opens")
.finish()
.await
.expect("a malformed call does not fail the stream");
assert_eq!(call_of(&streamed), call_of(&response));
}
#[tokio::test]
async fn a_reply_reports_its_finish_usage() {
let streamed = MockStreamingClient {
sse_bytes: Bytes::from_static(
b"data: {\"type\":\"delta\",\"text\":\"hi\"}\n\n\
data: {\"type\":\"stop\",\"usage\":{\"output_tokens\":3}}\n\n",
),
};
for usage in [
reply_usage(
Echo::unary(),
RecordingHttpClient::new(UNARY_BODY),
Mode::Unary,
)
.await,
reply_usage(Echo::streaming(), streamed, Mode::Streaming).await,
] {
assert_eq!(usage.output_tokens, Some(3), "{usage:?}");
}
}
#[tokio::test]
async fn the_driver_records_the_folded_responses_metadata() {
use tracing::subscriber::with_default;
let capture = TraceCapture::default();
let bound = Model::new(Echo::unary(), RecordingHttpClient::new(UNARY_BODY));
let response = with_default(capture.subscriber(), || {
futures::executor::block_on(bound.call(prompt()))
})
.expect("the reply decodes");
assert_eq!(response.model(), Some("echo-1"));
assert!(
capture
.values_of("gen_ai.response.model")
.contains(&json!("echo-1")),
"expected the response model on the span"
);
assert!(
!capture.values_of("gen_ai.usage.output_tokens").is_empty(),
"expected usage on the span"
);
}
#[test]
fn a_streamed_embedding_records_its_response_on_the_span() {
use tracing::subscriber::with_default;
const REPLY: &str = r#"{"object":"list","model":"text-embedding-3-small","data":[{"object":"embedding","index":0,"embedding":[0.1]}],"usage":{"prompt_tokens":2,"total_tokens":2}}"#;
let capture = TraceCapture::default();
let wire = crate::providers::openai::wire::OpenAIConfig::new("sk-test")
.embedding("text-embedding-3-small", None);
let model = Model::new(
wire,
SequencedHttpClient::new([MockHttpResponse::success(REPLY)]),
);
with_default(capture.subscriber(), || {
let stream = model
.stream(vec!["a".to_owned()])
.expect("the embedding opens");
futures::executor::block_on(stream.finish()).expect("the reply folds")
});
assert!(
capture
.values_of("gen_ai.response.model")
.contains(&json!("text-embedding-3-small")),
"expected the response model on the span"
);
}
fn text_of(response: &crate::completion::CompletionResponse) -> Option<&str> {
response.choice.first().and_then(|block| match block {
crate::message::AssistantContent::Text(text) => Some(text.text.as_str()),
_ => None,
})
}
#[test]
fn a_stream_the_driver_cannot_send_is_a_request_failure() {
#[derive(Clone, Debug)]
struct Multipart;
impl crate::completion::ReplayTarget for Multipart {
fn map_options(
&self,
_request: &crate::completion::CompletionRequest,
fields: crate::completion::options::OptionFields<'_>,
) -> crate::completion::options::OptionMap {
crate::test_utils::refuse_options(fields)
}
fn api(&self) -> crate::message::Api {
crate::message::Api::from_static("echo.chat")
}
fn provider(&self) -> &str {
"echo"
}
fn model(&self) -> &str {
""
}
fn accepts(&self, _model: &str) -> crate::completion::Accepts {
crate::completion::Accepts::ALL
}
}
impl Wire for Multipart {
type Op = Completion;
type Payload = Encoded;
type Frame = WireFrame;
type Event = Frame;
type Decoder<'id> = EchoDecoder<'id>;
type Reassembler = EchoDocument;
fn describe(&self) -> Descriptor<'_> {
Descriptor::new("echo").replay(self)
}
fn encode(&self, _request: CompletionRequest, _mode: Mode) -> Result<Encoded, EncodeError> {
let request = http::Request::post("https://echo.invalid/a")
.body(Body::Multipart(crate::http_client::MultipartForm::new()))?;
Ok(Encoded::new(request, Framing::Sse))
}
fn decoder<'id>(&self) -> Self::Decoder<'id> {
EchoDecoder::default()
}
}
let http = crate::test_utils::RecordingHttpClient::new("");
let items: Vec<_> = futures::executor::block_on(
stream(&Multipart, &http, prompt())
.expect("the request encodes")
.collect::<Vec<_>>(),
);
let [Err(error)] = items.as_slice() else {
panic!("the failure is the stream's only item: {items:?}");
};
assert_eq!(
error.to_string(),
"RequestError: a multipart request cannot open a streamed reply"
);
assert_eq!(error.kind(), crate::error::ErrorKind::Request);
assert!(!error.is_retryable());
assert!(http.requests().is_empty(), "nothing was sent");
}
async fn calls_ended_before_the_end<W>(
wire: W,
chunks: Vec<&'static str>,
) -> (Vec<CallId>, Vec<CallId>)
where
W: Wire<Op = Completion, Payload = crate::wire::Encoded, Frame = WireFrame> + 'static,
{
let paced = PacedStreamingClient {
chunks,
gap: std::time::Duration::from_millis(5),
};
let mut stream = Model::new(wire, paced)
.stream(prompt())
.expect("the stream opens");
let mut early = Vec::new();
while let Some(item) = stream.next().await {
if let Ok(crate::streaming::Item::Event(crate::streaming::StreamEvent::End {
content: crate::message::AssistantContent::ToolCall(call),
..
})) = item
&& stream.partial().finish_reason().is_none()
{
early.push(call.id);
}
}
let unfinished = stream
.partial()
.unfinished_calls()
.map(|call| call.id.clone())
.collect();
(early, unfinished)
}
#[tokio::test]
async fn an_unfinished_calls_end_comes_with_the_replys_end() {
use crate::providers::anthropic::AnthropicConfig;
use crate::providers::openai::wire::{OPENAI, OpenAIConfig};
let responses = vec![
"event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_1\",\"object\":\"response\",\"status\":\"in_progress\",\"model\":\"gpt-5.4\",\"output\":[]}}\n\n",
"event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"function_call\",\"id\":\"fc_1\",\"call_id\":\"call_1\",\"name\":\"add\",\"arguments\":\"\",\"status\":\"in_progress\"}}\n\n",
"event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"output_index\":0,\"item_id\":\"fc_1\",\"delta\":\"{\\\"x\\\":481\"}\n\n",
"event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"function_call\",\"id\":\"fc_1\",\"call_id\":\"call_1\",\"name\":\"add\",\"arguments\":\"{\\\"x\\\":481\",\"status\":\"incomplete\"}}\n\n",
"event: response.incomplete\ndata: {\"type\":\"response.incomplete\",\"response\":{\"id\":\"resp_1\",\"object\":\"response\",\"status\":\"incomplete\",\"model\":\"gpt-5.4\",\"incomplete_details\":{\"reason\":\"max_output_tokens\"},\"output\":[{\"type\":\"function_call\",\"id\":\"fc_1\",\"call_id\":\"call_1\",\"name\":\"add\",\"arguments\":\"{\\\"x\\\":481\",\"status\":\"incomplete\"}]}}\n\n",
];
let anthropic = vec![
"event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-6\",\"content\":[],\"stop_reason\":null,\"usage\":{\"input_tokens\":3,\"output_tokens\":1}}}\n\n",
"event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"add\",\"input\":{}}}\n\n",
"event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"x\\\":481\"}}\n\n",
"event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"max_tokens\"},\"usage\":{\"output_tokens\":16}}\n\n",
"event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",
];
let chat = vec![
"data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"add\",\"arguments\":\"{\\\"x\\\":481\"}}]},\"finish_reason\":null}]}\n\n",
"data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"length\"}]}\n\n",
"data: [DONE]\n\n",
];
let cases = [
(
"responses",
calls_ended_before_the_end(
OpenAIConfig::with_key(&OPENAI, "key").responses("gpt-5.4"),
responses,
)
.await,
),
(
"anthropic",
calls_ended_before_the_end(
AnthropicConfig::new("key").completion("claude-sonnet-4-6"),
anthropic,
)
.await,
),
(
"chat",
calls_ended_before_the_end(OpenAIConfig::with_key(&OPENAI, "key").chat("m"), chat)
.await,
),
];
for (wire, (early, unfinished)) in cases {
assert_eq!(unfinished.len(), 1, "{wire}: the cut call is unfinished");
let early_unfinished: Vec<&CallId> =
early.iter().filter(|id| unfinished.contains(id)).collect();
assert!(
early_unfinished.is_empty(),
"{wire}: an unfinished call ended before the reply did: {early_unfinished:?}"
);
}
}
#[tokio::test]
async fn a_streamed_reply_with_another_2xx_status_is_read() {
for status in [http::StatusCode::OK, http::StatusCode::ACCEPTED] {
let http = NonSuccessStreamingClient {
status,
headers: http::HeaderMap::new(),
body: Bytes::from_static(
b"data: {\"type\":\"delta\",\"text\":\"hi\"}\n\ndata: {\"type\":\"stop\",\"usage\":{\"output_tokens\":1}}\n\n",
),
};
let response = stream(&Echo::streaming().relaxed(), &http, prompt())
.expect("the stream opens")
.finish()
.await;
assert_eq!(
response.as_ref().ok().and_then(text_of),
Some("hi"),
"status {status}: {response:?}"
);
}
}