use crate::interceptor::{StreamChunkContext, StreamEndContext, StreamTerminalContext};
use crate::{Error, Result};
use bytes::Bytes;
use futures::future::FutureExt;
use futures::stream::Stream;
use futures::StreamExt;
use openai_client_base::models::{
ChatCompletionStreamResponseDelta, CreateChatCompletionStreamResponse,
};
use std::any::Any;
use std::collections::VecDeque;
use std::future::Future;
use std::panic::AssertUnwindSafe;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::{Duration, Instant};
use tokio::runtime::Handle;
use tokio::sync::mpsc::{self, OwnedPermit};
use tokio::sync::watch;
const STREAM_CALLBACK_QUEUE_CAPACITY: usize = 1;
const MAX_RETAINED_CALLBACK_FAILURES: usize = 64;
const MAX_CALLBACK_DIAGNOSTIC_CHARS: usize = 512;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StreamFailureKind {
Connection,
Parsing,
Buffer,
Other,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct StreamFailure {
pub kind: StreamFailureKind,
}
impl StreamFailure {
fn from_error(error: &Error) -> Self {
let kind = match error {
Error::StreamConnection { .. } | Error::Http(_) => StreamFailureKind::Connection,
Error::StreamParsing { .. } | Error::Json(_) => StreamFailureKind::Parsing,
Error::StreamBuffer { .. } => StreamFailureKind::Buffer,
_ => StreamFailureKind::Other,
};
Self { kind }
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StreamOutcome {
Completed,
StreamFailed(StreamFailure),
Cancelled,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StreamCallbackHook {
Chunk,
Terminal,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StreamCallbackFailureKind {
ReturnedError,
Panicked,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct StreamCallbackFailure {
pub hook: StreamCallbackHook,
pub interceptor_index: usize,
pub chunk_index: Option<usize>,
pub kind: StreamCallbackFailureKind,
pub diagnostic: String,
pub diagnostic_truncated: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct StreamCompletionReport {
pub outcome: StreamOutcome,
pub total_chunks: usize,
pub duration: Duration,
pub input_tokens: Option<i64>,
pub output_tokens: Option<i64>,
pub callback_attempts: usize,
pub callback_successes: usize,
pub callback_failure_count: usize,
pub callback_failures: Vec<StreamCallbackFailure>,
pub truncated_failure_details: usize,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
#[error("stream callback dispatcher ended before producing a completion report")]
pub struct CompletionUnavailable;
#[derive(Clone, Debug)]
pub struct StreamCompletion {
report: watch::Receiver<Option<Arc<StreamCompletionReport>>>,
}
impl StreamCompletion {
pub async fn wait(
&self,
) -> std::result::Result<Arc<StreamCompletionReport>, CompletionUnavailable> {
let mut report = self.report.clone();
loop {
let current = { report.borrow().clone() };
if let Some(value) = current {
return Ok(value);
}
if report.changed().await.is_err() {
return Err(CompletionUnavailable);
}
}
}
}
pub type BoxedChatStream = Pin<Box<dyn Stream<Item = Result<ChatCompletionChunk>> + Send>>;
#[derive(Debug, Clone)]
pub struct ChatCompletionChunk {
response: CreateChatCompletionStreamResponse,
}
impl ChatCompletionChunk {
#[must_use]
pub fn new(response: CreateChatCompletionStreamResponse) -> Self {
Self { response }
}
#[must_use]
pub fn content(&self) -> Option<&str> {
self.response
.choices
.first()
.and_then(|choice| choice.delta.content.as_ref().and_then(|c| c.as_deref()))
}
#[must_use]
pub fn role(&self) -> Option<&str> {
self.response
.choices
.first()
.and_then(|choice| choice.delta.role.as_ref())
.map(|role| match role {
openai_client_base::models::chat_completion_stream_response_delta::Role::System => {
"system"
}
openai_client_base::models::chat_completion_stream_response_delta::Role::User => {
"user"
}
openai_client_base::models::chat_completion_stream_response_delta::Role::Assistant => {
"assistant"
}
openai_client_base::models::chat_completion_stream_response_delta::Role::Tool => {
"tool"
}
openai_client_base::models::chat_completion_stream_response_delta::Role::Developer => {
"developer"
}
})
}
#[must_use]
pub fn tool_calls(
&self,
) -> Option<&Vec<openai_client_base::models::ChatCompletionMessageToolCallChunk>> {
self.response
.choices
.first()
.and_then(|choice| choice.delta.tool_calls.as_ref())
}
#[must_use]
pub fn finish_reason(&self) -> Option<&str> {
self.response.choices.first().and_then(|choice| choice.finish_reason.as_ref()).and_then(Option::as_ref).map(|reason| {
match reason {
openai_client_base::models::create_chat_completion_stream_response_choices_inner::FinishReason::Stop => "stop",
openai_client_base::models::create_chat_completion_stream_response_choices_inner::FinishReason::Length => "length",
openai_client_base::models::create_chat_completion_stream_response_choices_inner::FinishReason::ToolCalls => "tool_calls",
openai_client_base::models::create_chat_completion_stream_response_choices_inner::FinishReason::ContentFilter => "content_filter",
openai_client_base::models::create_chat_completion_stream_response_choices_inner::FinishReason::FunctionCall => "function_call",
}
})
}
#[must_use]
pub fn is_final(&self) -> bool {
self.finish_reason().is_some()
}
#[must_use]
pub fn raw_response(&self) -> &CreateChatCompletionStreamResponse {
&self.response
}
#[must_use]
pub fn delta(&self) -> Option<&ChatCompletionStreamResponseDelta> {
self.response
.choices
.first()
.map(|choice| choice.delta.as_ref())
}
}
pub struct ChatCompletionStream {
inner: Pin<Box<dyn Stream<Item = Result<ChatCompletionChunk>> + Send>>,
}
impl ChatCompletionStream {
pub fn new(response: reqwest::Response) -> Self {
let byte_stream = response.bytes_stream();
let stream = parse_sse_stream(byte_stream);
Self {
inner: Box::pin(stream),
}
}
pub async fn collect_content(mut self) -> Result<String> {
let mut content = String::new();
while let Some(chunk) = self.next().await {
let chunk = chunk?;
if let Some(text) = chunk.content() {
content.push_str(text);
}
}
Ok(content)
}
}
impl Stream for ChatCompletionStream {
type Item = Result<ChatCompletionChunk>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
self.inner.as_mut().poll_next(cx)
}
}
struct ChunkEvent {
chunk_json: String,
chunk_index: usize,
input_tokens: Option<i64>,
output_tokens: Option<i64>,
}
type ReserveFuture = Pin<
Box<
dyn Future<
Output = std::result::Result<OwnedPermit<ChunkEvent>, mpsc::error::SendError<()>>,
> + Send,
>,
>;
pub struct InterceptedStream<T: Send + Sync + 'static = ()> {
inner: Option<Pin<Box<dyn Stream<Item = Result<ChatCompletionChunk>> + Send>>>,
interceptors: Arc<crate::interceptor::InterceptorChain<T>>,
operation: String,
model: String,
request_json: String,
state: Arc<T>,
start_time: Instant,
chunk_index: usize,
input_tokens: Option<i64>,
output_tokens: Option<i64>,
started: bool,
queue_tx: Option<mpsc::Sender<ChunkEvent>>,
reserve_future: Option<ReserveFuture>,
reserved: Option<OwnedPermit<ChunkEvent>>,
terminal_tx: Option<watch::Sender<Option<StreamOutcome>>>,
report_tx: Option<watch::Sender<Option<Arc<StreamCompletionReport>>>>,
terminated: bool,
}
impl<T: Send + Sync + 'static> InterceptedStream<T> {
pub fn new(
inner: ChatCompletionStream,
interceptors: Arc<crate::interceptor::InterceptorChain<T>>,
operation: String,
model: String,
request_json: String,
state: T,
) -> Self {
Self {
inner: Some(Box::pin(inner)),
interceptors,
operation,
model,
request_json,
state: Arc::new(state),
start_time: Instant::now(),
chunk_index: 0,
input_tokens: None,
output_tokens: None,
started: false,
queue_tx: None,
reserve_future: None,
reserved: None,
terminal_tx: None,
report_tx: None,
terminated: false,
}
}
pub fn new_with_completion(
inner: ChatCompletionStream,
interceptors: Arc<crate::interceptor::InterceptorChain<T>>,
operation: String,
model: String,
request_json: String,
state: T,
) -> Result<(Self, StreamCompletion)> {
let runtime = Handle::try_current().map_err(|_| {
Error::Config(
"InterceptedStream::new_with_completion requires an active Tokio runtime".into(),
)
})?;
let mut stream = Self::new(inner, interceptors, operation, model, request_json, state);
let completion = stream.start_lifecycle(&runtime);
Ok((stream, completion))
}
fn start_lifecycle(&mut self, runtime: &Handle) -> StreamCompletion {
let (report_tx, report_rx) = watch::channel(None);
if self.interceptors.is_empty() {
self.started = true;
self.report_tx = Some(report_tx);
return StreamCompletion { report: report_rx };
}
let (queue_tx, queue_rx) = mpsc::channel(STREAM_CALLBACK_QUEUE_CAPACITY);
let (terminal_tx, terminal_rx) = watch::channel(None);
let interceptors = Arc::clone(&self.interceptors);
let operation = self.operation.clone();
let model = self.model.clone();
let request_json = self.request_json.clone();
let state = Arc::clone(&self.state);
let start_time = self.start_time;
runtime.spawn(async move {
dispatch_stream_callbacks(
queue_rx,
terminal_rx,
report_tx,
interceptors,
operation,
model,
request_json,
state,
start_time,
)
.await;
});
self.queue_tx = Some(queue_tx);
self.terminal_tx = Some(terminal_tx);
self.started = true;
StreamCompletion { report: report_rx }
}
fn finish(&mut self, outcome: StreamOutcome) {
self.terminated = true;
self.inner.take();
self.reserve_future.take();
self.reserved.take();
self.queue_tx.take();
if let Some(report_tx) = self.report_tx.take() {
let report = StreamCompletionReport {
outcome,
total_chunks: self.chunk_index,
duration: self.start_time.elapsed(),
input_tokens: self.input_tokens,
output_tokens: self.output_tokens,
callback_attempts: 0,
callback_successes: 0,
callback_failure_count: 0,
callback_failures: Vec::new(),
truncated_failure_details: 0,
};
let _ = report_tx.send(Some(Arc::new(report)));
}
if let Some(terminal_tx) = self.terminal_tx.take() {
terminal_tx.send_if_modified(|current| {
if current.is_none() {
*current = Some(outcome);
true
} else {
false
}
});
}
}
fn poll_inner(&mut self, cx: &mut Context<'_>) -> Poll<Option<Result<ChatCompletionChunk>>> {
self.inner
.as_mut()
.expect("live intercepted stream has an inner stream")
.as_mut()
.poll_next(cx)
}
pub async fn collect_content(mut self) -> Result<String> {
let mut content = String::new();
while let Some(chunk) = self.next().await {
let chunk = chunk?;
if let Some(text) = chunk.content() {
content.push_str(text);
}
}
Ok(content)
}
}
impl<T: Send + Sync + 'static> Drop for InterceptedStream<T> {
fn drop(&mut self) {
if !self.terminated {
self.finish(StreamOutcome::Cancelled);
}
}
}
impl<T: Send + Sync + 'static> Stream for InterceptedStream<T> {
type Item = Result<ChatCompletionChunk>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
if this.terminated {
return Poll::Ready(None);
}
if !this.started {
let Ok(runtime) = Handle::try_current() else {
this.terminated = true;
this.inner.take();
return Poll::Ready(Some(Err(Error::Config(
"InterceptedStream must be polled inside an active Tokio runtime".into(),
))));
};
this.start_lifecycle(&runtime);
}
if this.interceptors.is_empty() {
return match this.poll_inner(cx) {
Poll::Ready(Some(Ok(chunk))) => {
update_usage(this, &chunk);
this.chunk_index = this.chunk_index.saturating_add(1);
Poll::Ready(Some(Ok(chunk)))
}
Poll::Ready(Some(Err(error))) => {
let outcome = StreamOutcome::StreamFailed(StreamFailure::from_error(&error));
this.finish(outcome);
Poll::Ready(Some(Err(error)))
}
Poll::Ready(None) => {
this.finish(StreamOutcome::Completed);
Poll::Ready(None)
}
Poll::Pending => Poll::Pending,
};
}
if this.reserved.is_none() {
if this.reserve_future.is_none() {
let sender = this
.queue_tx
.as_ref()
.expect("started stream has queue sender")
.clone();
this.reserve_future = Some(Box::pin(sender.reserve_owned()));
}
match this
.reserve_future
.as_mut()
.expect("reserve future was just initialized")
.as_mut()
.poll(cx)
{
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(permit)) => {
this.reserve_future.take();
this.reserved = Some(permit);
}
Poll::Ready(Err(_)) => {
this.finish(StreamOutcome::StreamFailed(StreamFailure {
kind: StreamFailureKind::Other,
}));
return Poll::Ready(Some(Err(Error::Internal(
"stream callback dispatcher stopped unexpectedly".into(),
))));
}
}
}
match this.poll_inner(cx) {
Poll::Ready(Some(Ok(chunk))) => {
update_usage(this, &chunk);
let event = ChunkEvent {
chunk_json: serde_json::to_string(chunk.raw_response())
.unwrap_or_else(|_| "{}".to_string()),
chunk_index: this.chunk_index,
input_tokens: this.input_tokens,
output_tokens: this.output_tokens,
};
this.reserved
.take()
.expect("polling the inner stream requires a reserved callback slot")
.send(event);
this.chunk_index = this.chunk_index.saturating_add(1);
Poll::Ready(Some(Ok(chunk)))
}
Poll::Ready(Some(Err(error))) => {
let outcome = StreamOutcome::StreamFailed(StreamFailure::from_error(&error));
this.finish(outcome);
Poll::Ready(Some(Err(error)))
}
Poll::Ready(None) => {
this.finish(StreamOutcome::Completed);
Poll::Ready(None)
}
Poll::Pending => Poll::Pending,
}
}
}
fn update_usage<T: Send + Sync + 'static>(
stream: &mut InterceptedStream<T>,
chunk: &ChatCompletionChunk,
) {
if let Some(Some(usage)) = &chunk.raw_response().usage {
stream.input_tokens = Some(i64::from(usage.prompt_tokens));
stream.output_tokens = Some(i64::from(usage.completion_tokens));
}
}
async fn dispatch_stream_callbacks<T: Send + Sync + 'static>(
mut queue: mpsc::Receiver<ChunkEvent>,
mut terminal: watch::Receiver<Option<StreamOutcome>>,
report_tx: watch::Sender<Option<Arc<StreamCompletionReport>>>,
interceptors: Arc<crate::interceptor::InterceptorChain<T>>,
operation: String,
model: String,
request_json: String,
state: Arc<T>,
start_time: Instant,
) {
let mut report = StreamCompletionReport {
outcome: StreamOutcome::Completed,
total_chunks: 0,
duration: Duration::ZERO,
input_tokens: None,
output_tokens: None,
callback_attempts: 0,
callback_successes: 0,
callback_failure_count: 0,
callback_failures: Vec::new(),
truncated_failure_details: 0,
};
let outcome = loop {
tokio::select! {
changed = terminal.changed() => {
if changed.is_err() {
return;
}
let current = *terminal.borrow();
if let Some(outcome) = current {
break outcome;
}
}
event = queue.recv() => if let Some(event) = event {
dispatch_chunk_event(
&mut report,
&interceptors,
&operation,
&model,
&request_json,
&state,
event,
).await;
} else {
let current = *terminal.borrow();
if let Some(outcome) = current {
break outcome;
}
if terminal.changed().await.is_err() {
return;
}
let current = *terminal.borrow();
if let Some(outcome) = current {
break outcome;
}
}
}
};
while let Some(event) = queue.recv().await {
dispatch_chunk_event(
&mut report,
&interceptors,
&operation,
&model,
&request_json,
&state,
event,
)
.await;
}
report.outcome = outcome;
report.duration = start_time.elapsed();
dispatch_terminal_event(
&mut report,
&interceptors,
&operation,
&model,
&request_json,
&state,
outcome,
)
.await;
let _ = report_tx.send(Some(Arc::new(report)));
}
async fn dispatch_chunk_event<T: Send + Sync + 'static>(
report: &mut StreamCompletionReport,
interceptors: &crate::interceptor::InterceptorChain<T>,
operation: &str,
model: &str,
request_json: &str,
state: &T,
event: ChunkEvent,
) {
report.total_chunks = report.total_chunks.saturating_add(1);
if let Some(input_tokens) = event.input_tokens {
report.input_tokens = Some(input_tokens);
}
if let Some(output_tokens) = event.output_tokens {
report.output_tokens = Some(output_tokens);
}
let ctx = StreamChunkContext {
operation,
model,
request_json,
chunk_json: &event.chunk_json,
chunk_index: event.chunk_index,
state,
};
for (interceptor_index, interceptor) in interceptors.iter().enumerate() {
record_callback(
report,
StreamCallbackHook::Chunk,
interceptor_index,
Some(event.chunk_index),
|| interceptor.on_stream_chunk(&ctx),
)
.await;
}
}
async fn dispatch_terminal_event<T: Send + Sync + 'static>(
report: &mut StreamCompletionReport,
interceptors: &crate::interceptor::InterceptorChain<T>,
operation: &str,
model: &str,
request_json: &str,
state: &T,
outcome: StreamOutcome,
) {
let summary = StreamEndContext {
operation,
model,
request_json,
total_chunks: report.total_chunks,
duration: report.duration,
input_tokens: report.input_tokens,
output_tokens: report.output_tokens,
state,
};
let ctx = StreamTerminalContext { summary, outcome };
for (interceptor_index, interceptor) in interceptors.iter().enumerate() {
record_callback(
report,
StreamCallbackHook::Terminal,
interceptor_index,
None,
|| interceptor.on_stream_terminal(&ctx),
)
.await;
}
}
async fn record_callback<F, Fut>(
report: &mut StreamCompletionReport,
hook: StreamCallbackHook,
interceptor_index: usize,
chunk_index: Option<usize>,
callback: F,
) where
F: FnOnce() -> Fut + Send,
Fut: Future<Output = Result<()>> + Send,
{
report.callback_attempts = report.callback_attempts.saturating_add(1);
let callback_future = match std::panic::catch_unwind(AssertUnwindSafe(callback)) {
Ok(callback) => callback,
Err(payload) => {
let diagnostic = panic_diagnostic(payload.as_ref());
record_callback_failure(
report,
hook,
interceptor_index,
chunk_index,
StreamCallbackFailureKind::Panicked,
&diagnostic,
);
return;
}
};
match AssertUnwindSafe(callback_future).catch_unwind().await {
Ok(Ok(())) => report.callback_successes = report.callback_successes.saturating_add(1),
Ok(Err(error)) => {
let diagnostic = std::panic::catch_unwind(AssertUnwindSafe(|| error.to_string()))
.unwrap_or_else(|_| {
"callback returned an error (diagnostic formatting panicked)".into()
});
record_callback_failure(
report,
hook,
interceptor_index,
chunk_index,
StreamCallbackFailureKind::ReturnedError,
&diagnostic,
);
}
Err(payload) => {
let diagnostic = panic_diagnostic(payload.as_ref());
record_callback_failure(
report,
hook,
interceptor_index,
chunk_index,
StreamCallbackFailureKind::Panicked,
&diagnostic,
);
}
}
}
fn record_callback_failure(
report: &mut StreamCompletionReport,
hook: StreamCallbackHook,
interceptor_index: usize,
chunk_index: Option<usize>,
kind: StreamCallbackFailureKind,
diagnostic: &str,
) {
report.callback_failure_count = report.callback_failure_count.saturating_add(1);
if report.callback_failures.len() == MAX_RETAINED_CALLBACK_FAILURES {
report.truncated_failure_details = report.truncated_failure_details.saturating_add(1);
return;
}
let (diagnostic, diagnostic_truncated) = truncate_diagnostic(diagnostic);
report.callback_failures.push(StreamCallbackFailure {
hook,
interceptor_index,
chunk_index,
kind,
diagnostic,
diagnostic_truncated,
});
}
fn truncate_diagnostic(value: &str) -> (String, bool) {
let mut chars = value.chars();
let truncated: String = chars.by_ref().take(MAX_CALLBACK_DIAGNOSTIC_CHARS).collect();
if chars.next().is_some() {
(truncated, true)
} else {
(truncated, false)
}
}
fn panic_diagnostic(payload: &(dyn Any + Send)) -> String {
payload
.downcast_ref::<String>()
.cloned()
.or_else(|| {
payload
.downcast_ref::<&'static str>()
.map(|value| (*value).to_string())
})
.unwrap_or_else(|| "callback panicked with a non-string payload".into())
}
fn parse_sse_stream<S>(byte_stream: S) -> impl Stream<Item = Result<ChatCompletionChunk>> + Send
where
S: Stream<Item = reqwest::Result<Bytes>> + Send + 'static,
{
struct State<S> {
byte_stream: Pin<Box<S>>,
buffer: Vec<u8>,
swallow_lf: bool,
data_lines: Vec<String>,
ready: VecDeque<Result<ChatCompletionChunk>>,
terminated: bool,
}
futures::stream::unfold(
State {
byte_stream: Box::pin(byte_stream),
buffer: Vec::new(),
swallow_lf: false,
data_lines: Vec::new(),
ready: VecDeque::new(),
terminated: false,
},
|mut state| async move {
loop {
if let Some(item) = state.ready.pop_front() {
return Some((item, state));
}
if state.terminated {
return None;
}
match state.byte_stream.next().await {
Some(Ok(bytes)) => {
state.buffer.extend_from_slice(&bytes);
while let Some(line) =
take_sse_line(&mut state.buffer, &mut state.swallow_lf)
{
if process_sse_line(&line, &mut state.data_lines, &mut state.ready) {
state.terminated = true;
break;
}
}
}
Some(Err(error)) => {
state.ready.push_back(Err(Error::StreamConnection {
message: format!("Stream connection error: {error}"),
}));
state.terminated = true;
}
None => {
while let Some(line) =
take_sse_line(&mut state.buffer, &mut state.swallow_lf)
{
if process_sse_line(&line, &mut state.data_lines, &mut state.ready) {
state.terminated = true;
break;
}
}
state.buffer.clear();
state.data_lines.clear();
state.terminated = true;
}
}
}
},
)
.fuse()
}
fn take_sse_line(buffer: &mut Vec<u8>, swallow_lf: &mut bool) -> Option<Vec<u8>> {
if *swallow_lf {
if buffer.first() == Some(&b'\n') {
buffer.drain(..1);
*swallow_lf = false;
} else if !buffer.is_empty() {
*swallow_lf = false;
}
if buffer.is_empty() {
return None;
}
}
for index in 0..buffer.len() {
match buffer[index] {
b'\n' => {
let line = buffer[..index].to_vec();
buffer.drain(..=index);
return Some(line);
}
b'\r' => {
let line = buffer[..index].to_vec();
buffer.drain(..=index);
*swallow_lf = true;
return Some(line);
}
_ => {}
}
}
None
}
fn process_sse_line(
line_bytes: &[u8],
data_lines: &mut Vec<String>,
ready: &mut VecDeque<Result<ChatCompletionChunk>>,
) -> bool {
let line = String::from_utf8_lossy(line_bytes).into_owned();
if line.is_empty() {
if data_lines.is_empty() {
return false;
}
let data = std::mem::take(data_lines).join("\n");
if data.is_empty() {
return false;
}
if data == "[DONE]" {
return true;
}
match parse_sse_line(&format!("data: {data}")) {
Ok(Some(chunk)) => ready.push_back(Ok(chunk)),
Ok(None) => {}
Err(error) => {
ready.push_back(Err(error));
return true;
}
}
return false;
}
if line.starts_with(':') {
return false;
}
if let Some(data) = line.strip_prefix("data:") {
data_lines.push(data.strip_prefix(' ').unwrap_or(data).to_string());
}
false
}
fn parse_sse_line(line: &str) -> Result<Option<ChatCompletionChunk>> {
if line.is_empty() || line.starts_with(':') {
return Ok(None);
}
if let Some(data) = line.strip_prefix("data:") {
let data = data.strip_prefix(' ').unwrap_or(data);
if data == "[DONE]" {
return Ok(None);
}
let response: CreateChatCompletionStreamResponse =
serde_json::from_str(data).map_err(|e| Error::StreamParsing {
message: format!("Failed to deserialize chunk: {e}"),
chunk: data.to_string(),
})?;
return Ok(Some(ChatCompletionChunk::new(response)));
}
Ok(None)
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicBool, Ordering};
struct PendingBody {
bytes: Option<Bytes>,
dropped: Arc<AtomicBool>,
}
impl Stream for PendingBody {
type Item = reqwest::Result<Bytes>;
fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
self.bytes
.take()
.map_or_else(|| Poll::Pending, |bytes| Poll::Ready(Some(Ok(bytes))))
}
}
impl Drop for PendingBody {
fn drop(&mut self) {
self.dropped.store(true, Ordering::SeqCst);
}
}
struct TerminalFlag(Arc<AtomicBool>);
#[async_trait::async_trait]
impl crate::interceptor::Interceptor for TerminalFlag {
async fn on_stream_terminal(&self, _ctx: &StreamTerminalContext<'_>) -> Result<()> {
self.0.store(true, Ordering::SeqCst);
Ok(())
}
}
#[tokio::test]
async fn terminal_parser_result_releases_pending_http_body_immediately() {
for (payload, expected) in [
("data: {not-json}\n\n", Some(StreamFailureKind::Parsing)),
("data: [DONE]\n\n", None),
] {
let body_dropped = Arc::new(AtomicBool::new(false));
let terminal_called = Arc::new(AtomicBool::new(false));
let body = PendingBody {
bytes: Some(Bytes::from(payload)),
dropped: Arc::clone(&body_dropped),
};
let chat_stream = ChatCompletionStream {
inner: Box::pin(parse_sse_stream(body)),
};
let mut interceptors = crate::interceptor::InterceptorChain::new();
interceptors.add(Box::new(TerminalFlag(Arc::clone(&terminal_called))));
let (mut stream, completion) = InterceptedStream::new_with_completion(
chat_stream,
Arc::new(interceptors),
"chat".into(),
"mock-model".into(),
"{}".into(),
(),
)
.unwrap();
match expected {
Some(_) => assert!(matches!(
stream.next().await.unwrap(),
Err(Error::StreamParsing { .. })
)),
None => assert!(stream.next().await.is_none()),
}
assert!(body_dropped.load(Ordering::SeqCst));
assert!(stream.next().await.is_none());
assert!(stream.next().await.is_none());
let report = tokio::time::timeout(Duration::from_secs(1), completion.wait())
.await
.expect("terminal callback drain must finish")
.unwrap();
assert_eq!(
report.outcome,
expected.map_or(StreamOutcome::Completed, |kind| {
StreamOutcome::StreamFailed(StreamFailure { kind })
})
);
assert!(terminal_called.load(Ordering::SeqCst));
}
}
#[test]
fn test_parse_sse_line_with_content() {
let line = r#"data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1234567890,"model":"gpt-4","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]}"#;
let result = parse_sse_line(line).unwrap();
assert!(result.is_some());
let chunk = result.unwrap();
assert_eq!(chunk.content(), Some("Hello"));
assert_eq!(chunk.role(), Some("assistant"));
assert_eq!(chunk.finish_reason(), None);
assert_eq!(chunk.raw_response().choices[0].finish_reason, Some(None));
assert!(!chunk.is_final());
assert!(!crate::responses::ChatCompletionStreamResponseWrapper::new(
chunk.raw_response().clone()
)
.is_finished());
}
#[test]
fn test_parse_sse_line_with_missing_finish_reason() {
let line = r#"data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1234567890,"model":"gpt-4","choices":[{"index":0,"delta":{}}]}"#;
let chunk = parse_sse_line(line).unwrap().unwrap();
assert_eq!(chunk.finish_reason(), None);
assert_eq!(chunk.raw_response().choices[0].finish_reason, None);
assert!(!chunk.is_final());
}
#[test]
fn test_parse_sse_line_with_finish_reason() {
let line = r#"data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1234567890,"model":"gpt-4","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":null}"#;
let chunk = parse_sse_line(line).unwrap().unwrap();
assert_eq!(chunk.finish_reason(), Some("stop"));
assert!(chunk.is_final());
assert!(crate::responses::ChatCompletionStreamResponseWrapper::new(
chunk.raw_response().clone()
)
.is_finished());
}
#[test]
fn test_finish_reason_helper_maps_all_supported_reasons() {
for (wire, expected) in [
("stop", "stop"),
("length", "length"),
("tool_calls", "tool_calls"),
("content_filter", "content_filter"),
("function_call", "function_call"),
] {
let event = serde_json::json!({
"id": "chatcmpl-123",
"object": "chat.completion.chunk",
"created": 1,
"model": "gpt-4",
"choices": [{"index": 0, "delta": {}, "finish_reason": wire}],
});
let chunk = parse_sse_line(&format!("data: {event}")).unwrap().unwrap();
assert_eq!(chunk.finish_reason(), Some(expected));
}
}
#[test]
fn test_final_audio_update_has_no_finish_reason() {
let line = r#"data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1,"model":"gpt-4o-audio-preview","choices":[{"index":0,"delta":{"audio":{"expires_at":1700000000}}}]}"#;
let chunk = parse_sse_line(line).unwrap().unwrap();
assert_eq!(chunk.finish_reason(), None);
assert_eq!(chunk.raw_response().choices[0].finish_reason, None);
assert!(!chunk.is_final());
}
#[test]
fn test_empty_choices_without_usage_are_not_finished() {
let line = r#"data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1,"model":"gpt-4","choices":[]}"#;
let chunk = parse_sse_line(line).unwrap().unwrap();
assert_eq!(chunk.finish_reason(), None);
assert!(!chunk.is_final());
}
#[test]
fn test_usage_only_chunk_is_not_a_finished_choice() {
let line = r#"data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1234567890,"model":"gpt-4","choices":[],"usage":{"prompt_tokens":3,"completion_tokens":4,"total_tokens":7}}"#;
let chunk = parse_sse_line(line).unwrap().unwrap();
assert_eq!(chunk.finish_reason(), None);
assert!(!crate::responses::ChatCompletionStreamResponseWrapper::new(
chunk.raw_response().clone()
)
.is_finished());
}
#[test]
fn test_parse_sse_line_done_marker() {
let line = "data: [DONE]";
let result = parse_sse_line(line).unwrap();
assert!(result.is_none());
}
#[test]
fn test_parse_sse_line_empty() {
let line = "";
let result = parse_sse_line(line).unwrap();
assert!(result.is_none());
}
#[test]
fn test_parse_sse_line_comment() {
let line = ": this is a comment";
let result = parse_sse_line(line).unwrap();
assert!(result.is_none());
}
#[test]
fn test_parse_sse_line_removes_only_one_space_after_colon() {
let result = parse_sse_line("data: {not-json}").unwrap_err();
assert!(matches!(
result,
Error::StreamParsing { chunk, .. } if chunk == " {not-json}"
));
}
#[tokio::test]
async fn test_parse_sse_stream_handles_cr_only_while_connection_is_open() {
let bytes = Bytes::from(
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"cr-only\"},\"finish_reason\":null}]}\r\r",
);
let source = futures::stream::iter(vec![Ok::<Bytes, reqwest::Error>(bytes)])
.chain(futures::stream::pending());
let mut stream = Box::pin(parse_sse_stream(source));
let chunk = tokio::time::timeout(std::time::Duration::from_millis(250), stream.next())
.await
.expect("CR-only event should dispatch without waiting for more bytes")
.unwrap()
.unwrap();
assert_eq!(chunk.content(), Some("cr-only"));
}
#[tokio::test]
async fn test_parse_sse_stream_handles_split_crlf_before_lf_blank_line() {
let json = r#"{"id":"chatcmpl-123","object":"chat.completion.chunk","created":1,"model":"gpt-4","choices":[{"index":0,"delta":{"content":"split"},"finish_reason":null}]}"#;
let chunks = vec![
Ok::<Bytes, reqwest::Error>(Bytes::from(format!("data: {json}\r"))),
Ok(Bytes::from("\n")),
Ok(Bytes::from("\n")),
];
let source = futures::stream::iter(chunks).chain(futures::stream::pending());
let mut stream = Box::pin(parse_sse_stream(source));
let chunk = tokio::time::timeout(std::time::Duration::from_millis(250), stream.next())
.await
.expect("split CRLF followed by a blank LF line should dispatch promptly")
.unwrap()
.unwrap();
assert_eq!(chunk.content(), Some("split"));
}
#[tokio::test]
async fn test_parse_sse_stream_ignores_space_only_field_between_data_lines() {
let bytes = Bytes::from(
"data: {\n \ndata: \"id\":\"chatcmpl-123\",\ndata: \"object\":\"chat.completion.chunk\",\ndata: \"created\":1,\ndata: \"model\":\"gpt-4\",\ndata: \"choices\":[{\"index\":0,\"delta\":{\"content\":\"multiline\"},\"finish_reason\":null}]\ndata: }\n\n",
);
let mut stream = Box::pin(parse_sse_stream(futures::stream::iter(vec![Ok(bytes)])));
let chunk = stream.next().await.unwrap().unwrap();
assert_eq!(chunk.content(), Some("multiline"));
assert!(stream.next().await.is_none());
}
#[tokio::test]
async fn test_parse_sse_stream_handles_fragmented_crlf_and_usage_only_events() {
let first_event = format!(
"data: {{\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4\",\"choices\":[{{\"index\":0,\"delta\":{{\"content\":{}}},\"finish_reason\":null}}]}}\r\n\r\n",
serde_json::to_string("Hello 🦀").unwrap()
);
let crab = first_event
.as_bytes()
.windows(4)
.position(|window| window == "🦀".as_bytes())
.unwrap();
let crlf = first_event
.as_bytes()
.windows(4)
.position(|window| window == b"\r\n\r\n")
.unwrap();
let first_bytes = first_event.as_bytes();
let fragmented = vec![
Ok(Bytes::copy_from_slice(&first_bytes[..5]),),
Ok(Bytes::copy_from_slice(&first_bytes[5..crab + 2]),),
Ok(Bytes::copy_from_slice(&first_bytes[(crab + 2)..=crlf]),),
Ok(Bytes::copy_from_slice(&first_bytes[crlf + 1..crlf + 2]),),
Ok(Bytes::copy_from_slice(&first_bytes[crlf + 2..]),),
Ok(Bytes::from(
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\" done\"},\"finish_reason\":\"stop\"}]}\r\n\r\ndata: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4\",\"choices\":[],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":4,\"total_tokens\":7}}\r\n\r\ndata: [DONE]\r\n\r\n",
)),
];
let mut stream = Box::pin(parse_sse_stream(futures::stream::iter(fragmented)));
let first = stream.next().await.unwrap().unwrap();
let second = stream.next().await.unwrap().unwrap();
let usage = stream.next().await.unwrap().unwrap();
assert_eq!(first.content(), Some("Hello 🦀"));
assert_eq!(second.content(), Some(" done"));
assert_eq!(usage.raw_response().choices.as_slice(), &[]);
assert_eq!(
usage
.raw_response()
.usage
.as_ref()
.unwrap()
.as_ref()
.unwrap()
.total_tokens,
7
);
assert!(stream.next().await.is_none());
}
#[tokio::test]
async fn test_parse_sse_stream_reports_malformed_json() {
let bytes = Bytes::from("data: {not-json}\n\n");
let mut stream = Box::pin(parse_sse_stream(futures::stream::iter(vec![Ok(bytes)])));
assert!(matches!(
stream.next().await.unwrap(),
Err(Error::StreamParsing { .. })
));
assert!(stream.next().await.is_none());
}
#[tokio::test]
async fn test_parse_sse_stream_delivers_valid_event_before_parse_error() {
let bytes = Bytes::from(
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"kept\"},\"finish_reason\":null}]}\n\ndata: {not-json}\n\n",
);
let mut stream = Box::pin(parse_sse_stream(futures::stream::iter(vec![Ok(bytes)])));
assert_eq!(
stream.next().await.unwrap().unwrap().content(),
Some("kept")
);
assert!(matches!(
stream.next().await.unwrap(),
Err(Error::StreamParsing { .. })
));
assert!(stream.next().await.is_none());
assert!(stream.next().await.is_none());
}
#[tokio::test]
async fn test_parse_sse_stream_stops_at_done() {
let bytes = Bytes::from(
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"before\"},\"finish_reason\":null}]}\n\ndata: [DONE]\n\ndata: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"after\"},\"finish_reason\":null}]}\n\n",
);
let mut stream = Box::pin(parse_sse_stream(futures::stream::iter(vec![Ok(bytes)])));
assert_eq!(
stream.next().await.unwrap().unwrap().content(),
Some("before")
);
assert!(stream.next().await.is_none());
assert!(stream.next().await.is_none());
}
#[tokio::test]
async fn test_parse_sse_stream_discards_incomplete_trailing_event() {
let bytes = Bytes::from(
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"complete\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"missing blank line\"},\"finish_reason\":null}]}\ndata: {partial",
);
let mut stream = Box::pin(parse_sse_stream(futures::stream::iter(vec![Ok(bytes)])));
assert_eq!(
stream.next().await.unwrap().unwrap().content(),
Some("complete")
);
assert!(stream.next().await.is_none());
assert!(stream.next().await.is_none());
}
#[tokio::test]
async fn completion_wait_reports_unavailable_if_worker_sender_closes() {
let (sender, report) = watch::channel(None);
let completion = StreamCompletion { report };
drop(sender);
assert_eq!(completion.wait().await, Err(CompletionUnavailable));
}
}