use std::convert::Infallible;
use std::sync::Arc;
use axum::response::sse::{Event, Sse};
use futures_util::Stream;
use serde_json::{Value, json};
use switchyard_translation::{LlmStreamError, RawEventStream, WireFormat};
use crate::redaction::Redactor;
pub(crate) type SseFrameStream =
std::pin::Pin<Box<dyn Stream<Item = Result<Event, Infallible>> + Send>>;
pub(crate) fn frame_stream(
stream: RawEventStream,
target_format: WireFormat,
redactor: Arc<Redactor>,
) -> Sse<SseFrameStream> {
let framed = async_stream::stream! {
let mut stream = stream;
let mut failed = false;
while let Some(item) = futures_util::StreamExt::next(&mut stream).await {
let event = match item {
Ok(value) => match frame_event(target_format, value, &redactor) {
Ok(event) => event,
Err(error) => {
failed = true;
error_event(target_format, error.to_string(), &redactor)
}
},
Err(LlmStreamError::Upstream(value)) => {
failed = true;
frame_event(target_format, value.clone(), &redactor).unwrap_or_else(|error| {
tracing::warn!(error = %error, "in-band error event could not be framed");
error_event(target_format, value.to_string(), &redactor)
})
}
Err(LlmStreamError::Client(error)) => {
tracing::warn!(error = %error, "stream iteration failed");
failed = true;
error_event(target_format, error.to_string(), &redactor)
}
};
yield Ok(event);
if failed {
break;
}
}
if !failed && target_format == WireFormat::OpenAiChat {
yield Ok(Event::default().data("[DONE]"));
}
};
Sse::new(Box::pin(framed) as SseFrameStream)
}
fn frame_event(
target_format: WireFormat,
value: Value,
redactor: &Redactor,
) -> Result<Event, serde_json::Error> {
let data = redactor.json(serde_json::to_string(&value)?);
match target_format {
WireFormat::OpenAiChat => Ok(Event::default().data(data)),
WireFormat::AnthropicMessages | WireFormat::OpenAiResponses => {
let event_type = value
.get("type")
.and_then(Value::as_str)
.unwrap_or("message")
.to_string();
Ok(Event::default().event(event_type).data(data))
}
}
}
fn error_event(target_format: WireFormat, message: String, redactor: &Redactor) -> Event {
match target_format {
WireFormat::OpenAiChat => Event::default().data(
redactor.json(
json!({
"error": {
"message": message,
"type": "SwitchyardError",
}
})
.to_string(),
),
),
WireFormat::AnthropicMessages | WireFormat::OpenAiResponses => {
Event::default().event("error").data(
redactor.json(
json!({
"type": "error",
"error": {
"message": message,
"type": "SwitchyardError",
}
})
.to_string(),
),
)
}
}
}
#[cfg(test)]
mod tests {
use std::error::Error;
use axum::{body::to_bytes, response::IntoResponse};
use futures_util::stream;
use switchyard_protocol::LlmClientError;
use super::*;
type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
async fn chat_body(items: Vec<Result<Value, LlmStreamError>>) -> TestResult<String> {
let stream: RawEventStream = Box::pin(stream::iter(items));
let response = frame_stream(
stream,
WireFormat::OpenAiChat,
Arc::new(Redactor::default()),
)
.into_response();
Ok(String::from_utf8(
to_bytes(response.into_body(), usize::MAX).await?.to_vec(),
)?)
}
#[tokio::test]
async fn stream_error_terminates_without_done_marker() -> TestResult {
let failure = LlmClientError::General("boom".to_string());
let body = chat_body(vec![
Ok(json!({"id": "before"})),
Err(LlmStreamError::Client(failure)),
Ok(json!({"id": "after"})),
])
.await?;
assert!(body.contains("before"));
assert!(body.contains("boom"));
assert!(!body.contains("after"));
assert!(!body.contains("[DONE]"));
Ok(())
}
#[tokio::test]
async fn in_band_error_is_forwarded_verbatim_without_done_marker() -> TestResult {
let upstream_error = json!({
"error": {"code": "stream_failed", "message": "boom", "type": "upstream_stream_error"}
});
let body = chat_body(vec![
Ok(json!({"id": "before"})),
Err(LlmStreamError::Upstream(upstream_error)),
Ok(json!({"id": "after"})),
])
.await?;
assert!(body.contains("before"));
assert!(body.contains("stream_failed"));
assert!(body.contains("upstream_stream_error"));
assert!(!body.contains("SwitchyardError"));
assert!(!body.contains("after"));
assert!(!body.contains("[DONE]"));
Ok(())
}
}