use crate::error::LlmWasmError;
use crate::types::{StreamChunk, Usage};
use bytes::BytesMut;
use serde_json::Value;
use sse_core::{SseDecoder, SseEvent};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct StreamUsage {
pub input_tokens: u32,
pub output_tokens: u32,
}
#[derive(Debug)]
pub struct StreamDecoder {
sse: SseDecoder,
buf: BytesMut,
finished: bool,
usage: Option<StreamUsage>,
text: String,
}
impl From<StreamUsage> for Usage {
fn from(u: StreamUsage) -> Self {
Usage::new(u64::from(u.input_tokens), u64::from(u.output_tokens))
}
}
fn provider_error(err: &Value) -> LlmWasmError {
let message = match err {
Value::String(s) => s.clone(),
_ => err
.get("message")
.and_then(Value::as_str)
.unwrap_or("unknown error")
.to_string(),
};
let as_kind = |v: Option<&Value>| match v {
Some(Value::String(s)) if !s.is_empty() => Some(s.clone()),
Some(Value::Number(n)) => Some(n.to_string()),
_ => None,
};
let kind = as_kind(err.get("code")).or_else(|| as_kind(err.get("type")));
LlmWasmError::Provider { kind, message }
}
fn u32_at(v: &Value, path: &[&str]) -> Option<u32> {
let mut cur = v;
for p in path {
cur = cur.get(*p)?;
}
cur.as_u64().and_then(|n| u32::try_from(n).ok())
}
impl Default for StreamDecoder {
fn default() -> Self {
Self::new()
}
}
impl StreamDecoder {
pub fn new() -> Self {
Self {
sse: SseDecoder::new(),
buf: BytesMut::new(),
finished: false,
usage: None,
text: String::new(),
}
}
pub fn push(&mut self, bytes: &[u8]) -> Result<Vec<StreamChunk>, LlmWasmError> {
self.buf.extend_from_slice(bytes);
let mut out = Vec::new();
while let Some(ev) = self.sse.next(&mut self.buf) {
let ev = ev.map_err(|e| LlmWasmError::Serialization(format!("stream event: {e}")))?;
let SseEvent::Message(msg) = ev else { continue };
if self.finished {
continue;
}
let data = msg.data.trim();
if data.is_empty() {
continue;
}
if data == "[DONE]" {
self.finished = true;
out.push(StreamChunk {
delta: String::new(),
finished: true,
});
continue;
}
let v: Value = serde_json::from_str(data).map_err(|e| {
LlmWasmError::Serialization(format!("stream event is not JSON: {e}"))
})?;
if let Some(chunk) = self.handle(&msg.event, &v)? {
out.push(chunk);
}
}
Ok(out)
}
fn handle(&mut self, event: &str, v: &Value) -> Result<Option<StreamChunk>, LlmWasmError> {
let kind = v.get("type").and_then(Value::as_str).unwrap_or(event);
match kind {
"content_block_delta" => {
let text = v
.pointer("/delta/text")
.and_then(Value::as_str)
.unwrap_or("");
return Ok(self.emit(text, false));
}
"message_start" => {
if let Some(i) = u32_at(v, &["message", "usage", "input_tokens"]) {
self.usage
.get_or_insert_with(StreamUsage::default)
.input_tokens = i;
}
return Ok(None);
}
"message_delta" => {
if let Some(o) = u32_at(v, &["usage", "output_tokens"]) {
self.usage
.get_or_insert_with(StreamUsage::default)
.output_tokens = o;
}
return Ok(None);
}
"message_stop" => {
self.finished = true;
return Ok(Some(StreamChunk {
delta: String::new(),
finished: true,
}));
}
"error" => return Err(provider_error(v.get("error").unwrap_or(&Value::Null))),
_ => {}
}
if let Some(err) = v.get("error") {
return Err(provider_error(err));
}
if let Some(u) = v.get("usage").filter(|u| !u.is_null()) {
self.usage = Some(StreamUsage {
input_tokens: u32_at(u, &["prompt_tokens"]).unwrap_or(0),
output_tokens: u32_at(u, &["completion_tokens"]).unwrap_or(0),
});
}
let Some(choice) = v.get("choices").and_then(|c| c.get(0)) else {
return Ok(None);
};
let text = choice
.pointer("/delta/content")
.and_then(Value::as_str)
.unwrap_or("");
let done = choice.get("finish_reason").is_some_and(|f| !f.is_null());
Ok(self.emit(text, done))
}
fn emit(&mut self, text: &str, finished: bool) -> Option<StreamChunk> {
if text.is_empty() && !finished {
return None;
}
self.text.push_str(text);
Some(StreamChunk {
delta: text.to_string(),
finished,
})
}
pub fn is_finished(&self) -> bool {
self.finished
}
pub fn text(&self) -> &str {
&self.text
}
pub fn usage(&self) -> Option<StreamUsage> {
self.usage
}
}
#[cfg(test)]
mod tests {
use super::*;
const OPENAI: &str = "data: {\"id\":\"c1\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"finish_reason\":null}]}\n\n\
data: {\"id\":\"c1\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Bon\"},\"finish_reason\":null}]}\n\n\
data: {\"id\":\"c1\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"jour \u{1F44B}\"},\"finish_reason\":null}]}\n\n\
data: {\"id\":\"c1\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n\
data: {\"id\":\"c1\",\"choices\":[],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":3,\"total_tokens\":12}}\n\n\
data: [DONE]\n\n";
const ANTHROPIC: &str = "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"usage\":{\"input_tokens\":25,\"output_tokens\":1}}}\n\n\
event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n\
event: ping\ndata: {\"type\": \"ping\"}\n\n\
event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hello\"}}\n\n\
event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" there\"}}\n\n\
event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":15}}\n\n\
event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n";
fn run(stream: &[u8], piece: usize) -> (String, StreamDecoder) {
let mut d = StreamDecoder::new();
let mut text = String::new();
for part in stream.chunks(piece) {
for c in d.push(part).unwrap() {
text.push_str(&c.delta);
}
}
(text, d)
}
#[test]
fn openai_stream_any_split() {
for piece in [1, 2, 3, 7, 64, 10_000] {
let (text, d) = run(OPENAI.as_bytes(), piece);
assert_eq!(text, "Bonjour \u{1F44B}", "piece size {piece}");
assert!(d.is_finished());
assert_eq!(
d.usage(),
Some(StreamUsage {
input_tokens: 9,
output_tokens: 3
})
);
}
}
#[test]
fn anthropic_stream_any_split() {
for piece in [1, 5, 4096] {
let (text, d) = run(ANTHROPIC.as_bytes(), piece);
assert_eq!(text, "Hello there");
assert!(d.is_finished());
assert_eq!(
d.usage(),
Some(StreamUsage {
input_tokens: 25,
output_tokens: 15
})
);
assert_eq!(d.text(), "Hello there");
}
}
#[test]
fn crlf_and_comments_and_after_done() {
let s = ": keep-alive\r\ndata: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\r\n\r\ndata: [DONE]\r\n\r\ndata: {\"choices\":[{\"delta\":{\"content\":\"ignored\"}}]}\r\n\r\n";
let (text, d) = run(s.as_bytes(), 3);
assert_eq!(text, "a");
assert!(d.is_finished());
}
#[test]
fn errors_are_reported() {
let mut d = StreamDecoder::new();
assert!(d.push(b"data: not json\n\n").is_err());
let mut d = StreamDecoder::new();
let e = d.push(b"event: error\ndata: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded\"}}\n\n").unwrap_err();
assert!(e.to_string().contains("Overloaded"));
assert!(
matches!(e, LlmWasmError::Provider { kind: Some(ref k), ref message } if k == "overloaded_error" && message == "Overloaded")
);
let mut d = StreamDecoder::new();
let e = d.push(b"data: {\"error\":{\"message\":\"rate limited\",\"type\":\"requests\",\"code\":\"rate_limit_exceeded\"}}\n\n").unwrap_err();
assert!(e.to_string().contains("rate limited"));
assert!(
matches!(e, LlmWasmError::Provider { kind: Some(ref k), .. } if k == "rate_limit_exceeded")
);
let mut d = StreamDecoder::new();
let e = d
.push(b"data: {\"error\":{\"message\":\"x\",\"code\":null}}\n\n")
.unwrap_err();
assert!(matches!(e, LlmWasmError::Provider { kind: None, .. }));
}
#[test]
fn stream_usage_converts_to_usage() {
let u: Usage = StreamUsage {
input_tokens: 25,
output_tokens: 15,
}
.into();
assert_eq!(u, Usage::new(25, 15));
}
#[test]
fn incomplete_event_waits_for_more() {
let mut d = StreamDecoder::new();
assert!(d
.push(b"data: {\"choices\":[{\"delta\":{\"content\":\"x\"}}]}\n")
.unwrap()
.is_empty());
assert_eq!(d.push(b"\n").unwrap().len(), 1);
}
}