use std::io::{BufRead, BufReader, Read};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::Sender;
use serde_json::Value;
use crate::message::ToolCall;
use crate::{extract_json, Chunk, Error};
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub(crate) struct Tally {
pub input: u32,
pub output: u32,
pub cost_micros: u64,
}
fn stopped(cancel: Option<&AtomicBool>) -> bool {
cancel.is_some_and(|flag| flag.load(Ordering::Relaxed))
}
fn arguments(raw: &str) -> Value {
serde_json::from_str(raw)
.ok()
.or_else(|| extract_json(raw))
.unwrap_or_else(|| serde_json::json!({}))
}
fn send_done(tx: &Sender<Chunk>, tally: &Tally) {
let _ = tx.send(Chunk::Done {
input_tokens: tally.input,
output_tokens: tally.output,
});
}
fn reported(message: &str) -> Chunk {
Chunk::Failed(Error::Stream(message.to_string()))
}
pub(crate) fn responses<R: Read>(
reader: R,
tx: &Sender<Chunk>,
cancel: Option<&AtomicBool>,
) -> Tally {
let mut reader = BufReader::new(reader);
let (mut line, mut data) = (String::new(), String::new());
let mut tally = Tally::default();
let mut finished = false;
loop {
if stopped(cancel) {
let _ = tx.send(Chunk::Failed(Error::Cancelled));
return tally;
}
line.clear();
match reader.read_line(&mut line) {
Ok(0) => break,
Ok(_) => {}
Err(error) => {
let _ = tx.send(Chunk::Failed(Error::Stream(error.to_string())));
return tally;
}
}
let line = line.trim_end_matches(['\r', '\n']);
if line.is_empty() {
if !data.is_empty() {
if response_event(&data, tx, &mut tally) {
finished = true;
break;
}
data.clear();
}
} else if let Some(value) = line.strip_prefix("data:") {
let value = value.trim_start();
if value == "[DONE]" {
finished = true;
break;
}
if !data.is_empty() {
data.push('\n');
}
data.push_str(value);
}
}
if !finished && !data.is_empty() {
finished = response_event(&data, tx, &mut tally);
}
if !finished {
send_done(tx, &tally);
}
tally
}
fn response_event(data: &str, tx: &Sender<Chunk>, tally: &mut Tally) -> bool {
let Ok(value) = serde_json::from_str::<Value>(data) else {
return false;
};
match value.get("type").and_then(Value::as_str).unwrap_or("") {
"response.output_text.delta" => {
if let Some(text) = value
.get("delta")
.and_then(Value::as_str)
.filter(|text| !text.is_empty())
{
let _ = tx.send(Chunk::Text(text.to_string()));
}
}
"response.output_item.done" => {
let item = &value["item"];
if item.get("type").and_then(Value::as_str) == Some("function_call") {
let name = item.get("name").and_then(Value::as_str).unwrap_or_default();
if !name.is_empty() {
let id = item
.get("call_id")
.or_else(|| item.get("id"))
.and_then(Value::as_str)
.unwrap_or_default();
let raw = item
.get("arguments")
.and_then(Value::as_str)
.unwrap_or("{}");
let _ = tx.send(Chunk::Tool(ToolCall::new(name, arguments(raw)).with_id(id)));
}
}
}
"response.completed" | "response.done" => {
tally.input = value
.pointer("/response/usage/input_tokens")
.and_then(Value::as_u64)
.unwrap_or(0) as u32;
tally.output = value
.pointer("/response/usage/output_tokens")
.and_then(Value::as_u64)
.unwrap_or(0) as u32;
tally.cost_micros = value
.pointer("/response/usage/cost")
.and_then(Value::as_f64)
.map(|cost| (cost.max(0.0) * 1_000_000.0).round() as u64)
.unwrap_or(0);
send_done(tx, tally);
return true;
}
"response.failed" | "response.incomplete" | "error" => {
let message = value
.pointer("/response/error/message")
.or_else(|| value.pointer("/error/message"))
.or_else(|| value.get("message"))
.and_then(Value::as_str)
.unwrap_or("model returned an error");
let _ = tx.send(reported(message));
return true;
}
_ => {}
}
false
}
pub(crate) fn chat<R: Read>(reader: R, tx: &Sender<Chunk>, cancel: Option<&AtomicBool>) -> Tally {
let mut reader = BufReader::new(reader);
let mut line = String::new();
let mut tally = Tally::default();
let mut calls: Vec<(String, String, String)> = Vec::new();
loop {
if stopped(cancel) {
let _ = tx.send(Chunk::Failed(Error::Cancelled));
return tally;
}
line.clear();
match reader.read_line(&mut line) {
Ok(0) => break,
Ok(_) => {}
Err(error) => {
let _ = tx.send(Chunk::Failed(Error::Stream(error.to_string())));
return tally;
}
}
let Some(payload) = line.trim_end_matches(['\r', '\n']).strip_prefix("data:") else {
continue;
};
let payload = payload.trim_start();
if payload == "[DONE]" {
break;
}
let Ok(value) = serde_json::from_str::<Value>(payload) else {
continue;
};
if let Some(message) = value.pointer("/error/message").and_then(Value::as_str) {
let _ = tx.send(reported(message));
return tally;
}
if let Some(usage) = value.get("usage").filter(|usage| !usage.is_null()) {
tally.input = usage
.get("prompt_tokens")
.and_then(Value::as_u64)
.unwrap_or(tally.input as u64) as u32;
tally.output = usage
.get("completion_tokens")
.and_then(Value::as_u64)
.unwrap_or(tally.output as u64) as u32;
}
let Some(delta) = value.pointer("/choices/0/delta") else {
continue;
};
if let Some(text) = delta
.get("content")
.and_then(Value::as_str)
.filter(|text| !text.is_empty())
{
let _ = tx.send(Chunk::Text(text.to_string()));
}
for call in delta
.get("tool_calls")
.and_then(Value::as_array)
.into_iter()
.flatten()
{
let index = call.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
while calls.len() <= index {
calls.push((String::new(), String::new(), String::new()));
}
if let Some(id) = call
.get("id")
.and_then(Value::as_str)
.filter(|id| !id.is_empty())
{
calls[index].0 = id.to_string();
}
if let Some(name) = call
.pointer("/function/name")
.and_then(Value::as_str)
.filter(|name| !name.is_empty())
{
calls[index].1 = name.to_string();
}
if let Some(args) = call.pointer("/function/arguments").and_then(Value::as_str) {
calls[index].2.push_str(args);
}
}
}
for (id, name, raw) in calls {
if !name.is_empty() {
let _ = tx.send(Chunk::Tool(
ToolCall::new(name, arguments(&raw)).with_id(id),
));
}
}
send_done(tx, &tally);
tally
}
pub(crate) fn anthropic<R: Read>(
reader: R,
tx: &Sender<Chunk>,
cancel: Option<&AtomicBool>,
) -> Tally {
let mut reader = BufReader::new(reader);
let mut line = String::new();
let mut tally = Tally::default();
let mut tool: Option<(String, String, String)> = None;
let mut finished = false;
loop {
if stopped(cancel) {
let _ = tx.send(Chunk::Failed(Error::Cancelled));
return tally;
}
line.clear();
match reader.read_line(&mut line) {
Ok(0) => break,
Ok(_) => {}
Err(error) => {
let _ = tx.send(Chunk::Failed(Error::Stream(error.to_string())));
return tally;
}
}
let Some(payload) = line.trim_end_matches(['\r', '\n']).strip_prefix("data:") else {
continue;
};
let Ok(value) = serde_json::from_str::<Value>(payload.trim_start()) else {
continue;
};
match value.get("type").and_then(Value::as_str).unwrap_or("") {
"message_start" => {
tally.input = value
.pointer("/message/usage/input_tokens")
.and_then(Value::as_u64)
.unwrap_or(0) as u32;
}
"content_block_start"
if value.pointer("/content_block/type").and_then(Value::as_str)
== Some("tool_use") =>
{
tool = Some((
value
.pointer("/content_block/id")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string(),
value
.pointer("/content_block/name")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string(),
String::new(),
));
}
"content_block_delta" => match value.pointer("/delta/type").and_then(Value::as_str) {
Some("text_delta") => {
if let Some(text) = value
.pointer("/delta/text")
.and_then(Value::as_str)
.filter(|text| !text.is_empty())
{
let _ = tx.send(Chunk::Text(text.to_string()));
}
}
Some("input_json_delta") => {
if let (Some((_, _, buffer)), Some(part)) = (
tool.as_mut(),
value.pointer("/delta/partial_json").and_then(Value::as_str),
) {
buffer.push_str(part);
}
}
_ => {}
},
"content_block_stop" => {
if let Some((id, name, raw)) = tool.take() {
if !name.is_empty() {
let _ = tx.send(Chunk::Tool(
ToolCall::new(name, arguments(&raw)).with_id(id),
));
}
}
}
"message_delta" => {
tally.output = value
.pointer("/usage/output_tokens")
.and_then(Value::as_u64)
.unwrap_or(tally.output as u64) as u32;
}
"message_stop" => {
finished = true;
send_done(tx, &tally);
break;
}
"error" => {
let message = value
.pointer("/error/message")
.and_then(Value::as_str)
.unwrap_or("model returned an error");
let _ = tx.send(reported(message));
return tally;
}
_ => {}
}
}
if !finished {
send_done(tx, &tally);
}
tally
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::mpsc::channel;
fn tool_of(chunks: &[Chunk]) -> &ToolCall {
chunks
.iter()
.find_map(|chunk| match chunk {
Chunk::Tool(call) => Some(call),
_ => None,
})
.expect("a tool call")
}
#[test]
fn chat_reassembles_tool_arguments_and_keeps_the_call_id() {
let data = concat!(
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"weather\",\"arguments\":\"{\\\"city\\\":\"}}]}}]}\n\n",
"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Paris\\\"}\"}}]}}]}\n\n",
"data: [DONE]\n\n",
);
let (tx, rx) = channel();
chat(data.as_bytes(), &tx, None);
drop(tx);
let chunks: Vec<_> = rx.into_iter().collect();
let call = tool_of(&chunks);
assert_eq!(call.name, "weather");
assert_eq!(call.id, "call_a");
assert_eq!(call.args["city"], "Paris");
}
#[test]
fn responses_emits_text_tool_and_usage() {
let data = concat!(
"data: {\"type\":\"response.output_text.delta\",\"delta\":\"Hello\"}\n\n",
"data: {\"type\":\"response.output_item.done\",\"item\":{\"type\":\"function_call\",\"call_id\":\"fc_1\",\"name\":\"lookup\",\"arguments\":\"{\\\"term\\\":\\\"hola\\\"}\"}}\n\n",
"data: {\"type\":\"response.completed\",\"response\":{\"usage\":{\"input_tokens\":7,\"output_tokens\":3,\"cost\":0.00002}}}\n\n",
);
let (tx, rx) = channel();
let tally = responses(data.as_bytes(), &tx, None);
drop(tx);
let chunks: Vec<_> = rx.into_iter().collect();
assert_eq!(
tally,
Tally {
input: 7,
output: 3,
cost_micros: 20
}
);
assert!(chunks
.iter()
.any(|chunk| matches!(chunk, Chunk::Text(text) if text == "Hello")));
assert_eq!(tool_of(&chunks).id, "fc_1");
assert_eq!(tool_of(&chunks).args["term"], "hola");
assert!(matches!(chunks.last(), Some(Chunk::Done { .. })));
}
#[test]
fn anthropic_emits_text_tool_and_usage() {
let data = concat!(
"data: {\"type\":\"message_start\",\"message\":{\"usage\":{\"input_tokens\":11}}}\n",
"data: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"Bonjour\"}}\n",
"data: {\"type\":\"content_block_start\",\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_1\",\"name\":\"define\"}}\n",
"data: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"word\\\":\"}}\n",
"data: {\"type\":\"content_block_delta\",\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"salut\\\"}\"}}\n",
"data: {\"type\":\"content_block_stop\"}\n",
"data: {\"type\":\"message_delta\",\"usage\":{\"output_tokens\":5}}\n",
"data: {\"type\":\"message_stop\"}\n",
);
let (tx, rx) = channel();
let tally = anthropic(data.as_bytes(), &tx, None);
drop(tx);
let chunks: Vec<_> = rx.into_iter().collect();
assert_eq!(tally.input, 11);
assert_eq!(tally.output, 5);
assert_eq!(tool_of(&chunks).id, "toolu_1");
assert_eq!(tool_of(&chunks).args["word"], "salut");
assert!(matches!(chunks.last(), Some(Chunk::Done { .. })));
}
#[test]
fn truncated_streams_still_finish() {
let data = "data: {\"type\":\"response.output_text.delta\",\"delta\":\"partial\"}\n\n";
let (tx, rx) = channel();
responses(data.as_bytes(), &tx, None);
drop(tx);
let chunks: Vec<_> = rx.into_iter().collect();
assert!(matches!(chunks.first(), Some(Chunk::Text(text)) if text == "partial"));
assert!(matches!(
chunks.last(),
Some(Chunk::Done {
input_tokens: 0,
output_tokens: 0
})
));
}
#[test]
fn a_raised_cancel_flag_ends_every_decoder_with_cancelled() {
let cancel = AtomicBool::new(true);
let data = "data: {\"type\":\"message_stop\"}\n".as_bytes();
let cancelled = |decode: &dyn Fn(&Sender<Chunk>) -> Tally| {
let (tx, rx) = channel();
decode(&tx);
drop(tx);
matches!(rx.into_iter().last(), Some(Chunk::Failed(Error::Cancelled)))
};
assert!(cancelled(&|tx| responses(data, tx, Some(&cancel))));
assert!(cancelled(&|tx| chat(data, tx, Some(&cancel))));
assert!(cancelled(&|tx| anthropic(data, tx, Some(&cancel))));
}
#[test]
fn a_mid_stream_provider_error_is_reported_as_a_stream_failure() {
let (tx, rx) = channel();
chat(
"data: {\"error\":{\"message\":\"context length exceeded\"}}\n".as_bytes(),
&tx,
None,
);
drop(tx);
let chunks: Vec<_> = rx.into_iter().collect();
assert!(matches!(
chunks.first(),
Some(Chunk::Failed(Error::Stream(message))) if message.contains("context length")
));
}
}