#![allow(clippy::result_large_err)]
use std::sync::Arc;
use futures::stream::{Stream, StreamExt};
use http::header::ACCEPT;
use http::Method;
use serde_json::Value;
use super::per_call::{CallFailure, CallMeta};
use super::{raw, Client};
use crate::streaming::{sealed, InterceptedItem, InterceptedStream};
use crate::time::Instant;
use crate::{Error, MaybeSend, MaybeSync, Result};
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct JsonCallResponse {
pub body: Value,
pub meta: CallMeta,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct JsonEvent(pub Value);
impl sealed::Sealed for JsonEvent {}
impl InterceptedItem for JsonEvent {
fn chunk_json(&self) -> String {
self.0.to_string()
}
fn token_usage(&self) -> Option<(i64, i64)> {
let usage = self
.0
.get("response")
.and_then(|response| response.get("usage"))
.or_else(|| self.0.get("usage"))?;
token_usage(usage)
}
}
pub type JsonEventStream = crate::BoxStream<'static, Result<JsonEvent>>;
fn token_usage(usage: &Value) -> Option<(i64, i64)> {
let tokens = |names: [&str; 2]| {
names
.iter()
.find_map(|name| usage.get(*name).and_then(Value::as_i64))
};
Some((
tokens(["input_tokens", "prompt_tokens"])?,
tokens(["output_tokens", "completion_tokens"])?,
))
}
impl<T: Default + MaybeSend + MaybeSync + 'static> Client<T> {
pub async fn json_call(
&self,
operation: &str,
path: &str,
body: &Value,
) -> std::result::Result<JsonCallResponse, CallFailure> {
let model = model_of(body);
let request_json = body.to_string();
let mut state = T::default();
self.call_before_request(operation, &model, &request_json, &mut state)
.await?;
let start_time = Instant::now();
let sent = self
.raw_request(Method::POST, path)
.header(ACCEPT, "application/json")
.json(body)
.send()
.await;
let failure = match sent {
Ok(response) if response.status().is_success() => {
let meta = CallMeta::from_response(response.status().as_u16(), response.headers());
let decoded = match response.text().await {
Ok(text) => serde_json::from_str::<Value>(&text).map_err(Error::Json),
Err(error) => Err(Error::Http(error)),
};
match decoded {
Ok(body) => {
let tokens = body.get("usage").and_then(token_usage);
self.call_after_response(
&body,
operation,
&model,
&request_json,
&state,
start_time.elapsed(),
tokens.map(|(input, _)| input),
tokens.map(|(_, output)| output),
)
.await;
return Ok(JsonCallResponse { body, meta });
}
Err(error) => CallFailure::after_body(error, meta),
}
}
Ok(response) => {
let meta = CallMeta::from_response(response.status().as_u16(), response.headers());
let error = match raw::check_status(response).await {
Err(error) => error,
Ok(_) => Error::Internal("a failed response was a success".into()),
};
CallFailure::before_body_with(error, meta)
}
Err(error) => CallFailure::before_body_with(Error::Http(error), CallMeta::default()),
};
self.report_error(failure.error(), operation, &model, &request_json, &state)
.await;
Err(failure)
}
pub async fn json_event_call(
&self,
operation: &str,
path: &str,
body: &Value,
) -> std::result::Result<(JsonEventStream, CallMeta), CallFailure> {
let model = model_of(body);
let request_json = body.to_string();
let mut state = T::default();
self.call_before_request(operation, &model, &request_json, &mut state)
.await?;
let sent = self
.raw_request(Method::POST, path)
.header(ACCEPT, "text/event-stream")
.json(body)
.send()
.await;
let failure = match sent {
Ok(response) if response.status().is_success() => {
let meta = CallMeta::from_response(response.status().as_u16(), response.headers());
let events = crate::time::idle_timeout(
json_events(response),
self.config.stream_idle_timeout(),
);
if self.interceptors.is_empty() {
return Ok((events, meta));
}
let stream = InterceptedStream::new(
events,
Arc::clone(&self.interceptors),
operation.to_string(),
model,
request_json,
state,
);
return Ok((crate::box_stream(stream), meta));
}
Ok(response) => {
let meta = CallMeta::from_response(response.status().as_u16(), response.headers());
let error = match raw::check_status(response).await {
Err(error) => error,
Ok(_) => Error::Internal("a failed response was a success".into()),
};
CallFailure::before_body_with(error, meta)
}
Err(error) => CallFailure::before_body_with(Error::Http(error), CallMeta::default()),
};
self.report_error(failure.error(), operation, &model, &request_json, &state)
.await;
Err(failure)
}
}
fn model_of(body: &Value) -> String {
body.get("model")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string()
}
fn json_events(
response: timvw_rust_runtime::Response,
) -> impl Stream<Item = Result<JsonEvent>> + MaybeSend + 'static {
response.events().filter_map(|event| async move {
match event {
Ok(event) if event.data.trim().is_empty() => None,
Ok(event) => Some(
serde_json::from_str::<Value>(&event.data)
.map(JsonEvent)
.map_err(|error| Error::StreamParsing {
message: format!("Failed to deserialize event: {error}"),
chunk: event.data,
}),
),
Err(error) => Some(Err(Error::StreamConnection {
message: format!("Stream connection error: {error}"),
})),
}
})
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn reads_the_token_usage_of_both_apis() {
let responses = JsonEvent(json!({
"type": "response.completed",
"response": {"usage": {"input_tokens": 3, "output_tokens": 4}}
}));
assert_eq!(responses.token_usage(), Some((3, 4)));
let chat = JsonEvent(json!({"usage": {"prompt_tokens": 5, "completion_tokens": 6}}));
assert_eq!(chat.token_usage(), Some((5, 6)));
assert_eq!(JsonEvent(json!({"type": "x"})).token_usage(), None);
}
}