use std::collections::BTreeMap;
use std::fmt;
use std::pin::Pin;
use futures::future::BoxFuture;
use futures::{Stream, StreamExt};
use graphrt_core::event::Usage;
use graphrt_core::tool::ToolSpec;
use serde::{Deserialize, Serialize};
use crate::error::LlmError;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
#[non_exhaustive]
pub enum Role {
System,
User,
Assistant,
Tool,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
#[non_exhaustive]
pub enum ContentPart {
Text {
text: String,
},
Image {
url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
detail: Option<String>,
},
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ToolCall {
pub id: String,
pub name: String,
pub arguments: String,
}
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
#[error("tool call `{name}` has invalid JSON arguments: {reason}")]
pub struct InvalidToolArguments {
pub name: String,
pub reason: String,
}
impl ToolCall {
pub fn arguments_json(&self) -> Result<serde_json::Map<String, serde_json::Value>, InvalidToolArguments> {
if self.arguments.trim().is_empty() {
return Ok(serde_json::Map::new());
}
let invalid = |reason: String| InvalidToolArguments { name: self.name.clone(), reason };
match serde_json::from_str::<serde_json::Value>(&self.arguments) {
Ok(serde_json::Value::Object(map)) => Ok(map),
Ok(other) => Err(invalid(format!("expected a JSON object, got {}", json_kind(&other)))),
Err(e) => Err(invalid(e.to_string())),
}
}
}
fn json_kind(v: &serde_json::Value) -> &'static str {
match v {
serde_json::Value::Null => "null",
serde_json::Value::Bool(_) => "a boolean",
serde_json::Value::Number(_) => "a number",
serde_json::Value::String(_) => "a string",
serde_json::Value::Array(_) => "an array",
serde_json::Value::Object(_) => "an object",
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Message {
pub role: Role,
#[serde(default)]
pub content: Vec<ContentPart>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tool_calls: Vec<ToolCall>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tool_call_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_state: Option<serde_json::Value>,
}
impl Message {
fn with_text(role: Role, text: impl Into<String>) -> Self {
Self {
role,
content: vec![ContentPart::Text { text: text.into() }],
tool_calls: Vec::new(),
tool_call_id: None,
provider_state: None,
}
}
pub fn system(text: impl Into<String>) -> Self {
Self::with_text(Role::System, text)
}
pub fn user(text: impl Into<String>) -> Self {
Self::with_text(Role::User, text)
}
pub fn assistant(text: impl Into<String>) -> Self {
Self::with_text(Role::Assistant, text)
}
pub fn tool(call_id: impl Into<String>, text: impl Into<String>) -> Self {
let mut m = Self::with_text(Role::Tool, text);
m.tool_call_id = Some(call_id.into());
m
}
pub fn text(&self) -> String {
let mut out = String::new();
for part in &self.content {
if let ContentPart::Text { text } = part {
out.push_str(text);
}
}
out
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum ToolChoice {
Auto,
None,
Required,
Tool(String),
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
#[non_exhaustive]
pub enum ResponseFormat {
Text,
JsonObject,
JsonSchema {
name: String,
schema: serde_json::Value,
#[serde(default)]
strict: bool,
},
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct ChatRequest {
pub model: String,
pub messages: Vec<Message>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tools: Vec<ToolSpec>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tool_choice: Option<ToolChoice>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub temperature: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub top_p: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u32>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub stop: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub response_format: Option<ResponseFormat>,
#[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
pub extra: serde_json::Map<String, serde_json::Value>,
}
impl ChatRequest {
pub fn new(model: impl Into<String>, messages: Vec<Message>) -> Self {
Self { model: model.into(), messages, ..Self::default() }
}
pub fn with_tools(mut self, tools: Vec<ToolSpec>) -> Self {
self.tools = tools;
self
}
pub fn with_tool_choice(mut self, choice: ToolChoice) -> Self {
self.tool_choice = Some(choice);
self
}
pub fn with_temperature(mut self, temperature: f64) -> Self {
self.temperature = Some(temperature);
self
}
pub fn with_top_p(mut self, top_p: f64) -> Self {
self.top_p = Some(top_p);
self
}
pub fn with_max_tokens(mut self, max_tokens: u32) -> Self {
self.max_tokens = Some(max_tokens);
self
}
pub fn with_stop(mut self, stop: Vec<String>) -> Self {
self.stop = stop;
self
}
pub fn with_response_format(mut self, format: ResponseFormat) -> Self {
self.response_format = Some(format);
self
}
pub fn with_extra(mut self, key: impl Into<String>, value: serde_json::Value) -> Self {
self.extra.insert(key.into(), value);
self
}
}
pub const RESERVED_EXTRA_KEYS: &[&str] = &["model", "messages", "stream", "stream_options", "tools", "tool_choice"];
pub fn check_extra(req: &ChatRequest) -> Result<(), LlmError> {
for key in req.extra.keys() {
if RESERVED_EXTRA_KEYS.contains(&key.as_str()) {
return Err(LlmError::Config(format!(
"parameter `{key}` cannot be set through extra parameters; use the dedicated request field"
)));
}
}
Ok(())
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum FinishReason {
Stop,
Length,
ToolCalls,
ContentFilter,
Other(String),
}
impl FinishReason {
pub fn as_str(&self) -> &str {
match self {
FinishReason::Stop => "stop",
FinishReason::Length => "length",
FinishReason::ToolCalls => "tool_calls",
FinishReason::ContentFilter => "content_filter",
FinishReason::Other(s) => s,
}
}
}
#[derive(Clone, Debug, PartialEq)]
#[non_exhaustive]
pub enum ChatDelta {
Text(String),
Reasoning(String),
ToolCall {
index: usize,
id: Option<String>,
name: Option<String>,
arguments: String,
},
Usage(Usage),
Finish(FinishReason),
ProviderState(serde_json::Value),
Warning(String),
}
pub type ChatStream = Pin<Box<dyn Stream<Item = Result<ChatDelta, LlmError>> + Send>>;
pub trait ChatModel: Send + Sync + fmt::Debug {
fn chat(&self, request: ChatRequest) -> BoxFuture<'_, Result<ChatStream, LlmError>>;
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct ChatResponse {
pub text: String,
pub reasoning: String,
pub tool_calls: Vec<ToolCall>,
pub usage: Usage,
pub finish_reason: Option<FinishReason>,
pub provider_state: Option<serde_json::Value>,
pub warnings: Vec<String>,
}
impl ChatResponse {
pub fn to_message(&self) -> Message {
let content =
if self.text.is_empty() { Vec::new() } else { vec![ContentPart::Text { text: self.text.clone() }] };
Message {
role: Role::Assistant,
content,
tool_calls: self.tool_calls.clone(),
tool_call_id: None,
provider_state: self.provider_state.clone(),
}
}
}
#[derive(Default)]
struct PartialCall {
id: Option<String>,
name: String,
arguments: String,
}
pub async fn collect(mut stream: ChatStream, mut on_delta: impl FnMut(&ChatDelta)) -> Result<ChatResponse, LlmError> {
let mut response = ChatResponse::default();
let mut calls: BTreeMap<usize, PartialCall> = BTreeMap::new();
while let Some(delta) = stream.next().await {
let delta = delta?;
on_delta(&delta);
match delta {
ChatDelta::Text(t) => response.text.push_str(&t),
ChatDelta::Reasoning(t) => response.reasoning.push_str(&t),
ChatDelta::ToolCall { index, id, name, arguments } => {
let call = calls.entry(index).or_default();
if let Some(id) = id.filter(|s| !s.is_empty()) {
call.id = Some(id);
}
if let Some(name) = name.filter(|s| !s.is_empty()) {
call.name = name;
}
call.arguments.push_str(&arguments);
}
ChatDelta::Usage(u) => response.usage = u,
ChatDelta::Finish(r) => response.finish_reason = Some(r),
ChatDelta::ProviderState(s) => response.provider_state = Some(s),
ChatDelta::Warning(w) => response.warnings.push(w),
}
}
let mut tool_calls = Vec::with_capacity(calls.len());
for (index, call) in calls {
if call.name.is_empty() {
if call.id.is_none() && call.arguments.trim().is_empty() {
continue;
}
return Err(LlmError::Decode(format!("tool call #{index} has arguments or an id but no name")));
}
tool_calls.push(ToolCall {
id: call.id.unwrap_or_else(|| format!("call_{index}")),
name: call.name,
arguments: call.arguments,
});
}
response.tool_calls = tool_calls;
if response.usage.total_tokens == 0 {
response.usage.total_tokens = response.usage.prompt_tokens.saturating_add(response.usage.completion_tokens);
}
Ok(response)
}
#[cfg(test)]
mod tests {
use super::*;
fn stream(deltas: Vec<ChatDelta>) -> ChatStream {
Box::pin(futures::stream::iter(deltas.into_iter().map(Ok)))
}
#[tokio::test]
async fn interleaved_parallel_tool_calls_assemble_by_index() {
let deltas = vec![
ChatDelta::ToolCall {
index: 0,
id: Some("a".into()),
name: Some("search".into()),
arguments: "{\"q\":".into(),
},
ChatDelta::ToolCall {
index: 1,
id: Some("b".into()),
name: Some("fetch".into()),
arguments: "{\"url\":".into(),
},
ChatDelta::ToolCall { index: 0, id: None, name: None, arguments: "\"rust\"}".into() },
ChatDelta::ToolCall { index: 1, id: None, name: None, arguments: "\"x\"}".into() },
ChatDelta::Usage(Usage { prompt_tokens: 3, completion_tokens: 4, total_tokens: 0 }),
ChatDelta::Finish(FinishReason::ToolCalls),
];
let r = collect(stream(deltas), |_| {}).await.unwrap();
assert_eq!(r.tool_calls.len(), 2);
assert_eq!(r.tool_calls[0].arguments, "{\"q\":\"rust\"}");
assert_eq!(r.tool_calls[1].id, "b");
assert_eq!(r.tool_calls[1].arguments_json().unwrap()["url"], "x");
assert_eq!(r.usage.total_tokens, 7);
assert_eq!(r.finish_reason, Some(FinishReason::ToolCalls));
}
#[tokio::test]
async fn nameless_tool_call_is_an_error() {
let deltas = vec![ChatDelta::ToolCall { index: 0, id: Some("a".into()), name: None, arguments: "{}".into() }];
let err = collect(stream(deltas), |_| {}).await.unwrap_err();
assert!(matches!(err, LlmError::Decode(_)), "{err:?}");
let deltas = vec![ChatDelta::ToolCall { index: 0, id: None, name: None, arguments: String::new() }];
assert!(collect(stream(deltas), |_| {}).await.unwrap().tool_calls.is_empty());
}
#[test]
fn arguments_json_rejects_malformed_input() {
let call = |args: &str| ToolCall { id: "1".into(), name: "t".into(), arguments: args.into() };
assert!(call("").arguments_json().unwrap().is_empty());
assert_eq!(call("{\"a\":1}").arguments_json().unwrap()["a"], 1);
assert!(call("{\"a\":").arguments_json().is_err());
let err = call("[1]").arguments_json().unwrap_err();
assert!(err.to_string().contains("an array"), "{err}");
}
#[test]
fn reserved_extra_keys_are_rejected() {
let req = ChatRequest::new("m", vec![]).with_extra("stream", serde_json::json!(false));
assert!(matches!(check_extra(&req), Err(LlmError::Config(_))));
let req = ChatRequest::new("m", vec![]).with_extra("seed", serde_json::json!(1)).with_temperature(0.1);
assert!(check_extra(&req).is_ok());
assert_eq!(req.temperature, Some(0.1));
}
#[tokio::test]
async fn text_and_reasoning_accumulate() {
let deltas =
vec![ChatDelta::Reasoning("hm".into()), ChatDelta::Text("he".into()), ChatDelta::Text("llo".into())];
let mut seen = 0;
let r = collect(stream(deltas), |_| seen += 1).await.unwrap();
assert_eq!(seen, 3);
assert_eq!(r.text, "hello");
assert_eq!(r.reasoning, "hm");
assert!(r.tool_calls.is_empty());
}
}