1use std::collections::BTreeMap;
8use std::fmt;
9use std::pin::Pin;
10
11use futures::future::BoxFuture;
12use futures::{Stream, StreamExt};
13use graphrt_core::event::Usage;
14use graphrt_core::tool::ToolSpec;
15use serde::{Deserialize, Serialize};
16
17use crate::error::LlmError;
18
19#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
21#[serde(rename_all = "lowercase")]
22#[non_exhaustive]
23pub enum Role {
24 System,
25 User,
26 Assistant,
27 Tool,
28}
29
30#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
32#[serde(tag = "type", rename_all = "snake_case")]
33#[non_exhaustive]
34pub enum ContentPart {
35 Text {
36 text: String,
37 },
38 Image {
40 url: String,
41 #[serde(default, skip_serializing_if = "Option::is_none")]
42 detail: Option<String>,
43 },
44}
45
46#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
48pub struct ToolCall {
49 pub id: String,
50 pub name: String,
51 pub arguments: String,
52}
53
54#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
56#[error("tool call `{name}` has invalid JSON arguments: {reason}")]
57pub struct InvalidToolArguments {
58 pub name: String,
60 pub reason: String,
61}
62
63impl ToolCall {
64 pub fn arguments_json(&self) -> Result<serde_json::Map<String, serde_json::Value>, InvalidToolArguments> {
68 if self.arguments.trim().is_empty() {
69 return Ok(serde_json::Map::new());
70 }
71 let invalid = |reason: String| InvalidToolArguments { name: self.name.clone(), reason };
72 match serde_json::from_str::<serde_json::Value>(&self.arguments) {
73 Ok(serde_json::Value::Object(map)) => Ok(map),
74 Ok(other) => Err(invalid(format!("expected a JSON object, got {}", json_kind(&other)))),
75 Err(e) => Err(invalid(e.to_string())),
76 }
77 }
78}
79
80fn json_kind(v: &serde_json::Value) -> &'static str {
81 match v {
82 serde_json::Value::Null => "null",
83 serde_json::Value::Bool(_) => "a boolean",
84 serde_json::Value::Number(_) => "a number",
85 serde_json::Value::String(_) => "a string",
86 serde_json::Value::Array(_) => "an array",
87 serde_json::Value::Object(_) => "an object",
88 }
89}
90
91#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
93pub struct Message {
94 pub role: Role,
95 #[serde(default)]
96 pub content: Vec<ContentPart>,
97 #[serde(default, skip_serializing_if = "Vec::is_empty")]
99 pub tool_calls: Vec<ToolCall>,
100 #[serde(default, skip_serializing_if = "Option::is_none")]
102 pub tool_call_id: Option<String>,
103 #[serde(default, skip_serializing_if = "Option::is_none")]
106 pub provider_state: Option<serde_json::Value>,
107}
108
109impl Message {
110 fn with_text(role: Role, text: impl Into<String>) -> Self {
111 Self {
112 role,
113 content: vec![ContentPart::Text { text: text.into() }],
114 tool_calls: Vec::new(),
115 tool_call_id: None,
116 provider_state: None,
117 }
118 }
119
120 pub fn system(text: impl Into<String>) -> Self {
121 Self::with_text(Role::System, text)
122 }
123
124 pub fn user(text: impl Into<String>) -> Self {
125 Self::with_text(Role::User, text)
126 }
127
128 pub fn assistant(text: impl Into<String>) -> Self {
129 Self::with_text(Role::Assistant, text)
130 }
131
132 pub fn tool(call_id: impl Into<String>, text: impl Into<String>) -> Self {
134 let mut m = Self::with_text(Role::Tool, text);
135 m.tool_call_id = Some(call_id.into());
136 m
137 }
138
139 pub fn text(&self) -> String {
141 let mut out = String::new();
142 for part in &self.content {
143 if let ContentPart::Text { text } = part {
144 out.push_str(text);
145 }
146 }
147 out
148 }
149}
150
151#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
153#[serde(rename_all = "snake_case")]
154#[non_exhaustive]
155pub enum ToolChoice {
156 Auto,
157 None,
158 Required,
160 Tool(String),
162}
163
164#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
166#[serde(tag = "type", rename_all = "snake_case")]
167#[non_exhaustive]
168pub enum ResponseFormat {
169 Text,
170 JsonObject,
171 JsonSchema {
172 name: String,
173 schema: serde_json::Value,
174 #[serde(default)]
175 strict: bool,
176 },
177}
178
179#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
181pub struct ChatRequest {
182 pub model: String,
183 pub messages: Vec<Message>,
184 #[serde(default, skip_serializing_if = "Vec::is_empty")]
185 pub tools: Vec<ToolSpec>,
186 #[serde(default, skip_serializing_if = "Option::is_none")]
187 pub tool_choice: Option<ToolChoice>,
188 #[serde(default, skip_serializing_if = "Option::is_none")]
189 pub temperature: Option<f64>,
190 #[serde(default, skip_serializing_if = "Option::is_none")]
191 pub top_p: Option<f64>,
192 #[serde(default, skip_serializing_if = "Option::is_none")]
193 pub max_tokens: Option<u32>,
194 #[serde(default, skip_serializing_if = "Vec::is_empty")]
195 pub stop: Vec<String>,
196 #[serde(default, skip_serializing_if = "Option::is_none")]
197 pub response_format: Option<ResponseFormat>,
198 #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
200 pub extra: serde_json::Map<String, serde_json::Value>,
201}
202
203impl ChatRequest {
204 pub fn new(model: impl Into<String>, messages: Vec<Message>) -> Self {
205 Self { model: model.into(), messages, ..Self::default() }
206 }
207
208 pub fn with_tools(mut self, tools: Vec<ToolSpec>) -> Self {
210 self.tools = tools;
211 self
212 }
213
214 pub fn with_tool_choice(mut self, choice: ToolChoice) -> Self {
216 self.tool_choice = Some(choice);
217 self
218 }
219
220 pub fn with_temperature(mut self, temperature: f64) -> Self {
221 self.temperature = Some(temperature);
222 self
223 }
224
225 pub fn with_top_p(mut self, top_p: f64) -> Self {
226 self.top_p = Some(top_p);
227 self
228 }
229
230 pub fn with_max_tokens(mut self, max_tokens: u32) -> Self {
231 self.max_tokens = Some(max_tokens);
232 self
233 }
234
235 pub fn with_stop(mut self, stop: Vec<String>) -> Self {
236 self.stop = stop;
237 self
238 }
239
240 pub fn with_response_format(mut self, format: ResponseFormat) -> Self {
241 self.response_format = Some(format);
242 self
243 }
244
245 pub fn with_extra(mut self, key: impl Into<String>, value: serde_json::Value) -> Self {
247 self.extra.insert(key.into(), value);
248 self
249 }
250}
251
252pub const RESERVED_EXTRA_KEYS: &[&str] = &["model", "messages", "stream", "stream_options", "tools", "tool_choice"];
254
255pub fn check_extra(req: &ChatRequest) -> Result<(), LlmError> {
258 for key in req.extra.keys() {
259 if RESERVED_EXTRA_KEYS.contains(&key.as_str()) {
260 return Err(LlmError::Config(format!(
261 "parameter `{key}` cannot be set through extra parameters; use the dedicated request field"
262 )));
263 }
264 }
265 Ok(())
266}
267
268#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
270#[serde(rename_all = "snake_case")]
271#[non_exhaustive]
272pub enum FinishReason {
273 Stop,
274 Length,
275 ToolCalls,
276 ContentFilter,
277 Other(String),
278}
279
280impl FinishReason {
281 pub fn as_str(&self) -> &str {
283 match self {
284 FinishReason::Stop => "stop",
285 FinishReason::Length => "length",
286 FinishReason::ToolCalls => "tool_calls",
287 FinishReason::ContentFilter => "content_filter",
288 FinishReason::Other(s) => s,
289 }
290 }
291}
292
293#[derive(Clone, Debug, PartialEq)]
295#[non_exhaustive]
296pub enum ChatDelta {
297 Text(String),
299 Reasoning(String),
301 ToolCall {
304 index: usize,
305 id: Option<String>,
306 name: Option<String>,
307 arguments: String,
308 },
309 Usage(Usage),
311 Finish(FinishReason),
312 ProviderState(serde_json::Value),
314 Warning(String),
317}
318
319pub type ChatStream = Pin<Box<dyn Stream<Item = Result<ChatDelta, LlmError>> + Send>>;
321
322pub trait ChatModel: Send + Sync + fmt::Debug {
324 fn chat(&self, request: ChatRequest) -> BoxFuture<'_, Result<ChatStream, LlmError>>;
326}
327
328#[derive(Clone, Debug, Default, PartialEq)]
330pub struct ChatResponse {
331 pub text: String,
332 pub reasoning: String,
333 pub tool_calls: Vec<ToolCall>,
334 pub usage: Usage,
335 pub finish_reason: Option<FinishReason>,
336 pub provider_state: Option<serde_json::Value>,
337 pub warnings: Vec<String>,
339}
340
341impl ChatResponse {
342 pub fn to_message(&self) -> Message {
344 let content =
345 if self.text.is_empty() { Vec::new() } else { vec![ContentPart::Text { text: self.text.clone() }] };
346 Message {
347 role: Role::Assistant,
348 content,
349 tool_calls: self.tool_calls.clone(),
350 tool_call_id: None,
351 provider_state: self.provider_state.clone(),
352 }
353 }
354}
355
356#[derive(Default)]
357struct PartialCall {
358 id: Option<String>,
359 name: String,
360 arguments: String,
361}
362
363pub async fn collect(mut stream: ChatStream, mut on_delta: impl FnMut(&ChatDelta)) -> Result<ChatResponse, LlmError> {
374 let mut response = ChatResponse::default();
375 let mut calls: BTreeMap<usize, PartialCall> = BTreeMap::new();
376 while let Some(delta) = stream.next().await {
377 let delta = delta?;
378 on_delta(&delta);
379 match delta {
380 ChatDelta::Text(t) => response.text.push_str(&t),
381 ChatDelta::Reasoning(t) => response.reasoning.push_str(&t),
382 ChatDelta::ToolCall { index, id, name, arguments } => {
383 let call = calls.entry(index).or_default();
384 if let Some(id) = id.filter(|s| !s.is_empty()) {
385 call.id = Some(id);
386 }
387 if let Some(name) = name.filter(|s| !s.is_empty()) {
388 call.name = name;
389 }
390 call.arguments.push_str(&arguments);
391 }
392 ChatDelta::Usage(u) => response.usage = u,
393 ChatDelta::Finish(r) => response.finish_reason = Some(r),
394 ChatDelta::ProviderState(s) => response.provider_state = Some(s),
395 ChatDelta::Warning(w) => response.warnings.push(w),
396 }
397 }
398 let mut tool_calls = Vec::with_capacity(calls.len());
399 for (index, call) in calls {
400 if call.name.is_empty() {
401 if call.id.is_none() && call.arguments.trim().is_empty() {
402 continue;
404 }
405 return Err(LlmError::Decode(format!("tool call #{index} has arguments or an id but no name")));
406 }
407 tool_calls.push(ToolCall {
408 id: call.id.unwrap_or_else(|| format!("call_{index}")),
409 name: call.name,
410 arguments: call.arguments,
411 });
412 }
413 response.tool_calls = tool_calls;
414 if response.usage.total_tokens == 0 {
415 response.usage.total_tokens = response.usage.prompt_tokens.saturating_add(response.usage.completion_tokens);
416 }
417 Ok(response)
418}
419
420#[cfg(test)]
421mod tests {
422 use super::*;
423
424 fn stream(deltas: Vec<ChatDelta>) -> ChatStream {
425 Box::pin(futures::stream::iter(deltas.into_iter().map(Ok)))
426 }
427
428 #[tokio::test]
429 async fn interleaved_parallel_tool_calls_assemble_by_index() {
430 let deltas = vec![
431 ChatDelta::ToolCall {
432 index: 0,
433 id: Some("a".into()),
434 name: Some("search".into()),
435 arguments: "{\"q\":".into(),
436 },
437 ChatDelta::ToolCall {
438 index: 1,
439 id: Some("b".into()),
440 name: Some("fetch".into()),
441 arguments: "{\"url\":".into(),
442 },
443 ChatDelta::ToolCall { index: 0, id: None, name: None, arguments: "\"rust\"}".into() },
444 ChatDelta::ToolCall { index: 1, id: None, name: None, arguments: "\"x\"}".into() },
445 ChatDelta::Usage(Usage { prompt_tokens: 3, completion_tokens: 4, total_tokens: 0 }),
446 ChatDelta::Finish(FinishReason::ToolCalls),
447 ];
448 let r = collect(stream(deltas), |_| {}).await.unwrap();
449 assert_eq!(r.tool_calls.len(), 2);
450 assert_eq!(r.tool_calls[0].arguments, "{\"q\":\"rust\"}");
451 assert_eq!(r.tool_calls[1].id, "b");
452 assert_eq!(r.tool_calls[1].arguments_json().unwrap()["url"], "x");
453 assert_eq!(r.usage.total_tokens, 7);
454 assert_eq!(r.finish_reason, Some(FinishReason::ToolCalls));
455 }
456
457 #[tokio::test]
458 async fn nameless_tool_call_is_an_error() {
459 let deltas = vec![ChatDelta::ToolCall { index: 0, id: Some("a".into()), name: None, arguments: "{}".into() }];
460 let err = collect(stream(deltas), |_| {}).await.unwrap_err();
461 assert!(matches!(err, LlmError::Decode(_)), "{err:?}");
462 let deltas = vec![ChatDelta::ToolCall { index: 0, id: None, name: None, arguments: String::new() }];
464 assert!(collect(stream(deltas), |_| {}).await.unwrap().tool_calls.is_empty());
465 }
466
467 #[test]
468 fn arguments_json_rejects_malformed_input() {
469 let call = |args: &str| ToolCall { id: "1".into(), name: "t".into(), arguments: args.into() };
470 assert!(call("").arguments_json().unwrap().is_empty());
471 assert_eq!(call("{\"a\":1}").arguments_json().unwrap()["a"], 1);
472 assert!(call("{\"a\":").arguments_json().is_err());
473 let err = call("[1]").arguments_json().unwrap_err();
474 assert!(err.to_string().contains("an array"), "{err}");
475 }
476
477 #[test]
478 fn reserved_extra_keys_are_rejected() {
479 let req = ChatRequest::new("m", vec![]).with_extra("stream", serde_json::json!(false));
480 assert!(matches!(check_extra(&req), Err(LlmError::Config(_))));
481 let req = ChatRequest::new("m", vec![]).with_extra("seed", serde_json::json!(1)).with_temperature(0.1);
482 assert!(check_extra(&req).is_ok());
483 assert_eq!(req.temperature, Some(0.1));
484 }
485
486 #[tokio::test]
487 async fn text_and_reasoning_accumulate() {
488 let deltas =
489 vec![ChatDelta::Reasoning("hm".into()), ChatDelta::Text("he".into()), ChatDelta::Text("llo".into())];
490 let mut seen = 0;
491 let r = collect(stream(deltas), |_| seen += 1).await.unwrap();
492 assert_eq!(seen, 3);
493 assert_eq!(r.text, "hello");
494 assert_eq!(r.reasoning, "hm");
495 assert!(r.tool_calls.is_empty());
496 }
497}