1use std::collections::BTreeMap;
8use std::fmt;
9use std::pin::Pin;
10
11use futures::future::BoxFuture;
12use futures::{Stream, StreamExt};
13use graphrt_core::event::Usage;
14use graphrt_core::tool::ToolSpec;
15use serde::{Deserialize, Serialize};
16
17use crate::error::LlmError;
18
19#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
21#[serde(rename_all = "lowercase")]
22#[non_exhaustive]
23pub enum Role {
24 System,
25 User,
26 Assistant,
27 Tool,
28}
29
30#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
32#[serde(tag = "type", rename_all = "snake_case")]
33#[non_exhaustive]
34pub enum ContentPart {
35 Text {
36 text: String,
37 },
38 Image {
40 url: String,
41 #[serde(default, skip_serializing_if = "Option::is_none")]
42 detail: Option<String>,
43 },
44}
45
46#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
48pub struct ToolCall {
49 pub id: String,
50 pub name: String,
51 pub arguments: String,
52}
53
54#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
56#[error("tool call `{name}` has invalid JSON arguments: {reason}")]
57pub struct InvalidToolArguments {
58 pub name: String,
60 pub reason: String,
61}
62
63impl ToolCall {
64 pub fn arguments_json(&self) -> Result<serde_json::Map<String, serde_json::Value>, InvalidToolArguments> {
68 if self.arguments.trim().is_empty() {
69 return Ok(serde_json::Map::new());
70 }
71 let invalid = |reason: String| InvalidToolArguments { name: self.name.clone(), reason };
72 match serde_json::from_str::<serde_json::Value>(&self.arguments) {
73 Ok(serde_json::Value::Object(map)) => Ok(map),
74 Ok(other) => Err(invalid(format!("expected a JSON object, got {}", json_kind(&other)))),
75 Err(e) => Err(invalid(e.to_string())),
76 }
77 }
78}
79
80fn json_kind(v: &serde_json::Value) -> &'static str {
81 match v {
82 serde_json::Value::Null => "null",
83 serde_json::Value::Bool(_) => "a boolean",
84 serde_json::Value::Number(_) => "a number",
85 serde_json::Value::String(_) => "a string",
86 serde_json::Value::Array(_) => "an array",
87 serde_json::Value::Object(_) => "an object",
88 }
89}
90
91#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
93pub struct Message {
94 pub role: Role,
95 #[serde(default)]
96 pub content: Vec<ContentPart>,
97 #[serde(default, skip_serializing_if = "Vec::is_empty")]
99 pub tool_calls: Vec<ToolCall>,
100 #[serde(default, skip_serializing_if = "Option::is_none")]
102 pub tool_call_id: Option<String>,
103 #[serde(default, skip_serializing_if = "Option::is_none")]
106 pub provider_state: Option<serde_json::Value>,
107}
108
109impl Message {
110 fn with_text(role: Role, text: impl Into<String>) -> Self {
111 Self {
112 role,
113 content: vec![ContentPart::Text { text: text.into() }],
114 tool_calls: Vec::new(),
115 tool_call_id: None,
116 provider_state: None,
117 }
118 }
119
120 pub fn system(text: impl Into<String>) -> Self {
121 Self::with_text(Role::System, text)
122 }
123
124 pub fn user(text: impl Into<String>) -> Self {
125 Self::with_text(Role::User, text)
126 }
127
128 pub fn assistant(text: impl Into<String>) -> Self {
129 Self::with_text(Role::Assistant, text)
130 }
131
132 pub fn tool(call_id: impl Into<String>, text: impl Into<String>) -> Self {
134 let mut m = Self::with_text(Role::Tool, text);
135 m.tool_call_id = Some(call_id.into());
136 m
137 }
138
139 pub fn tool_with_images(
143 call_id: impl Into<String>,
144 text: impl Into<String>,
145 image_urls: impl IntoIterator<Item = String>,
146 ) -> Self {
147 let mut m = Self::tool(call_id, text);
148 m.content.extend(image_urls.into_iter().map(|url| ContentPart::Image { url, detail: None }));
149 m
150 }
151
152 pub fn images(&self) -> impl Iterator<Item = &ContentPart> {
154 self.content.iter().filter(|p| matches!(p, ContentPart::Image { .. }))
155 }
156
157 pub fn text(&self) -> String {
159 let mut out = String::new();
160 for part in &self.content {
161 if let ContentPart::Text { text } = part {
162 out.push_str(text);
163 }
164 }
165 out
166 }
167}
168
169#[cfg(feature = "openai")]
172pub(crate) fn tool_text_with_image_note(m: &Message) -> String {
173 let text = m.text();
174 let count = m.images().count();
175 if count == 0 {
176 return text;
177 }
178 let note = if count == 1 {
179 "[1 image attached after the tool results]".to_owned()
180 } else {
181 format!("[{count} images attached after the tool results]")
182 };
183 if text.is_empty() { note } else { format!("{text}\n{note}") }
184}
185
186#[cfg(any(feature = "openai", feature = "gemini", feature = "bedrock"))]
188pub(crate) fn tool_images_intro(name: Option<&str>, call_id: &str) -> String {
189 match name {
190 Some(name) => format!("Images returned by tool `{name}` (call {call_id}):"),
191 None => format!("Images returned by tool call {call_id}:"),
192 }
193}
194
195#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
197#[serde(rename_all = "snake_case")]
198#[non_exhaustive]
199pub enum ToolChoice {
200 Auto,
201 None,
202 Required,
204 Tool(String),
206}
207
208#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
210#[serde(tag = "type", rename_all = "snake_case")]
211#[non_exhaustive]
212pub enum ResponseFormat {
213 Text,
214 JsonObject,
215 JsonSchema {
216 name: String,
217 schema: serde_json::Value,
218 #[serde(default)]
219 strict: bool,
220 },
221}
222
223#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
225pub struct ChatRequest {
226 pub model: String,
227 pub messages: Vec<Message>,
228 #[serde(default, skip_serializing_if = "Vec::is_empty")]
229 pub tools: Vec<ToolSpec>,
230 #[serde(default, skip_serializing_if = "Option::is_none")]
231 pub tool_choice: Option<ToolChoice>,
232 #[serde(default, skip_serializing_if = "Option::is_none")]
233 pub temperature: Option<f64>,
234 #[serde(default, skip_serializing_if = "Option::is_none")]
235 pub top_p: Option<f64>,
236 #[serde(default, skip_serializing_if = "Option::is_none")]
237 pub max_tokens: Option<u32>,
238 #[serde(default, skip_serializing_if = "Vec::is_empty")]
239 pub stop: Vec<String>,
240 #[serde(default, skip_serializing_if = "Option::is_none")]
241 pub response_format: Option<ResponseFormat>,
242 #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
244 pub extra: serde_json::Map<String, serde_json::Value>,
245}
246
247impl ChatRequest {
248 pub fn new(model: impl Into<String>, messages: Vec<Message>) -> Self {
249 Self { model: model.into(), messages, ..Self::default() }
250 }
251
252 pub fn with_tools(mut self, tools: Vec<ToolSpec>) -> Self {
254 self.tools = tools;
255 self
256 }
257
258 pub fn with_tool_choice(mut self, choice: ToolChoice) -> Self {
260 self.tool_choice = Some(choice);
261 self
262 }
263
264 pub fn with_temperature(mut self, temperature: f64) -> Self {
265 self.temperature = Some(temperature);
266 self
267 }
268
269 pub fn with_top_p(mut self, top_p: f64) -> Self {
270 self.top_p = Some(top_p);
271 self
272 }
273
274 pub fn with_max_tokens(mut self, max_tokens: u32) -> Self {
275 self.max_tokens = Some(max_tokens);
276 self
277 }
278
279 pub fn with_stop(mut self, stop: Vec<String>) -> Self {
280 self.stop = stop;
281 self
282 }
283
284 pub fn with_response_format(mut self, format: ResponseFormat) -> Self {
285 self.response_format = Some(format);
286 self
287 }
288
289 pub fn with_extra(mut self, key: impl Into<String>, value: serde_json::Value) -> Self {
291 self.extra.insert(key.into(), value);
292 self
293 }
294}
295
296pub const RESERVED_EXTRA_KEYS: &[&str] = &["model", "messages", "stream", "stream_options", "tools", "tool_choice"];
298
299pub fn check_extra(req: &ChatRequest) -> Result<(), LlmError> {
302 for key in req.extra.keys() {
303 if RESERVED_EXTRA_KEYS.contains(&key.as_str()) {
304 return Err(LlmError::Config(format!(
305 "parameter `{key}` cannot be set through extra parameters; use the dedicated request field"
306 )));
307 }
308 }
309 Ok(())
310}
311
312#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
314#[serde(rename_all = "snake_case")]
315#[non_exhaustive]
316pub enum FinishReason {
317 Stop,
318 Length,
319 ToolCalls,
320 ContentFilter,
321 Other(String),
322}
323
324impl FinishReason {
325 pub fn as_str(&self) -> &str {
327 match self {
328 FinishReason::Stop => "stop",
329 FinishReason::Length => "length",
330 FinishReason::ToolCalls => "tool_calls",
331 FinishReason::ContentFilter => "content_filter",
332 FinishReason::Other(s) => s,
333 }
334 }
335}
336
337#[derive(Clone, Debug, PartialEq)]
339#[non_exhaustive]
340pub enum ChatDelta {
341 Text(String),
343 Reasoning(String),
345 ToolCall {
348 index: usize,
349 id: Option<String>,
350 name: Option<String>,
351 arguments: String,
352 },
353 Usage(Usage),
355 Finish(FinishReason),
356 ProviderState(serde_json::Value),
358 Warning(String),
361}
362
363pub type ChatStream = Pin<Box<dyn Stream<Item = Result<ChatDelta, LlmError>> + Send>>;
365
366pub trait ChatModel: Send + Sync + fmt::Debug {
368 fn chat(&self, request: ChatRequest) -> BoxFuture<'_, Result<ChatStream, LlmError>>;
370}
371
372#[derive(Clone, Debug, Default, PartialEq)]
374pub struct ChatResponse {
375 pub text: String,
376 pub reasoning: String,
377 pub tool_calls: Vec<ToolCall>,
378 pub usage: Usage,
379 pub finish_reason: Option<FinishReason>,
380 pub provider_state: Option<serde_json::Value>,
381 pub warnings: Vec<String>,
383}
384
385impl ChatResponse {
386 pub fn to_message(&self) -> Message {
388 let content =
389 if self.text.is_empty() { Vec::new() } else { vec![ContentPart::Text { text: self.text.clone() }] };
390 Message {
391 role: Role::Assistant,
392 content,
393 tool_calls: self.tool_calls.clone(),
394 tool_call_id: None,
395 provider_state: self.provider_state.clone(),
396 }
397 }
398}
399
400#[derive(Default)]
401struct PartialCall {
402 id: Option<String>,
403 name: String,
404 arguments: String,
405}
406
407pub async fn collect(mut stream: ChatStream, mut on_delta: impl FnMut(ChatDelta)) -> Result<ChatResponse, LlmError> {
438 let mut response = ChatResponse::default();
439 let mut calls: BTreeMap<usize, PartialCall> = BTreeMap::new();
440 while let Some(delta) = stream.next().await {
441 let delta = delta?;
442 match &delta {
445 ChatDelta::Text(t) => response.text.push_str(t),
446 ChatDelta::Reasoning(t) => response.reasoning.push_str(t),
447 ChatDelta::ToolCall { index, id, name, arguments } => {
448 let call = calls.entry(*index).or_default();
449 if let Some(id) = id.as_ref().filter(|s| !s.is_empty()) {
450 call.id = Some(id.clone());
451 }
452 if let Some(name) = name.as_ref().filter(|s| !s.is_empty()) {
453 call.name.clone_from(name);
454 }
455 call.arguments.push_str(arguments);
456 }
457 ChatDelta::Usage(u) => response.usage = *u,
458 ChatDelta::Finish(r) => response.finish_reason = Some(r.clone()),
459 ChatDelta::ProviderState(s) => response.provider_state = Some(s.clone()),
460 ChatDelta::Warning(w) => response.warnings.push(w.clone()),
461 }
462 on_delta(delta);
463 }
464 let mut tool_calls = Vec::with_capacity(calls.len());
465 for (index, call) in calls {
466 if call.name.is_empty() {
467 if call.id.is_none() && call.arguments.trim().is_empty() {
468 continue;
470 }
471 return Err(LlmError::Decode(format!("tool call #{index} has arguments or an id but no name")));
472 }
473 tool_calls.push(ToolCall {
474 id: call.id.unwrap_or_else(|| format!("call_{index}")),
475 name: call.name,
476 arguments: call.arguments,
477 });
478 }
479 response.tool_calls = tool_calls;
480 if response.usage.total_tokens == 0 {
481 response.usage.total_tokens = response.usage.prompt_tokens.saturating_add(response.usage.completion_tokens);
482 }
483 Ok(response)
484}
485
486#[cfg(test)]
487mod tests {
488 use super::*;
489
490 fn stream(deltas: Vec<ChatDelta>) -> ChatStream {
491 Box::pin(futures::stream::iter(deltas.into_iter().map(Ok)))
492 }
493
494 #[tokio::test]
495 async fn interleaved_parallel_tool_calls_assemble_by_index() {
496 let deltas = vec![
497 ChatDelta::ToolCall {
498 index: 0,
499 id: Some("a".into()),
500 name: Some("search".into()),
501 arguments: "{\"q\":".into(),
502 },
503 ChatDelta::ToolCall {
504 index: 1,
505 id: Some("b".into()),
506 name: Some("fetch".into()),
507 arguments: "{\"url\":".into(),
508 },
509 ChatDelta::ToolCall { index: 0, id: None, name: None, arguments: "\"rust\"}".into() },
510 ChatDelta::ToolCall { index: 1, id: None, name: None, arguments: "\"x\"}".into() },
511 ChatDelta::Usage(Usage { prompt_tokens: 3, completion_tokens: 4, total_tokens: 0 }),
512 ChatDelta::Finish(FinishReason::ToolCalls),
513 ];
514 let r = collect(stream(deltas), |_| {}).await.unwrap();
515 assert_eq!(r.tool_calls.len(), 2);
516 assert_eq!(r.tool_calls[0].arguments, "{\"q\":\"rust\"}");
517 assert_eq!(r.tool_calls[1].id, "b");
518 assert_eq!(r.tool_calls[1].arguments_json().unwrap()["url"], "x");
519 assert_eq!(r.usage.total_tokens, 7);
520 assert_eq!(r.finish_reason, Some(FinishReason::ToolCalls));
521 }
522
523 #[tokio::test]
524 async fn nameless_tool_call_is_an_error() {
525 let deltas = vec![ChatDelta::ToolCall { index: 0, id: Some("a".into()), name: None, arguments: "{}".into() }];
526 let err = collect(stream(deltas), |_| {}).await.unwrap_err();
527 assert!(matches!(err, LlmError::Decode(_)), "{err:?}");
528 let deltas = vec![ChatDelta::ToolCall { index: 0, id: None, name: None, arguments: String::new() }];
530 assert!(collect(stream(deltas), |_| {}).await.unwrap().tool_calls.is_empty());
531 }
532
533 #[test]
534 fn arguments_json_rejects_malformed_input() {
535 let call = |args: &str| ToolCall { id: "1".into(), name: "t".into(), arguments: args.into() };
536 assert!(call("").arguments_json().unwrap().is_empty());
537 assert_eq!(call("{\"a\":1}").arguments_json().unwrap()["a"], 1);
538 assert!(call("{\"a\":").arguments_json().is_err());
539 let err = call("[1]").arguments_json().unwrap_err();
540 assert!(err.to_string().contains("an array"), "{err}");
541 }
542
543 #[test]
544 fn reserved_extra_keys_are_rejected() {
545 let req = ChatRequest::new("m", vec![]).with_extra("stream", serde_json::json!(false));
546 assert!(matches!(check_extra(&req), Err(LlmError::Config(_))));
547 let req = ChatRequest::new("m", vec![]).with_extra("seed", serde_json::json!(1)).with_temperature(0.1);
548 assert!(check_extra(&req).is_ok());
549 assert_eq!(req.temperature, Some(0.1));
550 }
551
552 #[cfg(feature = "openai")]
553 #[test]
554 fn tool_messages_with_images() {
555 let m = Message::tool_with_images("c1", "shot", ["data:image/png;base64,AAAA".to_owned()]);
556 assert_eq!(m.text(), "shot");
557 assert_eq!(m.images().count(), 1);
558 assert_eq!(tool_text_with_image_note(&m), "shot\n[1 image attached after the tool results]");
559 let m = Message::tool_with_images("c1", "", vec!["https://x/a.png".to_owned(); 2]);
560 assert_eq!(tool_text_with_image_note(&m), "[2 images attached after the tool results]");
561 assert_eq!(tool_text_with_image_note(&Message::tool("c1", "plain")), "plain");
562 assert_eq!(tool_images_intro(Some("screenshot"), "c1"), "Images returned by tool `screenshot` (call c1):");
563 }
564
565 #[tokio::test]
566 async fn text_and_reasoning_accumulate() {
567 let deltas =
568 vec![ChatDelta::Reasoning("hm".into()), ChatDelta::Text("he".into()), ChatDelta::Text("llo".into())];
569 let mut seen = 0;
570 let r = collect(stream(deltas), |_| seen += 1).await.unwrap();
571 assert_eq!(seen, 3);
572 assert_eq!(r.text, "hello");
573 assert_eq!(r.reasoning, "hm");
574 assert!(r.tool_calls.is_empty());
575 }
576}