Skip to main content

graphrt_llm/
types.rs

1//! Provider-neutral chat types.
2//!
3//! Every provider translates [`ChatRequest`] into its wire format and its
4//! streaming response into [`ChatDelta`]s. [`collect`] folds a delta stream
5//! into a [`ChatResponse`].
6
7use std::collections::BTreeMap;
8use std::fmt;
9use std::pin::Pin;
10
11use futures::future::BoxFuture;
12use futures::{Stream, StreamExt};
13use graphrt_core::event::Usage;
14use graphrt_core::tool::ToolSpec;
15use serde::{Deserialize, Serialize};
16
17use crate::error::LlmError;
18
19/// Author of a message.
20#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
21#[serde(rename_all = "lowercase")]
22#[non_exhaustive]
23pub enum Role {
24    System,
25    User,
26    Assistant,
27    Tool,
28}
29
30/// One piece of message content.
31#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
32#[serde(tag = "type", rename_all = "snake_case")]
33#[non_exhaustive]
34pub enum ContentPart {
35    Text {
36        text: String,
37    },
38    /// An image by URL (`https://...` or a `data:<mime>;base64,...` URL).
39    Image {
40        url: String,
41        #[serde(default, skip_serializing_if = "Option::is_none")]
42        detail: Option<String>,
43    },
44}
45
46/// A tool call requested by the model. `arguments` is the raw JSON text.
47#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
48pub struct ToolCall {
49    pub id: String,
50    pub name: String,
51    pub arguments: String,
52}
53
54/// Tool-call arguments that are not a JSON object.
55#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
56#[error("tool call `{name}` has invalid JSON arguments: {reason}")]
57pub struct InvalidToolArguments {
58    /// Tool name of the offending call.
59    pub name: String,
60    pub reason: String,
61}
62
63impl ToolCall {
64    /// Arguments parsed as a JSON object. Empty text means "no arguments" and
65    /// yields an empty object; anything that is not a JSON object is an error
66    /// (it usually means the model output was truncated or malformed).
67    pub fn arguments_json(&self) -> Result<serde_json::Map<String, serde_json::Value>, InvalidToolArguments> {
68        if self.arguments.trim().is_empty() {
69            return Ok(serde_json::Map::new());
70        }
71        let invalid = |reason: String| InvalidToolArguments { name: self.name.clone(), reason };
72        match serde_json::from_str::<serde_json::Value>(&self.arguments) {
73            Ok(serde_json::Value::Object(map)) => Ok(map),
74            Ok(other) => Err(invalid(format!("expected a JSON object, got {}", json_kind(&other)))),
75            Err(e) => Err(invalid(e.to_string())),
76        }
77    }
78}
79
80fn json_kind(v: &serde_json::Value) -> &'static str {
81    match v {
82        serde_json::Value::Null => "null",
83        serde_json::Value::Bool(_) => "a boolean",
84        serde_json::Value::Number(_) => "a number",
85        serde_json::Value::String(_) => "a string",
86        serde_json::Value::Array(_) => "an array",
87        serde_json::Value::Object(_) => "an object",
88    }
89}
90
91/// A chat message.
92#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
93pub struct Message {
94    pub role: Role,
95    #[serde(default)]
96    pub content: Vec<ContentPart>,
97    /// Tool calls made by an assistant message.
98    #[serde(default, skip_serializing_if = "Vec::is_empty")]
99    pub tool_calls: Vec<ToolCall>,
100    /// For `tool` messages: the call this message answers.
101    #[serde(default, skip_serializing_if = "Option::is_none")]
102    pub tool_call_id: Option<String>,
103    /// Opaque provider state attached to an assistant message (e.g. signed
104    /// reasoning blocks) that must be sent back verbatim on the next turn.
105    #[serde(default, skip_serializing_if = "Option::is_none")]
106    pub provider_state: Option<serde_json::Value>,
107}
108
109impl Message {
110    fn with_text(role: Role, text: impl Into<String>) -> Self {
111        Self {
112            role,
113            content: vec![ContentPart::Text { text: text.into() }],
114            tool_calls: Vec::new(),
115            tool_call_id: None,
116            provider_state: None,
117        }
118    }
119
120    pub fn system(text: impl Into<String>) -> Self {
121        Self::with_text(Role::System, text)
122    }
123
124    pub fn user(text: impl Into<String>) -> Self {
125        Self::with_text(Role::User, text)
126    }
127
128    pub fn assistant(text: impl Into<String>) -> Self {
129        Self::with_text(Role::Assistant, text)
130    }
131
132    /// A tool result answering `call_id`.
133    pub fn tool(call_id: impl Into<String>, text: impl Into<String>) -> Self {
134        let mut m = Self::with_text(Role::Tool, text);
135        m.tool_call_id = Some(call_id.into());
136        m
137    }
138
139    /// A tool result answering `call_id` with images (`https://` or `data:`
140    /// URLs) after the text. Providers that cannot put images in a tool
141    /// result send them in a user turn right after the tool results.
142    pub fn tool_with_images(
143        call_id: impl Into<String>,
144        text: impl Into<String>,
145        image_urls: impl IntoIterator<Item = String>,
146    ) -> Self {
147        let mut m = Self::tool(call_id, text);
148        m.content.extend(image_urls.into_iter().map(|url| ContentPart::Image { url, detail: None }));
149        m
150    }
151
152    /// Image parts, in order.
153    pub fn images(&self) -> impl Iterator<Item = &ContentPart> {
154        self.content.iter().filter(|p| matches!(p, ContentPart::Image { .. }))
155    }
156
157    /// Concatenated text parts.
158    pub fn text(&self) -> String {
159        let mut out = String::new();
160        for part in &self.content {
161            if let ContentPart::Text { text } = part {
162                out.push_str(text);
163            }
164        }
165        out
166    }
167}
168
169/// Text of a tool message for providers whose tool results are text-only:
170/// the message text plus a note saying how many images follow.
171#[cfg(feature = "openai")]
172pub(crate) fn tool_text_with_image_note(m: &Message) -> String {
173    let text = m.text();
174    let count = m.images().count();
175    if count == 0 {
176        return text;
177    }
178    let note = if count == 1 {
179        "[1 image attached after the tool results]".to_owned()
180    } else {
181        format!("[{count} images attached after the tool results]")
182    };
183    if text.is_empty() { note } else { format!("{text}\n{note}") }
184}
185
186/// Text introducing the images of one tool result in the fallback user turn.
187#[cfg(any(feature = "openai", feature = "gemini", feature = "bedrock"))]
188pub(crate) fn tool_images_intro(name: Option<&str>, call_id: &str) -> String {
189    match name {
190        Some(name) => format!("Images returned by tool `{name}` (call {call_id}):"),
191        None => format!("Images returned by tool call {call_id}:"),
192    }
193}
194
195/// How the model may use tools.
196#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
197#[serde(rename_all = "snake_case")]
198#[non_exhaustive]
199pub enum ToolChoice {
200    Auto,
201    None,
202    /// The model must call some tool.
203    Required,
204    /// The model must call this tool.
205    Tool(String),
206}
207
208/// Requested output format.
209#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
210#[serde(tag = "type", rename_all = "snake_case")]
211#[non_exhaustive]
212pub enum ResponseFormat {
213    Text,
214    JsonObject,
215    JsonSchema {
216        name: String,
217        schema: serde_json::Value,
218        #[serde(default)]
219        strict: bool,
220    },
221}
222
223/// A provider-neutral chat request.
224#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
225pub struct ChatRequest {
226    pub model: String,
227    pub messages: Vec<Message>,
228    #[serde(default, skip_serializing_if = "Vec::is_empty")]
229    pub tools: Vec<ToolSpec>,
230    #[serde(default, skip_serializing_if = "Option::is_none")]
231    pub tool_choice: Option<ToolChoice>,
232    #[serde(default, skip_serializing_if = "Option::is_none")]
233    pub temperature: Option<f64>,
234    #[serde(default, skip_serializing_if = "Option::is_none")]
235    pub top_p: Option<f64>,
236    #[serde(default, skip_serializing_if = "Option::is_none")]
237    pub max_tokens: Option<u32>,
238    #[serde(default, skip_serializing_if = "Vec::is_empty")]
239    pub stop: Vec<String>,
240    #[serde(default, skip_serializing_if = "Option::is_none")]
241    pub response_format: Option<ResponseFormat>,
242    /// Provider-specific parameters merged into the request body as-is.
243    #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
244    pub extra: serde_json::Map<String, serde_json::Value>,
245}
246
247impl ChatRequest {
248    pub fn new(model: impl Into<String>, messages: Vec<Message>) -> Self {
249        Self { model: model.into(), messages, ..Self::default() }
250    }
251
252    /// Set the tools offered to the model.
253    pub fn with_tools(mut self, tools: Vec<ToolSpec>) -> Self {
254        self.tools = tools;
255        self
256    }
257
258    /// Set how the model may use tools.
259    pub fn with_tool_choice(mut self, choice: ToolChoice) -> Self {
260        self.tool_choice = Some(choice);
261        self
262    }
263
264    pub fn with_temperature(mut self, temperature: f64) -> Self {
265        self.temperature = Some(temperature);
266        self
267    }
268
269    pub fn with_top_p(mut self, top_p: f64) -> Self {
270        self.top_p = Some(top_p);
271        self
272    }
273
274    pub fn with_max_tokens(mut self, max_tokens: u32) -> Self {
275        self.max_tokens = Some(max_tokens);
276        self
277    }
278
279    pub fn with_stop(mut self, stop: Vec<String>) -> Self {
280        self.stop = stop;
281        self
282    }
283
284    pub fn with_response_format(mut self, format: ResponseFormat) -> Self {
285        self.response_format = Some(format);
286        self
287    }
288
289    /// Add a provider-specific body parameter.
290    pub fn with_extra(mut self, key: impl Into<String>, value: serde_json::Value) -> Self {
291        self.extra.insert(key.into(), value);
292        self
293    }
294}
295
296/// Body keys owned by the providers; [`ChatRequest::extra`] must not set them.
297pub const RESERVED_EXTRA_KEYS: &[&str] = &["model", "messages", "stream", "stream_options", "tools", "tool_choice"];
298
299/// Reject `extra` keys that would silently override provider-managed fields.
300/// Custom [`ChatModel`] implementations should call this before building a body.
301pub fn check_extra(req: &ChatRequest) -> Result<(), LlmError> {
302    for key in req.extra.keys() {
303        if RESERVED_EXTRA_KEYS.contains(&key.as_str()) {
304            return Err(LlmError::Config(format!(
305                "parameter `{key}` cannot be set through extra parameters; use the dedicated request field"
306            )));
307        }
308    }
309    Ok(())
310}
311
312/// Why generation stopped.
313#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
314#[serde(rename_all = "snake_case")]
315#[non_exhaustive]
316pub enum FinishReason {
317    Stop,
318    Length,
319    ToolCalls,
320    ContentFilter,
321    Other(String),
322}
323
324impl FinishReason {
325    /// Stable wire name.
326    pub fn as_str(&self) -> &str {
327        match self {
328            FinishReason::Stop => "stop",
329            FinishReason::Length => "length",
330            FinishReason::ToolCalls => "tool_calls",
331            FinishReason::ContentFilter => "content_filter",
332            FinishReason::Other(s) => s,
333        }
334    }
335}
336
337/// One streamed increment.
338#[derive(Clone, Debug, PartialEq)]
339#[non_exhaustive]
340pub enum ChatDelta {
341    /// Visible answer text.
342    Text(String),
343    /// Reasoning / thinking text.
344    Reasoning(String),
345    /// Fragment of a tool call. Fragments sharing `index` belong to the same call;
346    /// `id` and `name` usually arrive on the first fragment only.
347    ToolCall {
348        index: usize,
349        id: Option<String>,
350        name: Option<String>,
351        arguments: String,
352    },
353    /// Cumulative usage snapshot; the last one wins.
354    Usage(Usage),
355    Finish(FinishReason),
356    /// Opaque provider state for the assistant message; the last one wins.
357    ProviderState(serde_json::Value),
358    /// Something the provider adapter changed or dropped while translating the
359    /// request (e.g. unsupported JSON Schema keywords). Surfaced in node metadata.
360    Warning(String),
361}
362
363/// Stream of deltas returned by [`ChatModel::chat`].
364pub type ChatStream = Pin<Box<dyn Stream<Item = Result<ChatDelta, LlmError>> + Send>>;
365
366/// A chat-capable model endpoint.
367pub trait ChatModel: Send + Sync + fmt::Debug {
368    /// Start a streaming chat completion.
369    fn chat(&self, request: ChatRequest) -> BoxFuture<'_, Result<ChatStream, LlmError>>;
370}
371
372/// A fully collected response.
373#[derive(Clone, Debug, Default, PartialEq)]
374pub struct ChatResponse {
375    pub text: String,
376    pub reasoning: String,
377    pub tool_calls: Vec<ToolCall>,
378    pub usage: Usage,
379    pub finish_reason: Option<FinishReason>,
380    pub provider_state: Option<serde_json::Value>,
381    /// Adapter warnings (see [`ChatDelta::Warning`]).
382    pub warnings: Vec<String>,
383}
384
385impl ChatResponse {
386    /// The assistant message to append to the conversation.
387    pub fn to_message(&self) -> Message {
388        let content =
389            if self.text.is_empty() { Vec::new() } else { vec![ContentPart::Text { text: self.text.clone() }] };
390        Message {
391            role: Role::Assistant,
392            content,
393            tool_calls: self.tool_calls.clone(),
394            tool_call_id: None,
395            provider_state: self.provider_state.clone(),
396        }
397    }
398}
399
400#[derive(Default)]
401struct PartialCall {
402    id: Option<String>,
403    name: String,
404    arguments: String,
405}
406
407/// Collect a delta stream. `on_delta` receives every delta as it arrives, by
408/// value, after it has been added to the response (used for streaming chunks to
409/// the run's event stream): a text delta can become a chunk without copying.
410///
411/// Tool-call fragments are grouped by their `index`, so interleaved parallel
412/// calls are assembled correctly. A call that has an id or arguments but never
413/// received a name is a decode error rather than being dropped.
414///
415/// Detecting a truncated stream is the transport's job: the built-in providers
416/// fail with a retryable [`LlmError::Network`] when the connection closes
417/// before the provider's end-of-stream marker.
418///
419/// ```
420/// # use graphrt_llm::{ChatDelta, ChatStream, collect};
421/// # let runtime = tokio::runtime::Builder::new_current_thread().build()?;
422/// # runtime.block_on(async {
423/// let stream: ChatStream =
424///     Box::pin(futures::stream::iter(["Hel", "lo"].map(|t| Ok(ChatDelta::Text(t.into())))));
425/// let mut chunks = Vec::new();
426/// let response = collect(stream, |delta| {
427///     if let ChatDelta::Text(text) = delta {
428///         chunks.push(text); // moved, not copied
429///     }
430/// })
431/// .await?;
432/// assert_eq!((response.text.as_str(), chunks.len()), ("Hello", 2));
433/// # Ok::<(), graphrt_llm::LlmError>(())
434/// # })?;
435/// # Ok::<(), Box<dyn std::error::Error>>(())
436/// ```
437pub async fn collect(mut stream: ChatStream, mut on_delta: impl FnMut(ChatDelta)) -> Result<ChatResponse, LlmError> {
438    let mut response = ChatResponse::default();
439    let mut calls: BTreeMap<usize, PartialCall> = BTreeMap::new();
440    while let Some(delta) = stream.next().await {
441        let delta = delta?;
442        // Text is copied into the response and the delta handed on; the rare
443        // non-text deltas are cloned instead.
444        match &delta {
445            ChatDelta::Text(t) => response.text.push_str(t),
446            ChatDelta::Reasoning(t) => response.reasoning.push_str(t),
447            ChatDelta::ToolCall { index, id, name, arguments } => {
448                let call = calls.entry(*index).or_default();
449                if let Some(id) = id.as_ref().filter(|s| !s.is_empty()) {
450                    call.id = Some(id.clone());
451                }
452                if let Some(name) = name.as_ref().filter(|s| !s.is_empty()) {
453                    call.name.clone_from(name);
454                }
455                call.arguments.push_str(arguments);
456            }
457            ChatDelta::Usage(u) => response.usage = *u,
458            ChatDelta::Finish(r) => response.finish_reason = Some(r.clone()),
459            ChatDelta::ProviderState(s) => response.provider_state = Some(s.clone()),
460            ChatDelta::Warning(w) => response.warnings.push(w.clone()),
461        }
462        on_delta(delta);
463    }
464    let mut tool_calls = Vec::with_capacity(calls.len());
465    for (index, call) in calls {
466        if call.name.is_empty() {
467            if call.id.is_none() && call.arguments.trim().is_empty() {
468                // An empty fragment carries no information; nothing is lost.
469                continue;
470            }
471            return Err(LlmError::Decode(format!("tool call #{index} has arguments or an id but no name")));
472        }
473        tool_calls.push(ToolCall {
474            id: call.id.unwrap_or_else(|| format!("call_{index}")),
475            name: call.name,
476            arguments: call.arguments,
477        });
478    }
479    response.tool_calls = tool_calls;
480    if response.usage.total_tokens == 0 {
481        response.usage.total_tokens = response.usage.prompt_tokens.saturating_add(response.usage.completion_tokens);
482    }
483    Ok(response)
484}
485
486#[cfg(test)]
487mod tests {
488    use super::*;
489
490    fn stream(deltas: Vec<ChatDelta>) -> ChatStream {
491        Box::pin(futures::stream::iter(deltas.into_iter().map(Ok)))
492    }
493
494    #[tokio::test]
495    async fn interleaved_parallel_tool_calls_assemble_by_index() {
496        let deltas = vec![
497            ChatDelta::ToolCall {
498                index: 0,
499                id: Some("a".into()),
500                name: Some("search".into()),
501                arguments: "{\"q\":".into(),
502            },
503            ChatDelta::ToolCall {
504                index: 1,
505                id: Some("b".into()),
506                name: Some("fetch".into()),
507                arguments: "{\"url\":".into(),
508            },
509            ChatDelta::ToolCall { index: 0, id: None, name: None, arguments: "\"rust\"}".into() },
510            ChatDelta::ToolCall { index: 1, id: None, name: None, arguments: "\"x\"}".into() },
511            ChatDelta::Usage(Usage { prompt_tokens: 3, completion_tokens: 4, total_tokens: 0 }),
512            ChatDelta::Finish(FinishReason::ToolCalls),
513        ];
514        let r = collect(stream(deltas), |_| {}).await.unwrap();
515        assert_eq!(r.tool_calls.len(), 2);
516        assert_eq!(r.tool_calls[0].arguments, "{\"q\":\"rust\"}");
517        assert_eq!(r.tool_calls[1].id, "b");
518        assert_eq!(r.tool_calls[1].arguments_json().unwrap()["url"], "x");
519        assert_eq!(r.usage.total_tokens, 7);
520        assert_eq!(r.finish_reason, Some(FinishReason::ToolCalls));
521    }
522
523    #[tokio::test]
524    async fn nameless_tool_call_is_an_error() {
525        let deltas = vec![ChatDelta::ToolCall { index: 0, id: Some("a".into()), name: None, arguments: "{}".into() }];
526        let err = collect(stream(deltas), |_| {}).await.unwrap_err();
527        assert!(matches!(err, LlmError::Decode(_)), "{err:?}");
528        // An empty fragment is harmless.
529        let deltas = vec![ChatDelta::ToolCall { index: 0, id: None, name: None, arguments: String::new() }];
530        assert!(collect(stream(deltas), |_| {}).await.unwrap().tool_calls.is_empty());
531    }
532
533    #[test]
534    fn arguments_json_rejects_malformed_input() {
535        let call = |args: &str| ToolCall { id: "1".into(), name: "t".into(), arguments: args.into() };
536        assert!(call("").arguments_json().unwrap().is_empty());
537        assert_eq!(call("{\"a\":1}").arguments_json().unwrap()["a"], 1);
538        assert!(call("{\"a\":").arguments_json().is_err());
539        let err = call("[1]").arguments_json().unwrap_err();
540        assert!(err.to_string().contains("an array"), "{err}");
541    }
542
543    #[test]
544    fn reserved_extra_keys_are_rejected() {
545        let req = ChatRequest::new("m", vec![]).with_extra("stream", serde_json::json!(false));
546        assert!(matches!(check_extra(&req), Err(LlmError::Config(_))));
547        let req = ChatRequest::new("m", vec![]).with_extra("seed", serde_json::json!(1)).with_temperature(0.1);
548        assert!(check_extra(&req).is_ok());
549        assert_eq!(req.temperature, Some(0.1));
550    }
551
552    #[cfg(feature = "openai")]
553    #[test]
554    fn tool_messages_with_images() {
555        let m = Message::tool_with_images("c1", "shot", ["data:image/png;base64,AAAA".to_owned()]);
556        assert_eq!(m.text(), "shot");
557        assert_eq!(m.images().count(), 1);
558        assert_eq!(tool_text_with_image_note(&m), "shot\n[1 image attached after the tool results]");
559        let m = Message::tool_with_images("c1", "", vec!["https://x/a.png".to_owned(); 2]);
560        assert_eq!(tool_text_with_image_note(&m), "[2 images attached after the tool results]");
561        assert_eq!(tool_text_with_image_note(&Message::tool("c1", "plain")), "plain");
562        assert_eq!(tool_images_intro(Some("screenshot"), "c1"), "Images returned by tool `screenshot` (call c1):");
563    }
564
565    #[tokio::test]
566    async fn text_and_reasoning_accumulate() {
567        let deltas =
568            vec![ChatDelta::Reasoning("hm".into()), ChatDelta::Text("he".into()), ChatDelta::Text("llo".into())];
569        let mut seen = 0;
570        let r = collect(stream(deltas), |_| seen += 1).await.unwrap();
571        assert_eq!(seen, 3);
572        assert_eq!(r.text, "hello");
573        assert_eq!(r.reasoning, "hm");
574        assert!(r.tool_calls.is_empty());
575    }
576}