Skip to main content

graphrt_llm/
types.rs

1//! Provider-neutral chat types.
2//!
3//! Every provider translates [`ChatRequest`] into its wire format and its
4//! streaming response into [`ChatDelta`]s. [`collect`] folds a delta stream
5//! into a [`ChatResponse`].
6
7use std::collections::BTreeMap;
8use std::fmt;
9use std::pin::Pin;
10
11use futures::future::BoxFuture;
12use futures::{Stream, StreamExt};
13use graphrt_core::event::Usage;
14use graphrt_core::tool::ToolSpec;
15use serde::{Deserialize, Serialize};
16
17use crate::error::LlmError;
18
19/// Author of a message.
20#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
21#[serde(rename_all = "lowercase")]
22#[non_exhaustive]
23pub enum Role {
24    System,
25    User,
26    Assistant,
27    Tool,
28}
29
30/// One piece of message content.
31#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
32#[serde(tag = "type", rename_all = "snake_case")]
33#[non_exhaustive]
34pub enum ContentPart {
35    Text {
36        text: String,
37    },
38    /// An image by URL (`https://...` or a `data:<mime>;base64,...` URL).
39    Image {
40        url: String,
41        #[serde(default, skip_serializing_if = "Option::is_none")]
42        detail: Option<String>,
43    },
44}
45
46/// A tool call requested by the model. `arguments` is the raw JSON text.
47#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
48pub struct ToolCall {
49    pub id: String,
50    pub name: String,
51    pub arguments: String,
52}
53
54/// Tool-call arguments that are not a JSON object.
55#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
56#[error("tool call `{name}` has invalid JSON arguments: {reason}")]
57pub struct InvalidToolArguments {
58    /// Tool name of the offending call.
59    pub name: String,
60    pub reason: String,
61}
62
63impl ToolCall {
64    /// Arguments parsed as a JSON object. Empty text means "no arguments" and
65    /// yields an empty object; anything that is not a JSON object is an error
66    /// (it usually means the model output was truncated or malformed).
67    pub fn arguments_json(&self) -> Result<serde_json::Map<String, serde_json::Value>, InvalidToolArguments> {
68        if self.arguments.trim().is_empty() {
69            return Ok(serde_json::Map::new());
70        }
71        let invalid = |reason: String| InvalidToolArguments { name: self.name.clone(), reason };
72        match serde_json::from_str::<serde_json::Value>(&self.arguments) {
73            Ok(serde_json::Value::Object(map)) => Ok(map),
74            Ok(other) => Err(invalid(format!("expected a JSON object, got {}", json_kind(&other)))),
75            Err(e) => Err(invalid(e.to_string())),
76        }
77    }
78}
79
80fn json_kind(v: &serde_json::Value) -> &'static str {
81    match v {
82        serde_json::Value::Null => "null",
83        serde_json::Value::Bool(_) => "a boolean",
84        serde_json::Value::Number(_) => "a number",
85        serde_json::Value::String(_) => "a string",
86        serde_json::Value::Array(_) => "an array",
87        serde_json::Value::Object(_) => "an object",
88    }
89}
90
91/// A chat message.
92#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
93pub struct Message {
94    pub role: Role,
95    #[serde(default)]
96    pub content: Vec<ContentPart>,
97    /// Tool calls made by an assistant message.
98    #[serde(default, skip_serializing_if = "Vec::is_empty")]
99    pub tool_calls: Vec<ToolCall>,
100    /// For `tool` messages: the call this message answers.
101    #[serde(default, skip_serializing_if = "Option::is_none")]
102    pub tool_call_id: Option<String>,
103    /// Opaque provider state attached to an assistant message (e.g. signed
104    /// reasoning blocks) that must be sent back verbatim on the next turn.
105    #[serde(default, skip_serializing_if = "Option::is_none")]
106    pub provider_state: Option<serde_json::Value>,
107}
108
109impl Message {
110    fn with_text(role: Role, text: impl Into<String>) -> Self {
111        Self {
112            role,
113            content: vec![ContentPart::Text { text: text.into() }],
114            tool_calls: Vec::new(),
115            tool_call_id: None,
116            provider_state: None,
117        }
118    }
119
120    pub fn system(text: impl Into<String>) -> Self {
121        Self::with_text(Role::System, text)
122    }
123
124    pub fn user(text: impl Into<String>) -> Self {
125        Self::with_text(Role::User, text)
126    }
127
128    pub fn assistant(text: impl Into<String>) -> Self {
129        Self::with_text(Role::Assistant, text)
130    }
131
132    /// A tool result answering `call_id`.
133    pub fn tool(call_id: impl Into<String>, text: impl Into<String>) -> Self {
134        let mut m = Self::with_text(Role::Tool, text);
135        m.tool_call_id = Some(call_id.into());
136        m
137    }
138
139    /// Concatenated text parts.
140    pub fn text(&self) -> String {
141        let mut out = String::new();
142        for part in &self.content {
143            if let ContentPart::Text { text } = part {
144                out.push_str(text);
145            }
146        }
147        out
148    }
149}
150
151/// How the model may use tools.
152#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
153#[serde(rename_all = "snake_case")]
154#[non_exhaustive]
155pub enum ToolChoice {
156    Auto,
157    None,
158    /// The model must call some tool.
159    Required,
160    /// The model must call this tool.
161    Tool(String),
162}
163
164/// Requested output format.
165#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
166#[serde(tag = "type", rename_all = "snake_case")]
167#[non_exhaustive]
168pub enum ResponseFormat {
169    Text,
170    JsonObject,
171    JsonSchema {
172        name: String,
173        schema: serde_json::Value,
174        #[serde(default)]
175        strict: bool,
176    },
177}
178
179/// A provider-neutral chat request.
180#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
181pub struct ChatRequest {
182    pub model: String,
183    pub messages: Vec<Message>,
184    #[serde(default, skip_serializing_if = "Vec::is_empty")]
185    pub tools: Vec<ToolSpec>,
186    #[serde(default, skip_serializing_if = "Option::is_none")]
187    pub tool_choice: Option<ToolChoice>,
188    #[serde(default, skip_serializing_if = "Option::is_none")]
189    pub temperature: Option<f64>,
190    #[serde(default, skip_serializing_if = "Option::is_none")]
191    pub top_p: Option<f64>,
192    #[serde(default, skip_serializing_if = "Option::is_none")]
193    pub max_tokens: Option<u32>,
194    #[serde(default, skip_serializing_if = "Vec::is_empty")]
195    pub stop: Vec<String>,
196    #[serde(default, skip_serializing_if = "Option::is_none")]
197    pub response_format: Option<ResponseFormat>,
198    /// Provider-specific parameters merged into the request body as-is.
199    #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
200    pub extra: serde_json::Map<String, serde_json::Value>,
201}
202
203impl ChatRequest {
204    pub fn new(model: impl Into<String>, messages: Vec<Message>) -> Self {
205        Self { model: model.into(), messages, ..Self::default() }
206    }
207
208    /// Set the tools offered to the model.
209    pub fn with_tools(mut self, tools: Vec<ToolSpec>) -> Self {
210        self.tools = tools;
211        self
212    }
213
214    /// Set how the model may use tools.
215    pub fn with_tool_choice(mut self, choice: ToolChoice) -> Self {
216        self.tool_choice = Some(choice);
217        self
218    }
219
220    pub fn with_temperature(mut self, temperature: f64) -> Self {
221        self.temperature = Some(temperature);
222        self
223    }
224
225    pub fn with_top_p(mut self, top_p: f64) -> Self {
226        self.top_p = Some(top_p);
227        self
228    }
229
230    pub fn with_max_tokens(mut self, max_tokens: u32) -> Self {
231        self.max_tokens = Some(max_tokens);
232        self
233    }
234
235    pub fn with_stop(mut self, stop: Vec<String>) -> Self {
236        self.stop = stop;
237        self
238    }
239
240    pub fn with_response_format(mut self, format: ResponseFormat) -> Self {
241        self.response_format = Some(format);
242        self
243    }
244
245    /// Add a provider-specific body parameter.
246    pub fn with_extra(mut self, key: impl Into<String>, value: serde_json::Value) -> Self {
247        self.extra.insert(key.into(), value);
248        self
249    }
250}
251
252/// Body keys owned by the providers; [`ChatRequest::extra`] must not set them.
253pub const RESERVED_EXTRA_KEYS: &[&str] = &["model", "messages", "stream", "stream_options", "tools", "tool_choice"];
254
255/// Reject `extra` keys that would silently override provider-managed fields.
256/// Custom [`ChatModel`] implementations should call this before building a body.
257pub fn check_extra(req: &ChatRequest) -> Result<(), LlmError> {
258    for key in req.extra.keys() {
259        if RESERVED_EXTRA_KEYS.contains(&key.as_str()) {
260            return Err(LlmError::Config(format!(
261                "parameter `{key}` cannot be set through extra parameters; use the dedicated request field"
262            )));
263        }
264    }
265    Ok(())
266}
267
268/// Why generation stopped.
269#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
270#[serde(rename_all = "snake_case")]
271#[non_exhaustive]
272pub enum FinishReason {
273    Stop,
274    Length,
275    ToolCalls,
276    ContentFilter,
277    Other(String),
278}
279
280impl FinishReason {
281    /// Stable wire name.
282    pub fn as_str(&self) -> &str {
283        match self {
284            FinishReason::Stop => "stop",
285            FinishReason::Length => "length",
286            FinishReason::ToolCalls => "tool_calls",
287            FinishReason::ContentFilter => "content_filter",
288            FinishReason::Other(s) => s,
289        }
290    }
291}
292
293/// One streamed increment.
294#[derive(Clone, Debug, PartialEq)]
295#[non_exhaustive]
296pub enum ChatDelta {
297    /// Visible answer text.
298    Text(String),
299    /// Reasoning / thinking text.
300    Reasoning(String),
301    /// Fragment of a tool call. Fragments sharing `index` belong to the same call;
302    /// `id` and `name` usually arrive on the first fragment only.
303    ToolCall {
304        index: usize,
305        id: Option<String>,
306        name: Option<String>,
307        arguments: String,
308    },
309    /// Cumulative usage snapshot; the last one wins.
310    Usage(Usage),
311    Finish(FinishReason),
312    /// Opaque provider state for the assistant message; the last one wins.
313    ProviderState(serde_json::Value),
314    /// Something the provider adapter changed or dropped while translating the
315    /// request (e.g. unsupported JSON Schema keywords). Surfaced in node metadata.
316    Warning(String),
317}
318
319/// Stream of deltas returned by [`ChatModel::chat`].
320pub type ChatStream = Pin<Box<dyn Stream<Item = Result<ChatDelta, LlmError>> + Send>>;
321
322/// A chat-capable model endpoint.
323pub trait ChatModel: Send + Sync + fmt::Debug {
324    /// Start a streaming chat completion.
325    fn chat(&self, request: ChatRequest) -> BoxFuture<'_, Result<ChatStream, LlmError>>;
326}
327
328/// A fully collected response.
329#[derive(Clone, Debug, Default, PartialEq)]
330pub struct ChatResponse {
331    pub text: String,
332    pub reasoning: String,
333    pub tool_calls: Vec<ToolCall>,
334    pub usage: Usage,
335    pub finish_reason: Option<FinishReason>,
336    pub provider_state: Option<serde_json::Value>,
337    /// Adapter warnings (see [`ChatDelta::Warning`]).
338    pub warnings: Vec<String>,
339}
340
341impl ChatResponse {
342    /// The assistant message to append to the conversation.
343    pub fn to_message(&self) -> Message {
344        let content =
345            if self.text.is_empty() { Vec::new() } else { vec![ContentPart::Text { text: self.text.clone() }] };
346        Message {
347            role: Role::Assistant,
348            content,
349            tool_calls: self.tool_calls.clone(),
350            tool_call_id: None,
351            provider_state: self.provider_state.clone(),
352        }
353    }
354}
355
356#[derive(Default)]
357struct PartialCall {
358    id: Option<String>,
359    name: String,
360    arguments: String,
361}
362
363/// Collect a delta stream. `on_delta` sees every delta as it arrives (used for
364/// streaming chunks to the run's event stream).
365///
366/// Tool-call fragments are grouped by their `index`, so interleaved parallel
367/// calls are assembled correctly. A call that has an id or arguments but never
368/// received a name is a decode error rather than being dropped.
369///
370/// Detecting a truncated stream is the transport's job: the built-in providers
371/// fail with a retryable [`LlmError::Network`] when the connection closes
372/// before the provider's end-of-stream marker.
373pub async fn collect(mut stream: ChatStream, mut on_delta: impl FnMut(&ChatDelta)) -> Result<ChatResponse, LlmError> {
374    let mut response = ChatResponse::default();
375    let mut calls: BTreeMap<usize, PartialCall> = BTreeMap::new();
376    while let Some(delta) = stream.next().await {
377        let delta = delta?;
378        on_delta(&delta);
379        match delta {
380            ChatDelta::Text(t) => response.text.push_str(&t),
381            ChatDelta::Reasoning(t) => response.reasoning.push_str(&t),
382            ChatDelta::ToolCall { index, id, name, arguments } => {
383                let call = calls.entry(index).or_default();
384                if let Some(id) = id.filter(|s| !s.is_empty()) {
385                    call.id = Some(id);
386                }
387                if let Some(name) = name.filter(|s| !s.is_empty()) {
388                    call.name = name;
389                }
390                call.arguments.push_str(&arguments);
391            }
392            ChatDelta::Usage(u) => response.usage = u,
393            ChatDelta::Finish(r) => response.finish_reason = Some(r),
394            ChatDelta::ProviderState(s) => response.provider_state = Some(s),
395            ChatDelta::Warning(w) => response.warnings.push(w),
396        }
397    }
398    let mut tool_calls = Vec::with_capacity(calls.len());
399    for (index, call) in calls {
400        if call.name.is_empty() {
401            if call.id.is_none() && call.arguments.trim().is_empty() {
402                // An empty fragment carries no information; nothing is lost.
403                continue;
404            }
405            return Err(LlmError::Decode(format!("tool call #{index} has arguments or an id but no name")));
406        }
407        tool_calls.push(ToolCall {
408            id: call.id.unwrap_or_else(|| format!("call_{index}")),
409            name: call.name,
410            arguments: call.arguments,
411        });
412    }
413    response.tool_calls = tool_calls;
414    if response.usage.total_tokens == 0 {
415        response.usage.total_tokens = response.usage.prompt_tokens.saturating_add(response.usage.completion_tokens);
416    }
417    Ok(response)
418}
419
420#[cfg(test)]
421mod tests {
422    use super::*;
423
424    fn stream(deltas: Vec<ChatDelta>) -> ChatStream {
425        Box::pin(futures::stream::iter(deltas.into_iter().map(Ok)))
426    }
427
428    #[tokio::test]
429    async fn interleaved_parallel_tool_calls_assemble_by_index() {
430        let deltas = vec![
431            ChatDelta::ToolCall {
432                index: 0,
433                id: Some("a".into()),
434                name: Some("search".into()),
435                arguments: "{\"q\":".into(),
436            },
437            ChatDelta::ToolCall {
438                index: 1,
439                id: Some("b".into()),
440                name: Some("fetch".into()),
441                arguments: "{\"url\":".into(),
442            },
443            ChatDelta::ToolCall { index: 0, id: None, name: None, arguments: "\"rust\"}".into() },
444            ChatDelta::ToolCall { index: 1, id: None, name: None, arguments: "\"x\"}".into() },
445            ChatDelta::Usage(Usage { prompt_tokens: 3, completion_tokens: 4, total_tokens: 0 }),
446            ChatDelta::Finish(FinishReason::ToolCalls),
447        ];
448        let r = collect(stream(deltas), |_| {}).await.unwrap();
449        assert_eq!(r.tool_calls.len(), 2);
450        assert_eq!(r.tool_calls[0].arguments, "{\"q\":\"rust\"}");
451        assert_eq!(r.tool_calls[1].id, "b");
452        assert_eq!(r.tool_calls[1].arguments_json().unwrap()["url"], "x");
453        assert_eq!(r.usage.total_tokens, 7);
454        assert_eq!(r.finish_reason, Some(FinishReason::ToolCalls));
455    }
456
457    #[tokio::test]
458    async fn nameless_tool_call_is_an_error() {
459        let deltas = vec![ChatDelta::ToolCall { index: 0, id: Some("a".into()), name: None, arguments: "{}".into() }];
460        let err = collect(stream(deltas), |_| {}).await.unwrap_err();
461        assert!(matches!(err, LlmError::Decode(_)), "{err:?}");
462        // An empty fragment is harmless.
463        let deltas = vec![ChatDelta::ToolCall { index: 0, id: None, name: None, arguments: String::new() }];
464        assert!(collect(stream(deltas), |_| {}).await.unwrap().tool_calls.is_empty());
465    }
466
467    #[test]
468    fn arguments_json_rejects_malformed_input() {
469        let call = |args: &str| ToolCall { id: "1".into(), name: "t".into(), arguments: args.into() };
470        assert!(call("").arguments_json().unwrap().is_empty());
471        assert_eq!(call("{\"a\":1}").arguments_json().unwrap()["a"], 1);
472        assert!(call("{\"a\":").arguments_json().is_err());
473        let err = call("[1]").arguments_json().unwrap_err();
474        assert!(err.to_string().contains("an array"), "{err}");
475    }
476
477    #[test]
478    fn reserved_extra_keys_are_rejected() {
479        let req = ChatRequest::new("m", vec![]).with_extra("stream", serde_json::json!(false));
480        assert!(matches!(check_extra(&req), Err(LlmError::Config(_))));
481        let req = ChatRequest::new("m", vec![]).with_extra("seed", serde_json::json!(1)).with_temperature(0.1);
482        assert!(check_extra(&req).is_ok());
483        assert_eq!(req.temperature, Some(0.1));
484    }
485
486    #[tokio::test]
487    async fn text_and_reasoning_accumulate() {
488        let deltas =
489            vec![ChatDelta::Reasoning("hm".into()), ChatDelta::Text("he".into()), ChatDelta::Text("llo".into())];
490        let mut seen = 0;
491        let r = collect(stream(deltas), |_| seen += 1).await.unwrap();
492        assert_eq!(seen, 3);
493        assert_eq!(r.text, "hello");
494        assert_eq!(r.reasoning, "hm");
495        assert!(r.tool_calls.is_empty());
496    }
497}