Skip to main content

graphrt_llm/nodes/
llm.rs

1//! `llm` node: a single (streaming) model call.
2//!
3//! Outputs: `text`, `reasoning` (when present), `usage`, `finish_reason` and,
4//! with structured output enabled, `structured_output`. Answer tokens stream
5//! as chunks on field `text`, reasoning on field `reasoning`. With
6//! `reasoning_format: separated`, `<think>` sections are streamed on
7//! `reasoning` too, so the streamed text always equals the `text` output.
8//!
9//! Structured output is parsed from the answer and checked against the schema
10//! (top-level type, required properties, property types); a mismatch is a
11//! retryable `structured_output` error.
12
13use std::sync::Arc;
14
15use futures::future::BoxFuture;
16use graphrt_core::dsl::NodeDoc;
17use graphrt_core::node::{ConfigError, NodeResult, parse_config};
18use graphrt_core::value::Value;
19use graphrt_core::{Node, NodeContext, NodeError, NodeOutput, Selector};
20use serde::Deserialize;
21use serde_json::Value as Json;
22
23use super::common::{
24    ContextConfig, FinishCheck, MemoryConfig, ModelConfig, PromptTemplate, StreamMode, VisionConfig, build_messages,
25    check_finish, extract_json_where, invoke, model_for, split_think_tags, usage_value,
26};
27use super::schema;
28use crate::types::ResponseFormat;
29
30/// How `<think>` tags inside the answer are treated.
31#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
32#[serde(rename_all = "snake_case")]
33#[non_exhaustive]
34pub enum ReasoningFormat {
35    /// Keep tags in `text` as produced.
36    #[default]
37    Tagged,
38    /// Move tagged reasoning into `reasoning`.
39    Separated,
40}
41
42#[derive(Clone, Debug, Default, Deserialize)]
43struct StructuredOutput {
44    #[serde(default)]
45    enabled: Option<bool>,
46    #[serde(default)]
47    schema: Option<Json>,
48}
49
50/// Configuration of the `llm` node.
51#[derive(Clone, Debug, Deserialize)]
52pub struct LlmConfig {
53    pub model: ModelConfig,
54    #[serde(default)]
55    pub prompt_template: PromptTemplate,
56    #[serde(default)]
57    pub context: ContextConfig,
58    #[serde(default)]
59    pub vision: VisionConfig,
60    #[serde(default)]
61    pub memory: Option<MemoryConfig>,
62    #[serde(default)]
63    structured_output: Option<StructuredOutput>,
64    #[serde(default, alias = "structured_output_switch_on")]
65    structured_output_enabled: Option<bool>,
66    #[serde(default)]
67    pub reasoning_format: ReasoningFormat,
68}
69
70impl LlmConfig {
71    /// JSON Schema for structured output, when enabled.
72    fn output_schema(&self) -> Option<Json> {
73        let so = self.structured_output.as_ref()?;
74        let enabled = self.structured_output_enabled.or(so.enabled).unwrap_or(true);
75        if !enabled {
76            return None;
77        }
78        let schema = so.schema.clone()?;
79        // Accept both `{schema: {...}}` and a bare schema.
80        match schema.get("schema") {
81            Some(inner) if inner.is_object() => Some(inner.clone()),
82            _ => Some(schema),
83        }
84    }
85}
86
87/// The `llm` node.
88#[derive(Debug)]
89pub struct LlmNode {
90    config: LlmConfig,
91    schema: Option<Json>,
92}
93
94impl LlmNode {
95    pub fn from_doc(doc: &NodeDoc) -> Result<Self, ConfigError> {
96        let config: LlmConfig = parse_config(doc)?;
97        if config.prompt_template.messages().is_empty() && config.memory.is_none() {
98            return Err(ConfigError::new(doc, "prompt_template must contain at least one message"));
99        }
100        let schema = config.output_schema();
101        Ok(Self { config, schema })
102    }
103
104    async fn execute(&self, ctx: NodeContext) -> NodeResult {
105        let cfg = &self.config;
106        let pool = &ctx.pool;
107        let context = cfg.context.resolve(pool);
108        let images = cfg.vision.image_parts(pool)?;
109        let messages = build_messages(&cfg.prompt_template, pool, context.as_deref(), cfg.memory.as_ref(), images)?;
110        if messages.is_empty() {
111            return Err(NodeError::invalid_input("prompt rendered to no messages"));
112        }
113        let mut request = cfg.model.request(messages, pool)?;
114        if let Some(schema) = &self.schema {
115            request.response_format = Some(ResponseFormat::JsonSchema {
116                name: "structured_output".into(),
117                schema: schema.clone(),
118                strict: false,
119            });
120        }
121
122        let model = model_for(&ctx, &cfg.model)?;
123        let mode = match cfg.reasoning_format {
124            ReasoningFormat::Tagged => StreamMode::Raw,
125            ReasoningFormat::Separated => StreamMode::SeparateThink,
126        };
127        let response = invoke(&ctx, model.as_ref(), request, mode).await?;
128        let finish = check_finish(&response)?;
129
130        let (text, mut reasoning) = match cfg.reasoning_format {
131            ReasoningFormat::Tagged => (response.text.clone(), String::new()),
132            ReasoningFormat::Separated => split_think_tags(&response.text),
133        };
134        if !response.reasoning.is_empty() {
135            reasoning = if reasoning.is_empty() {
136                response.reasoning.clone()
137            } else {
138                format!("{}\n{reasoning}", response.reasoning)
139            };
140        }
141
142        let mut output = NodeOutput::new()
143            .with("text", text.as_str())
144            .with("usage", usage_value(&response.usage))
145            .with("finish_reason", response.finish_reason.as_ref().map(|r| r.as_str()).unwrap_or_default())
146            .usage(response.usage)
147            .meta("model", cfg.model.name.as_str())
148            .meta("provider", cfg.model.provider.as_str());
149        if !reasoning.is_empty() {
150            output = output.with("reasoning", reasoning);
151        }
152        let mut warnings: Vec<Value> = response.warnings.iter().map(|w| Value::from(w.as_str())).collect();
153        if let FinishCheck::Unknown(reason) = &finish {
154            warnings.push(Value::from(format!("unrecognized finish reason `{reason}`")));
155        }
156        if !warnings.is_empty() {
157            output = output.meta("warnings", Value::from(warnings));
158        }
159        if let Some(schema) = &self.schema {
160            let parsed = extract_json_where(&text, |v| schema::matches_top_level_type(v, schema)).ok_or_else(|| {
161                NodeError::new("structured_output", "model output contains no JSON value of the requested type")
162            })?;
163            schema::validate(&parsed, schema).map_err(|problem| {
164                NodeError::new("structured_output", format!("model output does not match the schema: {problem}"))
165            })?;
166            output = output.with("structured_output", Value::from_json(parsed));
167        }
168        Ok(output.into())
169    }
170}
171
172impl Node for LlmNode {
173    fn run(&self, ctx: NodeContext) -> BoxFuture<'_, NodeResult> {
174        Box::pin(self.execute(ctx))
175    }
176
177    fn inputs(&self) -> Vec<Selector> {
178        let cfg = &self.config;
179        let mut out = cfg.prompt_template.selectors();
180        out.extend(cfg.context.selectors());
181        out.extend(cfg.vision.selectors());
182        if let Some(memory) = &cfg.memory {
183            out.extend(memory.selectors());
184        }
185        out.extend(cfg.model.selectors());
186        out
187    }
188}
189
190/// Factory for the node registry.
191pub fn factory(doc: &NodeDoc) -> Result<Arc<dyn Node>, ConfigError> {
192    Ok(Arc::new(LlmNode::from_doc(doc)?))
193}
194
195#[cfg(test)]
196mod tests {
197    use super::*;
198    use crate::mock::MockModel;
199    use crate::registry::ModelRegistry;
200    use crate::types::{ChatDelta, ContentPart, Role};
201    use graphrt_core::event::Event;
202    use graphrt_core::{NodeOutcome, Services, VariablePool};
203    use std::sync::Mutex;
204
205    fn doc(config: Json) -> NodeDoc {
206        NodeDoc::new("llm", "llm", config)
207    }
208
209    fn ctx_with(model: Arc<MockModel>, pool: VariablePool) -> (NodeContext, Arc<Mutex<Vec<Event>>>) {
210        let services = Services::new().with(ModelRegistry::new().with("openai", model));
211        let events = Arc::new(Mutex::new(Vec::new()));
212        let sink_events = Arc::clone(&events);
213        let ctx = NodeContext::for_test("llm", pool)
214            .with_services(Arc::new(services))
215            .with_sink(Arc::new(move |e| sink_events.lock().unwrap().push(e)));
216        (ctx, events)
217    }
218
219    fn completed(r: NodeResult) -> NodeOutput {
220        match r.unwrap() {
221            NodeOutcome::Completed(o) => o,
222            NodeOutcome::Paused(p) => panic!("unexpected pause {p:?}"),
223        }
224    }
225
226    #[tokio::test]
227    async fn content_filter_is_an_error_and_unknown_reasons_are_reported() {
228        let node = LlmNode::from_doc(&doc(serde_json::json!({
229            "model": {"provider": "openai", "name": "m"},
230            "prompt_template": [{"role": "user", "text": "hi"}]
231        })))
232        .unwrap();
233        let run = |deltas: Vec<ChatDelta>| {
234            let model = Arc::new(MockModel::new(vec![deltas]));
235            let (ctx, _) = ctx_with(model, VariablePool::new());
236            node.run(ctx)
237        };
238        let err = run(vec![ChatDelta::Finish(crate::types::FinishReason::ContentFilter)]).await.unwrap_err();
239        assert_eq!(err.kind, "content_filter");
240        assert!(!err.retryable);
241        let err = run(vec![ChatDelta::Finish(crate::types::FinishReason::Other("malformed_function_call".into()))])
242            .await
243            .unwrap_err();
244        assert_eq!(err.kind, "model_decode");
245        assert!(err.retryable);
246        let out = completed(
247            run(vec![
248                ChatDelta::Warning("dropped `format`".into()),
249                ChatDelta::Text("ok".into()),
250                ChatDelta::Finish(crate::types::FinishReason::Other("language".into())),
251            ])
252            .await,
253        );
254        let warnings = out.metadata["warnings"].to_text();
255        assert!(warnings.contains("dropped `format`") && warnings.contains("language"), "{warnings}");
256    }
257
258    #[tokio::test]
259    async fn streams_and_outputs() {
260        let model = Arc::new(MockModel::new(vec![vec![
261            ChatDelta::Reasoning("why".into()),
262            ChatDelta::Text("Hel".into()),
263            ChatDelta::Text("lo".into()),
264            ChatDelta::Finish(crate::types::FinishReason::Stop),
265        ]]));
266        let mut pool = VariablePool::new();
267        pool.insert("sys", "query", Value::from("hi there"));
268        let node = LlmNode::from_doc(&doc(serde_json::json!({
269            "model": {"provider": "langgenius/openai/openai", "name": "gpt", "completion_params": {"temperature": 0.2}},
270            "prompt_template": [{"role": "system", "text": "be nice"}, {"role": "user", "text": "{{#sys.query#}}"}]
271        })))
272        .unwrap();
273        let (ctx, events) = ctx_with(Arc::clone(&model), pool);
274        let out = completed(node.run(ctx).await);
275        assert_eq!(out.outputs["text"], Value::from("Hello"));
276        assert_eq!(out.outputs["reasoning"], Value::from("why"));
277        assert_eq!(out.outputs["finish_reason"], Value::from("stop"));
278        let chunks: Vec<(String, String)> = events
279            .lock()
280            .unwrap()
281            .iter()
282            .filter_map(|e| match e {
283                Event::NodeChunk { field, chunk, .. } => Some((field.clone(), chunk.clone())),
284                _ => None,
285            })
286            .collect();
287        assert_eq!(chunks.len(), 3);
288        assert_eq!(chunks[0].0, "reasoning");
289        let req = &model.requests()[0];
290        assert_eq!(req.model, "gpt");
291        assert_eq!(req.temperature, Some(0.2));
292        assert_eq!(req.messages[0].role, Role::System);
293        assert!(matches!(&req.messages[1].content[0], ContentPart::Text { text } if text == "hi there"));
294    }
295
296    #[tokio::test]
297    async fn structured_output_and_separated_reasoning() {
298        let model = Arc::new(MockModel::new(vec![MockModel::text("<think>plan</think>{\"n\": 3}")]));
299        let node = LlmNode::from_doc(&doc(serde_json::json!({
300            "model": {"provider": "openai", "name": "gpt"},
301            "prompt_template": [{"role": "user", "text": "give n"}],
302            "structured_output_switch_on": true,
303            "structured_output": {"schema": {"type": "object", "properties": {"n": {"type": "integer"}}}},
304            "reasoning_format": "separated"
305        })))
306        .unwrap();
307        let (ctx, _) = ctx_with(Arc::clone(&model), VariablePool::new());
308        let out = completed(node.run(ctx).await);
309        assert_eq!(out.outputs["structured_output"].get_path(&["n"]), Some(Value::Int(3)));
310        assert_eq!(out.outputs["reasoning"], Value::from("plan"));
311        assert!(matches!(model.requests()[0].response_format, Some(ResponseFormat::JsonSchema { .. })));
312    }
313
314    #[tokio::test]
315    async fn separated_reasoning_streams_without_think_tags() {
316        let model = Arc::new(MockModel::new(vec![vec![
317            ChatDelta::Text("<thi".into()),
318            ChatDelta::Text("nk>plan</th".into()),
319            ChatDelta::Text("ink>\nHel".into()),
320            ChatDelta::Text("lo".into()),
321            ChatDelta::Finish(crate::types::FinishReason::Stop),
322        ]]));
323        let node = LlmNode::from_doc(&doc(serde_json::json!({
324            "model": {"provider": "openai", "name": "gpt"},
325            "prompt_template": [{"role": "user", "text": "hi"}],
326            "reasoning_format": "separated"
327        })))
328        .unwrap();
329        let (ctx, events) = ctx_with(model, VariablePool::new());
330        let out = completed(node.run(ctx).await);
331        let mut streamed = String::new();
332        let mut reasoning = String::new();
333        for e in events.lock().unwrap().iter() {
334            if let Event::NodeChunk { field, chunk, .. } = e {
335                match field.as_str() {
336                    "text" => streamed.push_str(chunk),
337                    _ => reasoning.push_str(chunk),
338                }
339            }
340        }
341        assert_eq!(streamed, "Hello");
342        assert_eq!(out.outputs["text"], Value::from(streamed.as_str()));
343        assert_eq!(reasoning, "plan");
344    }
345
346    #[tokio::test]
347    async fn structured_output_is_validated() {
348        let model = Arc::new(MockModel::new(vec![MockModel::text("see [1] and {\"m\": 1}")]));
349        let node = LlmNode::from_doc(&doc(serde_json::json!({
350            "model": {"provider": "openai", "name": "gpt"},
351            "prompt_template": [{"role": "user", "text": "give n"}],
352            "structured_output": {"schema": {"type": "object", "required": ["n"], "properties": {"n": {"type": "integer"}}}}
353        })))
354        .unwrap();
355        let (ctx, _) = ctx_with(model, VariablePool::new());
356        let err = node.run(ctx).await.unwrap_err();
357        assert_eq!(err.kind, "structured_output");
358        assert!(err.message.contains("`n`"), "{}", err.message);
359        assert!(err.retryable);
360    }
361
362    #[tokio::test]
363    async fn invalid_completion_params_fail_the_node() {
364        let model = Arc::new(MockModel::new(vec![MockModel::text("x")]));
365        let node = LlmNode::from_doc(&doc(serde_json::json!({
366            "model": {"provider": "openai", "name": "gpt", "completion_params": {"temperature": "abc"}},
367            "prompt_template": [{"role": "user", "text": "hi"}]
368        })))
369        .unwrap();
370        let (ctx, _) = ctx_with(Arc::clone(&model), VariablePool::new());
371        let err = node.run(ctx).await.unwrap_err();
372        assert_eq!(err.kind, "invalid_input");
373        assert!(err.message.contains("temperature"));
374        assert!(model.requests().is_empty());
375    }
376
377    #[test]
378    fn inputs_cover_every_reference() {
379        let node = LlmNode::from_doc(&doc(serde_json::json!({
380            "model": {"provider": "openai", "name": "gpt", "completion_params": {"temperature": "{{#cfg.t#}}"}},
381            "prompt_template": [{"role": "user", "text": "{{#start.q#}}"}],
382            "context": {"enabled": true, "variable_selector": ["kb", "result"]},
383            "vision": {"enabled": true, "configs": {"variable_selector": ["start", "files"]}},
384            "memory": {"window": {"enabled": false}}
385        })))
386        .unwrap();
387        let inputs: Vec<String> = node.inputs().iter().map(ToString::to_string).collect();
388        for expected in ["start.q", "kb.result", "start.files", "sys.history", "sys.query", "cfg.t"] {
389            assert!(inputs.iter().any(|i| i == expected), "missing {expected} in {inputs:?}");
390        }
391    }
392
393    #[tokio::test]
394    async fn model_errors_map_to_node_errors() {
395        let model = Arc::new(MockModel::new(vec![]));
396        model.push_error(crate::error::LlmError::Server { status: 503, message: "down".into() });
397        let node = LlmNode::from_doc(&doc(serde_json::json!({
398            "model": {"provider": "openai", "name": "gpt"},
399            "prompt_template": {"text": "completion style"}
400        })))
401        .unwrap();
402        let (ctx, _) = ctx_with(model, VariablePool::new());
403        let err = node.run(ctx).await.unwrap_err();
404        assert!(err.retryable);
405        assert_eq!(err.kind, "model_server");
406    }
407
408    #[tokio::test]
409    async fn missing_registry_is_a_fatal_error() {
410        let node = LlmNode::from_doc(&doc(serde_json::json!({
411            "model": {"provider": "openai", "name": "gpt"},
412            "prompt_template": [{"role": "user", "text": "x"}]
413        })))
414        .unwrap();
415        let err = node.run(NodeContext::for_test("llm", VariablePool::new())).await.unwrap_err();
416        assert!(!err.retryable);
417    }
418
419    #[test]
420    fn rejects_empty_prompt() {
421        let err = LlmNode::from_doc(&doc(serde_json::json!({"model": {"provider": "p", "name": "m"}}))).unwrap_err();
422        assert!(err.message.contains("prompt_template"));
423    }
424}