Skip to main content

graphrt_llm/nodes/
llm.rs

1//! `llm` node: a single (streaming) model call.
2//!
3//! Outputs: `text`, `reasoning` (when present), `usage`, `finish_reason` and,
4//! with structured output enabled, `structured_output`. Answer tokens stream
5//! as chunks on field `text`, reasoning on field `reasoning`. With
6//! `reasoning_format: separated`, `<think>` sections are streamed on
7//! `reasoning` too, so the streamed text always equals the `text` output.
8//!
9//! Structured output is parsed from the answer and checked against the schema
10//! (top-level type, required properties, property types); a mismatch is a
11//! retryable `structured_output` error.
12
13use std::sync::Arc;
14
15use futures::future::BoxFuture;
16use graphrt_core::dsl::NodeDoc;
17use graphrt_core::node::{ConfigError, NodeResult, parse_config};
18use graphrt_core::value::Value;
19use graphrt_core::{Node, NodeContext, NodeError, NodeOutput, Selector};
20use serde::Deserialize;
21use serde_json::Value as Json;
22
23use super::common::{
24    ContextConfig, FinishCheck, MemoryConfig, ModelConfig, PromptTemplate, StreamMode, VisionConfig, build_messages,
25    check_finish, extract_json_where, finish_reason_value, invoke, model_for, split_think_tags, usage_value,
26};
27use super::schema;
28use crate::types::{ChatResponse, ResponseFormat};
29
30/// How `<think>` tags inside the answer are treated.
31#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
32#[serde(rename_all = "snake_case")]
33#[non_exhaustive]
34pub enum ReasoningFormat {
35    /// Keep tags in `text` as produced.
36    #[default]
37    Tagged,
38    /// Move tagged reasoning into `reasoning`.
39    Separated,
40}
41
42#[derive(Clone, Debug, Default, Deserialize)]
43struct StructuredOutput {
44    #[serde(default)]
45    enabled: Option<bool>,
46    #[serde(default)]
47    schema: Option<Json>,
48}
49
50/// Configuration of the `llm` node.
51#[derive(Clone, Debug, Deserialize)]
52pub struct LlmConfig {
53    pub model: ModelConfig,
54    #[serde(default)]
55    pub prompt_template: PromptTemplate,
56    #[serde(default)]
57    pub context: ContextConfig,
58    #[serde(default)]
59    pub vision: VisionConfig,
60    #[serde(default)]
61    pub memory: Option<MemoryConfig>,
62    #[serde(default)]
63    structured_output: Option<StructuredOutput>,
64    #[serde(default, alias = "structured_output_switch_on")]
65    structured_output_enabled: Option<bool>,
66    #[serde(default)]
67    pub reasoning_format: ReasoningFormat,
68}
69
70impl LlmConfig {
71    /// JSON Schema for structured output, when enabled.
72    fn output_schema(&self) -> Option<Json> {
73        let so = self.structured_output.as_ref()?;
74        let enabled = self.structured_output_enabled.or(so.enabled).unwrap_or(true);
75        if !enabled {
76            return None;
77        }
78        let schema = so.schema.clone()?;
79        // Accept both `{schema: {...}}` and a bare schema.
80        match schema.get("schema") {
81            Some(inner) if inner.is_object() => Some(inner.clone()),
82            _ => Some(schema),
83        }
84    }
85}
86
87/// The `llm` node.
88#[derive(Debug)]
89pub struct LlmNode {
90    config: LlmConfig,
91    schema: Option<Json>,
92    /// `model` and `provider` metadata, built once instead of per run.
93    model_name: Value,
94    provider: Value,
95}
96
97impl LlmNode {
98    pub fn from_doc(doc: &NodeDoc) -> Result<Self, ConfigError> {
99        let config: LlmConfig = parse_config(doc)?;
100        if config.prompt_template.messages().is_empty() && config.memory.is_none() {
101            return Err(ConfigError::new(doc, "prompt_template must contain at least one message"));
102        }
103        let schema = config.output_schema();
104        let model_name = Value::from(config.model.name.as_str());
105        let provider = Value::from(config.model.provider.as_str());
106        Ok(Self { config, schema, model_name, provider })
107    }
108
109    async fn execute(&self, ctx: NodeContext) -> NodeResult {
110        let cfg = &self.config;
111        let pool = &ctx.pool;
112        let context = cfg.context.resolve(pool);
113        let images = cfg.vision.image_parts(pool)?;
114        let messages = build_messages(&cfg.prompt_template, pool, context.as_deref(), cfg.memory.as_ref(), images)?;
115        if messages.is_empty() {
116            return Err(NodeError::invalid_input("prompt rendered to no messages"));
117        }
118        let mut request = cfg.model.request(messages, pool)?;
119        if let Some(schema) = &self.schema {
120            request.response_format = Some(ResponseFormat::JsonSchema {
121                name: "structured_output".into(),
122                schema: schema.clone(),
123                strict: false,
124            });
125        }
126
127        let model = model_for(&ctx, &cfg.model)?;
128        let mode = match cfg.reasoning_format {
129            ReasoningFormat::Tagged => StreamMode::Raw,
130            ReasoningFormat::Separated => StreamMode::SeparateThink,
131        };
132        let response = invoke(&ctx, model.as_ref(), request, mode).await?;
133        let finish = check_finish(&response)?;
134        let finish_reason = finish_reason_value(response.finish_reason.as_ref());
135        let ChatResponse { text: answer, reasoning: model_reasoning, usage, warnings, .. } = response;
136
137        let (text, mut reasoning) = match cfg.reasoning_format {
138            ReasoningFormat::Tagged => (answer, String::new()),
139            ReasoningFormat::Separated => split_think_tags(&answer),
140        };
141        if !model_reasoning.is_empty() {
142            reasoning = if reasoning.is_empty() { model_reasoning } else { format!("{model_reasoning}\n{reasoning}") };
143        }
144
145        let mut output = NodeOutput::new()
146            .with("text", text.as_str())
147            .with("usage", usage_value(&usage))
148            .with("finish_reason", finish_reason)
149            .usage(usage)
150            .meta("model", self.model_name.clone())
151            .meta("provider", self.provider.clone());
152        if !reasoning.is_empty() {
153            output = output.with("reasoning", reasoning);
154        }
155        let mut warnings: Vec<Value> = warnings.into_iter().map(Value::from).collect();
156        if let FinishCheck::Unknown(reason) = &finish {
157            warnings.push(Value::from(format!("unrecognized finish reason `{reason}`")));
158        }
159        if !warnings.is_empty() {
160            output = output.meta("warnings", Value::from(warnings));
161        }
162        if let Some(schema) = &self.schema {
163            let parsed = extract_json_where(&text, |v| schema::matches_top_level_type(v, schema)).ok_or_else(|| {
164                NodeError::new("structured_output", "model output contains no JSON value of the requested type")
165            })?;
166            schema::validate(&parsed, schema).map_err(|problem| {
167                NodeError::new("structured_output", format!("model output does not match the schema: {problem}"))
168            })?;
169            output = output.with("structured_output", Value::from_json(parsed));
170        }
171        Ok(output.into())
172    }
173}
174
175impl Node for LlmNode {
176    fn run(&self, ctx: NodeContext) -> BoxFuture<'_, NodeResult> {
177        Box::pin(self.execute(ctx))
178    }
179
180    fn inputs(&self) -> Vec<Selector> {
181        let cfg = &self.config;
182        let mut out = cfg.prompt_template.selectors();
183        out.extend(cfg.context.selectors());
184        out.extend(cfg.vision.selectors());
185        if let Some(memory) = &cfg.memory {
186            out.extend(memory.selectors());
187        }
188        out.extend(cfg.model.selectors());
189        out
190    }
191}
192
193/// Factory for the node registry.
194pub fn factory(doc: &NodeDoc) -> Result<Arc<dyn Node>, ConfigError> {
195    Ok(Arc::new(LlmNode::from_doc(doc)?))
196}
197
198#[cfg(test)]
199mod tests {
200    use super::*;
201    use crate::mock::MockModel;
202    use crate::registry::ModelRegistry;
203    use crate::types::{ChatDelta, ContentPart, Role};
204    use graphrt_core::event::Event;
205    use graphrt_core::{NodeOutcome, Services, VariablePool};
206    use std::sync::Mutex;
207
208    fn doc(config: Json) -> NodeDoc {
209        NodeDoc::new("llm", "llm", config)
210    }
211
212    fn ctx_with(model: Arc<MockModel>, pool: VariablePool) -> (NodeContext, Arc<Mutex<Vec<Event>>>) {
213        let services = Services::new().with(ModelRegistry::new().with("openai", model));
214        let events = Arc::new(Mutex::new(Vec::new()));
215        let sink_events = Arc::clone(&events);
216        let ctx = NodeContext::for_test("llm", pool)
217            .with_services(Arc::new(services))
218            .with_sink(Arc::new(move |e| sink_events.lock().unwrap().push(e)));
219        (ctx, events)
220    }
221
222    fn completed(r: NodeResult) -> NodeOutput {
223        match r.unwrap() {
224            NodeOutcome::Completed(o) => o,
225            NodeOutcome::Paused(p) => panic!("unexpected pause {p:?}"),
226        }
227    }
228
229    #[tokio::test]
230    async fn content_filter_is_an_error_and_unknown_reasons_are_reported() {
231        let node = LlmNode::from_doc(&doc(serde_json::json!({
232            "model": {"provider": "openai", "name": "m"},
233            "prompt_template": [{"role": "user", "text": "hi"}]
234        })))
235        .unwrap();
236        let run = |deltas: Vec<ChatDelta>| {
237            let model = Arc::new(MockModel::new(vec![deltas]));
238            let (ctx, _) = ctx_with(model, VariablePool::new());
239            node.run(ctx)
240        };
241        let err = run(vec![ChatDelta::Finish(crate::types::FinishReason::ContentFilter)]).await.unwrap_err();
242        assert_eq!(err.kind, "content_filter");
243        assert!(!err.retryable);
244        let err = run(vec![ChatDelta::Finish(crate::types::FinishReason::Other("malformed_function_call".into()))])
245            .await
246            .unwrap_err();
247        assert_eq!(err.kind, "model_decode");
248        assert!(err.retryable);
249        let out = completed(
250            run(vec![
251                ChatDelta::Warning("dropped `format`".into()),
252                ChatDelta::Text("ok".into()),
253                ChatDelta::Finish(crate::types::FinishReason::Other("language".into())),
254            ])
255            .await,
256        );
257        let warnings = out.metadata["warnings"].to_text();
258        assert!(warnings.contains("dropped `format`") && warnings.contains("language"), "{warnings}");
259    }
260
261    #[tokio::test]
262    async fn streams_and_outputs() {
263        let model = Arc::new(MockModel::new(vec![vec![
264            ChatDelta::Reasoning("why".into()),
265            ChatDelta::Text("Hel".into()),
266            ChatDelta::Text("lo".into()),
267            ChatDelta::Finish(crate::types::FinishReason::Stop),
268        ]]));
269        let mut pool = VariablePool::new();
270        pool.insert("sys", "query", Value::from("hi there"));
271        let node = LlmNode::from_doc(&doc(serde_json::json!({
272            "model": {"provider": "langgenius/openai/openai", "name": "gpt", "completion_params": {"temperature": 0.2}},
273            "prompt_template": [{"role": "system", "text": "be nice"}, {"role": "user", "text": "{{#sys.query#}}"}]
274        })))
275        .unwrap();
276        let (ctx, events) = ctx_with(Arc::clone(&model), pool);
277        let out = completed(node.run(ctx).await);
278        assert_eq!(out.outputs["text"], Value::from("Hello"));
279        assert_eq!(out.outputs["reasoning"], Value::from("why"));
280        assert_eq!(out.outputs["finish_reason"], Value::from("stop"));
281        let chunks: Vec<(String, String)> = events
282            .lock()
283            .unwrap()
284            .iter()
285            .filter_map(|e| match e {
286                Event::NodeChunk { field, chunk, .. } => Some((field.to_string(), chunk.clone())),
287                _ => None,
288            })
289            .collect();
290        assert_eq!(chunks.len(), 3);
291        assert_eq!(chunks[0].0, "reasoning");
292        let req = &model.requests()[0];
293        assert_eq!(req.model, "gpt");
294        assert_eq!(req.temperature, Some(0.2));
295        assert_eq!(req.messages[0].role, Role::System);
296        assert!(matches!(&req.messages[1].content[0], ContentPart::Text { text } if text == "hi there"));
297    }
298
299    #[tokio::test]
300    async fn structured_output_and_separated_reasoning() {
301        let model = Arc::new(MockModel::new(vec![MockModel::text("<think>plan</think>{\"n\": 3}")]));
302        let node = LlmNode::from_doc(&doc(serde_json::json!({
303            "model": {"provider": "openai", "name": "gpt"},
304            "prompt_template": [{"role": "user", "text": "give n"}],
305            "structured_output_switch_on": true,
306            "structured_output": {"schema": {"type": "object", "properties": {"n": {"type": "integer"}}}},
307            "reasoning_format": "separated"
308        })))
309        .unwrap();
310        let (ctx, _) = ctx_with(Arc::clone(&model), VariablePool::new());
311        let out = completed(node.run(ctx).await);
312        assert_eq!(out.outputs["structured_output"].get_path(&["n"]), Some(Value::Int(3)));
313        assert_eq!(out.outputs["reasoning"], Value::from("plan"));
314        assert!(matches!(model.requests()[0].response_format, Some(ResponseFormat::JsonSchema { .. })));
315    }
316
317    #[tokio::test]
318    async fn separated_reasoning_streams_without_think_tags() {
319        let model = Arc::new(MockModel::new(vec![vec![
320            ChatDelta::Text("<thi".into()),
321            ChatDelta::Text("nk>plan</th".into()),
322            ChatDelta::Text("ink>\nHel".into()),
323            ChatDelta::Text("lo".into()),
324            ChatDelta::Finish(crate::types::FinishReason::Stop),
325        ]]));
326        let node = LlmNode::from_doc(&doc(serde_json::json!({
327            "model": {"provider": "openai", "name": "gpt"},
328            "prompt_template": [{"role": "user", "text": "hi"}],
329            "reasoning_format": "separated"
330        })))
331        .unwrap();
332        let (ctx, events) = ctx_with(model, VariablePool::new());
333        let out = completed(node.run(ctx).await);
334        let mut streamed = String::new();
335        let mut reasoning = String::new();
336        for e in events.lock().unwrap().iter() {
337            if let Event::NodeChunk { field, chunk, .. } = e {
338                match field.as_ref() {
339                    "text" => streamed.push_str(chunk),
340                    _ => reasoning.push_str(chunk),
341                }
342            }
343        }
344        assert_eq!(streamed, "Hello");
345        assert_eq!(out.outputs["text"], Value::from(streamed.as_str()));
346        assert_eq!(reasoning, "plan");
347    }
348
349    #[tokio::test]
350    async fn structured_output_is_validated() {
351        let model = Arc::new(MockModel::new(vec![MockModel::text("see [1] and {\"m\": 1}")]));
352        let node = LlmNode::from_doc(&doc(serde_json::json!({
353            "model": {"provider": "openai", "name": "gpt"},
354            "prompt_template": [{"role": "user", "text": "give n"}],
355            "structured_output": {"schema": {"type": "object", "required": ["n"], "properties": {"n": {"type": "integer"}}}}
356        })))
357        .unwrap();
358        let (ctx, _) = ctx_with(model, VariablePool::new());
359        let err = node.run(ctx).await.unwrap_err();
360        assert_eq!(err.kind, "structured_output");
361        assert!(err.message.contains("`n`"), "{}", err.message);
362        assert!(err.retryable);
363    }
364
365    #[tokio::test]
366    async fn invalid_completion_params_fail_the_node() {
367        let model = Arc::new(MockModel::new(vec![MockModel::text("x")]));
368        let node = LlmNode::from_doc(&doc(serde_json::json!({
369            "model": {"provider": "openai", "name": "gpt", "completion_params": {"temperature": "abc"}},
370            "prompt_template": [{"role": "user", "text": "hi"}]
371        })))
372        .unwrap();
373        let (ctx, _) = ctx_with(Arc::clone(&model), VariablePool::new());
374        let err = node.run(ctx).await.unwrap_err();
375        assert_eq!(err.kind, "invalid_input");
376        assert!(err.message.contains("temperature"));
377        assert!(model.requests().is_empty());
378    }
379
380    #[test]
381    fn inputs_cover_every_reference() {
382        let node = LlmNode::from_doc(&doc(serde_json::json!({
383            "model": {"provider": "openai", "name": "gpt", "completion_params": {"temperature": "{{#cfg.t#}}"}},
384            "prompt_template": [{"role": "user", "text": "{{#start.q#}}"}],
385            "context": {"enabled": true, "variable_selector": ["kb", "result"]},
386            "vision": {"enabled": true, "configs": {"variable_selector": ["start", "files"]}},
387            "memory": {"window": {"enabled": false}}
388        })))
389        .unwrap();
390        let inputs: Vec<String> = node.inputs().iter().map(ToString::to_string).collect();
391        for expected in ["start.q", "kb.result", "start.files", "sys.history", "sys.query", "cfg.t"] {
392            assert!(inputs.iter().any(|i| i == expected), "missing {expected} in {inputs:?}");
393        }
394    }
395
396    #[tokio::test]
397    async fn model_errors_map_to_node_errors() {
398        let model = Arc::new(MockModel::new(vec![]));
399        model.push_error(crate::error::LlmError::Server { status: 503, message: "down".into() });
400        let node = LlmNode::from_doc(&doc(serde_json::json!({
401            "model": {"provider": "openai", "name": "gpt"},
402            "prompt_template": {"text": "completion style"}
403        })))
404        .unwrap();
405        let (ctx, _) = ctx_with(model, VariablePool::new());
406        let err = node.run(ctx).await.unwrap_err();
407        assert!(err.retryable);
408        assert_eq!(err.kind, "model_server");
409    }
410
411    #[tokio::test]
412    async fn missing_registry_is_a_fatal_error() {
413        let node = LlmNode::from_doc(&doc(serde_json::json!({
414            "model": {"provider": "openai", "name": "gpt"},
415            "prompt_template": [{"role": "user", "text": "x"}]
416        })))
417        .unwrap();
418        let err = node.run(NodeContext::for_test("llm", VariablePool::new())).await.unwrap_err();
419        assert!(!err.retryable);
420    }
421
422    #[test]
423    fn rejects_empty_prompt() {
424        let err = LlmNode::from_doc(&doc(serde_json::json!({"model": {"provider": "p", "name": "m"}}))).unwrap_err();
425        assert!(err.message.contains("prompt_template"));
426    }
427}