1use std::sync::Arc;
14
15use futures::future::BoxFuture;
16use graphrt_core::dsl::NodeDoc;
17use graphrt_core::node::{ConfigError, NodeResult, parse_config};
18use graphrt_core::value::Value;
19use graphrt_core::{Node, NodeContext, NodeError, NodeOutput, Selector};
20use serde::Deserialize;
21use serde_json::Value as Json;
22
23use super::common::{
24 ContextConfig, FinishCheck, MemoryConfig, ModelConfig, PromptTemplate, StreamMode, VisionConfig, build_messages,
25 check_finish, extract_json_where, invoke, model_for, split_think_tags, usage_value,
26};
27use super::schema;
28use crate::types::ResponseFormat;
29
30#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
32#[serde(rename_all = "snake_case")]
33#[non_exhaustive]
34pub enum ReasoningFormat {
35 #[default]
37 Tagged,
38 Separated,
40}
41
42#[derive(Clone, Debug, Default, Deserialize)]
43struct StructuredOutput {
44 #[serde(default)]
45 enabled: Option<bool>,
46 #[serde(default)]
47 schema: Option<Json>,
48}
49
50#[derive(Clone, Debug, Deserialize)]
52pub struct LlmConfig {
53 pub model: ModelConfig,
54 #[serde(default)]
55 pub prompt_template: PromptTemplate,
56 #[serde(default)]
57 pub context: ContextConfig,
58 #[serde(default)]
59 pub vision: VisionConfig,
60 #[serde(default)]
61 pub memory: Option<MemoryConfig>,
62 #[serde(default)]
63 structured_output: Option<StructuredOutput>,
64 #[serde(default, alias = "structured_output_switch_on")]
65 structured_output_enabled: Option<bool>,
66 #[serde(default)]
67 pub reasoning_format: ReasoningFormat,
68}
69
70impl LlmConfig {
71 fn output_schema(&self) -> Option<Json> {
73 let so = self.structured_output.as_ref()?;
74 let enabled = self.structured_output_enabled.or(so.enabled).unwrap_or(true);
75 if !enabled {
76 return None;
77 }
78 let schema = so.schema.clone()?;
79 match schema.get("schema") {
81 Some(inner) if inner.is_object() => Some(inner.clone()),
82 _ => Some(schema),
83 }
84 }
85}
86
87#[derive(Debug)]
89pub struct LlmNode {
90 config: LlmConfig,
91 schema: Option<Json>,
92}
93
94impl LlmNode {
95 pub fn from_doc(doc: &NodeDoc) -> Result<Self, ConfigError> {
96 let config: LlmConfig = parse_config(doc)?;
97 if config.prompt_template.messages().is_empty() && config.memory.is_none() {
98 return Err(ConfigError::new(doc, "prompt_template must contain at least one message"));
99 }
100 let schema = config.output_schema();
101 Ok(Self { config, schema })
102 }
103
104 async fn execute(&self, ctx: NodeContext) -> NodeResult {
105 let cfg = &self.config;
106 let pool = &ctx.pool;
107 let context = cfg.context.resolve(pool);
108 let images = cfg.vision.image_parts(pool)?;
109 let messages = build_messages(&cfg.prompt_template, pool, context.as_deref(), cfg.memory.as_ref(), images)?;
110 if messages.is_empty() {
111 return Err(NodeError::invalid_input("prompt rendered to no messages"));
112 }
113 let mut request = cfg.model.request(messages, pool)?;
114 if let Some(schema) = &self.schema {
115 request.response_format = Some(ResponseFormat::JsonSchema {
116 name: "structured_output".into(),
117 schema: schema.clone(),
118 strict: false,
119 });
120 }
121
122 let model = model_for(&ctx, &cfg.model)?;
123 let mode = match cfg.reasoning_format {
124 ReasoningFormat::Tagged => StreamMode::Raw,
125 ReasoningFormat::Separated => StreamMode::SeparateThink,
126 };
127 let response = invoke(&ctx, model.as_ref(), request, mode).await?;
128 let finish = check_finish(&response)?;
129
130 let (text, mut reasoning) = match cfg.reasoning_format {
131 ReasoningFormat::Tagged => (response.text.clone(), String::new()),
132 ReasoningFormat::Separated => split_think_tags(&response.text),
133 };
134 if !response.reasoning.is_empty() {
135 reasoning = if reasoning.is_empty() {
136 response.reasoning.clone()
137 } else {
138 format!("{}\n{reasoning}", response.reasoning)
139 };
140 }
141
142 let mut output = NodeOutput::new()
143 .with("text", text.as_str())
144 .with("usage", usage_value(&response.usage))
145 .with("finish_reason", response.finish_reason.as_ref().map(|r| r.as_str()).unwrap_or_default())
146 .usage(response.usage)
147 .meta("model", cfg.model.name.as_str())
148 .meta("provider", cfg.model.provider.as_str());
149 if !reasoning.is_empty() {
150 output = output.with("reasoning", reasoning);
151 }
152 let mut warnings: Vec<Value> = response.warnings.iter().map(|w| Value::from(w.as_str())).collect();
153 if let FinishCheck::Unknown(reason) = &finish {
154 warnings.push(Value::from(format!("unrecognized finish reason `{reason}`")));
155 }
156 if !warnings.is_empty() {
157 output = output.meta("warnings", Value::from(warnings));
158 }
159 if let Some(schema) = &self.schema {
160 let parsed = extract_json_where(&text, |v| schema::matches_top_level_type(v, schema)).ok_or_else(|| {
161 NodeError::new("structured_output", "model output contains no JSON value of the requested type")
162 })?;
163 schema::validate(&parsed, schema).map_err(|problem| {
164 NodeError::new("structured_output", format!("model output does not match the schema: {problem}"))
165 })?;
166 output = output.with("structured_output", Value::from_json(parsed));
167 }
168 Ok(output.into())
169 }
170}
171
172impl Node for LlmNode {
173 fn run(&self, ctx: NodeContext) -> BoxFuture<'_, NodeResult> {
174 Box::pin(self.execute(ctx))
175 }
176
177 fn inputs(&self) -> Vec<Selector> {
178 let cfg = &self.config;
179 let mut out = cfg.prompt_template.selectors();
180 out.extend(cfg.context.selectors());
181 out.extend(cfg.vision.selectors());
182 if let Some(memory) = &cfg.memory {
183 out.extend(memory.selectors());
184 }
185 out.extend(cfg.model.selectors());
186 out
187 }
188}
189
190pub fn factory(doc: &NodeDoc) -> Result<Arc<dyn Node>, ConfigError> {
192 Ok(Arc::new(LlmNode::from_doc(doc)?))
193}
194
195#[cfg(test)]
196mod tests {
197 use super::*;
198 use crate::mock::MockModel;
199 use crate::registry::ModelRegistry;
200 use crate::types::{ChatDelta, ContentPart, Role};
201 use graphrt_core::event::Event;
202 use graphrt_core::{NodeOutcome, Services, VariablePool};
203 use std::sync::Mutex;
204
205 fn doc(config: Json) -> NodeDoc {
206 NodeDoc::new("llm", "llm", config)
207 }
208
209 fn ctx_with(model: Arc<MockModel>, pool: VariablePool) -> (NodeContext, Arc<Mutex<Vec<Event>>>) {
210 let services = Services::new().with(ModelRegistry::new().with("openai", model));
211 let events = Arc::new(Mutex::new(Vec::new()));
212 let sink_events = Arc::clone(&events);
213 let ctx = NodeContext::for_test("llm", pool)
214 .with_services(Arc::new(services))
215 .with_sink(Arc::new(move |e| sink_events.lock().unwrap().push(e)));
216 (ctx, events)
217 }
218
219 fn completed(r: NodeResult) -> NodeOutput {
220 match r.unwrap() {
221 NodeOutcome::Completed(o) => o,
222 NodeOutcome::Paused(p) => panic!("unexpected pause {p:?}"),
223 }
224 }
225
226 #[tokio::test]
227 async fn content_filter_is_an_error_and_unknown_reasons_are_reported() {
228 let node = LlmNode::from_doc(&doc(serde_json::json!({
229 "model": {"provider": "openai", "name": "m"},
230 "prompt_template": [{"role": "user", "text": "hi"}]
231 })))
232 .unwrap();
233 let run = |deltas: Vec<ChatDelta>| {
234 let model = Arc::new(MockModel::new(vec![deltas]));
235 let (ctx, _) = ctx_with(model, VariablePool::new());
236 node.run(ctx)
237 };
238 let err = run(vec![ChatDelta::Finish(crate::types::FinishReason::ContentFilter)]).await.unwrap_err();
239 assert_eq!(err.kind, "content_filter");
240 assert!(!err.retryable);
241 let err = run(vec![ChatDelta::Finish(crate::types::FinishReason::Other("malformed_function_call".into()))])
242 .await
243 .unwrap_err();
244 assert_eq!(err.kind, "model_decode");
245 assert!(err.retryable);
246 let out = completed(
247 run(vec![
248 ChatDelta::Warning("dropped `format`".into()),
249 ChatDelta::Text("ok".into()),
250 ChatDelta::Finish(crate::types::FinishReason::Other("language".into())),
251 ])
252 .await,
253 );
254 let warnings = out.metadata["warnings"].to_text();
255 assert!(warnings.contains("dropped `format`") && warnings.contains("language"), "{warnings}");
256 }
257
258 #[tokio::test]
259 async fn streams_and_outputs() {
260 let model = Arc::new(MockModel::new(vec![vec![
261 ChatDelta::Reasoning("why".into()),
262 ChatDelta::Text("Hel".into()),
263 ChatDelta::Text("lo".into()),
264 ChatDelta::Finish(crate::types::FinishReason::Stop),
265 ]]));
266 let mut pool = VariablePool::new();
267 pool.insert("sys", "query", Value::from("hi there"));
268 let node = LlmNode::from_doc(&doc(serde_json::json!({
269 "model": {"provider": "langgenius/openai/openai", "name": "gpt", "completion_params": {"temperature": 0.2}},
270 "prompt_template": [{"role": "system", "text": "be nice"}, {"role": "user", "text": "{{#sys.query#}}"}]
271 })))
272 .unwrap();
273 let (ctx, events) = ctx_with(Arc::clone(&model), pool);
274 let out = completed(node.run(ctx).await);
275 assert_eq!(out.outputs["text"], Value::from("Hello"));
276 assert_eq!(out.outputs["reasoning"], Value::from("why"));
277 assert_eq!(out.outputs["finish_reason"], Value::from("stop"));
278 let chunks: Vec<(String, String)> = events
279 .lock()
280 .unwrap()
281 .iter()
282 .filter_map(|e| match e {
283 Event::NodeChunk { field, chunk, .. } => Some((field.clone(), chunk.clone())),
284 _ => None,
285 })
286 .collect();
287 assert_eq!(chunks.len(), 3);
288 assert_eq!(chunks[0].0, "reasoning");
289 let req = &model.requests()[0];
290 assert_eq!(req.model, "gpt");
291 assert_eq!(req.temperature, Some(0.2));
292 assert_eq!(req.messages[0].role, Role::System);
293 assert!(matches!(&req.messages[1].content[0], ContentPart::Text { text } if text == "hi there"));
294 }
295
296 #[tokio::test]
297 async fn structured_output_and_separated_reasoning() {
298 let model = Arc::new(MockModel::new(vec![MockModel::text("<think>plan</think>{\"n\": 3}")]));
299 let node = LlmNode::from_doc(&doc(serde_json::json!({
300 "model": {"provider": "openai", "name": "gpt"},
301 "prompt_template": [{"role": "user", "text": "give n"}],
302 "structured_output_switch_on": true,
303 "structured_output": {"schema": {"type": "object", "properties": {"n": {"type": "integer"}}}},
304 "reasoning_format": "separated"
305 })))
306 .unwrap();
307 let (ctx, _) = ctx_with(Arc::clone(&model), VariablePool::new());
308 let out = completed(node.run(ctx).await);
309 assert_eq!(out.outputs["structured_output"].get_path(&["n"]), Some(Value::Int(3)));
310 assert_eq!(out.outputs["reasoning"], Value::from("plan"));
311 assert!(matches!(model.requests()[0].response_format, Some(ResponseFormat::JsonSchema { .. })));
312 }
313
314 #[tokio::test]
315 async fn separated_reasoning_streams_without_think_tags() {
316 let model = Arc::new(MockModel::new(vec![vec![
317 ChatDelta::Text("<thi".into()),
318 ChatDelta::Text("nk>plan</th".into()),
319 ChatDelta::Text("ink>\nHel".into()),
320 ChatDelta::Text("lo".into()),
321 ChatDelta::Finish(crate::types::FinishReason::Stop),
322 ]]));
323 let node = LlmNode::from_doc(&doc(serde_json::json!({
324 "model": {"provider": "openai", "name": "gpt"},
325 "prompt_template": [{"role": "user", "text": "hi"}],
326 "reasoning_format": "separated"
327 })))
328 .unwrap();
329 let (ctx, events) = ctx_with(model, VariablePool::new());
330 let out = completed(node.run(ctx).await);
331 let mut streamed = String::new();
332 let mut reasoning = String::new();
333 for e in events.lock().unwrap().iter() {
334 if let Event::NodeChunk { field, chunk, .. } = e {
335 match field.as_str() {
336 "text" => streamed.push_str(chunk),
337 _ => reasoning.push_str(chunk),
338 }
339 }
340 }
341 assert_eq!(streamed, "Hello");
342 assert_eq!(out.outputs["text"], Value::from(streamed.as_str()));
343 assert_eq!(reasoning, "plan");
344 }
345
346 #[tokio::test]
347 async fn structured_output_is_validated() {
348 let model = Arc::new(MockModel::new(vec![MockModel::text("see [1] and {\"m\": 1}")]));
349 let node = LlmNode::from_doc(&doc(serde_json::json!({
350 "model": {"provider": "openai", "name": "gpt"},
351 "prompt_template": [{"role": "user", "text": "give n"}],
352 "structured_output": {"schema": {"type": "object", "required": ["n"], "properties": {"n": {"type": "integer"}}}}
353 })))
354 .unwrap();
355 let (ctx, _) = ctx_with(model, VariablePool::new());
356 let err = node.run(ctx).await.unwrap_err();
357 assert_eq!(err.kind, "structured_output");
358 assert!(err.message.contains("`n`"), "{}", err.message);
359 assert!(err.retryable);
360 }
361
362 #[tokio::test]
363 async fn invalid_completion_params_fail_the_node() {
364 let model = Arc::new(MockModel::new(vec![MockModel::text("x")]));
365 let node = LlmNode::from_doc(&doc(serde_json::json!({
366 "model": {"provider": "openai", "name": "gpt", "completion_params": {"temperature": "abc"}},
367 "prompt_template": [{"role": "user", "text": "hi"}]
368 })))
369 .unwrap();
370 let (ctx, _) = ctx_with(Arc::clone(&model), VariablePool::new());
371 let err = node.run(ctx).await.unwrap_err();
372 assert_eq!(err.kind, "invalid_input");
373 assert!(err.message.contains("temperature"));
374 assert!(model.requests().is_empty());
375 }
376
377 #[test]
378 fn inputs_cover_every_reference() {
379 let node = LlmNode::from_doc(&doc(serde_json::json!({
380 "model": {"provider": "openai", "name": "gpt", "completion_params": {"temperature": "{{#cfg.t#}}"}},
381 "prompt_template": [{"role": "user", "text": "{{#start.q#}}"}],
382 "context": {"enabled": true, "variable_selector": ["kb", "result"]},
383 "vision": {"enabled": true, "configs": {"variable_selector": ["start", "files"]}},
384 "memory": {"window": {"enabled": false}}
385 })))
386 .unwrap();
387 let inputs: Vec<String> = node.inputs().iter().map(ToString::to_string).collect();
388 for expected in ["start.q", "kb.result", "start.files", "sys.history", "sys.query", "cfg.t"] {
389 assert!(inputs.iter().any(|i| i == expected), "missing {expected} in {inputs:?}");
390 }
391 }
392
393 #[tokio::test]
394 async fn model_errors_map_to_node_errors() {
395 let model = Arc::new(MockModel::new(vec![]));
396 model.push_error(crate::error::LlmError::Server { status: 503, message: "down".into() });
397 let node = LlmNode::from_doc(&doc(serde_json::json!({
398 "model": {"provider": "openai", "name": "gpt"},
399 "prompt_template": {"text": "completion style"}
400 })))
401 .unwrap();
402 let (ctx, _) = ctx_with(model, VariablePool::new());
403 let err = node.run(ctx).await.unwrap_err();
404 assert!(err.retryable);
405 assert_eq!(err.kind, "model_server");
406 }
407
408 #[tokio::test]
409 async fn missing_registry_is_a_fatal_error() {
410 let node = LlmNode::from_doc(&doc(serde_json::json!({
411 "model": {"provider": "openai", "name": "gpt"},
412 "prompt_template": [{"role": "user", "text": "x"}]
413 })))
414 .unwrap();
415 let err = node.run(NodeContext::for_test("llm", VariablePool::new())).await.unwrap_err();
416 assert!(!err.retryable);
417 }
418
419 #[test]
420 fn rejects_empty_prompt() {
421 let err = LlmNode::from_doc(&doc(serde_json::json!({"model": {"provider": "p", "name": "m"}}))).unwrap_err();
422 assert!(err.message.contains("prompt_template"));
423 }
424}