1use std::sync::Arc;
14
15use futures::future::BoxFuture;
16use graphrt_core::dsl::NodeDoc;
17use graphrt_core::node::{ConfigError, NodeResult, parse_config};
18use graphrt_core::value::Value;
19use graphrt_core::{Node, NodeContext, NodeError, NodeOutput, Selector};
20use serde::Deserialize;
21use serde_json::Value as Json;
22
23use super::common::{
24 ContextConfig, FinishCheck, MemoryConfig, ModelConfig, PromptTemplate, StreamMode, VisionConfig, build_messages,
25 check_finish, extract_json_where, finish_reason_value, invoke, model_for, split_think_tags, usage_value,
26};
27use super::schema;
28use crate::types::{ChatResponse, ResponseFormat};
29
30#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
32#[serde(rename_all = "snake_case")]
33#[non_exhaustive]
34pub enum ReasoningFormat {
35 #[default]
37 Tagged,
38 Separated,
40}
41
42#[derive(Clone, Debug, Default, Deserialize)]
43struct StructuredOutput {
44 #[serde(default)]
45 enabled: Option<bool>,
46 #[serde(default)]
47 schema: Option<Json>,
48}
49
50#[derive(Clone, Debug, Deserialize)]
52pub struct LlmConfig {
53 pub model: ModelConfig,
54 #[serde(default)]
55 pub prompt_template: PromptTemplate,
56 #[serde(default)]
57 pub context: ContextConfig,
58 #[serde(default)]
59 pub vision: VisionConfig,
60 #[serde(default)]
61 pub memory: Option<MemoryConfig>,
62 #[serde(default)]
63 structured_output: Option<StructuredOutput>,
64 #[serde(default, alias = "structured_output_switch_on")]
65 structured_output_enabled: Option<bool>,
66 #[serde(default)]
67 pub reasoning_format: ReasoningFormat,
68}
69
70impl LlmConfig {
71 fn output_schema(&self) -> Option<Json> {
73 let so = self.structured_output.as_ref()?;
74 let enabled = self.structured_output_enabled.or(so.enabled).unwrap_or(true);
75 if !enabled {
76 return None;
77 }
78 let schema = so.schema.clone()?;
79 match schema.get("schema") {
81 Some(inner) if inner.is_object() => Some(inner.clone()),
82 _ => Some(schema),
83 }
84 }
85}
86
87#[derive(Debug)]
89pub struct LlmNode {
90 config: LlmConfig,
91 schema: Option<Json>,
92 model_name: Value,
94 provider: Value,
95}
96
97impl LlmNode {
98 pub fn from_doc(doc: &NodeDoc) -> Result<Self, ConfigError> {
99 let config: LlmConfig = parse_config(doc)?;
100 if config.prompt_template.messages().is_empty() && config.memory.is_none() {
101 return Err(ConfigError::new(doc, "prompt_template must contain at least one message"));
102 }
103 let schema = config.output_schema();
104 let model_name = Value::from(config.model.name.as_str());
105 let provider = Value::from(config.model.provider.as_str());
106 Ok(Self { config, schema, model_name, provider })
107 }
108
109 async fn execute(&self, ctx: NodeContext) -> NodeResult {
110 let cfg = &self.config;
111 let pool = &ctx.pool;
112 let context = cfg.context.resolve(pool);
113 let images = cfg.vision.image_parts(pool)?;
114 let messages = build_messages(&cfg.prompt_template, pool, context.as_deref(), cfg.memory.as_ref(), images)?;
115 if messages.is_empty() {
116 return Err(NodeError::invalid_input("prompt rendered to no messages"));
117 }
118 let mut request = cfg.model.request(messages, pool)?;
119 if let Some(schema) = &self.schema {
120 request.response_format = Some(ResponseFormat::JsonSchema {
121 name: "structured_output".into(),
122 schema: schema.clone(),
123 strict: false,
124 });
125 }
126
127 let model = model_for(&ctx, &cfg.model)?;
128 let mode = match cfg.reasoning_format {
129 ReasoningFormat::Tagged => StreamMode::Raw,
130 ReasoningFormat::Separated => StreamMode::SeparateThink,
131 };
132 let response = invoke(&ctx, model.as_ref(), request, mode).await?;
133 let finish = check_finish(&response)?;
134 let finish_reason = finish_reason_value(response.finish_reason.as_ref());
135 let ChatResponse { text: answer, reasoning: model_reasoning, usage, warnings, .. } = response;
136
137 let (text, mut reasoning) = match cfg.reasoning_format {
138 ReasoningFormat::Tagged => (answer, String::new()),
139 ReasoningFormat::Separated => split_think_tags(&answer),
140 };
141 if !model_reasoning.is_empty() {
142 reasoning = if reasoning.is_empty() { model_reasoning } else { format!("{model_reasoning}\n{reasoning}") };
143 }
144
145 let mut output = NodeOutput::new()
146 .with("text", text.as_str())
147 .with("usage", usage_value(&usage))
148 .with("finish_reason", finish_reason)
149 .usage(usage)
150 .meta("model", self.model_name.clone())
151 .meta("provider", self.provider.clone());
152 if !reasoning.is_empty() {
153 output = output.with("reasoning", reasoning);
154 }
155 let mut warnings: Vec<Value> = warnings.into_iter().map(Value::from).collect();
156 if let FinishCheck::Unknown(reason) = &finish {
157 warnings.push(Value::from(format!("unrecognized finish reason `{reason}`")));
158 }
159 if !warnings.is_empty() {
160 output = output.meta("warnings", Value::from(warnings));
161 }
162 if let Some(schema) = &self.schema {
163 let parsed = extract_json_where(&text, |v| schema::matches_top_level_type(v, schema)).ok_or_else(|| {
164 NodeError::new("structured_output", "model output contains no JSON value of the requested type")
165 })?;
166 schema::validate(&parsed, schema).map_err(|problem| {
167 NodeError::new("structured_output", format!("model output does not match the schema: {problem}"))
168 })?;
169 output = output.with("structured_output", Value::from_json(parsed));
170 }
171 Ok(output.into())
172 }
173}
174
175impl Node for LlmNode {
176 fn run(&self, ctx: NodeContext) -> BoxFuture<'_, NodeResult> {
177 Box::pin(self.execute(ctx))
178 }
179
180 fn inputs(&self) -> Vec<Selector> {
181 let cfg = &self.config;
182 let mut out = cfg.prompt_template.selectors();
183 out.extend(cfg.context.selectors());
184 out.extend(cfg.vision.selectors());
185 if let Some(memory) = &cfg.memory {
186 out.extend(memory.selectors());
187 }
188 out.extend(cfg.model.selectors());
189 out
190 }
191}
192
193pub fn factory(doc: &NodeDoc) -> Result<Arc<dyn Node>, ConfigError> {
195 Ok(Arc::new(LlmNode::from_doc(doc)?))
196}
197
198#[cfg(test)]
199mod tests {
200 use super::*;
201 use crate::mock::MockModel;
202 use crate::registry::ModelRegistry;
203 use crate::types::{ChatDelta, ContentPart, Role};
204 use graphrt_core::event::Event;
205 use graphrt_core::{NodeOutcome, Services, VariablePool};
206 use std::sync::Mutex;
207
208 fn doc(config: Json) -> NodeDoc {
209 NodeDoc::new("llm", "llm", config)
210 }
211
212 fn ctx_with(model: Arc<MockModel>, pool: VariablePool) -> (NodeContext, Arc<Mutex<Vec<Event>>>) {
213 let services = Services::new().with(ModelRegistry::new().with("openai", model));
214 let events = Arc::new(Mutex::new(Vec::new()));
215 let sink_events = Arc::clone(&events);
216 let ctx = NodeContext::for_test("llm", pool)
217 .with_services(Arc::new(services))
218 .with_sink(Arc::new(move |e| sink_events.lock().unwrap().push(e)));
219 (ctx, events)
220 }
221
222 fn completed(r: NodeResult) -> NodeOutput {
223 match r.unwrap() {
224 NodeOutcome::Completed(o) => o,
225 NodeOutcome::Paused(p) => panic!("unexpected pause {p:?}"),
226 }
227 }
228
229 #[tokio::test]
230 async fn content_filter_is_an_error_and_unknown_reasons_are_reported() {
231 let node = LlmNode::from_doc(&doc(serde_json::json!({
232 "model": {"provider": "openai", "name": "m"},
233 "prompt_template": [{"role": "user", "text": "hi"}]
234 })))
235 .unwrap();
236 let run = |deltas: Vec<ChatDelta>| {
237 let model = Arc::new(MockModel::new(vec![deltas]));
238 let (ctx, _) = ctx_with(model, VariablePool::new());
239 node.run(ctx)
240 };
241 let err = run(vec![ChatDelta::Finish(crate::types::FinishReason::ContentFilter)]).await.unwrap_err();
242 assert_eq!(err.kind, "content_filter");
243 assert!(!err.retryable);
244 let err = run(vec![ChatDelta::Finish(crate::types::FinishReason::Other("malformed_function_call".into()))])
245 .await
246 .unwrap_err();
247 assert_eq!(err.kind, "model_decode");
248 assert!(err.retryable);
249 let out = completed(
250 run(vec![
251 ChatDelta::Warning("dropped `format`".into()),
252 ChatDelta::Text("ok".into()),
253 ChatDelta::Finish(crate::types::FinishReason::Other("language".into())),
254 ])
255 .await,
256 );
257 let warnings = out.metadata["warnings"].to_text();
258 assert!(warnings.contains("dropped `format`") && warnings.contains("language"), "{warnings}");
259 }
260
261 #[tokio::test]
262 async fn streams_and_outputs() {
263 let model = Arc::new(MockModel::new(vec![vec![
264 ChatDelta::Reasoning("why".into()),
265 ChatDelta::Text("Hel".into()),
266 ChatDelta::Text("lo".into()),
267 ChatDelta::Finish(crate::types::FinishReason::Stop),
268 ]]));
269 let mut pool = VariablePool::new();
270 pool.insert("sys", "query", Value::from("hi there"));
271 let node = LlmNode::from_doc(&doc(serde_json::json!({
272 "model": {"provider": "langgenius/openai/openai", "name": "gpt", "completion_params": {"temperature": 0.2}},
273 "prompt_template": [{"role": "system", "text": "be nice"}, {"role": "user", "text": "{{#sys.query#}}"}]
274 })))
275 .unwrap();
276 let (ctx, events) = ctx_with(Arc::clone(&model), pool);
277 let out = completed(node.run(ctx).await);
278 assert_eq!(out.outputs["text"], Value::from("Hello"));
279 assert_eq!(out.outputs["reasoning"], Value::from("why"));
280 assert_eq!(out.outputs["finish_reason"], Value::from("stop"));
281 let chunks: Vec<(String, String)> = events
282 .lock()
283 .unwrap()
284 .iter()
285 .filter_map(|e| match e {
286 Event::NodeChunk { field, chunk, .. } => Some((field.to_string(), chunk.clone())),
287 _ => None,
288 })
289 .collect();
290 assert_eq!(chunks.len(), 3);
291 assert_eq!(chunks[0].0, "reasoning");
292 let req = &model.requests()[0];
293 assert_eq!(req.model, "gpt");
294 assert_eq!(req.temperature, Some(0.2));
295 assert_eq!(req.messages[0].role, Role::System);
296 assert!(matches!(&req.messages[1].content[0], ContentPart::Text { text } if text == "hi there"));
297 }
298
299 #[tokio::test]
300 async fn structured_output_and_separated_reasoning() {
301 let model = Arc::new(MockModel::new(vec![MockModel::text("<think>plan</think>{\"n\": 3}")]));
302 let node = LlmNode::from_doc(&doc(serde_json::json!({
303 "model": {"provider": "openai", "name": "gpt"},
304 "prompt_template": [{"role": "user", "text": "give n"}],
305 "structured_output_switch_on": true,
306 "structured_output": {"schema": {"type": "object", "properties": {"n": {"type": "integer"}}}},
307 "reasoning_format": "separated"
308 })))
309 .unwrap();
310 let (ctx, _) = ctx_with(Arc::clone(&model), VariablePool::new());
311 let out = completed(node.run(ctx).await);
312 assert_eq!(out.outputs["structured_output"].get_path(&["n"]), Some(Value::Int(3)));
313 assert_eq!(out.outputs["reasoning"], Value::from("plan"));
314 assert!(matches!(model.requests()[0].response_format, Some(ResponseFormat::JsonSchema { .. })));
315 }
316
317 #[tokio::test]
318 async fn separated_reasoning_streams_without_think_tags() {
319 let model = Arc::new(MockModel::new(vec![vec![
320 ChatDelta::Text("<thi".into()),
321 ChatDelta::Text("nk>plan</th".into()),
322 ChatDelta::Text("ink>\nHel".into()),
323 ChatDelta::Text("lo".into()),
324 ChatDelta::Finish(crate::types::FinishReason::Stop),
325 ]]));
326 let node = LlmNode::from_doc(&doc(serde_json::json!({
327 "model": {"provider": "openai", "name": "gpt"},
328 "prompt_template": [{"role": "user", "text": "hi"}],
329 "reasoning_format": "separated"
330 })))
331 .unwrap();
332 let (ctx, events) = ctx_with(model, VariablePool::new());
333 let out = completed(node.run(ctx).await);
334 let mut streamed = String::new();
335 let mut reasoning = String::new();
336 for e in events.lock().unwrap().iter() {
337 if let Event::NodeChunk { field, chunk, .. } = e {
338 match field.as_ref() {
339 "text" => streamed.push_str(chunk),
340 _ => reasoning.push_str(chunk),
341 }
342 }
343 }
344 assert_eq!(streamed, "Hello");
345 assert_eq!(out.outputs["text"], Value::from(streamed.as_str()));
346 assert_eq!(reasoning, "plan");
347 }
348
349 #[tokio::test]
350 async fn structured_output_is_validated() {
351 let model = Arc::new(MockModel::new(vec![MockModel::text("see [1] and {\"m\": 1}")]));
352 let node = LlmNode::from_doc(&doc(serde_json::json!({
353 "model": {"provider": "openai", "name": "gpt"},
354 "prompt_template": [{"role": "user", "text": "give n"}],
355 "structured_output": {"schema": {"type": "object", "required": ["n"], "properties": {"n": {"type": "integer"}}}}
356 })))
357 .unwrap();
358 let (ctx, _) = ctx_with(model, VariablePool::new());
359 let err = node.run(ctx).await.unwrap_err();
360 assert_eq!(err.kind, "structured_output");
361 assert!(err.message.contains("`n`"), "{}", err.message);
362 assert!(err.retryable);
363 }
364
365 #[tokio::test]
366 async fn invalid_completion_params_fail_the_node() {
367 let model = Arc::new(MockModel::new(vec![MockModel::text("x")]));
368 let node = LlmNode::from_doc(&doc(serde_json::json!({
369 "model": {"provider": "openai", "name": "gpt", "completion_params": {"temperature": "abc"}},
370 "prompt_template": [{"role": "user", "text": "hi"}]
371 })))
372 .unwrap();
373 let (ctx, _) = ctx_with(Arc::clone(&model), VariablePool::new());
374 let err = node.run(ctx).await.unwrap_err();
375 assert_eq!(err.kind, "invalid_input");
376 assert!(err.message.contains("temperature"));
377 assert!(model.requests().is_empty());
378 }
379
380 #[test]
381 fn inputs_cover_every_reference() {
382 let node = LlmNode::from_doc(&doc(serde_json::json!({
383 "model": {"provider": "openai", "name": "gpt", "completion_params": {"temperature": "{{#cfg.t#}}"}},
384 "prompt_template": [{"role": "user", "text": "{{#start.q#}}"}],
385 "context": {"enabled": true, "variable_selector": ["kb", "result"]},
386 "vision": {"enabled": true, "configs": {"variable_selector": ["start", "files"]}},
387 "memory": {"window": {"enabled": false}}
388 })))
389 .unwrap();
390 let inputs: Vec<String> = node.inputs().iter().map(ToString::to_string).collect();
391 for expected in ["start.q", "kb.result", "start.files", "sys.history", "sys.query", "cfg.t"] {
392 assert!(inputs.iter().any(|i| i == expected), "missing {expected} in {inputs:?}");
393 }
394 }
395
396 #[tokio::test]
397 async fn model_errors_map_to_node_errors() {
398 let model = Arc::new(MockModel::new(vec![]));
399 model.push_error(crate::error::LlmError::Server { status: 503, message: "down".into() });
400 let node = LlmNode::from_doc(&doc(serde_json::json!({
401 "model": {"provider": "openai", "name": "gpt"},
402 "prompt_template": {"text": "completion style"}
403 })))
404 .unwrap();
405 let (ctx, _) = ctx_with(model, VariablePool::new());
406 let err = node.run(ctx).await.unwrap_err();
407 assert!(err.retryable);
408 assert_eq!(err.kind, "model_server");
409 }
410
411 #[tokio::test]
412 async fn missing_registry_is_a_fatal_error() {
413 let node = LlmNode::from_doc(&doc(serde_json::json!({
414 "model": {"provider": "openai", "name": "gpt"},
415 "prompt_template": [{"role": "user", "text": "x"}]
416 })))
417 .unwrap();
418 let err = node.run(NodeContext::for_test("llm", VariablePool::new())).await.unwrap_err();
419 assert!(!err.retryable);
420 }
421
422 #[test]
423 fn rejects_empty_prompt() {
424 let err = LlmNode::from_doc(&doc(serde_json::json!({"model": {"provider": "p", "name": "m"}}))).unwrap_err();
425 assert!(err.message.contains("prompt_template"));
426 }
427}