Skip to main content

graphrt_llm/nodes/
common.rs

1//! Configuration and prompt assembly shared by the model-calling nodes.
2
3use std::sync::Arc;
4
5use graphrt_core::event::Usage;
6use graphrt_core::template::{Part, Template};
7use graphrt_core::value::{FileKind, Value};
8use graphrt_core::{NodeContext, NodeError, Selector, VariablePool};
9use serde::Deserialize;
10use serde_json::Value as Json;
11
12use crate::registry::ModelRegistry;
13use crate::types::{
14    ChatDelta, ChatModel, ChatRequest, ChatResponse, ContentPart, FinishReason, Message, RESERVED_EXTRA_KEYS, Role,
15    collect,
16};
17
18/// Literal placeholder replaced by the configured context value.
19pub const CONTEXT_PLACEHOLDER: &str = "{{#context#}}";
20
21/// `model: {provider, name, completion_params}`.
22#[derive(Clone, Debug, Deserialize)]
23pub struct ModelConfig {
24    pub provider: String,
25    pub name: String,
26    #[serde(default)]
27    pub completion_params: serde_json::Map<String, Json>,
28}
29
30/// Parameters never forwarded to providers (UI-only settings).
31const IGNORED_PARAMS: &[&str] = &["mode"];
32
33/// A resolved completion parameter.
34enum Param {
35    /// Not set (explicit `null`, or an empty literal string).
36    Unset,
37    Value(Json),
38}
39
40impl ModelConfig {
41    /// Build a request for this model, applying completion parameters. Parameter
42    /// values that are templates (`"{{#start.temp#}}"`) are resolved per run;
43    /// the configuration itself is never modified.
44    ///
45    /// A parameter whose value has the wrong type (e.g. `temperature: "hot"`),
46    /// or a template that resolves to nothing, is an input error naming the
47    /// parameter instead of being dropped silently.
48    pub fn request(&self, messages: Vec<Message>, pool: &VariablePool) -> Result<ChatRequest, NodeError> {
49        let mut req = ChatRequest::new(self.name.clone(), messages);
50        for (key, raw) in &self.completion_params {
51            if IGNORED_PARAMS.contains(&key.as_str()) {
52                continue;
53            }
54            let value = match resolve_param(key, raw, pool)? {
55                Param::Unset => continue,
56                Param::Value(v) => v,
57            };
58            let bad = |expected: &str| {
59                NodeError::invalid_input(format!("completion parameter `{key}` must be {expected}, got {value}"))
60            };
61            match key.as_str() {
62                "temperature" => req.temperature = Some(value.as_f64().ok_or_else(|| bad("a number"))?),
63                "top_p" => req.top_p = Some(value.as_f64().ok_or_else(|| bad("a number"))?),
64                "max_tokens" | "max_completion_tokens" => {
65                    let n = value
66                        .as_u64()
67                        .and_then(|n| u32::try_from(n).ok())
68                        .ok_or_else(|| bad("a non-negative integer that fits in 32 bits"))?;
69                    req.max_tokens = Some(n);
70                }
71                "stop" => {
72                    req.stop = match &value {
73                        Json::String(s) => vec![s.clone()],
74                        Json::Array(items) => items
75                            .iter()
76                            .map(|i| i.as_str().map(str::to_owned))
77                            .collect::<Option<Vec<_>>>()
78                            .ok_or_else(|| bad("a string or a list of strings"))?,
79                        _ => return Err(bad("a string or a list of strings")),
80                    };
81                }
82                reserved if RESERVED_EXTRA_KEYS.contains(&reserved) => {
83                    return Err(NodeError::invalid_input(format!(
84                        "completion parameter `{key}` is managed by the node and cannot be set"
85                    )));
86                }
87                _ => {
88                    req.extra.insert(key.clone(), value);
89                }
90            }
91        }
92        Ok(req)
93    }
94
95    /// Selectors referenced by templated completion parameters.
96    pub fn selectors(&self) -> Vec<Selector> {
97        self.completion_params
98            .values()
99            .filter_map(Json::as_str)
100            .filter(|s| s.contains("{{#"))
101            .flat_map(|s| Template::parse(s).selectors().cloned().collect::<Vec<_>>())
102            .collect()
103    }
104}
105
106/// Resolve a completion parameter: templates are rendered, numeric/boolean strings coerced.
107fn resolve_param(key: &str, raw: &Json, pool: &VariablePool) -> Result<Param, NodeError> {
108    let text = match raw {
109        Json::Null => return Ok(Param::Unset),
110        Json::String(s) => s,
111        other => return Ok(Param::Value(other.clone())),
112    };
113    let is_template = text.contains("{{#");
114    let resolved = if is_template { Template::parse(text).resolve(pool) } else { Value::from(text.as_str()) };
115    match resolved {
116        Value::String(s) => {
117            let t = s.trim();
118            if t.is_empty() {
119                if is_template {
120                    return Err(NodeError::invalid_input(format!(
121                        "completion parameter `{key}` resolved to an empty value (`{text}`)"
122                    )));
123                }
124                return Ok(Param::Unset);
125            }
126            Ok(Param::Value(if let Ok(i) = t.parse::<i64>() {
127                Json::from(i)
128            } else if let Ok(f) = t.parse::<f64>() {
129                Json::from(f)
130            } else if t == "true" || t == "false" {
131                Json::Bool(t == "true")
132            } else {
133                Json::String(s.to_string())
134            }))
135        }
136        Value::Null => Err(NodeError::invalid_input(format!(
137            "completion parameter `{key}` references a missing variable (`{text}`)"
138        ))),
139        other => Ok(Param::Value(other.to_json())),
140    }
141}
142
143/// One prompt message template.
144#[derive(Clone, Debug, Deserialize)]
145pub struct PromptMessage {
146    #[serde(default = "user_role")]
147    pub role: Role,
148    #[serde(default)]
149    pub text: Template,
150}
151
152fn user_role() -> Role {
153    Role::User
154}
155
156/// `prompt_template`: a list of chat messages, or a single completion prompt.
157#[derive(Clone, Debug, Deserialize)]
158#[serde(untagged)]
159pub enum PromptTemplate {
160    Chat(Vec<PromptMessage>),
161    Completion(PromptMessage),
162}
163
164impl Default for PromptTemplate {
165    fn default() -> Self {
166        PromptTemplate::Chat(Vec::new())
167    }
168}
169
170impl PromptTemplate {
171    pub fn messages(&self) -> Vec<&PromptMessage> {
172        match self {
173            PromptTemplate::Chat(m) => m.iter().collect(),
174            PromptTemplate::Completion(m) => vec![m],
175        }
176    }
177
178    /// Selectors referenced by the message templates.
179    pub fn selectors(&self) -> Vec<Selector> {
180        self.messages().iter().flat_map(|m| m.text.selectors().cloned()).collect()
181    }
182}
183
184/// `context: {enabled, variable_selector}`.
185#[derive(Clone, Debug, Default, Deserialize)]
186pub struct ContextConfig {
187    #[serde(default)]
188    pub enabled: bool,
189    #[serde(default)]
190    pub variable_selector: Option<Selector>,
191}
192
193impl ContextConfig {
194    /// Context text, if enabled and present.
195    pub fn resolve(&self, pool: &VariablePool) -> Option<String> {
196        if !self.enabled {
197            return None;
198        }
199        let value = pool.get(self.variable_selector.as_ref()?)?;
200        Some(context_text(&value))
201    }
202
203    pub fn selectors(&self) -> Vec<Selector> {
204        if !self.enabled {
205            return Vec::new();
206        }
207        self.variable_selector.iter().cloned().collect()
208    }
209}
210
211/// Retrieved-document lists render as their `content` fields; other values as text.
212fn context_text(value: &Value) -> String {
213    match value {
214        Value::Array(items) => items
215            .iter()
216            .map(|item| match item.as_object().and_then(|o| o.get("content")) {
217                Some(content) => content.to_text(),
218                None => item.to_text(),
219            })
220            .collect::<Vec<_>>()
221            .join("\n"),
222        other => other.to_text(),
223    }
224}
225
226#[derive(Clone, Debug, Default, Deserialize)]
227pub struct VisionDetail {
228    #[serde(default)]
229    pub variable_selector: Option<Selector>,
230    #[serde(default)]
231    pub detail: Option<String>,
232}
233
234/// `vision: {enabled, configs: {variable_selector, detail}}` (or the fields inline).
235#[derive(Clone, Debug, Default, Deserialize)]
236pub struct VisionConfig {
237    #[serde(default)]
238    pub enabled: bool,
239    #[serde(default)]
240    pub configs: Option<VisionDetail>,
241    #[serde(default)]
242    pub variable_selector: Option<Selector>,
243    #[serde(default)]
244    pub detail: Option<String>,
245}
246
247impl VisionConfig {
248    fn selector(&self) -> Selector {
249        self.configs
250            .as_ref()
251            .and_then(|c| c.variable_selector.clone())
252            .or_else(|| self.variable_selector.clone())
253            .unwrap_or_else(|| Selector::of("sys", "files"))
254    }
255
256    /// Image parts for the configured files (default selector `sys.files`).
257    ///
258    /// Non-image files are ignored. An image without a URL (e.g. a host
259    /// storage reference that was not resolved) is an input error rather than
260    /// being dropped silently.
261    pub fn image_parts(&self, pool: &VariablePool) -> Result<Vec<ContentPart>, NodeError> {
262        if !self.enabled {
263            return Ok(Vec::new());
264        }
265        let detail = self.configs.as_ref().and_then(|c| c.detail.clone()).or_else(|| self.detail.clone());
266        let files: Vec<Value> = match pool.get(&self.selector()) {
267            Some(Value::Array(items)) => items.iter().cloned().collect(),
268            Some(v @ Value::File(_)) => vec![v],
269            _ => Vec::new(),
270        };
271        let mut parts = Vec::new();
272        for file in files.iter().filter_map(Value::as_file) {
273            let is_image =
274                file.kind == FileKind::Image || file.mime_type.as_deref().is_some_and(|m| m.starts_with("image/"));
275            if !is_image {
276                continue;
277            }
278            let url = file.url.clone().filter(|u| !u.is_empty()).ok_or_else(|| {
279                NodeError::invalid_input(format!(
280                    "image `{}` has no URL; resolve host files to URLs before sending them to a model",
281                    file.name
282                ))
283            })?;
284            parts.push(ContentPart::Image { url, detail: detail.clone() });
285        }
286        Ok(parts)
287    }
288
289    pub fn selectors(&self) -> Vec<Selector> {
290        if self.enabled { vec![self.selector()] } else { Vec::new() }
291    }
292}
293
294#[derive(Clone, Debug, Default, Deserialize)]
295pub struct WindowConfig {
296    #[serde(default)]
297    pub enabled: bool,
298    #[serde(default)]
299    pub size: Option<usize>,
300}
301
302/// Conversation memory.
303///
304/// History is read from `variable_selector` (default `sys.history`): an array of
305/// `{role, content}` objects (or a JSON string encoding one), oldest first.
306/// Roles are `user`/`human`, `assistant`/`ai` and `system`. With
307/// `window.enabled`, only the last `window.size` turns (two messages per turn)
308/// are kept. The current query is appended as a user message, rendered from
309/// `query_prompt_template` (default `{{#sys.query#}}`). Without an explicit
310/// `query_prompt_template`, the query is not appended when a prompt template
311/// message already references `sys.query` (it would otherwise be sent twice).
312#[derive(Clone, Debug, Default, Deserialize)]
313pub struct MemoryConfig {
314    #[serde(default)]
315    pub window: Option<WindowConfig>,
316    #[serde(default)]
317    pub variable_selector: Option<Selector>,
318    #[serde(default)]
319    pub query_prompt_template: Option<Template>,
320}
321
322impl MemoryConfig {
323    fn history_selector(&self) -> Selector {
324        self.variable_selector.clone().unwrap_or_else(|| Selector::of("sys", "history"))
325    }
326
327    /// Conversation history. A missing history is empty; a history of the
328    /// wrong shape is an input error.
329    pub fn history(&self, pool: &VariablePool) -> Result<Vec<Message>, NodeError> {
330        let selector = self.history_selector();
331        let items: Vec<Value> = match pool.get(&selector) {
332            None | Some(Value::Null) => return Ok(Vec::new()),
333            Some(Value::Array(items)) => items.iter().cloned().collect(),
334            Some(Value::String(text)) if text.trim().is_empty() => return Ok(Vec::new()),
335            Some(Value::String(text)) => match serde_json::from_str::<Json>(&text) {
336                Ok(Json::Array(items)) => items.into_iter().map(Value::from_json).collect(),
337                _ => {
338                    return Err(NodeError::invalid_input(format!(
339                        "memory history `{selector}` is a string that is not a JSON array"
340                    )));
341                }
342            },
343            Some(other) => {
344                return Err(NodeError::invalid_input(format!(
345                    "memory history `{selector}` must be an array of {{role, content}} objects, got {}",
346                    other.type_name()
347                )));
348            }
349        };
350        let mut messages = Vec::with_capacity(items.len());
351        for (i, item) in items.iter().enumerate() {
352            let bad = |why: &str| NodeError::invalid_input(format!("memory history item #{i}: {why}"));
353            let obj = item.as_object().ok_or_else(|| bad("expected a {role, content} object"))?;
354            let role = match obj.get("role").and_then(Value::as_str) {
355                Some("user" | "human") => Role::User,
356                Some("assistant" | "ai") => Role::Assistant,
357                Some("system") => Role::System,
358                Some(other) => return Err(bad(&format!("unsupported role `{other}`"))),
359                None => return Err(bad("missing `role`")),
360            };
361            let content = obj.get("content").map(Value::to_text).unwrap_or_default();
362            messages.push(Message { role, ..Message::user(content) });
363        }
364        if let Some(window) = self.window.as_ref().filter(|w| w.enabled) {
365            let keep = window.size.unwrap_or(10).saturating_mul(2);
366            if messages.len() > keep {
367                messages.drain(..messages.len() - keep);
368            }
369        }
370        Ok(messages)
371    }
372
373    pub fn query(&self, pool: &VariablePool) -> Option<String> {
374        let text = match &self.query_prompt_template {
375            Some(t) => t.render(pool),
376            None => pool.get(&Selector::of("sys", "query"))?.to_text(),
377        };
378        if text.is_empty() { None } else { Some(text) }
379    }
380
381    /// Selectors read by the memory (history and query).
382    pub fn selectors(&self) -> Vec<Selector> {
383        let mut out = vec![self.history_selector()];
384        match &self.query_prompt_template {
385            Some(t) => out.extend(t.selectors().cloned()),
386            None => out.push(Selector::of("sys", "query")),
387        }
388        out
389    }
390}
391
392/// Render a template, substituting the literal context placeholder. Context and
393/// variable values are inserted as data and never parsed again, so content such
394/// as `{{#env.KEY#}}` inside retrieved text stays literal.
395pub fn render_with_context(template: &Template, pool: &VariablePool, context: Option<&str>) -> String {
396    let mut out = String::new();
397    for part in template.parts() {
398        match part {
399            Part::Text(t) => {
400                if t.contains(CONTEXT_PLACEHOLDER) {
401                    out.push_str(&t.replace(CONTEXT_PLACEHOLDER, context.unwrap_or_default()));
402                } else {
403                    out.push_str(t);
404                }
405            }
406            Part::Var(sel) => {
407                if let Some(v) = pool.get(sel) {
408                    out.push_str(&v.to_text());
409                }
410            }
411        }
412    }
413    out
414}
415
416/// Assemble the message list: leading system messages, history, remaining
417/// template messages, the memory query, then vision images on the last user message.
418pub fn build_messages(
419    template: &PromptTemplate,
420    pool: &VariablePool,
421    context: Option<&str>,
422    memory: Option<&MemoryConfig>,
423    images: Vec<ContentPart>,
424) -> Result<Vec<Message>, NodeError> {
425    let rendered: Vec<Message> = template
426        .messages()
427        .into_iter()
428        .map(|m| Message { role: m.role, ..Message::user(render_with_context(&m.text, pool, context)) })
429        .collect();
430    let split = rendered.iter().take_while(|m| m.role == Role::System).count();
431    let mut messages: Vec<Message> = rendered.iter().take(split).cloned().collect();
432    if let Some(mem) = memory {
433        messages.extend(mem.history(pool)?);
434    }
435    messages.extend(rendered.into_iter().skip(split));
436    // The current query is appended after the history unless the prompt template already
437    // contains it (a message referencing `sys.query`); otherwise it would be sent twice.
438    // An explicit `query_prompt_template` is always appended.
439    let template_has_query = template.messages().iter().any(|m| m.text.selectors().any(is_sys_query));
440    if let Some(mem) = memory {
441        if mem.query_prompt_template.is_some() || !template_has_query {
442            if let Some(query) = mem.query(pool) {
443                messages.push(Message::user(query));
444            }
445        }
446    }
447    if !images.is_empty() {
448        match messages.iter_mut().rev().find(|m| m.role == Role::User) {
449            Some(last_user) => last_user.content.extend(images),
450            None => messages.push(Message { content: images, ..Message::user("") }),
451        }
452    }
453    Ok(messages)
454}
455
456fn is_sys_query(selector: &Selector) -> bool {
457    selector.namespace() == "sys" && selector.name() == "query"
458}
459
460/// Look up the configured provider.
461pub fn model_for(ctx: &NodeContext, config: &ModelConfig) -> Result<Arc<dyn ChatModel>, NodeError> {
462    let registry = ctx.require_service::<ModelRegistry>("model registry (graphrt_llm::ModelRegistry)")?;
463    Ok(registry.get(&config.provider)?)
464}
465
466/// What a model call streams to the run's event stream.
467#[derive(Clone, Copy, Debug, PartialEq, Eq)]
468pub enum StreamMode {
469    /// Nothing is streamed.
470    Off,
471    /// Answer text on field `text`, reasoning on field `reasoning`, as produced.
472    Raw,
473    /// Like [`StreamMode::Raw`], but `<think>...</think>` sections inside the
474    /// answer are routed to `reasoning`, so the streamed `text` matches the
475    /// node's separated `text` output.
476    SeparateThink,
477}
478
479/// Run a request, streaming chunks according to `mode`.
480pub async fn invoke(
481    ctx: &NodeContext,
482    model: &dyn ChatModel,
483    request: ChatRequest,
484    mode: StreamMode,
485) -> Result<ChatResponse, NodeError> {
486    let stream = model.chat(request).await?;
487    let mut splitter = ThinkSplitter::default();
488    let response = collect(stream, |delta| match (mode, delta) {
489        (StreamMode::Off, _) => {}
490        (StreamMode::Raw, ChatDelta::Text(t)) if !t.is_empty() => ctx.chunk("text", t.clone()),
491        (StreamMode::SeparateThink, ChatDelta::Text(t)) => {
492            for (field, piece) in splitter.push(t) {
493                ctx.chunk(field.as_str(), piece);
494            }
495        }
496        (_, ChatDelta::Reasoning(t)) if !t.is_empty() => ctx.chunk("reasoning", t.clone()),
497        _ => {}
498    })
499    .await?;
500    if mode == StreamMode::SeparateThink {
501        for (field, piece) in splitter.finish() {
502            ctx.chunk(field.as_str(), piece);
503        }
504    }
505    Ok(response)
506}
507
508/// Finish reasons that mean the model produced a broken turn (worth a retry).
509const MALFORMED_FINISH: &[&str] = &["malformed_function_call", "unexpected_tool_call", "too_many_tool_calls"];
510
511/// What a node should do with a response's finish reason.
512#[derive(Clone, Debug, PartialEq, Eq)]
513pub enum FinishCheck {
514    /// A normal end (stop, length, tool calls, or none reported).
515    Normal,
516    /// An unrecognized provider reason: keep the answer but report it.
517    Unknown(String),
518}
519
520/// Turn finish reasons that mean "no usable answer" into node errors: a content
521/// filter block is fatal (`content_filter`), a malformed turn is retryable
522/// (`model_decode`). Unknown reasons are returned for the caller to report.
523pub fn check_finish(response: &ChatResponse) -> Result<FinishCheck, NodeError> {
524    match &response.finish_reason {
525        None | Some(FinishReason::Stop | FinishReason::Length | FinishReason::ToolCalls) => Ok(FinishCheck::Normal),
526        Some(FinishReason::ContentFilter) => Err(NodeError::fatal(
527            "content_filter",
528            "the provider's safety filter blocked the response; no answer was produced",
529        )),
530        Some(FinishReason::Other(reason)) if MALFORMED_FINISH.contains(&reason.as_str()) => {
531            Err(NodeError::new("model_decode", format!("the model produced a malformed turn ({reason})")))
532        }
533        Some(FinishReason::Other(reason)) => Ok(FinishCheck::Unknown(reason.clone())),
534        #[allow(unreachable_patterns)]
535        Some(other) => Ok(FinishCheck::Unknown(other.as_str().to_owned())),
536    }
537}
538
539/// Output field a streamed piece belongs to.
540#[derive(Clone, Copy, Debug, PartialEq, Eq)]
541pub enum StreamField {
542    Text,
543    Reasoning,
544}
545
546impl StreamField {
547    pub fn as_str(self) -> &'static str {
548        match self {
549            StreamField::Text => "text",
550            StreamField::Reasoning => "reasoning",
551        }
552    }
553}
554
555const THINK_OPEN: &str = "<think>";
556const THINK_CLOSE: &str = "</think>";
557
558/// Splits streamed answer text into answer and `<think>` reasoning pieces,
559/// handling tags split across chunks. Leading whitespace of the answer is
560/// dropped, matching [`split_think_tags`].
561#[derive(Debug, Default)]
562pub struct ThinkSplitter {
563    /// Text that may be the start of a tag and is held back until decided.
564    pending: String,
565    in_think: bool,
566    answer_started: bool,
567}
568
569impl ThinkSplitter {
570    /// Feed a chunk; returns the pieces that can be emitted now.
571    pub fn push(&mut self, chunk: &str) -> Vec<(StreamField, String)> {
572        self.pending.push_str(chunk);
573        let mut out = Vec::new();
574        loop {
575            let tag = if self.in_think { THINK_CLOSE } else { THINK_OPEN };
576            if let Some(pos) = self.pending.find(tag) {
577                let before: String = self.pending.drain(..pos).collect();
578                self.emit(&mut out, before);
579                self.pending.drain(..tag.len());
580                self.in_think = !self.in_think;
581                continue;
582            }
583            // Hold back the longest suffix that could still become `tag`.
584            let keep = longest_tag_prefix_suffix(&self.pending, tag);
585            let emit_len = self.pending.len() - keep;
586            let ready: String = self.pending.drain(..emit_len).collect();
587            self.emit(&mut out, ready);
588            break;
589        }
590        out
591    }
592
593    /// Flush held-back text at the end of the stream.
594    pub fn finish(&mut self) -> Vec<(StreamField, String)> {
595        let mut out = Vec::new();
596        let rest = std::mem::take(&mut self.pending);
597        self.emit(&mut out, rest);
598        out
599    }
600
601    fn emit(&mut self, out: &mut Vec<(StreamField, String)>, mut text: String) {
602        if self.in_think {
603            if !text.is_empty() {
604                out.push((StreamField::Reasoning, text));
605            }
606            return;
607        }
608        if !self.answer_started {
609            let trimmed = text.trim_start();
610            if trimmed.is_empty() {
611                return;
612            }
613            text = trimmed.to_owned();
614            self.answer_started = true;
615        }
616        if !text.is_empty() {
617            out.push((StreamField::Text, text));
618        }
619    }
620}
621
622/// Length of the longest suffix of `text` that is a proper prefix of `tag`.
623fn longest_tag_prefix_suffix(text: &str, tag: &str) -> usize {
624    (1..tag.len()).rev().find(|&n| tag.get(..n).is_some_and(|prefix| text.ends_with(prefix))).unwrap_or(0)
625}
626
627/// Usage as an output object.
628pub fn usage_value(usage: &Usage) -> Value {
629    Value::from_json(serde_json::json!({
630        "prompt_tokens": usage.prompt_tokens,
631        "completion_tokens": usage.completion_tokens,
632        "total_tokens": usage.total_tokens,
633    }))
634}
635
636/// Remove `<think>...</think>` sections, returning `(answer, reasoning)`.
637pub fn split_think_tags(text: &str) -> (String, String) {
638    let mut answer = String::new();
639    let mut reasoning = String::new();
640    let mut rest = text;
641    while let Some(start) = rest.find(THINK_OPEN) {
642        answer.push_str(&rest[..start]);
643        let after = &rest[start + THINK_OPEN.len()..];
644        match after.find(THINK_CLOSE) {
645            Some(end) => {
646                reasoning.push_str(after[..end].trim());
647                rest = &after[end + THINK_CLOSE.len()..];
648            }
649            None => {
650                reasoning.push_str(after.trim());
651                rest = "";
652            }
653        }
654    }
655    answer.push_str(rest);
656    (answer.trim_start().to_owned(), reasoning)
657}
658
659/// Maximum number of candidate positions [`extract_json`] tries. Each attempt
660/// scans at most the remaining text once, so total work is linear in the
661/// input size (times this constant).
662const MAX_JSON_CANDIDATES: usize = 32;
663
664/// Extract the first JSON object or array from model output, tolerating code
665/// fences, think tags and surrounding prose. Brace matching ignores braces
666/// inside string literals.
667pub fn extract_json(text: &str) -> Option<Json> {
668    extract_json_where(text, |v| v.is_object() || v.is_array())
669}
670
671/// Like [`extract_json`], returning the first candidate accepted by `accept`.
672pub fn extract_json_where(text: &str, accept: impl Fn(&Json) -> bool) -> Option<Json> {
673    let (answer, _) = split_think_tags(text);
674    let trimmed = answer.trim();
675    if let Ok(v) = serde_json::from_str::<Json>(trimmed) {
676        if accept(&v) {
677            return Some(v);
678        }
679    }
680    let bytes = trimmed.as_bytes();
681    let mut search_from = 0;
682    let mut attempts = 0;
683    while let Some(offset) = trimmed.get(search_from..)?.find(['{', '[']) {
684        attempts += 1;
685        if attempts > MAX_JSON_CANDIDATES {
686            return None;
687        }
688        let start = search_from + offset;
689        if let Some(candidate) = matching_close(bytes, start).and_then(|end| trimmed.get(start..=end)) {
690            if let Ok(v) = serde_json::from_str::<Json>(candidate) {
691                if accept(&v) {
692                    return Some(v);
693                }
694            }
695        }
696        search_from = start + 1;
697    }
698    None
699}
700
701fn matching_close(bytes: &[u8], start: usize) -> Option<usize> {
702    let mut depth: usize = 0;
703    let mut in_string = false;
704    let mut escaped = false;
705    for (i, &b) in bytes.iter().enumerate().skip(start) {
706        if in_string {
707            match b {
708                _ if escaped => escaped = false,
709                b'\\' => escaped = true,
710                b'"' => in_string = false,
711                _ => {}
712            }
713            continue;
714        }
715        match b {
716            b'"' => in_string = true,
717            b'{' | b'[' => depth += 1,
718            b'}' | b']' => {
719                depth = depth.checked_sub(1)?;
720                if depth == 0 {
721                    return Some(i);
722                }
723            }
724            _ => {}
725        }
726    }
727    None
728}
729
730#[cfg(test)]
731mod tests {
732    use super::*;
733    use graphrt_core::value::File;
734
735    #[test]
736    fn json_extraction() {
737        assert_eq!(extract_json("```json\n{\"a\":1}\n```").unwrap()["a"], 1);
738        assert_eq!(extract_json("<think>{x}</think>Sure: {\"s\":\"}{\",\"b\":2} done").unwrap()["b"], 2);
739        assert!(extract_json("no json here").is_none());
740        let v = extract_json_where("see [1], then {\"a\": 2}", Json::is_object).unwrap();
741        assert_eq!(v["a"], 2);
742    }
743
744    #[test]
745    fn json_extraction_is_bounded_on_hostile_input() {
746        let hostile = "{".repeat(200_000);
747        let started = std::time::Instant::now();
748        assert!(extract_json(&hostile).is_none());
749        assert!(started.elapsed() < std::time::Duration::from_secs(2), "{:?}", started.elapsed());
750    }
751
752    #[test]
753    fn think_tags() {
754        let (a, r) = split_think_tags("<think>plan</think>\nanswer");
755        assert_eq!(a, "answer");
756        assert_eq!(r, "plan");
757    }
758
759    fn split_stream(chunks: &[&str]) -> (String, String) {
760        let mut s = ThinkSplitter::default();
761        let mut pieces = Vec::new();
762        for c in chunks {
763            pieces.extend(s.push(c));
764        }
765        pieces.extend(s.finish());
766        let mut text = String::new();
767        let mut reasoning = String::new();
768        for (field, piece) in pieces {
769            match field {
770                StreamField::Text => text.push_str(&piece),
771                StreamField::Reasoning => reasoning.push_str(&piece),
772            }
773        }
774        (text, reasoning)
775    }
776
777    #[test]
778    fn think_splitter_handles_tags_split_across_chunks() {
779        let (text, reasoning) = split_stream(&["<thi", "nk>pl", "an</th", "ink>\n", "Hel", "lo <", "b>"]);
780        assert_eq!(text, "Hello <b>");
781        assert_eq!(reasoning, "plan");
782        let whole = "<think>plan</think>\nHello <b>";
783        assert_eq!(text, split_think_tags(whole).0);
784        let (text, reasoning) = split_stream(&["no tags at all"]);
785        assert_eq!((text.as_str(), reasoning.as_str()), ("no tags at all", ""));
786        let (text, _) = split_stream(&["a <", "/think> b"]);
787        assert_eq!(text, "a </think> b");
788    }
789
790    #[test]
791    fn context_is_not_reparsed() {
792        let mut pool = VariablePool::new();
793        pool.insert("env", "KEY", Value::Secret(Arc::from("s3cret")));
794        pool.insert("sys", "query", Value::from("q"));
795        let t = Template::parse("ctx: {{#context#}} q: {{#sys.query#}}");
796        let out = render_with_context(&t, &pool, Some("evil {{#env.KEY#}}"));
797        assert_eq!(out, "ctx: evil {{#env.KEY#}} q: q");
798        let t = Template::parse("{{#env.KEY#}}");
799        assert!(!render_with_context(&t, &pool, None).contains("s3cret"));
800    }
801
802    fn model_config(params: Json) -> ModelConfig {
803        serde_json::from_value(serde_json::json!({"provider": "p", "name": "m", "completion_params": params})).unwrap()
804    }
805
806    #[test]
807    fn params_resolve_without_mutation() {
808        let mut pool = VariablePool::new();
809        pool.insert("start", "t", Value::from("0.3"));
810        let cfg = model_config(serde_json::json!(
811            {"temperature": "{{#start.t#}}", "max_tokens": 20, "stop": "x", "seed": "7", "mode": "chat", "top_p": null}
812        ));
813        let req = cfg.request(vec![], &pool).unwrap();
814        assert_eq!(req.temperature, Some(0.3));
815        assert_eq!(req.max_tokens, Some(20));
816        assert_eq!(req.top_p, None);
817        assert_eq!(req.stop, vec!["x".to_owned()]);
818        assert_eq!(req.extra.get("seed"), Some(&Json::from(7)));
819        assert!(!req.extra.contains_key("mode"));
820        assert_eq!(cfg.completion_params["temperature"], "{{#start.t#}}");
821        assert_eq!(cfg.selectors(), vec![Selector::of("start", "t")]);
822    }
823
824    #[test]
825    fn invalid_params_are_errors() {
826        let pool = VariablePool::new();
827        for params in [
828            serde_json::json!({"temperature": "hot"}),
829            serde_json::json!({"max_tokens": -1}),
830            serde_json::json!({"max_tokens": 1e12}),
831            serde_json::json!({"stop": [1, 2]}),
832            serde_json::json!({"temperature": "{{#start.missing#}}"}),
833            serde_json::json!({"stream": false}),
834        ] {
835            let err = model_config(params.clone()).request(vec![], &pool).unwrap_err();
836            assert_eq!(err.kind, "invalid_input", "{params}");
837            assert!(!err.retryable);
838        }
839    }
840
841    #[test]
842    fn messages_with_memory_and_vision() {
843        let mut pool = VariablePool::new();
844        pool.insert("sys", "query", Value::from("now"));
845        pool.insert(
846            "sys",
847            "history",
848            Value::from_json(serde_json::json!([
849                {"role": "user", "content": "old1"}, {"role": "assistant", "content": "old2"},
850                {"role": "user", "content": "old3"}, {"role": "assistant", "content": "old4"}
851            ])),
852        );
853        let image = File {
854            marker: true,
855            name: "a.png".into(),
856            kind: FileKind::Image,
857            url: Some("https://x/a.png".into()),
858            ..File::default()
859        };
860        pool.insert("sys", "files", Value::array([Value::from(image)]));
861        let template: PromptTemplate = serde_json::from_value(serde_json::json!([
862            {"role": "system", "text": "sys"}, {"role": "user", "text": "tmpl"}
863        ]))
864        .unwrap();
865        let memory: MemoryConfig =
866            serde_json::from_value(serde_json::json!({"window": {"enabled": true, "size": 1}})).unwrap();
867        let vision = VisionConfig { enabled: true, ..VisionConfig::default() };
868        let msgs = build_messages(&template, &pool, None, Some(&memory), vision.image_parts(&pool).unwrap()).unwrap();
869        let texts: Vec<String> = msgs.iter().map(Message::text).collect();
870        assert_eq!(texts, vec!["sys", "old3", "old4", "tmpl", "now"]);
871        assert!(matches!(msgs[4].content.last(), Some(ContentPart::Image { .. })));
872    }
873
874    #[test]
875    fn memory_does_not_duplicate_a_templated_query() {
876        let mut pool = VariablePool::new();
877        pool.insert("sys", "query", Value::from("now"));
878        pool.insert("sys", "history", Value::from_json(serde_json::json!([{"role": "user", "content": "old"}])));
879        let memory = MemoryConfig::default();
880        let with_query: PromptTemplate = serde_json::from_value(serde_json::json!([
881            {"role": "system", "text": "sys"}, {"role": "user", "text": "Q: {{#sys.query#}}"}
882        ]))
883        .unwrap();
884        let msgs = build_messages(&with_query, &pool, None, Some(&memory), Vec::new()).unwrap();
885        let texts: Vec<String> = msgs.iter().map(Message::text).collect();
886        assert_eq!(texts, vec!["sys", "old", "Q: now"], "the query is sent once, after the history");
887
888        let without_query: PromptTemplate =
889            serde_json::from_value(serde_json::json!([{"role": "system", "text": "sys"}])).unwrap();
890        let msgs = build_messages(&without_query, &pool, None, Some(&memory), Vec::new()).unwrap();
891        let texts: Vec<String> = msgs.iter().map(Message::text).collect();
892        assert_eq!(texts, vec!["sys", "old", "now"], "memory appends the query when the template lacks it");
893
894        let explicit: MemoryConfig =
895            serde_json::from_value(serde_json::json!({"query_prompt_template": "again: {{#sys.query#}}"})).unwrap();
896        let msgs = build_messages(&with_query, &pool, None, Some(&explicit), Vec::new()).unwrap();
897        assert_eq!(msgs.last().map(Message::text).as_deref(), Some("again: now"), "explicit query template wins");
898    }
899
900    #[test]
901    fn history_shapes() {
902        let memory = MemoryConfig::default();
903        let mut pool = VariablePool::new();
904        pool.insert("sys", "history", Value::from(r#"[{"role": "human", "content": "hi"}]"#));
905        assert_eq!(memory.history(&pool).unwrap().len(), 1, "JSON-string history is parsed");
906        pool.insert("sys", "history", Value::from_json(serde_json::json!([{"role": "tool", "content": "x"}])));
907        assert!(memory.history(&pool).is_err());
908        pool.insert("sys", "history", Value::Int(3));
909        assert!(memory.history(&pool).is_err());
910        assert!(memory.history(&VariablePool::new()).unwrap().is_empty());
911    }
912
913    #[test]
914    fn image_without_url_is_an_error() {
915        let mut pool = VariablePool::new();
916        let image = File { marker: true, name: "a.png".into(), kind: FileKind::Image, ..File::default() };
917        pool.insert("sys", "files", Value::array([Value::from(image)]));
918        let vision = VisionConfig { enabled: true, ..VisionConfig::default() };
919        assert!(vision.image_parts(&pool).is_err());
920    }
921}