Skip to main content

graphrt_llm/nodes/
common.rs

1//! Configuration and prompt assembly shared by the model-calling nodes.
2
3use std::sync::{Arc, LazyLock};
4
5use graphrt_core::event::Usage;
6use graphrt_core::template::{Part, Template};
7use graphrt_core::value::{FileKind, Value};
8use graphrt_core::{NodeContext, NodeError, Selector, VariablePool};
9use serde::Deserialize;
10use serde_json::Value as Json;
11
12use crate::registry::ModelRegistry;
13use crate::types::{
14    ChatDelta, ChatModel, ChatRequest, ChatResponse, ContentPart, FinishReason, Message, RESERVED_EXTRA_KEYS, Role,
15    collect,
16};
17
18/// Literal placeholder replaced by the configured context value.
19pub const CONTEXT_PLACEHOLDER: &str = "{{#context#}}";
20
21/// `model: {provider, name, completion_params}`.
22#[derive(Clone, Debug, Deserialize)]
23pub struct ModelConfig {
24    pub provider: String,
25    pub name: String,
26    #[serde(default)]
27    pub completion_params: serde_json::Map<String, Json>,
28}
29
30/// Parameters never forwarded to providers (UI-only settings).
31const IGNORED_PARAMS: &[&str] = &["mode"];
32
33/// A resolved completion parameter.
34enum Param {
35    /// Not set (explicit `null`, or an empty literal string).
36    Unset,
37    Value(Json),
38}
39
40impl ModelConfig {
41    /// Build a request for this model, applying completion parameters. Parameter
42    /// values that are templates (`"{{#start.temp#}}"`) are resolved per run;
43    /// the configuration itself is never modified.
44    ///
45    /// A parameter whose value has the wrong type (e.g. `temperature: "hot"`),
46    /// or a template that resolves to nothing, is an input error naming the
47    /// parameter instead of being dropped silently.
48    pub fn request(&self, messages: Vec<Message>, pool: &VariablePool) -> Result<ChatRequest, NodeError> {
49        let mut req = ChatRequest::new(self.name.clone(), messages);
50        for (key, raw) in &self.completion_params {
51            if IGNORED_PARAMS.contains(&key.as_str()) {
52                continue;
53            }
54            let value = match resolve_param(key, raw, pool)? {
55                Param::Unset => continue,
56                Param::Value(v) => v,
57            };
58            let bad = |expected: &str| {
59                NodeError::invalid_input(format!("completion parameter `{key}` must be {expected}, got {value}"))
60            };
61            match key.as_str() {
62                "temperature" => req.temperature = Some(value.as_f64().ok_or_else(|| bad("a number"))?),
63                "top_p" => req.top_p = Some(value.as_f64().ok_or_else(|| bad("a number"))?),
64                "max_tokens" | "max_completion_tokens" => {
65                    let n = value
66                        .as_u64()
67                        .and_then(|n| u32::try_from(n).ok())
68                        .ok_or_else(|| bad("a non-negative integer that fits in 32 bits"))?;
69                    req.max_tokens = Some(n);
70                }
71                "stop" => {
72                    req.stop = match &value {
73                        Json::String(s) => vec![s.clone()],
74                        Json::Array(items) => items
75                            .iter()
76                            .map(|i| i.as_str().map(str::to_owned))
77                            .collect::<Option<Vec<_>>>()
78                            .ok_or_else(|| bad("a string or a list of strings"))?,
79                        _ => return Err(bad("a string or a list of strings")),
80                    };
81                }
82                reserved if RESERVED_EXTRA_KEYS.contains(&reserved) => {
83                    return Err(NodeError::invalid_input(format!(
84                        "completion parameter `{key}` is managed by the node and cannot be set"
85                    )));
86                }
87                _ => {
88                    req.extra.insert(key.clone(), value);
89                }
90            }
91        }
92        Ok(req)
93    }
94
95    /// Selectors referenced by templated completion parameters.
96    pub fn selectors(&self) -> Vec<Selector> {
97        self.completion_params
98            .values()
99            .filter_map(Json::as_str)
100            .filter(|s| s.contains("{{#"))
101            .flat_map(|s| Template::parse(s).selectors().cloned().collect::<Vec<_>>())
102            .collect()
103    }
104}
105
106/// Resolve a completion parameter: templates are rendered, numeric/boolean strings coerced.
107fn resolve_param(key: &str, raw: &Json, pool: &VariablePool) -> Result<Param, NodeError> {
108    let text = match raw {
109        Json::Null => return Ok(Param::Unset),
110        Json::String(s) => s,
111        other => return Ok(Param::Value(other.clone())),
112    };
113    let is_template = text.contains("{{#");
114    let resolved = if is_template { Template::parse(text).resolve(pool) } else { Value::from(text.as_str()) };
115    match resolved {
116        Value::String(s) => {
117            let t = s.trim();
118            if t.is_empty() {
119                if is_template {
120                    return Err(NodeError::invalid_input(format!(
121                        "completion parameter `{key}` resolved to an empty value (`{text}`)"
122                    )));
123                }
124                return Ok(Param::Unset);
125            }
126            Ok(Param::Value(if let Ok(i) = t.parse::<i64>() {
127                Json::from(i)
128            } else if let Ok(f) = t.parse::<f64>() {
129                Json::from(f)
130            } else if t == "true" || t == "false" {
131                Json::Bool(t == "true")
132            } else {
133                Json::String(s.to_string())
134            }))
135        }
136        Value::Null => Err(NodeError::invalid_input(format!(
137            "completion parameter `{key}` references a missing variable (`{text}`)"
138        ))),
139        other => Ok(Param::Value(other.to_json())),
140    }
141}
142
143/// One prompt message template.
144#[derive(Clone, Debug, Deserialize)]
145pub struct PromptMessage {
146    #[serde(default = "user_role")]
147    pub role: Role,
148    #[serde(default)]
149    pub text: Template,
150}
151
152fn user_role() -> Role {
153    Role::User
154}
155
156/// `prompt_template`: a list of chat messages, or a single completion prompt.
157#[derive(Clone, Debug, Deserialize)]
158#[serde(untagged)]
159pub enum PromptTemplate {
160    Chat(Vec<PromptMessage>),
161    Completion(PromptMessage),
162}
163
164impl Default for PromptTemplate {
165    fn default() -> Self {
166        PromptTemplate::Chat(Vec::new())
167    }
168}
169
170impl PromptTemplate {
171    pub fn messages(&self) -> &[PromptMessage] {
172        match self {
173            PromptTemplate::Chat(m) => m,
174            PromptTemplate::Completion(m) => std::slice::from_ref(m),
175        }
176    }
177
178    /// Selectors referenced by the message templates.
179    pub fn selectors(&self) -> Vec<Selector> {
180        self.messages().iter().flat_map(|m| m.text.selectors().cloned()).collect()
181    }
182}
183
184/// `context: {enabled, variable_selector}`.
185#[derive(Clone, Debug, Default, Deserialize)]
186pub struct ContextConfig {
187    #[serde(default)]
188    pub enabled: bool,
189    #[serde(default)]
190    pub variable_selector: Option<Selector>,
191}
192
193impl ContextConfig {
194    /// Context text, if enabled and present.
195    pub fn resolve(&self, pool: &VariablePool) -> Option<String> {
196        if !self.enabled {
197            return None;
198        }
199        let value = pool.get(self.variable_selector.as_ref()?)?;
200        Some(context_text(&value))
201    }
202
203    pub fn selectors(&self) -> Vec<Selector> {
204        if !self.enabled {
205            return Vec::new();
206        }
207        self.variable_selector.iter().cloned().collect()
208    }
209}
210
211/// Retrieved-document lists render as their `content` fields; other values as text.
212fn context_text(value: &Value) -> String {
213    match value {
214        Value::Array(items) => items
215            .iter()
216            .map(|item| match item.as_object().and_then(|o| o.get("content")) {
217                Some(content) => content.to_text(),
218                None => item.to_text(),
219            })
220            .collect::<Vec<_>>()
221            .join("\n"),
222        other => other.to_text(),
223    }
224}
225
226#[derive(Clone, Debug, Default, Deserialize)]
227pub struct VisionDetail {
228    #[serde(default)]
229    pub variable_selector: Option<Selector>,
230    #[serde(default)]
231    pub detail: Option<String>,
232}
233
234/// `vision: {enabled, configs: {variable_selector, detail}}` (or the fields inline).
235#[derive(Clone, Debug, Default, Deserialize)]
236pub struct VisionConfig {
237    #[serde(default)]
238    pub enabled: bool,
239    #[serde(default)]
240    pub configs: Option<VisionDetail>,
241    #[serde(default)]
242    pub variable_selector: Option<Selector>,
243    #[serde(default)]
244    pub detail: Option<String>,
245}
246
247impl VisionConfig {
248    fn selector(&self) -> Selector {
249        self.configs
250            .as_ref()
251            .and_then(|c| c.variable_selector.clone())
252            .or_else(|| self.variable_selector.clone())
253            .unwrap_or_else(|| Selector::of("sys", "files"))
254    }
255
256    /// Image parts for the configured files (default selector `sys.files`).
257    ///
258    /// Non-image files are ignored. An image without a URL (e.g. a host
259    /// storage reference that was not resolved) is an input error rather than
260    /// being dropped silently.
261    pub fn image_parts(&self, pool: &VariablePool) -> Result<Vec<ContentPart>, NodeError> {
262        if !self.enabled {
263            return Ok(Vec::new());
264        }
265        let detail = self.configs.as_ref().and_then(|c| c.detail.clone()).or_else(|| self.detail.clone());
266        let files: Vec<Value> = match pool.get(&self.selector()) {
267            Some(Value::Array(items)) => items.iter().cloned().collect(),
268            Some(v @ Value::File(_)) => vec![v],
269            _ => Vec::new(),
270        };
271        let mut parts = Vec::new();
272        for file in files.iter().filter_map(Value::as_file) {
273            let is_image =
274                file.kind == FileKind::Image || file.mime_type.as_deref().is_some_and(|m| m.starts_with("image/"));
275            if !is_image {
276                continue;
277            }
278            let url = file.url.clone().filter(|u| !u.is_empty()).ok_or_else(|| {
279                NodeError::invalid_input(format!(
280                    "image `{}` has no URL; resolve host files to URLs before sending them to a model",
281                    file.name
282                ))
283            })?;
284            parts.push(ContentPart::Image { url, detail: detail.clone() });
285        }
286        Ok(parts)
287    }
288
289    pub fn selectors(&self) -> Vec<Selector> {
290        if self.enabled { vec![self.selector()] } else { Vec::new() }
291    }
292}
293
294#[derive(Clone, Debug, Default, Deserialize)]
295pub struct WindowConfig {
296    #[serde(default)]
297    pub enabled: bool,
298    #[serde(default)]
299    pub size: Option<usize>,
300}
301
302/// Conversation memory.
303///
304/// History is read from `variable_selector` (default `sys.history`): an array of
305/// `{role, content}` objects (or a JSON string encoding one), oldest first.
306/// Roles are `user`/`human`, `assistant`/`ai` and `system`. With
307/// `window.enabled`, only the last `window.size` turns (two messages per turn)
308/// are kept. The current query is appended as a user message, rendered from
309/// `query_prompt_template` (default `{{#sys.query#}}`). Without an explicit
310/// `query_prompt_template`, the query is not appended when a prompt template
311/// message already references `sys.query` (it would otherwise be sent twice).
312#[derive(Clone, Debug, Default, Deserialize)]
313pub struct MemoryConfig {
314    #[serde(default)]
315    pub window: Option<WindowConfig>,
316    #[serde(default)]
317    pub variable_selector: Option<Selector>,
318    #[serde(default)]
319    pub query_prompt_template: Option<Template>,
320}
321
322impl MemoryConfig {
323    fn history_selector(&self) -> Selector {
324        self.variable_selector.clone().unwrap_or_else(|| Selector::of("sys", "history"))
325    }
326
327    /// Conversation history. A missing history is empty; a history of the
328    /// wrong shape is an input error.
329    pub fn history(&self, pool: &VariablePool) -> Result<Vec<Message>, NodeError> {
330        let selector = self.history_selector();
331        let items: Vec<Value> = match pool.get(&selector) {
332            None | Some(Value::Null) => return Ok(Vec::new()),
333            Some(Value::Array(items)) => items.iter().cloned().collect(),
334            Some(Value::String(text)) if text.trim().is_empty() => return Ok(Vec::new()),
335            Some(Value::String(text)) => match serde_json::from_str::<Json>(&text) {
336                Ok(Json::Array(items)) => items.into_iter().map(Value::from_json).collect(),
337                _ => {
338                    return Err(NodeError::invalid_input(format!(
339                        "memory history `{selector}` is a string that is not a JSON array"
340                    )));
341                }
342            },
343            Some(other) => {
344                return Err(NodeError::invalid_input(format!(
345                    "memory history `{selector}` must be an array of {{role, content}} objects, got {}",
346                    other.type_name()
347                )));
348            }
349        };
350        let mut messages = Vec::with_capacity(items.len());
351        for (i, item) in items.iter().enumerate() {
352            let bad = |why: &str| NodeError::invalid_input(format!("memory history item #{i}: {why}"));
353            let obj = item.as_object().ok_or_else(|| bad("expected a {role, content} object"))?;
354            let role = match obj.get("role").and_then(Value::as_str) {
355                Some("user" | "human") => Role::User,
356                Some("assistant" | "ai") => Role::Assistant,
357                Some("system") => Role::System,
358                Some(other) => return Err(bad(&format!("unsupported role `{other}`"))),
359                None => return Err(bad("missing `role`")),
360            };
361            let content = obj.get("content").map(Value::to_text).unwrap_or_default();
362            messages.push(Message { role, ..Message::user(content) });
363        }
364        if let Some(window) = self.window.as_ref().filter(|w| w.enabled) {
365            let keep = window.size.unwrap_or(10).saturating_mul(2);
366            if messages.len() > keep {
367                messages.drain(..messages.len() - keep);
368            }
369        }
370        Ok(messages)
371    }
372
373    pub fn query(&self, pool: &VariablePool) -> Option<String> {
374        let text = match &self.query_prompt_template {
375            Some(t) => t.render(pool),
376            None => pool.get(&Selector::of("sys", "query"))?.to_text(),
377        };
378        if text.is_empty() { None } else { Some(text) }
379    }
380
381    /// Selectors read by the memory (history and query).
382    pub fn selectors(&self) -> Vec<Selector> {
383        let mut out = vec![self.history_selector()];
384        match &self.query_prompt_template {
385            Some(t) => out.extend(t.selectors().cloned()),
386            None => out.push(Selector::of("sys", "query")),
387        }
388        out
389    }
390}
391
392/// Render a template, substituting the literal context placeholder. Context and
393/// variable values are inserted as data and never parsed again, so content such
394/// as `{{#env.KEY#}}` inside retrieved text stays literal.
395pub fn render_with_context(template: &Template, pool: &VariablePool, context: Option<&str>) -> String {
396    let mut out = String::new();
397    for part in template.parts() {
398        match part {
399            Part::Text(t) => {
400                if t.contains(CONTEXT_PLACEHOLDER) {
401                    out.push_str(&t.replace(CONTEXT_PLACEHOLDER, context.unwrap_or_default()));
402                } else {
403                    out.push_str(t);
404                }
405            }
406            Part::Var(sel) => match pool.get(sel) {
407                // Strings (the common case) are copied once, straight into the prompt.
408                Some(Value::String(s)) => out.push_str(&s),
409                Some(v) => out.push_str(&v.to_text()),
410                None => {}
411            },
412        }
413    }
414    out
415}
416
417/// Assemble the message list: leading system messages, history, remaining
418/// template messages, the memory query, then vision images on the last user message.
419pub fn build_messages(
420    template: &PromptTemplate,
421    pool: &VariablePool,
422    context: Option<&str>,
423    memory: Option<&MemoryConfig>,
424    images: Vec<ContentPart>,
425) -> Result<Vec<Message>, NodeError> {
426    let prompts = template.messages();
427    let render =
428        |m: &PromptMessage| Message { role: m.role, ..Message::user(render_with_context(&m.text, pool, context)) };
429    let split = prompts.iter().take_while(|m| m.role == Role::System).count();
430    let (system, rest) = prompts.split_at(split);
431    let history = match memory {
432        Some(mem) => mem.history(pool)?,
433        None => Vec::new(),
434    };
435    // Room for the memory query and an images-only message.
436    let mut messages = Vec::with_capacity(prompts.len() + history.len() + 2);
437    messages.extend(system.iter().map(render));
438    messages.extend(history);
439    messages.extend(rest.iter().map(render));
440    // The current query is appended after the history unless the prompt template already
441    // contains it (a message referencing `sys.query`); otherwise it would be sent twice.
442    // An explicit `query_prompt_template` is always appended.
443    let template_has_query = template.messages().iter().any(|m| m.text.selectors().any(is_sys_query));
444    if let Some(mem) = memory {
445        if mem.query_prompt_template.is_some() || !template_has_query {
446            if let Some(query) = mem.query(pool) {
447                messages.push(Message::user(query));
448            }
449        }
450    }
451    if !images.is_empty() {
452        match messages.iter_mut().rev().find(|m| m.role == Role::User) {
453            Some(last_user) => last_user.content.extend(images),
454            None => messages.push(Message { content: images, ..Message::user("") }),
455        }
456    }
457    Ok(messages)
458}
459
460fn is_sys_query(selector: &Selector) -> bool {
461    selector.namespace() == "sys" && selector.name() == "query"
462}
463
464/// Look up the configured provider.
465pub fn model_for(ctx: &NodeContext, config: &ModelConfig) -> Result<Arc<dyn ChatModel>, NodeError> {
466    let registry = ctx.require_service::<ModelRegistry>("model registry (graphrt_llm::ModelRegistry)")?;
467    Ok(registry.get(&config.provider)?)
468}
469
470/// What a model call streams to the run's event stream.
471#[derive(Clone, Copy, Debug, PartialEq, Eq)]
472pub enum StreamMode {
473    /// Nothing is streamed.
474    Off,
475    /// Answer text on field `text`, reasoning on field `reasoning`, as produced.
476    Raw,
477    /// Like [`StreamMode::Raw`], but `<think>...</think>` sections inside the
478    /// answer are routed to `reasoning`, so the streamed `text` matches the
479    /// node's separated `text` output.
480    SeparateThink,
481}
482
483/// Run a request, streaming chunks according to `mode`.
484pub async fn invoke(
485    ctx: &NodeContext,
486    model: &dyn ChatModel,
487    request: ChatRequest,
488    mode: StreamMode,
489) -> Result<ChatResponse, NodeError> {
490    let stream = model.chat(request).await?;
491    let mut splitter = ThinkSplitter::default();
492    let response = collect(stream, |delta| match (mode, delta) {
493        (StreamMode::Off, _) => {}
494        (StreamMode::Raw, ChatDelta::Text(t)) if !t.is_empty() => ctx.chunk_shared(StreamField::Text.shared(), t),
495        (StreamMode::SeparateThink, ChatDelta::Text(t)) => {
496            for (field, piece) in splitter.push(&t) {
497                ctx.chunk_shared(field.shared(), piece);
498            }
499        }
500        (_, ChatDelta::Reasoning(t)) if !t.is_empty() => ctx.chunk_shared(StreamField::Reasoning.shared(), t),
501        _ => {}
502    })
503    .await?;
504    if mode == StreamMode::SeparateThink {
505        for (field, piece) in splitter.finish() {
506            ctx.chunk_shared(field.shared(), piece);
507        }
508    }
509    Ok(response)
510}
511
512/// Finish reasons that mean the model produced a broken turn (worth a retry).
513const MALFORMED_FINISH: &[&str] = &["malformed_function_call", "unexpected_tool_call", "too_many_tool_calls"];
514
515/// What a node should do with a response's finish reason.
516#[derive(Clone, Debug, PartialEq, Eq)]
517pub enum FinishCheck {
518    /// A normal end (stop, length, tool calls, or none reported).
519    Normal,
520    /// An unrecognized provider reason: keep the answer but report it.
521    Unknown(String),
522}
523
524/// Turn finish reasons that mean "no usable answer" into node errors: a content
525/// filter block is fatal (`content_filter`), a malformed turn is retryable
526/// (`model_decode`). Unknown reasons are returned for the caller to report.
527pub fn check_finish(response: &ChatResponse) -> Result<FinishCheck, NodeError> {
528    match &response.finish_reason {
529        None | Some(FinishReason::Stop | FinishReason::Length | FinishReason::ToolCalls) => Ok(FinishCheck::Normal),
530        Some(FinishReason::ContentFilter) => Err(NodeError::fatal(
531            "content_filter",
532            "the provider's safety filter blocked the response; no answer was produced",
533        )),
534        Some(FinishReason::Other(reason)) if MALFORMED_FINISH.contains(&reason.as_str()) => {
535            Err(NodeError::new("model_decode", format!("the model produced a malformed turn ({reason})")))
536        }
537        Some(FinishReason::Other(reason)) => Ok(FinishCheck::Unknown(reason.clone())),
538        #[allow(unreachable_patterns)]
539        Some(other) => Ok(FinishCheck::Unknown(other.as_str().to_owned())),
540    }
541}
542
543/// Output field a streamed piece belongs to.
544#[derive(Clone, Copy, Debug, PartialEq, Eq)]
545pub enum StreamField {
546    Text,
547    Reasoning,
548}
549
550static TEXT_FIELD: LazyLock<Arc<str>> = LazyLock::new(|| Arc::from("text"));
551static REASONING_FIELD: LazyLock<Arc<str>> = LazyLock::new(|| Arc::from("reasoning"));
552
553impl StreamField {
554    pub fn as_str(self) -> &'static str {
555        match self {
556            StreamField::Text => "text",
557            StreamField::Reasoning => "reasoning",
558        }
559    }
560
561    /// The field name as a shared string for [`NodeContext::chunk_shared`]:
562    /// one allocation per process instead of one per chunk.
563    pub fn shared(self) -> &'static Arc<str> {
564        match self {
565            StreamField::Text => &TEXT_FIELD,
566            StreamField::Reasoning => &REASONING_FIELD,
567        }
568    }
569}
570
571const THINK_OPEN: &str = "<think>";
572const THINK_CLOSE: &str = "</think>";
573
574/// Splits streamed answer text into answer and `<think>` reasoning pieces,
575/// handling tags split across chunks. Leading whitespace of the answer is
576/// dropped, matching [`split_think_tags`].
577#[derive(Debug, Default)]
578pub struct ThinkSplitter {
579    /// Text that may be the start of a tag and is held back until decided.
580    pending: String,
581    in_think: bool,
582    answer_started: bool,
583}
584
585impl ThinkSplitter {
586    /// Feed a chunk; returns the pieces that can be emitted now.
587    pub fn push(&mut self, chunk: &str) -> Vec<(StreamField, String)> {
588        self.pending.push_str(chunk);
589        let mut out = Vec::new();
590        loop {
591            let tag = if self.in_think { THINK_CLOSE } else { THINK_OPEN };
592            if let Some(pos) = self.pending.find(tag) {
593                let before: String = self.pending.drain(..pos).collect();
594                self.emit(&mut out, before);
595                self.pending.drain(..tag.len());
596                self.in_think = !self.in_think;
597                continue;
598            }
599            // Hold back the longest suffix that could still become `tag`.
600            let keep = longest_tag_prefix_suffix(&self.pending, tag);
601            let emit_len = self.pending.len() - keep;
602            let ready: String = self.pending.drain(..emit_len).collect();
603            self.emit(&mut out, ready);
604            break;
605        }
606        out
607    }
608
609    /// Flush held-back text at the end of the stream.
610    pub fn finish(&mut self) -> Vec<(StreamField, String)> {
611        let mut out = Vec::new();
612        let rest = std::mem::take(&mut self.pending);
613        self.emit(&mut out, rest);
614        out
615    }
616
617    fn emit(&mut self, out: &mut Vec<(StreamField, String)>, mut text: String) {
618        if self.in_think {
619            if !text.is_empty() {
620                out.push((StreamField::Reasoning, text));
621            }
622            return;
623        }
624        if !self.answer_started {
625            let trimmed = text.trim_start();
626            if trimmed.is_empty() {
627                return;
628            }
629            text = trimmed.to_owned();
630            self.answer_started = true;
631        }
632        if !text.is_empty() {
633            out.push((StreamField::Text, text));
634        }
635    }
636}
637
638/// Length of the longest suffix of `text` that is a proper prefix of `tag`.
639fn longest_tag_prefix_suffix(text: &str, tag: &str) -> usize {
640    (1..tag.len()).rev().find(|&n| tag.get(..n).is_some_and(|prefix| text.ends_with(prefix))).unwrap_or(0)
641}
642
643/// The `finish_reason` output: the reason's wire name, or `""` when none was
644/// reported. The standard reasons share one allocation per process.
645pub fn finish_reason_value(reason: Option<&FinishReason>) -> Value {
646    static STANDARD: LazyLock<[Value; 5]> =
647        LazyLock::new(|| ["", "stop", "length", "tool_calls", "content_filter"].map(Value::from));
648    let slot = match reason {
649        None => 0,
650        Some(FinishReason::Stop) => 1,
651        Some(FinishReason::Length) => 2,
652        Some(FinishReason::ToolCalls) => 3,
653        Some(FinishReason::ContentFilter) => 4,
654        Some(other) => return Value::from(other.as_str()),
655    };
656    STANDARD.get(slot).cloned().unwrap_or_default()
657}
658
659/// The `usage` output object.
660pub fn usage_value(usage: &Usage) -> Value {
661    // Built directly rather than through `serde_json::json!`, which allocated a
662    // second map (and its keys) only to convert it.
663    let count = |n: u64| i64::try_from(n).map_or_else(|_| Value::Float(n as f64), Value::Int);
664    let mut map = graphrt_core::value::Map::with_capacity(3);
665    map.insert("prompt_tokens".to_owned(), count(usage.prompt_tokens));
666    map.insert("completion_tokens".to_owned(), count(usage.completion_tokens));
667    map.insert("total_tokens".to_owned(), count(usage.total_tokens));
668    Value::Object(Arc::new(map))
669}
670
671/// Remove `<think>...</think>` sections, returning `(answer, reasoning)`.
672pub fn split_think_tags(text: &str) -> (String, String) {
673    let mut answer = String::new();
674    let mut reasoning = String::new();
675    let mut rest = text;
676    while let Some(start) = rest.find(THINK_OPEN) {
677        answer.push_str(&rest[..start]);
678        let after = &rest[start + THINK_OPEN.len()..];
679        match after.find(THINK_CLOSE) {
680            Some(end) => {
681                reasoning.push_str(after[..end].trim());
682                rest = &after[end + THINK_CLOSE.len()..];
683            }
684            None => {
685                reasoning.push_str(after.trim());
686                rest = "";
687            }
688        }
689    }
690    answer.push_str(rest);
691    (answer.trim_start().to_owned(), reasoning)
692}
693
694/// Maximum number of candidate positions [`extract_json`] tries. Each attempt
695/// scans at most the remaining text once, so total work is linear in the
696/// input size (times this constant).
697const MAX_JSON_CANDIDATES: usize = 32;
698
699/// Extract the first JSON object or array from model output, tolerating code
700/// fences, think tags and surrounding prose. Brace matching ignores braces
701/// inside string literals.
702pub fn extract_json(text: &str) -> Option<Json> {
703    extract_json_where(text, |v| v.is_object() || v.is_array())
704}
705
706/// Like [`extract_json`], returning the first candidate accepted by `accept`.
707pub fn extract_json_where(text: &str, accept: impl Fn(&Json) -> bool) -> Option<Json> {
708    let (answer, _) = split_think_tags(text);
709    let trimmed = answer.trim();
710    if let Ok(v) = serde_json::from_str::<Json>(trimmed) {
711        if accept(&v) {
712            return Some(v);
713        }
714    }
715    let bytes = trimmed.as_bytes();
716    let mut search_from = 0;
717    let mut attempts = 0;
718    while let Some(offset) = trimmed.get(search_from..)?.find(['{', '[']) {
719        attempts += 1;
720        if attempts > MAX_JSON_CANDIDATES {
721            return None;
722        }
723        let start = search_from + offset;
724        if let Some(candidate) = matching_close(bytes, start).and_then(|end| trimmed.get(start..=end)) {
725            if let Ok(v) = serde_json::from_str::<Json>(candidate) {
726                if accept(&v) {
727                    return Some(v);
728                }
729            }
730        }
731        search_from = start + 1;
732    }
733    None
734}
735
736fn matching_close(bytes: &[u8], start: usize) -> Option<usize> {
737    let mut depth: usize = 0;
738    let mut in_string = false;
739    let mut escaped = false;
740    for (i, &b) in bytes.iter().enumerate().skip(start) {
741        if in_string {
742            match b {
743                _ if escaped => escaped = false,
744                b'\\' => escaped = true,
745                b'"' => in_string = false,
746                _ => {}
747            }
748            continue;
749        }
750        match b {
751            b'"' => in_string = true,
752            b'{' | b'[' => depth += 1,
753            b'}' | b']' => {
754                depth = depth.checked_sub(1)?;
755                if depth == 0 {
756                    return Some(i);
757                }
758            }
759            _ => {}
760        }
761    }
762    None
763}
764
765#[cfg(test)]
766mod tests {
767    use super::*;
768    use graphrt_core::value::File;
769
770    #[test]
771    fn json_extraction() {
772        assert_eq!(extract_json("```json\n{\"a\":1}\n```").unwrap()["a"], 1);
773        assert_eq!(extract_json("<think>{x}</think>Sure: {\"s\":\"}{\",\"b\":2} done").unwrap()["b"], 2);
774        assert!(extract_json("no json here").is_none());
775        let v = extract_json_where("see [1], then {\"a\": 2}", Json::is_object).unwrap();
776        assert_eq!(v["a"], 2);
777    }
778
779    #[test]
780    fn json_extraction_is_bounded_on_hostile_input() {
781        let hostile = "{".repeat(200_000);
782        let started = std::time::Instant::now();
783        assert!(extract_json(&hostile).is_none());
784        assert!(started.elapsed() < std::time::Duration::from_secs(2), "{:?}", started.elapsed());
785    }
786
787    #[test]
788    fn think_tags() {
789        let (a, r) = split_think_tags("<think>plan</think>\nanswer");
790        assert_eq!(a, "answer");
791        assert_eq!(r, "plan");
792    }
793
794    fn split_stream(chunks: &[&str]) -> (String, String) {
795        let mut s = ThinkSplitter::default();
796        let mut pieces = Vec::new();
797        for c in chunks {
798            pieces.extend(s.push(c));
799        }
800        pieces.extend(s.finish());
801        let mut text = String::new();
802        let mut reasoning = String::new();
803        for (field, piece) in pieces {
804            match field {
805                StreamField::Text => text.push_str(&piece),
806                StreamField::Reasoning => reasoning.push_str(&piece),
807            }
808        }
809        (text, reasoning)
810    }
811
812    #[test]
813    fn think_splitter_handles_tags_split_across_chunks() {
814        let (text, reasoning) = split_stream(&["<thi", "nk>pl", "an</th", "ink>\n", "Hel", "lo <", "b>"]);
815        assert_eq!(text, "Hello <b>");
816        assert_eq!(reasoning, "plan");
817        let whole = "<think>plan</think>\nHello <b>";
818        assert_eq!(text, split_think_tags(whole).0);
819        let (text, reasoning) = split_stream(&["no tags at all"]);
820        assert_eq!((text.as_str(), reasoning.as_str()), ("no tags at all", ""));
821        let (text, _) = split_stream(&["a <", "/think> b"]);
822        assert_eq!(text, "a </think> b");
823    }
824
825    #[test]
826    fn context_is_not_reparsed() {
827        let mut pool = VariablePool::new();
828        pool.insert("env", "KEY", Value::Secret(Arc::from("s3cret")));
829        pool.insert("sys", "query", Value::from("q"));
830        let t = Template::parse("ctx: {{#context#}} q: {{#sys.query#}}");
831        let out = render_with_context(&t, &pool, Some("evil {{#env.KEY#}}"));
832        assert_eq!(out, "ctx: evil {{#env.KEY#}} q: q");
833        let t = Template::parse("{{#env.KEY#}}");
834        assert!(!render_with_context(&t, &pool, None).contains("s3cret"));
835    }
836
837    fn model_config(params: Json) -> ModelConfig {
838        serde_json::from_value(serde_json::json!({"provider": "p", "name": "m", "completion_params": params})).unwrap()
839    }
840
841    #[test]
842    fn params_resolve_without_mutation() {
843        let mut pool = VariablePool::new();
844        pool.insert("start", "t", Value::from("0.3"));
845        let cfg = model_config(serde_json::json!(
846            {"temperature": "{{#start.t#}}", "max_tokens": 20, "stop": "x", "seed": "7", "mode": "chat", "top_p": null}
847        ));
848        let req = cfg.request(vec![], &pool).unwrap();
849        assert_eq!(req.temperature, Some(0.3));
850        assert_eq!(req.max_tokens, Some(20));
851        assert_eq!(req.top_p, None);
852        assert_eq!(req.stop, vec!["x".to_owned()]);
853        assert_eq!(req.extra.get("seed"), Some(&Json::from(7)));
854        assert!(!req.extra.contains_key("mode"));
855        assert_eq!(cfg.completion_params["temperature"], "{{#start.t#}}");
856        assert_eq!(cfg.selectors(), vec![Selector::of("start", "t")]);
857    }
858
859    #[test]
860    fn invalid_params_are_errors() {
861        let pool = VariablePool::new();
862        for params in [
863            serde_json::json!({"temperature": "hot"}),
864            serde_json::json!({"max_tokens": -1}),
865            serde_json::json!({"max_tokens": 1e12}),
866            serde_json::json!({"stop": [1, 2]}),
867            serde_json::json!({"temperature": "{{#start.missing#}}"}),
868            serde_json::json!({"stream": false}),
869        ] {
870            let err = model_config(params.clone()).request(vec![], &pool).unwrap_err();
871            assert_eq!(err.kind, "invalid_input", "{params}");
872            assert!(!err.retryable);
873        }
874    }
875
876    #[test]
877    fn messages_with_memory_and_vision() {
878        let mut pool = VariablePool::new();
879        pool.insert("sys", "query", Value::from("now"));
880        pool.insert(
881            "sys",
882            "history",
883            Value::from_json(serde_json::json!([
884                {"role": "user", "content": "old1"}, {"role": "assistant", "content": "old2"},
885                {"role": "user", "content": "old3"}, {"role": "assistant", "content": "old4"}
886            ])),
887        );
888        let image = File {
889            marker: true,
890            name: "a.png".into(),
891            kind: FileKind::Image,
892            url: Some("https://x/a.png".into()),
893            ..File::default()
894        };
895        pool.insert("sys", "files", Value::array([Value::from(image)]));
896        let template: PromptTemplate = serde_json::from_value(serde_json::json!([
897            {"role": "system", "text": "sys"}, {"role": "user", "text": "tmpl"}
898        ]))
899        .unwrap();
900        let memory: MemoryConfig =
901            serde_json::from_value(serde_json::json!({"window": {"enabled": true, "size": 1}})).unwrap();
902        let vision = VisionConfig { enabled: true, ..VisionConfig::default() };
903        let msgs = build_messages(&template, &pool, None, Some(&memory), vision.image_parts(&pool).unwrap()).unwrap();
904        let texts: Vec<String> = msgs.iter().map(Message::text).collect();
905        assert_eq!(texts, vec!["sys", "old3", "old4", "tmpl", "now"]);
906        assert!(matches!(msgs[4].content.last(), Some(ContentPart::Image { .. })));
907    }
908
909    #[test]
910    fn memory_does_not_duplicate_a_templated_query() {
911        let mut pool = VariablePool::new();
912        pool.insert("sys", "query", Value::from("now"));
913        pool.insert("sys", "history", Value::from_json(serde_json::json!([{"role": "user", "content": "old"}])));
914        let memory = MemoryConfig::default();
915        let with_query: PromptTemplate = serde_json::from_value(serde_json::json!([
916            {"role": "system", "text": "sys"}, {"role": "user", "text": "Q: {{#sys.query#}}"}
917        ]))
918        .unwrap();
919        let msgs = build_messages(&with_query, &pool, None, Some(&memory), Vec::new()).unwrap();
920        let texts: Vec<String> = msgs.iter().map(Message::text).collect();
921        assert_eq!(texts, vec!["sys", "old", "Q: now"], "the query is sent once, after the history");
922
923        let without_query: PromptTemplate =
924            serde_json::from_value(serde_json::json!([{"role": "system", "text": "sys"}])).unwrap();
925        let msgs = build_messages(&without_query, &pool, None, Some(&memory), Vec::new()).unwrap();
926        let texts: Vec<String> = msgs.iter().map(Message::text).collect();
927        assert_eq!(texts, vec!["sys", "old", "now"], "memory appends the query when the template lacks it");
928
929        let explicit: MemoryConfig =
930            serde_json::from_value(serde_json::json!({"query_prompt_template": "again: {{#sys.query#}}"})).unwrap();
931        let msgs = build_messages(&with_query, &pool, None, Some(&explicit), Vec::new()).unwrap();
932        assert_eq!(msgs.last().map(Message::text).as_deref(), Some("again: now"), "explicit query template wins");
933    }
934
935    #[test]
936    fn history_shapes() {
937        let memory = MemoryConfig::default();
938        let mut pool = VariablePool::new();
939        pool.insert("sys", "history", Value::from(r#"[{"role": "human", "content": "hi"}]"#));
940        assert_eq!(memory.history(&pool).unwrap().len(), 1, "JSON-string history is parsed");
941        pool.insert("sys", "history", Value::from_json(serde_json::json!([{"role": "tool", "content": "x"}])));
942        assert!(memory.history(&pool).is_err());
943        pool.insert("sys", "history", Value::Int(3));
944        assert!(memory.history(&pool).is_err());
945        assert!(memory.history(&VariablePool::new()).unwrap().is_empty());
946    }
947
948    #[test]
949    fn image_without_url_is_an_error() {
950        let mut pool = VariablePool::new();
951        let image = File { marker: true, name: "a.png".into(), kind: FileKind::Image, ..File::default() };
952        pool.insert("sys", "files", Value::array([Value::from(image)]));
953        let vision = VisionConfig { enabled: true, ..VisionConfig::default() };
954        assert!(vision.image_parts(&pool).is_err());
955    }
956}