dynamo-parsers 1.3.0

// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

// Reference implementation:
// https://github.com/vllm-project/vllm/blob/main/vllm/tool_parsers/gemma4_tool_parser.py
//
// Gemma 4 tool-call grammar (custom, non-JSON):
//
//     <|tool_call>call:func_name{key:<|"|>value<|"|>,num:42}<tool_call|>
//
// `<|"|>`-delimited strings, bare unquoted keys, nested objects/arrays,
// multiple calls concatenated without separators.

use std::sync::OnceLock;

use regex::Regex;
use serde_json::{Map, Value};
use uuid::Uuid;

use super::super::ToolDefinition;
use super::super::response::{CalledFunction, ToolCallResponse, ToolCallType};

pub(crate) const TOOL_CALL_START: &str = "<|tool_call>";
pub(crate) const TOOL_CALL_END: &str = "<tool_call|>";
pub(crate) const STRING_DELIM: &str = "<|\"|>";
pub(crate) const CALL_PREFIX: &str = "call:";

static TOOL_CALL_REGEX: OnceLock<Regex> = OnceLock::new();

/// Captures the function-name + raw-args body of a single complete tool call.
/// `(?s)` enables dot-all so nested arg bodies that span newlines parse correctly.
fn tool_call_regex() -> &'static Regex {
    TOOL_CALL_REGEX.get_or_init(|| {
        let pattern = format!(
            r"(?s){}{}(?P<name>[\w\-\.]+)\{{(?P<args>.*?)\}}{}",
            regex::escape(TOOL_CALL_START),
            regex::escape(CALL_PREFIX),
            regex::escape(TOOL_CALL_END),
        );
        Regex::new(&pattern).expect("Failed to compile gemma4 tool call regex")
    })
}

fn parse_gemma_call_parts(
    name: &str,
    args_raw: &str,
    tools: Option<&[ToolDefinition]>,
) -> anyhow::Result<ToolCallResponse> {
    let name = name.to_string();
    if let Some(tools) = tools
        && !tools.iter().any(|t| t.name == name)
    {
        tracing::warn!(
            "Tool '{}' is not defined in the tools list (Gemma 4 parser).",
            name
        );
    }

    let args_value = match parse_args_object(args_raw) {
        Ok(v) => v,
        Err(e) => {
            tracing::warn!(
                "Failed to parse Gemma 4 args for '{}': {}. Falling back to empty object.",
                name,
                e
            );
            Value::Object(Map::new())
        }
    };
    let arguments = serde_json::to_string(&args_value)?;

    Ok(ToolCallResponse {
        id: format!("call-{}", Uuid::new_v4()),
        tp: ToolCallType::Function,
        function: CalledFunction { name, arguments },
    })
}

fn find_balanced_args_end(input: &str, open_brace: usize) -> Option<usize> {
    debug_assert_eq!(input.as_bytes().get(open_brace), Some(&b'{'));
    let mut cursor = open_brace;
    let mut depth = 0usize;
    let mut in_string = false;

    while cursor < input.len() {
        let rest = &input[cursor..];
        if rest.starts_with(STRING_DELIM) {
            in_string = !in_string;
            cursor += STRING_DELIM.len();
            continue;
        }

        let ch = rest.chars().next()?;
        if !in_string {
            match ch {
                '{' => depth += 1,
                '}' => {
                    depth = depth.checked_sub(1)?;
                    if depth == 0 {
                        return Some(cursor);
                    }
                }
                _ => {}
            }
        }
        cursor += ch.len_utf8();
    }

    None
}

fn parse_recoverable_call_at(
    input: &str,
    allow_missing_start: bool,
    allow_missing_end: bool,
) -> Option<(&str, &str, usize)> {
    let after_start_offset = if let Some(rest) = input.strip_prefix(TOOL_CALL_START) {
        input.len() - rest.len()
    } else if allow_missing_start && input.starts_with(CALL_PREFIX) {
        0
    } else {
        return None;
    };

    let after_start = &input[after_start_offset..];
    let after_prefix = after_start.strip_prefix(CALL_PREFIX)?;
    let name_len = after_prefix.find('{').filter(|idx| *idx > 0)?;
    let name = &after_prefix[..name_len];
    if !name
        .chars()
        .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
    {
        return None;
    }

    let open_brace = after_start_offset + CALL_PREFIX.len() + name_len;
    let close_brace = find_balanced_args_end(input, open_brace)?;
    let args_start = open_brace + 1;
    let args_raw = &input[args_start..close_brace];
    let after_args = &input[close_brace + 1..];

    if after_args.starts_with(TOOL_CALL_END) {
        return Some((name, args_raw, close_brace + 1 + TOOL_CALL_END.len()));
    }

    if allow_missing_end && after_args.trim().is_empty() {
        return Some((name, args_raw, close_brace + 1));
    }

    None
}

fn is_call_prefix_boundary(input: &str, idx: usize) -> bool {
    idx == 0
        || input[..idx]
            .chars()
            .next_back()
            .is_none_or(|ch| !(ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.')))
}

fn find_call_prefix_at_boundary(input: &str, from: usize) -> Option<usize> {
    let mut cursor = from;
    while cursor < input.len() {
        let rel = input[cursor..].find(CALL_PREFIX)?;
        let idx = cursor + rel;
        if is_call_prefix_boundary(input, idx) {
            return Some(idx);
        }
        cursor = idx + CALL_PREFIX.len();
    }
    None
}

/// Detect whether `chunk` contains the start of a Gemma 4 tool call, including
/// partial-prefix matches at the chunk boundary so streaming pipelines can hold
/// off emitting bytes that may belong to a tool-call marker.
pub fn detect_tool_call_start_gemma4(chunk: &str) -> bool {
    if chunk.contains(TOOL_CALL_START) {
        return true;
    }

    let mut cursor = 0usize;
    while let Some(idx) = find_call_prefix_at_boundary(chunk, cursor) {
        let candidate = &chunk[idx..];
        if parse_recoverable_call_at(candidate, true, true).is_some()
            || has_bare_call_body_start(candidate)
        {
            return true;
        }
        cursor = idx + CALL_PREFIX.len();
    }

    for i in 1..TOOL_CALL_START.len() {
        if TOOL_CALL_START.is_char_boundary(i) && chunk.ends_with(&TOOL_CALL_START[..i]) {
            return true;
        }
    }

    false
}

fn has_bare_call_body_start(input: &str) -> bool {
    let Some(after_prefix) = input.strip_prefix(CALL_PREFIX) else {
        return false;
    };
    let Some(open_brace) = after_prefix.find('{') else {
        return false;
    };
    if open_brace == 0 {
        return false;
    }
    after_prefix[..open_brace]
        .chars()
        .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
}

pub fn split_partial_call_prefix_gemma4(chunk: &str) -> Option<(&str, &str)> {
    for i in 1..=CALL_PREFIX.len() {
        if !CALL_PREFIX.is_char_boundary(i) || !chunk.ends_with(&CALL_PREFIX[..i]) {
            continue;
        }
        let partial_start = chunk.len() - i;
        if is_call_prefix_boundary(chunk, partial_start) {
            return Some((&chunk[..partial_start], &chunk[partial_start..]));
        }
    }
    None
}

/// Returns the position immediately after the last *complete* tool-call match
/// in `chunk`, or `None` if no complete call has arrived yet (caller should
/// keep accumulating). The regex requires `}<tool_call|>` adjacency, so a bare
/// `<tool_call|>` literal embedded inside a `<|"|>` string value does not
/// false-trigger a "section complete" signal here — matches upstream.
pub fn find_tool_call_end_position_gemma4(chunk: &str) -> Option<usize> {
    let mut cursor = 0usize;
    let mut last_end = None;

    while cursor < chunk.len() {
        let next_start = chunk[cursor..]
            .find(TOOL_CALL_START)
            .map(|rel| (cursor + rel, false, TOOL_CALL_START.len()));
        let next_bare =
            find_call_prefix_at_boundary(chunk, cursor).map(|idx| (idx, true, CALL_PREFIX.len()));
        let Some((rel_start, allow_missing_start, marker_len)) = [next_start, next_bare]
            .into_iter()
            .flatten()
            .min_by_key(|(idx, _, _)| *idx)
        else {
            break;
        };

        if let Some((_, _, consumed)) =
            parse_recoverable_call_at(&chunk[rel_start..], allow_missing_start, false)
        {
            let mut end = rel_start + consumed;
            while chunk[end..].starts_with(TOOL_CALL_END) {
                end += TOOL_CALL_END.len();
            }
            last_end = Some(end);
            cursor = end;
        } else {
            cursor = rel_start + marker_len;
        }
    }

    last_end
}

fn push_recovered_call(
    calls: &mut Vec<ToolCallResponse>,
    first_tool_start: &mut Option<usize>,
    absolute_start: usize,
    recovered: (&str, &str, usize),
    tools: Option<&[ToolDefinition]>,
    reason: &'static str,
) -> anyhow::Result<()> {
    if first_tool_start.is_none_or(|idx| absolute_start < idx) {
        *first_tool_start = Some(absolute_start);
    }
    tracing::warn!(
        why = reason,
        recovered_calls = 1,
        recovered_bytes = recovered.2,
        "gemma4 recovery: recovered complete call body from damaged wrapper"
    );
    calls.push(parse_gemma_call_parts(recovered.0, recovered.1, tools)?);
    Ok(())
}

fn recover_calls_in_span(
    span: &str,
    span_offset: usize,
    allow_missing_end: bool,
    tools: Option<&[ToolDefinition]>,
    calls: &mut Vec<ToolCallResponse>,
    first_tool_start: &mut Option<usize>,
) -> anyhow::Result<()> {
    let mut cursor = 0usize;

    while cursor < span.len() {
        let next_start = span[cursor..]
            .find(TOOL_CALL_START)
            .map(|rel| (cursor + rel, false, TOOL_CALL_START.len()));
        let next_bare =
            find_call_prefix_at_boundary(span, cursor).map(|idx| (idx, true, CALL_PREFIX.len()));
        let Some((rel_start, allow_missing_start, marker_len)) = [next_start, next_bare]
            .into_iter()
            .flatten()
            .min_by_key(|(idx, _, _)| *idx)
        else {
            break;
        };

        let parsed = parse_recoverable_call_at(
            &span[rel_start..],
            allow_missing_start,
            allow_missing_end && !allow_missing_start,
        );

        if let Some(recovered) = parsed {
            let reason = if allow_missing_start {
                "missing_start_recovery"
            } else {
                "missing_end_recovery"
            };
            push_recovered_call(
                calls,
                first_tool_start,
                span_offset + rel_start,
                recovered,
                tools,
                reason,
            )?;
            cursor = rel_start + recovered.2;
        } else {
            cursor = rel_start + marker_len;
        }
    }

    Ok(())
}

/// Parse a Gemma 4 model response into structured tool calls + leftover text.
///
/// Returns `(parsed_tool_calls, normal_text_content)`. `normal_text` is the
/// text BEFORE the first `<|tool_call>` start marker, trimmed of whitespace.
/// Text between calls, after the last call, and truncated-incomplete-call
/// tails are all dropped — matching upstream vLLM
/// (`vllm/tool_parsers/gemma4_tool_parser.py::extract_tool_calls`) which
/// computes `content = model_output[:content_end].strip()` where
/// `content_end = model_output.find(self.tool_call_start_token)`.
///
/// Per `tests/parity/README.md`: vLLM and SGLang both drop trailing text
/// after the wrapper across XML-style families; this aligns Dynamo to that
/// behavior. Cases: TOOLCALLING.batch.{2.c, 8.b, 8.c, 8.d}.
///
/// The `}<tool_call|>` adjacency requirement in the regex means embedded
/// `<tool_call|>` literals inside string-typed args (e.g. a `description`
/// field documenting the tool-call format) don't truncate the match
/// prematurely.
pub fn try_tool_call_parse_gemma4(
    message: &str,
    tools: Option<&[ToolDefinition]>,
) -> anyhow::Result<(Vec<ToolCallResponse>, Option<String>)> {
    let regex = tool_call_regex();
    let mut calls = Vec::new();
    let mut first_tool_start = None;
    let mut cursor = 0usize;

    for caps in regex.captures_iter(message) {
        if let Some(m) = caps.get(0) {
            recover_calls_in_span(
                &message[cursor..m.start()],
                cursor,
                false,
                tools,
                &mut calls,
                &mut first_tool_start,
            )?;
            first_tool_start.get_or_insert(m.start());
            cursor = m.end();
        }
        let name = caps.name("name").map(|m| m.as_str()).unwrap_or_default();
        if name.is_empty() {
            continue;
        }
        let args_raw = caps.name("args").map(|m| m.as_str()).unwrap_or("");

        calls.push(parse_gemma_call_parts(name, args_raw, tools)?);
    }

    recover_calls_in_span(
        &message[cursor..],
        cursor,
        true,
        tools,
        &mut calls,
        &mut first_tool_start,
    )?;

    // No-leak contract for `normal_text`:
    //   - Success path (≥1 call extracted): prefix BEFORE the first
    //     `<|tool_call>` start marker — mirrors vLLM's
    //     `content = model_output[:content_end].strip()`. Inter-call,
    //     trailing, and truncation tails after a successful call are dropped.
    //   - Recovery path (zero calls extracted): if the message contains ANY
    //     gemma4 markup token (`<|tool_call>`, `<tool_call|>`, `<|"|>`),
    //     return empty — Dynamo intentionally diverges from vLLM's
    //     exception-fallback (which echoes raw bytes) so tool-call markup
    //     never leaks into normal_text on malformed / truncated / orphan-
    //     close / no-body inputs. Cases flagged by the parity table's `↯`:
    //     TOOLCALLING.batch.{4.a, 4.b, 4.c, 4.d, 5.a, 5.b, 5.c, 6.c}.
    //   - Plain-text path (zero calls AND no markup): return the message
    //     as-is.
    let has_markup = message.contains(TOOL_CALL_START)
        || message.contains(TOOL_CALL_END)
        || message.contains(STRING_DELIM);
    let normal_text = if calls.is_empty() {
        if has_markup {
            // Recovery: malformed/truncated/orphan-close/no-body shapes.
            // Suppress the whole message so tool-call markup doesn't leak.
            let preview: String = message.chars().take(120).collect();
            tracing::warn!(
                why = "no_calls_with_markup",
                stripped_bytes = message.len(),
                has_start = message.contains(TOOL_CALL_START),
                has_end = message.contains(TOOL_CALL_END),
                has_string_delim = message.contains(STRING_DELIM),
                "gemma4 strip (recovery): zero calls extracted but gemma4 markup present (<|tool_call>, <tool_call|>, <|\"|>); suppressing entire message to prevent leak into normal_text. preview={:?}",
                preview
            );
            String::new()
        } else {
            // No markup at all → plain text passes through unchanged. No strip.
            message.trim().to_string()
        }
    } else {
        // Success: prefix-only contract — drop everything from the first
        // `<|tool_call>` onward (inter-call text, trailing narration,
        // truncation tails). Mirrors vLLM's
        // `content = model_output[:content_end].strip()`.
        match first_tool_start {
            Some(idx) => {
                let stripped = &message[idx..];
                let preview: String = stripped.chars().take(120).collect();
                tracing::debug!(
                    why = "prefix_only_contract",
                    n_calls = calls.len(),
                    kept_prefix_bytes = idx,
                    stripped_bytes = stripped.len(),
                    "gemma4 strip (success): kept prefix before first <|tool_call>; dropped parsed-call(s) + any inter-call / trailing narration. preview={:?}",
                    preview
                );
                message[..idx].trim().to_string()
            }
            None => String::new(),
        }
    };

    Ok((calls, Some(normal_text)))
}

// ---------------------------------------------------------------------------
// Recursive-descent parser for the Gemma 4 argument grammar
// ---------------------------------------------------------------------------
//
// Grammar (informal):
//
//   args     = (entry ("," entry)*)?
//   entry    = key ":" value
//   key      = bare-identifier (no quoting in Gemma 4 emit)
//   value    = string | number | bool | null | object | array
//   string   = "<|\"|>" .* "<|\"|>"
//   number   = -? [0-9]+ ( "." [0-9]+ )?
//   bool     = "true" | "false"
//   null     = "null" | "none" | "nil"
//   object   = "{" args "}"
//   array    = "[" (value ("," value)*)? "]"
//
// We parse straight into `serde_json::Value` so the rest of the pipeline sees
// the same shape every other parser produces.

struct Cursor<'a> {
    src: &'a str,
    pos: usize,
}

impl<'a> Cursor<'a> {
    fn new(src: &'a str) -> Self {
        Self { src, pos: 0 }
    }

    fn rest(&self) -> &'a str {
        &self.src[self.pos..]
    }

    fn eof(&self) -> bool {
        self.pos >= self.src.len()
    }

    fn skip_whitespace(&mut self) {
        let bytes = self.src.as_bytes();
        while self.pos < bytes.len() && bytes[self.pos].is_ascii_whitespace() {
            self.pos += 1;
        }
    }

    fn peek_byte(&self) -> Option<u8> {
        self.src.as_bytes().get(self.pos).copied()
    }

    fn consume_byte(&mut self, b: u8) -> bool {
        if self.peek_byte() == Some(b) {
            self.pos += 1;
            true
        } else {
            false
        }
    }
}

pub(crate) fn parse_args_object(input: &str) -> anyhow::Result<Value> {
    let mut cur = Cursor::new(input);
    cur.skip_whitespace();
    let val = parse_object_body(&mut cur)?;
    cur.skip_whitespace();
    if !cur.eof() {
        anyhow::bail!(
            "trailing characters after Gemma 4 args object at offset {}: {:?}",
            cur.pos,
            cur.rest()
        );
    }
    Ok(val)
}

fn parse_object_body(cur: &mut Cursor) -> anyhow::Result<Value> {
    let mut map = Map::new();
    cur.skip_whitespace();
    if cur.eof() || cur.peek_byte() == Some(b'}') {
        return Ok(Value::Object(map));
    }
    loop {
        cur.skip_whitespace();
        let key = parse_key(cur)?;
        cur.skip_whitespace();
        if !cur.consume_byte(b':') {
            anyhow::bail!("expected ':' after key '{}' at offset {}", key, cur.pos);
        }
        cur.skip_whitespace();
        // `key:` with no value emits `{"key": ""}` (matches upstream).
        let value = match cur.peek_byte() {
            None | Some(b',') | Some(b'}') => Value::String(String::new()),
            _ => parse_value(cur)?,
        };
        map.insert(key, value);
        cur.skip_whitespace();
        if !cur.consume_byte(b',') {
            break;
        }
    }
    Ok(Value::Object(map))
}

fn parse_key(cur: &mut Cursor) -> anyhow::Result<String> {
    let bytes = cur.src.as_bytes();
    let start = cur.pos;
    while cur.pos < bytes.len() {
        let b = bytes[cur.pos];
        if b.is_ascii_alphanumeric() || b == b'_' || b == b'-' || b == b'.' {
            cur.pos += 1;
        } else {
            break;
        }
    }
    if cur.pos == start {
        anyhow::bail!("expected bare key at offset {}", start);
    }
    Ok(cur.src[start..cur.pos].to_string())
}

/// Consume `keyword` ASCII-case-insensitively, only when the next byte is not
/// a word character (so `nullable` doesn't match `null` + leftover `able`).
fn try_consume_keyword(cur: &mut Cursor, keyword: &str) -> bool {
    let bytes = cur.src.as_bytes();
    let kw = keyword.as_bytes();
    let end = cur.pos + kw.len();
    if end > bytes.len() {
        return false;
    }
    if !bytes[cur.pos..end].eq_ignore_ascii_case(kw) {
        return false;
    }
    if let Some(&next) = bytes.get(end)
        && (next.is_ascii_alphanumeric() || next == b'_')
    {
        return false;
    }
    cur.pos = end;
    true
}

fn parse_value(cur: &mut Cursor) -> anyhow::Result<Value> {
    cur.skip_whitespace();

    // Delimited string `<|"|>...<|"|>`. If the closing delimiter is missing
    // (model truncation), take everything after the opener as the value.
    if cur.rest().starts_with(STRING_DELIM) {
        cur.pos += STRING_DELIM.len();
        let body_start = cur.pos;
        match cur.src[body_start..].find(STRING_DELIM) {
            Some(end_rel) => {
                let body_end = body_start + end_rel;
                let s = cur.src[body_start..body_end].to_string();
                cur.pos = body_end + STRING_DELIM.len();
                return Ok(Value::String(s));
            }
            None => {
                let s = cur.src[body_start..].to_string();
                cur.pos = cur.src.len();
                return Ok(Value::String(s));
            }
        }
    }

    // Object
    if cur.consume_byte(b'{') {
        let v = parse_object_body(cur)?;
        cur.skip_whitespace();
        if !cur.consume_byte(b'}') {
            anyhow::bail!("expected '}}' to close object at offset {}", cur.pos);
        }
        return Ok(v);
    }

    // Array
    if cur.consume_byte(b'[') {
        return parse_array(cur);
    }

    // Booleans + null aliases (case-insensitive).
    if try_consume_keyword(cur, "true") {
        return Ok(Value::Bool(true));
    }
    if try_consume_keyword(cur, "false") {
        return Ok(Value::Bool(false));
    }
    if try_consume_keyword(cur, "null")
        || try_consume_keyword(cur, "none")
        || try_consume_keyword(cur, "nil")
    {
        return Ok(Value::Null);
    }

    // Number
    parse_number(cur)
}

fn parse_array(cur: &mut Cursor) -> anyhow::Result<Value> {
    let mut items = Vec::new();
    cur.skip_whitespace();
    if cur.consume_byte(b']') {
        return Ok(Value::Array(items));
    }
    loop {
        cur.skip_whitespace();
        items.push(parse_value(cur)?);
        cur.skip_whitespace();
        if cur.consume_byte(b']') {
            return Ok(Value::Array(items));
        }
        if !cur.consume_byte(b',') {
            anyhow::bail!("expected ',' or ']' in array at offset {}", cur.pos);
        }
    }
}

fn parse_number(cur: &mut Cursor) -> anyhow::Result<Value> {
    let start = cur.pos;
    let bytes = cur.src.as_bytes();
    if cur.peek_byte() == Some(b'-') {
        cur.pos += 1;
    }
    let int_start = cur.pos;
    while cur.pos < bytes.len() && bytes[cur.pos].is_ascii_digit() {
        cur.pos += 1;
    }
    if cur.pos == int_start {
        anyhow::bail!(
            "expected value at offset {} but got: {:?}",
            start,
            &cur.src[start..]
        );
    }
    let mut is_float = false;
    if cur.peek_byte() == Some(b'.') {
        is_float = true;
        cur.pos += 1;
        while cur.pos < bytes.len() && bytes[cur.pos].is_ascii_digit() {
            cur.pos += 1;
        }
    }
    let lex = &cur.src[start..cur.pos];
    if is_float {
        let f: f64 = lex.parse()?;
        Ok(serde_json::json!(f))
    } else {
        let i: i64 = lex.parse()?;
        Ok(serde_json::json!(i))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn extract_first(input: &str) -> (String, Value) {
        let (calls, _) = try_tool_call_parse_gemma4(input, None).unwrap();
        assert_eq!(calls.len(), 1, "expected exactly one tool call");
        let args: Value = serde_json::from_str(&calls[0].function.arguments).unwrap();
        (calls[0].function.name.clone(), args)
    }

    #[test] // detection helper
    fn detect_full_and_partial_start() {
        assert!(detect_tool_call_start_gemma4("<|tool_call>"));
        assert!(detect_tool_call_start_gemma4("blah <|tool_call>"));
        assert!(detect_tool_call_start_gemma4(
            "call:get_weather{location:<|\"|>NYC<|\"|>}<tool_call|>"
        ));
        assert!(detect_tool_call_start_gemma4("call:get_weather{"));
        assert!(detect_tool_call_start_gemma4("<|tool_"));
        assert!(detect_tool_call_start_gemma4("<|"));
        assert!(!detect_tool_call_start_gemma4("I will call: you tomorrow"));
        assert!(!detect_tool_call_start_gemma4("nothing here"));
        assert!(!detect_tool_call_start_gemma4("toolcall"));
    }

    #[test] // end-position helper
    fn find_end_returns_position_after_last_marker() {
        let text = "<|tool_call>call:f{}<tool_call|>more";
        let pos = find_tool_call_end_position_gemma4(text).unwrap();
        assert_eq!(&text[pos..], "more");
        assert_eq!(
            find_tool_call_end_position_gemma4("<|tool_call>call:f{"),
            None
        );
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.1 in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.yaml.
    #[test] // TOOLCALLING.batch.1 — single string argument
    fn parse_single_string_argument() {
        let input = r#"<|tool_call>call:get_weather{location:<|"|>Tokyo<|"|>}<tool_call|>"#;
        let (name, args) = extract_first(input);
        assert_eq!(name, "get_weather");
        assert_eq!(args["location"], "Tokyo");
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.1, TOOLCALLING.batch.7.a in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.7.yaml, tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.yaml.
    #[test] // TOOLCALLING.batch.1, TOOLCALLING.batch.7 — multiple typed arguments
    fn parse_multiple_typed_arguments() {
        let input = r#"<|tool_call>call:f{loc:<|"|>San Francisco, CA<|"|>,unit:<|"|>celsius<|"|>,count:42,flag:true,nope:null}<tool_call|>"#;
        let (name, args) = extract_first(input);
        assert_eq!(name, "f");
        assert_eq!(args["loc"], "San Francisco, CA");
        assert_eq!(args["unit"], "celsius");
        assert_eq!(args["count"], 42);
        assert_eq!(args["flag"], true);
        assert_eq!(args["nope"], Value::Null);
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.6.a in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.6.yaml.
    #[test] // TOOLCALLING.batch.6 — empty argument object
    fn parse_no_arg_call() {
        let input = "<|tool_call>call:get_time{}<tool_call|>";
        let (name, args) = extract_first(input);
        assert_eq!(name, "get_time");
        assert!(args.as_object().unwrap().is_empty());
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.7.d in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.7.yaml.
    #[test] // TOOLCALLING.batch.7 — nested object value
    fn parse_nested_object_value() {
        let input = r#"<|tool_call>call:f{cfg:{ssl:true,pool:{min:5,max:20}}}<tool_call|>"#;
        let (_name, args) = extract_first(input);
        assert_eq!(args["cfg"]["ssl"], true);
        assert_eq!(args["cfg"]["pool"]["min"], 5);
        assert_eq!(args["cfg"]["pool"]["max"], 20);
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.7.a in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.7.yaml.
    #[test] // TOOLCALLING.batch.7 — array of strings
    fn parse_array_of_strings() {
        let input = r#"<|tool_call>call:f{tags:[<|"|>a<|"|>,<|"|>b<|"|>,<|"|>c<|"|>]}<tool_call|>"#;
        let (_name, args) = extract_first(input);
        assert_eq!(args["tags"], serde_json::json!(["a", "b", "c"]));
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.7.a in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.7.yaml.
    #[test] // TOOLCALLING.batch.7 — array of mixed primitives
    fn parse_array_of_mixed_primitives() {
        let input = "<|tool_call>call:f{xs:[1,2,3.5,true,false,null]}<tool_call|>";
        let (_name, args) = extract_first(input);
        assert_eq!(args["xs"][0], 1);
        assert_eq!(args["xs"][1], 2);
        assert!((args["xs"][2].as_f64().unwrap() - 3.5).abs() < 1e-9);
        assert_eq!(args["xs"][3], true);
        assert_eq!(args["xs"][4], false);
        assert_eq!(args["xs"][5], Value::Null);
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.2.b in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.2.yaml.
    #[test] // TOOLCALLING.batch.2 — multiple parallel calls, zero spacing
    fn parse_multiple_parallel_calls() {
        let input = concat!(
            "<|tool_call>call:a{x:1}<tool_call|>",
            "<|tool_call>call:b{y:<|\"|>two<|\"|>}<tool_call|>",
            "<|tool_call>call:c{}<tool_call|>",
        );
        let (calls, normal) = try_tool_call_parse_gemma4(input, None).unwrap();
        assert_eq!(calls.len(), 3);
        assert_eq!(calls[0].function.name, "a");
        assert_eq!(calls[1].function.name, "b");
        assert_eq!(calls[2].function.name, "c");
        assert_eq!(normal, Some(String::new()));
    }

    #[test] // TOOLCALLING.batch.8.c — prefix narration preserved, trailing dropped (matches vLLM)
    fn parse_with_surrounding_text() {
        let input = r#"Sure thing. <|tool_call>call:f{x:1}<tool_call|> All set."#;
        let (calls, normal) = try_tool_call_parse_gemma4(input, None).unwrap();
        assert_eq!(calls.len(), 1);
        assert_eq!(normal, Some("Sure thing.".to_string()));
    }

    #[test] // TOOLCALLING.batch.3 — no tool calls at all
    fn parse_no_tool_calls() {
        let (calls, normal) = try_tool_call_parse_gemma4("just plain prose here", None).unwrap();
        assert_eq!(calls.len(), 0);
        assert_eq!(normal, Some("just plain prose here".to_string()));
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.5.c in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.5.yaml.
    #[test] // TOOLCALLING.batch.5 — complete prior call survives; truncated tail dropped (matches vLLM)
    fn truncated_tail_dropped_complete_prior_survives() {
        let input = concat!(
            "<|tool_call>call:complete{x:1}<tool_call|>",
            "<|tool_call>call:partial{y:<|\"|>incomp", // no end marker
        );
        let (calls, normal) = try_tool_call_parse_gemma4(input, None).unwrap();
        assert_eq!(calls.len(), 1);
        assert_eq!(calls[0].function.name, "complete");
        // vLLM computes content as text BEFORE the first start marker; here that
        // is empty (message starts with `<|tool_call>`), so normal_text is "".
        // The truncated tail bytes are dropped, not echoed.
        assert_eq!(normal, Some(String::new()));
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.4.a in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.4.yaml.
    #[test] // TOOLCALLING.batch.4 — malformed args body falls back to empty object, call still emitted
    fn malformed_args_falls_back_to_empty_object() {
        let input = "<|tool_call>call:f{garbage no colons here}<tool_call|>";
        let (calls, _) = try_tool_call_parse_gemma4(input, None).unwrap();
        assert_eq!(calls.len(), 1);
        assert_eq!(calls[0].function.name, "f");
        let args: Value = serde_json::from_str(&calls[0].function.arguments).unwrap();
        assert!(args.as_object().unwrap().is_empty());
    }

    #[test] // TOOLCALLING.fmt.1 — function names with hyphens, dots, underscores
    fn parse_function_names_with_special_chars() {
        for (input, expected_name) in [
            (
                "<|tool_call>call:list-tasklists{}<tool_call|>",
                "list-tasklists",
            ),
            (
                "<|tool_call>call:mcp__portal__search-doc{}<tool_call|>",
                "mcp__portal__search-doc",
            ),
            (
                "<|tool_call>call:my.namespaced.fn{}<tool_call|>",
                "my.namespaced.fn",
            ),
        ] {
            let (calls, _) = try_tool_call_parse_gemma4(input, None).unwrap();
            assert_eq!(calls.len(), 1, "input: {input}");
            assert_eq!(calls[0].function.name, expected_name);
        }
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.7.b in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.7.yaml.
    #[test] // TOOLCALLING.batch.7 — angle brackets / HTML inside string values
    fn parse_html_in_string_value() {
        let input = r#"<|tool_call>call:render{html:<|"|><div class="x"><h1>Hi</h1></div><|"|>}<tool_call|>"#;
        let (_name, args) = extract_first(input);
        assert_eq!(args["html"], "<div class=\"x\"><h1>Hi</h1></div>");
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.7.b in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.7.yaml.
    #[test] // TOOLCALLING.batch.7 — newlines inside string values
    fn parse_newlines_in_string_value() {
        let input = "<|tool_call>call:f{body:<|\"|>line1\nline2\nline3<|\"|>}<tool_call|>";
        let (_name, args) = extract_first(input);
        assert_eq!(args["body"], "line1\nline2\nline3");
    }

    #[test] // TOOLCALLING.fmt.2 — whitespace tolerance inside args
    fn parse_with_internal_whitespace() {
        let input = r#"<|tool_call>call:f{ x : 1 , y : <|"|>z<|"|> }<tool_call|>"#;
        let (_name, args) = extract_first(input);
        assert_eq!(args["x"], 1);
        assert_eq!(args["y"], "z");
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.7.a in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.7.yaml.
    #[test] // TOOLCALLING.batch.7 — negative numbers and floats
    fn parse_signed_numbers_and_floats() {
        let input = "<|tool_call>call:f{a:-1,b:-2.5,c:0,d:0.0}<tool_call|>";
        let (_name, args) = extract_first(input);
        assert_eq!(args["a"], -1);
        assert!((args["b"].as_f64().unwrap() - -2.5).abs() < 1e-9);
        assert_eq!(args["c"], 0);
        assert!((args["d"].as_f64().unwrap()).abs() < 1e-9);
    }

    #[test] // TOOLCALLING.fmt.1 — tool validation warns but doesn't drop
    fn parse_with_tool_validation() {
        let input = r#"<|tool_call>call:get_weather{x:1}<tool_call|>"#;
        let tools = vec![ToolDefinition {
            name: "get_weather".to_string(),
            parameters: None,
            strict: None,
        }];
        let (calls, _) = try_tool_call_parse_gemma4(input, Some(&tools)).unwrap();
        assert_eq!(calls.len(), 1);
        assert_eq!(calls[0].function.name, "get_weather");
    }

    #[test] // TOOLCALLING.fmt.4 — empty wrapper between calls
    fn parse_empty_text_between_calls() {
        let input = concat!(
            "<|tool_call>call:a{}<tool_call|>",
            "<|tool_call>call:b{}<tool_call|>",
        );
        let (calls, normal) = try_tool_call_parse_gemma4(input, None).unwrap();
        assert_eq!(calls.len(), 2);
        assert_eq!(normal, Some(String::new()));
    }

    // Argument-grammar unit tests (parse_args_object directly)

    #[test] // TOOLCALLING.batch.6 — empty args at the grammar entry point
    fn args_grammar_empty() {
        let v = parse_args_object("").unwrap();
        assert_eq!(v, serde_json::json!({}));
    }

    #[test] // TOOLCALLING.batch.7 — string with comma/colon/brace literals
    fn args_grammar_string_with_special_chars() {
        let v = parse_args_object(r#"x:<|"|>has,comma:and{brace}<|"|>"#).unwrap();
        assert_eq!(v["x"], "has,comma:and{brace}");
    }

    #[test] // TOOLCALLING.batch.7 — deeply nested object value
    fn args_grammar_deeply_nested() {
        let v = parse_args_object("a:{b:{c:{d:{e:1}}}}").unwrap();
        assert_eq!(v["a"]["b"]["c"]["d"]["e"], 1);
    }

    #[test] // TOOLCALLING.batch.7 — array of objects
    fn args_grammar_array_of_objects() {
        let v = parse_args_object(r#"items:[{n:<|"|>x<|"|>},{n:<|"|>y<|"|>}]"#).unwrap();
        assert_eq!(v["items"][0]["n"], "x");
        assert_eq!(v["items"][1]["n"], "y");
    }

    #[test] // TOOLCALLING.batch.4, TOOLCALLING.batch.5 — truncated string-arg recovery (mirrors upstream)
    fn args_grammar_unterminated_string_takes_remainder() {
        // Upstream Gemma 4 parser: when the closing <|"|> is missing,
        // everything after the opening delimiter becomes the value. We mirror
        // that so a truncated trailing arg doesn't lose all earlier args via
        // the entire-args-object error fallback.
        let v = parse_args_object(r#"x:<|"|>oops"#).unwrap();
        assert_eq!(v["x"], "oops");
    }

    #[test] // TOOLCALLING.batch.4 — empty value mid-args (upstream test_empty_value)
    fn args_grammar_empty_value_yields_empty_string() {
        let v = parse_args_object("x:,y:1").unwrap();
        assert_eq!(v["x"], "");
        assert_eq!(v["y"], 1);
    }

    #[test] // TOOLCALLING.batch.4 — empty value at end-of-args (upstream test_empty_value)
    fn args_grammar_trailing_empty_value() {
        let v = parse_args_object("x:1,y:").unwrap();
        assert_eq!(v["x"], 1);
        assert_eq!(v["y"], "");
    }

    #[test] // TOOLCALLING.batch.7 — typed-value null variants, case-insensitive (upstream `value_str.lower() in (...)`)
    fn args_grammar_null_aliases_case_insensitive() {
        for variant in [
            "null", "NULL", "Null", "none", "NONE", "None", "nil", "NIL", "Nil",
        ] {
            let body = format!("x:{variant}");
            let v = parse_args_object(&body).unwrap();
            assert_eq!(v["x"], Value::Null, "variant: {variant}");
        }
    }

    #[test] // TOOLCALLING.batch.4 — keyword-prefix must not partial-match (`nullable` vs `null`)
    fn args_grammar_keyword_prefix_not_consumed() {
        // `nullable` must not be parsed as `null` + leftover `able`.
        let _ = parse_args_object("x:nullable").unwrap_err();
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.5.c in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.5.yaml.
    #[test] // TOOLCALLING.batch.5 — missing end-marker, markup suppressed (no-leak contract)
    fn incomplete_tool_call_suppresses_markup() {
        let input = "<|tool_call>call:foo{x:1";
        let (calls, normal) = try_tool_call_parse_gemma4(input, None).unwrap();
        assert_eq!(calls.len(), 0);
        // No calls extracted → return the prefix BEFORE the first `<|tool_call>`
        // start marker (here: empty). Dynamo intentionally diverges from vLLM's
        // exception fallback (which echoes the raw bytes) so tool-call markup
        // never leaks into normal_text on the recovery path.
        assert_eq!(normal, Some(String::new()));
    }

    #[test] // TOOLCALLING.batch.7 — `<tool_call|>` literal inside a string-typed argument
    // must not truncate the call. The extraction regex requires
    // `}<tool_call|>` adjacency, so a bare embedded marker is safe.
    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.7.b in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.7.yaml.
    fn embedded_tool_call_marker_in_string_value() {
        let input = r#"<|tool_call>call:render{html:<|"|><tool_call|> example<|"|>}<tool_call|>"#;
        let (calls, _) = try_tool_call_parse_gemma4(input, None).unwrap();
        assert_eq!(calls.len(), 1);
        assert_eq!(calls[0].function.name, "render");
        let args: Value = serde_json::from_str(&calls[0].function.arguments).unwrap();
        assert_eq!(args["html"], "<tool_call|> example");
    }

    #[test] // find_tool_call_end_position must respect the regex's
    // `}<tool_call|>` adjacency, not just bare `<tool_call|>` occurrences.
    fn find_end_position_skips_embedded_marker() {
        let input = r#"<|tool_call>call:render{html:<|"|><tool_call|>x<|"|>}<tool_call|> trailing"#;
        let pos = find_tool_call_end_position_gemma4(input).unwrap();
        assert_eq!(&input[pos..], " trailing");
    }

    #[test] // TOOLCALLING.batch.8 — paired reasoning span + tool call in the same emission
    // (in production the reasoning parser runs first and strips `<|channel>`,
    // but the tool-call parser must remain correct if it sees the full
    // emission, e.g. when reasoning parsing is disabled).
    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.8.a in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.8.yaml.
    fn paired_reasoning_and_tool_call_in_same_emission() {
        let input = concat!(
            "<|channel>thought\nthinking about the request<channel|>",
            "<|tool_call>call:get_weather{location:<|\"|>Tokyo<|\"|>}<tool_call|>",
        );
        let (calls, normal) = try_tool_call_parse_gemma4(input, None).unwrap();
        assert_eq!(calls.len(), 1);
        assert_eq!(calls[0].function.name, "get_weather");
        let args: Value = serde_json::from_str(&calls[0].function.arguments).unwrap();
        assert_eq!(args["location"], "Tokyo");
        // The reasoning span is preserved as normal_text — the tool-call parser
        // doesn't try to interpret `<|channel>` markers itself.
        assert!(normal.unwrap().contains("<|channel>thought"));
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.9 in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.yaml.
    #[test] // TOOLCALLING.batch.9 — empty input, no tool calls
    fn empty_input_yields_zero_calls_empty_content() {
        let (calls, normal) = try_tool_call_parse_gemma4("", None).unwrap();
        assert_eq!(calls.len(), 0);
        assert_eq!(normal, Some(String::new()));
    }

    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.9 in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.yaml.
    #[test] // TOOLCALLING.batch.9 — `null` argument values round-trip as JSON null
    fn null_argument_values_preserved() {
        let input = "<|tool_call>call:f{x:null,y:none,z:nil}<tool_call|>";
        let (calls, _) = try_tool_call_parse_gemma4(input, None).unwrap();
        assert_eq!(calls.len(), 1);
        let args: Value = serde_json::from_str(&calls[0].function.arguments).unwrap();
        assert_eq!(args["x"], Value::Null);
        assert_eq!(args["y"], Value::Null);
        assert_eq!(args["z"], Value::Null);
    }

    #[test] // TOOLCALLING.batch.10 — duplicate calls to the same function name. Both must
    // appear with distinct IDs; client decides whether duplicate invocation
    // is intended.
    // DEPRECATED(parser-fixture-duplicate): Duplicate of YAML fixture coverage: TOOLCALLING.batch.10 in tests/parity/toolcalling/fixtures/gemma4/TOOLCALLING.batch.yaml.
    fn duplicate_tool_call_same_name() {
        let input = concat!(
            "<|tool_call>call:get_weather{location:<|\"|>Tokyo<|\"|>}<tool_call|>",
            "<|tool_call>call:get_weather{location:<|\"|>NYC<|\"|>}<tool_call|>",
        );
        let (calls, _) = try_tool_call_parse_gemma4(input, None).unwrap();
        assert_eq!(calls.len(), 2);
        assert_eq!(calls[0].function.name, "get_weather");
        assert_eq!(calls[1].function.name, "get_weather");
        assert_ne!(
            calls[0].id, calls[1].id,
            "duplicate calls need distinct IDs"
        );
        let args0: Value = serde_json::from_str(&calls[0].function.arguments).unwrap();
        let args1: Value = serde_json::from_str(&calls[1].function.arguments).unwrap();
        assert_eq!(args0["location"], "Tokyo");
        assert_eq!(args1["location"], "NYC");
    }

    // ----- Explicit N/A coverage notes (per lib/parsers/TOOLCALLING_CASES.md) -----
    //
    // TOOLCALLING.stream.3  — Streaming token-by-token assembly: indirect. The Gemma 4
    //           parser exposes only the synchronous extraction path; the
    //           streaming jail (`tools.rs::try_tool_call_parse_stream`) drives
    //           it via `detect_tool_call_start_gemma4` and
    //           `find_tool_call_end_position_gemma4`, both covered by helper
    //           tests above. Same pattern as kimi_k2.
    // FRONTEND.tool_choice — `tool_choice` (auto / required / named / none): universal gap
    //           in the repo as of 2026-04 — the cross-parser suites at
    //           `lib/llm/tests/{tool_choice.rs,parallel_tool_call_integration.rs,
    //           tool_choice_finish_reasons.rs}` run `hermes` only. Adding
    //           Gemma 4 to those suites is a separate parametrization PR.
    // PIPELINE.finish_reason — `finish_reason` semantics: same universal gap; lives in
    //           `lib/llm/tests/tool_choice_finish_reasons.rs`, hermes-only
    //           today.
    // TOOLCALLING.xml.1 / TOOLCALLING.xml.2 — XML-family only. N/A — Gemma 4 uses a custom
    //           non-JSON, non-XML grammar.
    // TOOLCALLING.harmony.1 / TOOLCALLING.harmony.2 — Harmony only. N/A.
}