/**
* std/llm/tool_parse — text-tool dialect composition.
*
* Rust delimits byte ranges and parses byte-oriented literals. This module
* owns what those ranges mean: dialect vocabulary, recovery order, visible
* prose, canonical replay, and the rule that every consumed non-call is
* recorded as dropped.
*/
import {
ARGUMENT_ALIASES,
BACKTICK_FENCE,
HTML_ENTITIES,
MARKUP_DIALECTS,
NAME_ALIASES,
TAGGED_DIALECT,
TILDE_FENCE,
tool_fence_info_intent,
} from "std/llm/dialects"
import { tool_parse_envelope } from "std/llm/tool_parse_envelope"
import {
bind_verbatim,
has_counted_verbatim,
json_call,
json_chunks,
json_fields,
} from "std/llm/tool_parse_json_support"
import { has_response_protocol_fragment } from "std/llm/tool_parse_protocol"
import { parse_provider_dialect } from "std/llm/tool_parse_provider"
import {
answer_block,
call_block,
canonical,
empty_parse,
prose_block,
protocol_violation,
provider_result,
registry_names,
} from "std/llm/tool_parse_result"
import { tool_parse_scan_spec } from "std/llm/tool_parse_scan_spec"
import { tool_shape_findings, tool_shape_violations } from "std/llm/tool_shape"
const TEXT_UNIT_KINDS = ["text", "fenced_line", "harmony_line"]
fn __tool_parse_tag_spellings(canonical: string) -> list<string> {
for dialect in TAGGED_DIALECT {
if dialect.canonical == canonical {
return dialect.spellings
}
}
return []
}
fn __tool_parse_schema_type(tools, tool_name: string, parameter_name: string) {
if type_of(tools) != "dict" {
return nil
}
for entry in tools?.tools ?? [] {
if to_string(entry?.name ?? "") == tool_name {
return entry?.parameters?.[parameter_name]?.type
}
}
return nil
}
fn __tool_parse_unframe_markup_value(value: string) -> string {
let framed = value
if starts_with(framed, "\r\n") {
framed = framed.slice(2, len(framed))
} else if starts_with(framed, "\n") {
framed = framed.slice(1, len(framed))
}
if ends_with(framed, "\r\n") {
framed = framed.slice(0, len(framed) - 2)
} else if ends_with(framed, "\n") {
framed = framed.slice(0, len(framed) - 1)
}
return framed
}
fn __tool_parse_markup_value(raw: string, schema_type) {
const framed = __tool_parse_unframe_markup_value(raw)
if schema_type == nil || schema_type == "string" {
return framed
}
return try {
json_parse(trim(framed))
} catch (_) {
framed
}
}
fn __tool_parse_markup(body: string, tools) -> dict {
const trimmed = trim(body)
let opener = ""
let close_tag = ""
let style = ""
if starts_with(trimmed, "<function=") {
opener = "<function="
close_tag = "</function>"
style = "`<function=...>`"
} else if starts_with(trimmed, "<invoke name=") {
opener = "<invoke name="
close_tag = "</invoke>"
style = "`<invoke name=...>`"
} else {
return {matched: false}
}
const rest = trimmed.slice(len(opener), len(trimmed))
const gt = rest.index_of(">")
if gt < 0 {
const subject = opener == "<function=" ? "a `<function=`" : "an `<invoke name=`"
return {
matched: true,
ok: false,
error: "TOOL CALL TRUNCATED: "
+ subject
+ " open tag was never closed with `>` — the response appears to have been cut off. "
+ "The call was NOT executed; re-emit the complete call.",
}
}
const name = trim(regex_replace("^\"|\"$", "", rest.slice(0, gt)))
if len(regex_captures("^[A-Za-z0-9_.-]+$", name)) == 0 {
return {matched: false}
}
const known = registry_names(tools)
if !known.contains(name) {
return {
matched: true,
ok: false,
error: "Unknown tool '"
+ name
+ "' in chat-template "
+ style
+ " tool-call markup. Available tools: ["
+ known.slice(0, 20).join(", ")
+ "]",
}
}
const after_open = rest.slice(gt + 1, len(rest))
const close_at = after_open.index_of(close_tag)
const inner = close_at >= 0 ? after_open.slice(0, close_at) : after_open
const parameter_pattern =
"(?s)<parameter(?:=([A-Za-z0-9_][A-Za-z0-9_.-]*)|\\s+name=\"([^\"]+)\"(?:\\s+[A-Za-z_][A-Za-z0-9_.-]*=\"[^\"]*\")*)\\s*>(.*?)</parameter>"
let arguments: dict = {}
for capture in regex_captures(parameter_pattern, inner) {
const groups = capture?.groups ?? []
const key = to_string(groups[0] ?? groups[1] ?? "")
const raw = to_string(groups[2] ?? "")
arguments[key] = __tool_parse_markup_value(raw, __tool_parse_schema_type(tools, name, key))
}
const leftover = regex_replace(parameter_pattern, "", inner)
if contains(leftover, "<parameter") {
return {
matched: true,
ok: false,
error: "TOOL CALL TRUNCATED: a `<parameter ...>` block in the "
+ style
+ " markup for `"
+ name
+ "` was never closed with `</parameter>` — the response appears to have been cut off. "
+ "The call was NOT executed; re-emit the complete call.",
}
}
if contains(leftover, "<function=") || contains(leftover, "<invoke name=") {
return {
matched: true,
ok: false,
error: "The "
+ style
+ " markup block for `"
+ name
+ "` contained more than one call; emit one call per <tool_call> block.",
}
}
if len(keys(arguments)) == 0 && starts_with(trim(leftover), "{") {
const json_source = trim(leftover)
const json_len = __host_tool_balanced_json_len(json_source)
if json_len == 0 {
return {
matched: true,
ok: false,
error: "TOOL CALL TRUNCATED: the JSON arguments object in the "
+ style
+ " markup for `"
+ name
+ "` was never closed — the response appears to have been cut off. "
+ "The call was NOT executed; re-emit the complete call.",
}
}
arguments = try {
json_parse(json_source.slice(0, json_len))
} catch (error) {
return {
matched: true,
ok: false,
error: "The "
+ style
+ " markup for `"
+ name
+ "` had a JSON arguments object that did not parse: "
+ to_string(error)
+ ". The call was NOT executed.",
}
}
if type_of(arguments) != "dict" {
return {
matched: true,
ok: false,
error: "JSON arguments for tool '"
+ name
+ "' in "
+ style
+ " markup must be an object, got `"
+ json_stringify(arguments)
+ "`.",
}
}
arguments = __host_tool_decode_entities(arguments, HTML_ENTITIES)
}
return {
matched: true,
ok: true,
call: {id: "tc_fnmarkup_" + name, name: name, arguments: arguments},
}
}
fn __tool_parse_nested_xml(body: string, tools) -> dict {
const trimmed = trim(body)
const captures = regex_captures("^<([A-Za-z_][A-Za-z0-9_.-]*)>\\s*", trimmed)
if len(captures) == 0 {
return {matched: false}
}
const name = to_string(captures[0]?.groups?.[0] ?? "")
const after_open = trim(trimmed.slice(captures[0].end, len(trimmed)))
if !starts_with(after_open, "{") {
return {matched: false}
}
const known = registry_names(tools)
if !known.contains(name) {
return {
matched: true,
ok: false,
error: "Unknown tool '"
+ name
+ "' in nested XML tool-call body. Available tools: ["
+ known.slice(0, 20).join(", ")
+ "]",
}
}
const object_len = __host_tool_balanced_json_len(after_open)
if object_len == 0 {
return {
matched: true,
ok: false,
error: "<tool_call><"
+ name
+ "> body did not contain a complete JSON object. Emit `<tool_call>"
+ name
+ "({ ... })</tool_call>` instead.",
}
}
let arguments = try {
json_parse(after_open.slice(0, object_len))
} catch (error) {
return {
matched: true,
ok: false,
error: "<tool_call><"
+ name
+ "> body did not parse as a JSON object: "
+ to_string(error)
+ ". Emit `<tool_call>"
+ name
+ "({ ... })</tool_call>` instead.",
}
}
if type_of(arguments) != "dict" {
return {
matched: true,
ok: false,
error: "Nested XML arguments for tool '"
+ name
+ "' must be a JSON object, got `"
+ json_stringify(arguments)
+ "`.",
}
}
arguments = __host_tool_decode_entities(arguments, HTML_ENTITIES)
return {matched: true, ok: true, call: {id: "tc_xml_" + name, name: name, arguments: arguments}}
}
fn __tool_parse_narration(body: string, tools) -> dict {
let rest = trim(body)
let prose: list<string> = []
for tag in ["assistant_prose", "assistantprose", "thinking", "reasoning"] {
const open = "<" + tag + ">"
const close = "</" + tag + ">"
if starts_with(rest, open) {
const close_at = rest.index_of(close)
if close_at >= 0 {
const text = trim(rest.slice(len(open), close_at))
if text != "" {
prose = prose.appending(text)
}
rest = trim(rest.slice(close_at + len(close), len(rest)))
}
}
}
if len(prose) > 0 {
if rest == "" {
return {matched: true, ok: true, call: nil, prose: prose}
}
const parsed = __tool_parse_call_from_body(rest, tools)
return {matched: true, ok: true, call: parsed?.ok ?? false ? parsed?.call : nil, prose: prose}
}
if rest != ""
&& !starts_with(rest, "<")
&& !starts_with(rest, "{")
&& !starts_with(rest, "[") {
const sniffed = __host_tool_scan_bare_calls(rest, tools)
if len(sniffed?.calls ?? []) == 0 && len(sniffed?.errors ?? []) == 0 {
return {matched: true, ok: true, call: nil, prose: [rest]}
}
}
return {matched: false}
}
fn __tool_parse_json_call_from_body(source: string, tools) -> dict {
const decoded = try {
json_parse(source)
} catch (error) {
return {
ok: false,
error: "<tool_call> body looked like JSON but did not parse: "
+ to_string(error)
+ ". Emit either `name({ ... })` or JSON with `name` and `arguments`.",
}
}
const item = if type_of(decoded) == "list" {
if len(decoded) != 1 {
return {
ok: false,
error: "<tool_call> JSON array contained "
+ to_string(len(decoded))
+ " calls; emit one call per <tool_call> block.",
}
}
decoded[0]
} else {
decoded
}
if type_of(item) != "dict" {
return {
ok: false,
error: "<tool_call> JSON body must be an object, got `" + json_stringify(item) + "`.",
}
}
const name = trim(to_string(item?.name ?? item?.tool_name ?? item?.function?.name ?? ""))
if name == "" {
return {ok: false, error: "<tool_call> JSON body did not contain a tool name"}
}
const known = registry_names(tools)
if !known.contains(name) {
return {
ok: false,
error: "Unknown tool '" + name + "'. Available tools: [" + known.join(", ") + "]",
}
}
let arguments = item?.arguments
?? item?.parameters
?? item?.args
?? item?.function?.arguments
?? {}
if type_of(arguments) == "string" {
arguments = try {
json_parse(arguments)
} catch (error) {
return {
ok: false,
error: "Could not parse JSON string arguments for tool '"
+ name
+ "': "
+ to_string(error),
}
}
}
if type_of(arguments) != "dict" {
return {
ok: false,
error: "Tool '"
+ name
+ "' arguments must be a JSON object, got `"
+ json_stringify(arguments)
+ "`.",
}
}
return {
ok: true,
call: {
id: to_string(item?.id ?? "tc_json"),
name: name,
arguments: __host_tool_decode_entities(arguments, HTML_ENTITIES),
},
}
}
fn __tool_parse_call_from_body(
body: string,
tools,
head_name: string = "",
head_sep: string = "",
direct_candidate = nil,
) -> dict {
const trimmed = trim(body)
if trimmed == "" {
return {
ok: false,
error: "<tool_call> body did not contain a bare `name({ ... })` expression. Got: \"\"",
}
}
if head_name != "" && registry_names(tools).contains(head_name) {
const direct = if direct_candidate != nil {
direct_candidate
} else if head_sep == "(" {
__host_tool_parse_call_expr(trimmed, head_name)
} else if head_sep == "{" {
const parsed = __host_tool_parse_object_literal(
trimmed.slice(len(head_name), len(trimmed)),
head_name,
)
parsed + {consumed: len(head_name) + (parsed?.consumed ?? 0)}
} else {
{ok: false}
}
if direct?.ok ?? false {
const trailing = trim(trimmed.slice(direct.consumed, len(trimmed)))
if trailing == "" {
return {
ok: true,
call: {id: "tc_0", name: head_name, arguments: direct.arguments ?? direct.value ?? {}},
}
}
}
}
const markup = __tool_parse_markup(trimmed, tools)
if markup?.matched ?? false {
return markup
}
const nested = __tool_parse_nested_xml(trimmed, tools)
if nested?.matched ?? false {
return nested
}
if starts_with(trimmed, "{") || starts_with(trimmed, "[") {
return __tool_parse_json_call_from_body(trimmed, tools)
}
const narration = __tool_parse_narration(trimmed, tools)
if narration?.matched ?? false {
return narration
}
const sniffed = __host_tool_scan_bare_calls(trimmed, tools)
if len(sniffed?.errors ?? []) > 0 {
return {ok: false, error: sniffed.errors[0]}
}
if len(sniffed?.calls ?? []) == 1 {
return {ok: true, call: sniffed.calls[0]}
}
if len(sniffed?.calls ?? []) > 1 {
return {
ok: false,
error: "<tool_call> body contained "
+ to_string(len(sniffed.calls))
+ " calls; emit one call per <tool_call> block.",
}
}
return {
ok: false,
error: "<tool_call> body did not contain a bare `name({ ... })` expression. Got: "
+ json_stringify(
trimmed,
),
}
}
fn __tool_parse_consume_stray(unit, tools, state: dict, bare_candidate = nil) -> dict {
const text = trim(to_string(unit?.text ?? ""))
if text == "" {
return state
}
if text == "<tool_call>" {
return state
}
// Protocol-looking response tags in narration are examples or malformed
// wrappers, not visible assistant prose. Keep the old tagged-parser
// diagnostic at this boundary; a real response block arrives as a `block`
// unit and never takes this path.
let response_tag_at = -1
for spelling in __tool_parse_tag_spellings("user_response") {
const found = text.index_of("<" + spelling)
if found >= 0 && (response_tag_at < 0 || found < response_tag_at) {
response_tag_at = found
}
}
if response_tag_at >= 0 {
const fragment = text.slice(response_tag_at, len(text))
return state
+ {
violations: state.violations.appending(
protocol_violation(
"stray_text",
"Stray text outside response tags: "
+ json_stringify(fragment)
+ ". Wrap all prose in <assistant_prose>...</assistant_prose> or "
+ "<user_response>...</user_response>, and every tool call in "
+ "<tool_call>...</tool_call>.",
fragment,
),
),
dropped: state.dropped.appending({text: fragment, reason: "salvaged_as_prose"}),
}
}
const reason = unit?.kind == "fenced_line" ? "fenced_narration" : "salvaged_as_prose"
const sniffed = bare_candidate ?? __host_tool_scan_bare_calls(text, tools)
const next = state
if len(sniffed?.calls ?? []) > 0 {
let calls = next.calls
let canonical = next.canonical
for call in sniffed.calls {
calls = calls.appending(call)
canonical = canonical.appending(call_block(call))
}
const sniffed_prose = trim(to_string(sniffed?.prose ?? ""))
let prose = next.prose
if sniffed_prose != "" {
prose = prose.appending(sniffed_prose)
canonical = canonical.appending(prose_block(sniffed_prose))
}
return next
+ {
calls: calls,
canonical: canonical,
prose: prose,
recovered: next.recovered + len(sniffed.calls),
}
}
if len(sniffed?.errors ?? []) > 0 {
return next
+ {
errors: next.errors + sniffed.errors,
dropped: next.dropped.appending({text: text, reason: reason}),
}
}
if has_response_protocol_fragment(text) {
return next
+ {
violations: next.violations.appending(
protocol_violation(
"stray_text",
"Stray text outside response tags: "
+ json_stringify(text)
+ ". Wrap all prose in <assistant_prose>...</assistant_prose> or "
+ "<user_response>...</user_response>, and every tool call in "
+ "<tool_call>...</tool_call>.",
text,
),
),
recovered: next.recovered + 1,
dropped: next.dropped.appending({text: text, reason: reason}),
}
}
return next
+ {
prose: next.prose.appending(text),
canonical: next.canonical.appending(prose_block(text)),
dropped: next.dropped.appending({text: text, reason: reason}),
}
}
fn __tool_parse_pending_unit(unit, state: dict) -> dict {
if state.pending_response == nil {
return {state: state, consumed: false}
}
const kind = to_string(unit?.kind ?? "")
if TEXT_UNIT_KINDS.contains(kind) {
const body = trim(to_string(unit?.text ?? ""))
if body == "" {
return {state: state, consumed: true}
}
return {
state: state
+ {
prose: state.prose.appending(body),
canonical: state.canonical.appending(prose_block(body)),
},
consumed: true,
}
}
const pending = to_string(state.pending_response)
return {
state: state
+ {
pending_response: nil,
unclosed_response_prose: __tool_parse_tag_spellings("user_response").contains(pending),
violations: state.violations.appending(
protocol_violation(
"unclosed_response",
"Unclosed <"
+ pending
+ "> block. Close it with </"
+ pending
+ "> or remove it; only <tool_call>, <assistant_prose>, "
+ "<user_response>, and <done> are accepted.",
),
),
},
consumed: false,
}
}
fn __tool_parse_block_unit(unit, tools, state: dict, direct_candidate = nil) -> dict {
const kind = to_string(unit?.kind ?? "")
const tag = to_string(unit?.tag ?? "")
if __tool_parse_tag_spellings("assistant_prose").contains(tag) {
const body = trim(to_string(unit?.body ?? ""))
if body == "" {
return state
}
return state
+ {
prose: state.prose.appending(body),
canonical: state.canonical.appending(prose_block(body)),
}
}
if __tool_parse_tag_spellings("user_response").contains(tag) {
const body = trim(to_string(unit?.body ?? ""))
if body == "" {
return state
}
return state
+ {
answers: state.answers.appending(body),
canonical: state.canonical.appending(answer_block(body)),
}
}
if __tool_parse_tag_spellings("done").contains(tag) {
const body = trim(to_string(unit?.body ?? ""))
if body == "" {
return state
+ {
violations: state.violations.appending(
protocol_violation(
"empty_done",
"<done> block is empty. Emit the configured done sentinel "
+ "(default `##DONE##`) inside the block.",
),
),
}
}
return state
+ {done: body, canonical: state.canonical.appending("<done>" + body + "</done>")}
}
const parsed = __tool_parse_call_from_body(
to_string(unit?.body ?? ""),
tools,
to_string(unit?.head_name ?? ""),
to_string(unit?.head_sep ?? ""),
direct_candidate,
)
if parsed?.ok ?? false {
let calls = state.calls
let canonical = state.canonical
let prose = state.prose
if parsed?.call != nil {
calls = calls.appending(parsed.call)
canonical = canonical.appending(call_block(parsed.call))
}
for narration in parsed?.prose ?? [] {
prose = prose.appending(narration)
canonical = canonical.appending(prose_block(narration))
}
return state + {calls: calls, canonical: canonical, prose: prose}
}
let message = to_string(parsed?.error ?? "tool-call parse failed")
if kind == "unclosed_block"
&& !(parsed?.matched ?? false)
&& !starts_with(message, "TOOL CALL TRUNCATED:") {
const head = to_string(unit?.head_name ?? "")
message = if head != "" {
"TOOL CALL TRUNCATED: `<tool_call>` opening `"
+ head
+ "(...)` was never closed — the response appears to have been cut off mid-call "
+ "(likely the model hit its max output token limit). Re-emit the complete "
+ "`<tool_call>"
+ head
+ "({ ... })</tool_call>` block; for very large arguments, split the work into smaller calls."
} else {
"TOOL CALL TRUNCATED: a `<tool_call>` block was opened but never closed — "
+ "the response appears to have been cut off mid-call."
}
}
return state + {errors: state.errors.appending(message)}
}
fn __tool_parse_tag_unit(unit, state: dict) -> dict {
const raw = trim(to_string(unit?.raw ?? unit?.text ?? ""))
const name = to_string(unit?.name ?? "")
if __tool_parse_tag_spellings("assistant_prose").contains(name)
|| __tool_parse_tag_spellings(
"user_response",
)
.contains(name)
|| __tool_parse_tag_spellings("done").contains(name) {
return state + {pending_response: name}
}
if starts_with(raw, "</") {
return state
+ {
violations: state.violations.appending(
protocol_violation(
"stray_text",
"Stray text outside response tags: "
+ json_stringify(raw)
+ ". Wrap all prose in <assistant_prose>...</assistant_prose> or "
+ "<user_response>...</user_response>, and every tool call in "
+ "<tool_call>...</tool_call>.",
raw,
),
),
recovered: state.recovered + 1,
dropped: state.dropped.appending({text: raw, reason: "salvaged_as_prose"}),
}
}
return state
+ {
violations: state.violations.appending(
protocol_violation(
"unknown_tag",
"Unknown top-level tag "
+ json_stringify(raw)
+ ". Use <tool_call>, <assistant_prose>, <user_response>, or <done> "
+ "— no other tags are accepted at the top level.",
raw,
),
),
dropped: state.dropped.appending({text: raw, reason: "salvaged_as_prose"}),
}
}
fn __tool_parse_markup_unit(unit, tools, state: dict) -> dict {
const parsed = __tool_parse_markup(to_string(unit?.text ?? ""), tools)
if parsed?.ok ?? false {
const name = to_string(parsed.call?.name ?? "")
return state
+ {
calls: state.calls.appending(parsed.call),
canonical: state.canonical.appending(call_block(parsed.call)),
violations: state.violations.appending(
protocol_violation(
"recovered_dialect",
"Tool call `"
+ name
+ "` was emitted as chat-template function markup instead of "
+ "`<tool_call>"
+ name
+ "({ ... })</tool_call>`. Recovered this turn so work moves forward; "
+ "emit the canonical form on subsequent turns.",
),
),
}
}
if parsed?.matched ?? false {
return state
+ {
errors: state.errors.appending(to_string(parsed?.error ?? "function-markup parse failed")),
}
}
return __tool_parse_consume_stray(unit, tools, state)
}
fn __tool_parse_tagged_unit(unit, tools, state: dict, candidates = nil) -> dict {
let next = state
if state.pending_response != nil {
const pending = __tool_parse_pending_unit(unit, state)
if pending.consumed {
return pending.state
}
next = pending.state
}
const kind = to_string(unit?.kind ?? "")
if TEXT_UNIT_KINDS.contains(kind) {
return __tool_parse_consume_stray(unit, tools, next, candidates?.bare)
}
if kind == "harmony_skip" || kind == "wrapper" || kind == "stray_close" {
return next
}
if kind == "angle_call" {
const call = {
id: "tc_angle_" + to_string(unit.name),
name: unit.name,
arguments: unit?.arguments ?? {},
}
return next
+ {
calls: next.calls.appending(call),
canonical: next.canonical.appending(call_block(call)),
violations: next.violations.appending(
protocol_violation(
"recovered_dialect",
"Recovered angle-wrapped tool call; emit it inside <tool_call>...</tool_call>.",
),
),
}
}
if kind == "markup" {
return __tool_parse_markup_unit(unit, tools, next)
}
if kind == "block" || kind == "unclosed_block" || kind == "reserved_block" {
return __tool_parse_block_unit(unit, tools, next, candidates?.direct)
}
if kind == "tag" {
return __tool_parse_tag_unit(unit, next)
}
return __tool_parse_consume_stray(unit, tools, next)
}
fn __tool_parse_tagged_result(state: dict, response_tags: list<string>) -> dict {
const prose = state.prose.join("\n\n")
const answer = if len(state.answers) > 0 {
state.answers.join("\n")
} else if response_tags.contains(to_string(state.pending_response ?? "")) {
prose
} else {
nil
}
const visible_prose = if len(state.violations) > 0
&& len(state.calls) == 0
&& answer == nil
&& !state.unclosed_response_prose {
""
} else {
answer ?? prose
}
const recovered_count = if state.recovered > len(state.calls) {
len(state.calls)
} else {
state.recovered
}
return {
calls: state.calls,
tool_calls: state.calls,
tool_parse_errors: state.errors,
protocol_violations: state.violations,
recovered_from_stray_count: recovered_count,
prose: visible_prose,
user_response: answer,
done_marker: state.done,
canonical_text: __host_tool_render_parts(state.canonical),
dropped: state.dropped,
}
}
/**
* True when `text` opens a fence that names itself a tool block.
*
* Deliberately narrower than the fenced-JSON parser's own accept set: only the
* `"canonical"` intent counts, never the drift spellings. See
* `tool_fence_info_intent` — under a tagged-text pin the model was never asked
* for a fence, so a ```json block is documentation until proven otherwise, and
* dispatching it would run an example.
*
* This is a gate, not a parse. What the fence contains is still decided by
* `__tool_parse_json`.
*/
fn __tool_parse_has_canonical_tool_fence(text: string) -> bool {
for line in text.split("\n") {
const trimmed = trim(line)
const marker = if starts_with(trimmed, BACKTICK_FENCE) {
BACKTICK_FENCE
} else if starts_with(trimmed, TILDE_FENCE) {
TILDE_FENCE
} else {
""
}
if marker != ""
&& tool_fence_info_intent(trimmed.slice(len(marker), len(trimmed))) == "canonical" {
return true
}
}
return false
}
/**
* The tagged parser's fast path for a text unit outside any pending response.
*
* It differs from `__tool_parse_consume_stray` on purpose: no response-tag
* salvage (a pending response has already been ruled out here) and a `bare`
* canonical part that replays the sniffed call together with its prose.
*
* Fence recovery lives here and not in `__tool_parse_consume_stray` because
* that helper also runs while a response block is still open, where a fence is
* as likely to be quoted material as an attempted call. Recovery is only
* offered where the fence is unambiguously the model's own top-level output.
*
* It is offered only after the bare sniff comes up empty, so no text that
* already produced a call parses differently than before. Fenced text is
* accumulated on `state.fenced` rather than decided here: whether it dispatches
* depends on what the rest of the turn produced, which is not known until the
* unit loop ends.
*/
fn __tool_parse_stray_fast(unit, candidates, state: dict) -> dict {
const kind = to_string(unit?.kind ?? "")
const stray_text = trim(to_string(unit?.text ?? ""))
if stray_text == "" || stray_text == "<tool_call>" {
return state
}
const drop_reason = kind == "fenced_line" ? "fenced_narration" : "salvaged_as_prose"
const sniffed = candidates?.bare ?? {calls: [], errors: [], prose: stray_text}
if len(sniffed?.calls ?? []) > 0 {
const sniffed_prose = trim(to_string(sniffed?.prose ?? ""))
const prose = if sniffed_prose == "" {
state.prose
} else {
state.prose.appending(sniffed_prose)
}
return state
+ {
calls: state.calls + sniffed.calls,
canonical: state.canonical.appending(
{kind: "bare", calls: sniffed.calls, prose: sniffed_prose},
),
prose: prose,
recovered: state.recovered + len(sniffed.calls),
}
}
// No call in the taught grammar, but a fence that names itself a tool block.
if __tool_parse_has_canonical_tool_fence(stray_text) {
return state + {fenced: state.fenced.appending(stray_text)}
}
if len(sniffed?.errors ?? []) > 0 {
return state
+ {
errors: state.errors + sniffed.errors,
dropped: state.dropped.appending({text: stray_text, reason: drop_reason}),
}
}
if has_response_protocol_fragment(stray_text) {
return state
+ {
violations: state.violations.appending(
protocol_violation(
"stray_text",
"Stray text outside response tags: "
+ json_stringify(stray_text)
+ ". Wrap all prose in <assistant_prose>...</assistant_prose> or "
+ "<user_response>...</user_response>, and every tool call in "
+ "<tool_call>...</tool_call>.",
stray_text,
),
),
recovered: state.recovered + 1,
dropped: state.dropped.appending({text: stray_text, reason: drop_reason}),
}
}
return state
+ {
prose: state.prose.appending(stray_text),
canonical: state.canonical.appending({kind: "prose", text: stray_text}),
dropped: state.dropped.appending({text: stray_text, reason: drop_reason}),
}
}
/**
* Decide what a turn's ```tool fences meant, once the taught grammar has had
* its say.
*
* A fence that names itself a tool block is an explicit statement of intent, so
* its body is parsed as a call and accepted — but only when the turn produced
* no call in the taught grammar. A turn that already called correctly is not
* improved by also mining its fences, and mixing the two would make the accept
* set depend on unit ordering.
*
* Either way the fence is REPORTED. Declining to dispatch is not licence to go
* quiet: a silent drop is the defect this exists to remove (burin-code#6388),
* and a conservative gate that reintroduced one would be worse than no gate.
*/
fn __tool_parse_recover_fenced_calls(blocks: list<string>, tools, state: dict) -> dict {
const fenced_text = blocks.join("\n")
const already_called = len(state.calls) > 0
const fenced_calls = if already_called {
[]
} else {
__tool_parse_json(fenced_text, tools, false)?.calls ?? []
}
if len(fenced_calls) > 0 {
let canonical = state.canonical
for call in fenced_calls {
canonical = canonical.appending(call_block(call))
}
return state
+ {
calls: state.calls + fenced_calls,
canonical: canonical,
recovered: state.recovered + len(fenced_calls),
violations: state.violations.appending(
protocol_violation(
"recovered_dialect",
"Recovered a tool call from a ```tool fence while `tool_format` is `text`; "
+ "accepted this turn, but emit it inside <tool_call>...</tool_call> "
+ "next turn.",
),
),
}
}
const reason = if already_called {
" because this turn already made a call in the taught grammar"
} else {
" because its body did not parse as a tool call"
}
return state
+ {
prose: state.prose.appending(fenced_text),
canonical: state.canonical.appending({kind: "prose", text: fenced_text}),
dropped: state.dropped.appending({text: fenced_text, reason: "fenced_narration"}),
violations: state.violations.appending(
protocol_violation(
"wrong_tool_format",
"A ```tool fence was present while `tool_format` is `text` but was not dispatched"
+ reason
+ "; emit calls inside <tool_call>...</tool_call>.",
),
),
}
}
fn __tool_parse_tagged(text: string, tools) -> dict {
const provider_dialect = parse_provider_dialect(text, tools)
if provider_dialect != nil {
return provider_dialect
}
const scanned = __host_tool_scan_units(text, tool_parse_scan_spec(tools))
const bare_candidates = __host_tool_scan_bare_units(scanned.units, tools)
const known = registry_names(tools)
const assistant_tags = __tool_parse_tag_spellings("assistant_prose")
const response_tags = __tool_parse_tag_spellings("user_response")
let calls: list<dict> = []
let errors: list<string> = []
let violations: list = []
let prose_parts: list<string> = []
let answers: list<string> = []
let done = nil
let canonical: list<string> = []
let dropped: list<dict> = []
let recovered = 0
let pending_response = nil
let unclosed_response_prose = false
// Stray text carrying a ```tool fence. Held aside rather than dropped as
// narration: if the turn produces no call in the taught grammar, a call
// written in the fenced-JSON grammar is still a call the model meant to make,
// and discarding it silently is the burin-code#6388 failure. The fence label
// is what makes this unambiguous — prose *about* a tool is not written inside
// a fence that announces itself as a tool block.
let fenced_call_blocks: list<string> = []
for index in range(0, len(scanned.units)) {
const unit = scanned.units[index]
const candidates = bare_candidates[index]
const kind = unit?.kind ?? ""
const tag = unit?.tag ?? ""
if pending_response == nil && TEXT_UNIT_KINDS.contains(kind) {
const stray = __tool_parse_stray_fast(
unit,
candidates,
{
calls: calls,
errors: errors,
violations: violations,
prose: prose_parts,
canonical: canonical,
dropped: dropped,
recovered: recovered,
fenced: fenced_call_blocks,
},
)
calls = stray.calls
errors = stray.errors
violations = stray.violations
prose_parts = stray.prose
canonical = stray.canonical
dropped = stray.dropped
recovered = stray.recovered
fenced_call_blocks = stray.fenced
continue
}
if pending_response == nil && kind == "block" {
const body = trim(unit?.body ?? "")
if assistant_tags.contains(tag) {
if body != "" {
prose_parts = prose_parts.appending(body)
canonical = canonical.appending({kind: "prose", text: body})
}
continue
}
if response_tags.contains(tag) {
if body != "" {
answers = answers.appending(body)
canonical = canonical.appending({kind: "answer", text: body})
}
continue
}
const head_name = unit?.head_name ?? ""
const direct = candidates?.direct
if head_name != "" && known.contains(head_name) && (direct?.ok ?? false) {
const trailing = trim(body.slice(direct.consumed, len(body)))
if trailing == "" {
const call = {
id: "tc_0",
name: head_name,
arguments: direct.arguments ?? direct.value ?? {},
}
calls = calls.appending(call)
canonical = canonical.appending(
{kind: "call", name: call.name, arguments: call.arguments ?? {}},
)
continue
}
}
}
const state = __tool_parse_tagged_unit(
unit,
tools,
{
calls: calls,
errors: errors,
violations: violations,
prose: prose_parts,
answers: answers,
done: done,
canonical: canonical,
dropped: dropped,
recovered: recovered,
pending_response: pending_response,
unclosed_response_prose: unclosed_response_prose,
},
candidates,
)
calls = state.calls
errors = state.errors
violations = state.violations
prose_parts = state.prose
answers = state.answers
done = state.done
canonical = state.canonical
dropped = state.dropped
recovered = state.recovered
pending_response = state.pending_response
unclosed_response_prose = state.unclosed_response_prose
}
if len(fenced_call_blocks) > 0 {
const recovered_state = __tool_parse_recover_fenced_calls(
fenced_call_blocks,
tools,
{
calls: calls,
canonical: canonical,
prose: prose_parts,
dropped: dropped,
violations: violations,
recovered: recovered,
},
)
calls = recovered_state.calls
canonical = recovered_state.canonical
prose_parts = recovered_state.prose
dropped = recovered_state.dropped
violations = recovered_state.violations
recovered = recovered_state.recovered
}
return __tool_parse_tagged_result(
{
calls: calls,
errors: errors,
violations: violations,
prose: prose_parts,
answers: answers,
done: done,
canonical: canonical,
dropped: dropped,
recovered: recovered,
pending_response: pending_response,
unclosed_response_prose: unclosed_response_prose,
},
response_tags,
)
}
/**
* `recover_heredoc` breaks the one cycle between the two grammars.
*
* The recovery paths call each other: this function hands `<tool_call>` markup
* to `__tool_parse_tagged`, and `__tool_parse_recover_fenced_calls` hands a
* ```tool fence back here. Left open that is a loop, and its entry test is
* looser than what could stop it — `contains(text, "<tool_call>")` fires on the
* literal substring, while only the SCANNER decides whether that substring is a
* tag or narration. Termination must not depend on the two agreeing.
*
* So the fence path passes `false`: text arriving from
* `__tool_parse_recover_fenced_calls` has already been through the tagged
* parser once, and a second heredoc pass cannot find a call the first did not.
* Nothing is lost and the cycle cannot close.
*
* The guard is deliberately NOT symmetric. Disabling fence recovery in the
* tagged parser called from here would silently drop the fence in a mixed turn
* on a json pin — this branch delegates the WHOLE text to tagged, so tagged's
* fence recovery is the only thing that sees it. That would reintroduce exactly
* the silent drop this change exists to remove. One direction is enough.
*/
fn __tool_parse_json(text: string, tools = nil, recover_heredoc = true) -> dict {
const envelope = tool_parse_envelope(text)
if envelope?.matched ?? false {
if envelope?.result != nil {
if len(envelope.result?.tool_parse_errors ?? []) > 0 {
return envelope.result + {prose: trim(text)}
}
return envelope.result
}
return empty_parse()
+ {tool_parse_errors: ["chat-template envelope parse failed"], prose: text}
}
if recover_heredoc && contains(text, "<tool_call>") {
// Accept the heredoc grammar here rather than discarding it. This route is
// measurably exposed: burin-code's default eval model `local-qwen3.6`
// resolves to llamacpp `qwen3.6-35b-a3b-ud-q4-k-xl`, whose capability row is
// `preferred = json` with `parity = text_only`, and a probe on it parses 1
// fenced call, 0 heredoc calls, and 0 errors — a silent total drop of
// exactly the grammar most open models reach for. The two grammars are
// mutually unambiguous, so forgiving one costs no soundness.
const recovered = __tool_parse_tagged(text, tools)
const violation = protocol_violation(
"recovered_dialect",
"Recovered `<tool_call>...</tool_call>` markup while `tool_format` is `json`; "
+ "accepted this turn, but wrap each call as a strict JSON object inside a "
+ "```tool fence next turn.",
)
if len(recovered?.calls ?? []) > 0 {
return recovered
+ {
protocol_violations: (recovered?.protocol_violations ?? []).appending(violation),
}
}
return empty_parse()
+ {
protocol_violations: [
protocol_violation(
"wrong_tool_format",
"protocol_violation: `<tool_call>...</tool_call>` markup was emitted while "
+ "`tool_format` is `json`; wrap each call as a strict JSON object inside "
+ "a ```tool fence instead.",
),
],
prose: trim(text),
}
}
const chunked = json_chunks(text)
let calls: list<dict> = []
let errors: list<string> = []
for body in chunked.bodies {
const source = trim(body)
const stream = __host_tool_json_stream(source)
let body_calls: list<dict> = []
for row in stream?.values ?? [] {
const normalized = json_call(row?.value, false)
if normalized?.ok ?? false {
body_calls = body_calls.appending(normalized.call)
} else {
errors = errors.appending(normalized.error)
}
}
const trailing = trim(source.slice(stream?.end ?? 0, len(source)))
if len(body_calls) > 0
&& (has_counted_verbatim(body_calls)
|| starts_with(trailing, "<<")) {
const bound = bind_verbatim(body_calls, trailing)
if bound?.ok ?? false {
body_calls = bound.calls
} else {
body_calls = []
errors = errors.appending(bound.error)
}
}
calls = calls + body_calls
if len(stream?.values ?? []) == 0 {
const missing_error = if stream?.eof ?? false {
"Unterminated ```tool fence opened on line 1 (1-based): the block reached "
+ "end-of-output with an incomplete JSON object. Re-emit the whole call inside "
+ "a ```tool ... ``` block and close it with a line that is exactly ```."
} else {
__tool_parse_json_error(
to_string(stream?.error ?? "Tool JSON block did not contain an object.")
+ " (near `"
+ source.slice(0, 80)
+ "`)",
)
}
errors = errors.appending(missing_error)
} else if stream?.error != nil && !starts_with(trailing, "<<") {
errors = errors.appending(
"The ```tool block is not valid JSON: "
+ to_string(stream.error)
+ " (near `"
+ source.slice(0, 80)
+ "`)"
+ ". Pass multi-line or code-bearing fields as ordinary JSON string values "
+ "(escape newlines as \\n, quotes as \\\", backslashes as \\\\); "
+ "backticks need no escaping.",
)
} else if stream?.eof ?? false {
errors = errors.appending(
"Unterminated ```tool fence opened on line 1 (1-based): the block reached "
+ "end-of-output with an incomplete JSON object. Re-emit the whole call inside "
+ "a ```tool ... ``` block and close it with a line that is exactly ```.",
)
}
}
const canonical = calls.map(fn(call) { return call_block(call) }).join("\n")
return empty_parse()
+ {
calls: calls,
tool_calls: calls,
tool_parse_errors: errors,
protocol_violations: chunked.violations,
canonical_text: canonical,
prose: chunked.prose,
}
}
fn __tool_parse_json_error(detail: string) -> string {
return "The ```tool block is not valid JSON: "
+ detail
+ ". Pass multi-line or code-bearing fields as ordinary JSON string values "
+ "(escape newlines as \\n, quotes as \\\", backslashes as \\\\); "
+ "backticks need no escaping."
}
/**
* Parse one model text response.
*
* @effects: []
* @errors: []
* @api_stability: experimental
*/
pub fn parse_tool_calls(payload: dict) -> dict {
const text = to_string(payload?.text ?? "")
const format = to_string(payload?.tool_format ?? "")
// `adaptive` was an opt-in union parser (#5272) deleted in the dialect
// cutover (#5700). Keep failing closed so an explicit pin cannot silently
// demote to tagged text and hide teach/accept drift (burin-code#4809).
if format == "adaptive" {
return empty_parse()
+ {
tool_parse_errors: [
"`tool_format=adaptive` is not implemented. Pin `json` (fenced JSON, "
+ "global default) or `text` (heredoc `<tool_call>`), or use `native` "
+ "when the provider tool channel is reliable.",
],
prose: trim(text),
}
}
const parsed = if format == "json" {
__tool_parse_json(text, payload?.tools)
} else {
__tool_parse_tagged(text, payload?.tools)
}
const unparsed = tool_shape_violations(tool_shape_findings(parsed?.dropped ?? []))
return parsed + {protocol_violations: (parsed?.protocol_violations ?? []) + unparsed}
}