/**
* std/llm/tool_parse — text-tool dialect composition.
*
* Rust delimits byte ranges and parses byte-oriented literals. This module
* owns what those ranges mean: dialect vocabulary, recovery order, visible
* prose, canonical replay, and the rule that every consumed non-call is
* recorded as dropped.
*/
import {
BACKTICK_FENCE,
TAGGED_DIALECT,
TILDE_FENCE,
tool_fence_info_intent,
} from "std/llm/dialects"
import { __tool_parse_call_from_body, __tool_parse_markup } from "std/llm/tool_parse_body"
import { tool_envelope_alternate_opener_at, tool_parse_envelope } from "std/llm/tool_parse_envelope"
import {
bind_verbatim,
has_counted_verbatim,
json_call,
json_chunks,
} from "std/llm/tool_parse_json_support"
import { has_response_protocol_fragment } from "std/llm/tool_parse_protocol"
import { parse_provider_dialect } from "std/llm/tool_parse_provider"
import {
answer_block,
call_block,
canonical,
empty_parse,
prose_block,
protocol_violation,
registry_names,
} from "std/llm/tool_parse_result"
import { tool_parse_scan_spec } from "std/llm/tool_parse_scan_spec"
import { tool_shape_findings, tool_shape_violations } from "std/llm/tool_shape"
const TEXT_UNIT_KINDS = ["text", "fenced_line", "harmony_line"]
fn __tool_parse_tag_spellings(canonical: string) -> list<string> {
for dialect in TAGGED_DIALECT {
if dialect.canonical == canonical {
return dialect.spellings
}
}
return []
}
fn __tool_parse_consume_stray(unit, tools, state: dict, bare_candidate = nil) -> dict {
const text = trim(to_string(unit?.text ?? ""))
if text == "" {
return state
}
if text == "<tool_call>" {
return state
}
// Protocol-looking response tags in narration are examples or malformed
// wrappers, not visible assistant prose. Keep the old tagged-parser
// diagnostic at this boundary; a real response block arrives as a `block`
// unit and never takes this path.
let response_tag_at = -1
for spelling in __tool_parse_tag_spellings("user_response") {
const found = text.index_of("<" + spelling)
if found >= 0 && (response_tag_at < 0 || found < response_tag_at) {
response_tag_at = found
}
}
if response_tag_at >= 0 {
const fragment = text.slice(response_tag_at, len(text))
return state
+ {
violations: state.violations.appending(
protocol_violation(
"stray_text",
"Stray text outside response tags: "
+ json_stringify(fragment)
+ ". Wrap all prose in <assistant_prose>...</assistant_prose> or "
+ "<user_response>...</user_response>, and every tool call in "
+ "<tool_call>...</tool_call>.",
fragment,
),
),
dropped: state.dropped.appending({text: fragment, reason: "salvaged_as_prose"}),
}
}
const reason = unit?.kind == "fenced_line" ? "fenced_narration" : "salvaged_as_prose"
const sniffed = bare_candidate ?? __host_tool_scan_bare_calls(text, tools)
const next = state
if len(sniffed?.calls ?? []) > 0 {
let calls = next.calls
let canonical = next.canonical
for call in sniffed.calls {
calls = calls.appending(call)
canonical = canonical.appending(call_block(call))
}
const sniffed_prose = trim(to_string(sniffed?.prose ?? ""))
let prose = next.prose
if sniffed_prose != "" {
prose = prose.appending(sniffed_prose)
canonical = canonical.appending(prose_block(sniffed_prose))
}
return next
+ {
calls: calls,
canonical: canonical,
prose: prose,
// Diagnostics raised beside a recovered call are still diagnostics. See
// the same carry-through in `__tool_parse_stray_fast`.
errors: next.errors + (sniffed?.errors ?? []),
recovered: next.recovered + len(sniffed.calls),
}
}
if len(sniffed?.errors ?? []) > 0 {
return next
+ {
errors: next.errors + sniffed.errors,
dropped: next.dropped.appending({text: text, reason: reason}),
}
}
if has_response_protocol_fragment(text) {
return next
+ {
violations: next.violations.appending(
protocol_violation(
"stray_text",
"Stray text outside response tags: "
+ json_stringify(text)
+ ". Wrap all prose in <assistant_prose>...</assistant_prose> or "
+ "<user_response>...</user_response>, and every tool call in "
+ "<tool_call>...</tool_call>.",
text,
),
),
recovered: next.recovered + 1,
dropped: next.dropped.appending({text: text, reason: reason}),
}
}
return next
+ {
prose: next.prose.appending(text),
canonical: next.canonical.appending(prose_block(text)),
dropped: next.dropped.appending({text: text, reason: reason}),
}
}
fn __tool_parse_pending_unit(unit, state: dict) -> dict {
if state.pending_response == nil {
return {state: state, consumed: false}
}
const kind = to_string(unit?.kind ?? "")
if TEXT_UNIT_KINDS.contains(kind) {
const body = trim(to_string(unit?.text ?? ""))
if body == "" {
return {state: state, consumed: true}
}
return {
state: state
+ {
prose: state.prose.appending(body),
canonical: state.canonical.appending(prose_block(body)),
},
consumed: true,
}
}
const pending = to_string(state.pending_response)
return {
state: state
+ {
pending_response: nil,
unclosed_response_prose: __tool_parse_tag_spellings("user_response").contains(pending),
violations: state.violations.appending(
protocol_violation(
"unclosed_response",
"Unclosed <"
+ pending
+ "> block. Close it with </"
+ pending
+ "> or remove it; only <tool_call>, <assistant_prose>, "
+ "<user_response>, and <done> are accepted.",
),
),
},
consumed: false,
}
}
fn __tool_parse_block_unit(unit, tools, state: dict, direct_candidate = nil) -> dict {
const kind = to_string(unit?.kind ?? "")
const tag = to_string(unit?.tag ?? "")
if __tool_parse_tag_spellings("assistant_prose").contains(tag) {
const body = trim(to_string(unit?.body ?? ""))
if body == "" {
return state
}
return state
+ {
prose: state.prose.appending(body),
canonical: state.canonical.appending(prose_block(body)),
}
}
if __tool_parse_tag_spellings("user_response").contains(tag) {
const body = trim(to_string(unit?.body ?? ""))
if body == "" {
return state
}
return state
+ {
answers: state.answers.appending(body),
canonical: state.canonical.appending(answer_block(body)),
}
}
if __tool_parse_tag_spellings("done").contains(tag) {
const body = trim(to_string(unit?.body ?? ""))
if body == "" {
return state
+ {
violations: state.violations.appending(
protocol_violation(
"empty_done",
"<done> block is empty. Emit the configured done sentinel "
+ "(default `##DONE##`) inside the block.",
),
),
}
}
return state
+ {done: body, canonical: state.canonical.appending("<done>" + body + "</done>")}
}
const parsed = __tool_parse_call_from_body(
to_string(unit?.body ?? ""),
tools,
to_string(unit?.head_name ?? ""),
to_string(unit?.head_sep ?? ""),
direct_candidate,
)
if parsed?.ok ?? false {
let calls = state.calls
let canonical = state.canonical
let prose = state.prose
let violations = state.violations
if parsed?.call != nil {
calls = calls.appending(parsed.call)
canonical = canonical.appending(call_block(parsed.call))
}
for narration in parsed?.prose ?? [] {
prose = prose.appending(narration)
canonical = canonical.appending(prose_block(narration))
}
// Forgiving a missing close still has to say so. A silent accept would
// teach the model that the close tag is optional and would blind every
// consumer that reads parse outcomes to detect dialect drift.
if kind == "unclosed_block" && parsed?.call != nil {
violations = violations.appending(
protocol_violation(
"recovered_dialect",
"Recovered a tool call from a `<tool_call>` block that was never closed. "
+ "Close every block with `</tool_call>`.",
),
)
}
return state
+ {calls: calls, canonical: canonical, prose: prose, violations: violations}
}
let message = to_string(parsed?.error ?? "tool-call parse failed")
if kind == "unclosed_block"
&& !(parsed?.matched ?? false)
&& !starts_with(message, "TOOL CALL TRUNCATED:") {
const head = to_string(unit?.head_name ?? "")
message = if head != "" {
"TOOL CALL TRUNCATED: `<tool_call>` opening `"
+ head
+ "(...)` was never closed — the response appears to have been cut off mid-call "
+ "(likely the model hit its max output token limit). Re-emit the complete "
+ "`<tool_call>"
+ head
+ "({ ... })</tool_call>` block; for very large arguments, split the work into smaller calls."
} else {
"TOOL CALL TRUNCATED: a `<tool_call>` block was opened but never closed — "
+ "the response appears to have been cut off mid-call."
}
}
return state + {errors: state.errors.appending(message)}
}
fn __tool_parse_tag_unit(unit, state: dict) -> dict {
const raw = trim(to_string(unit?.raw ?? unit?.text ?? ""))
const name = to_string(unit?.name ?? "")
if __tool_parse_tag_spellings("assistant_prose").contains(name)
|| __tool_parse_tag_spellings(
"user_response",
)
.contains(name)
|| __tool_parse_tag_spellings("done").contains(name) {
return state + {pending_response: name}
}
if starts_with(raw, "</") {
return state
+ {
violations: state.violations.appending(
protocol_violation(
"stray_text",
"Stray text outside response tags: "
+ json_stringify(raw)
+ ". Wrap all prose in <assistant_prose>...</assistant_prose> or "
+ "<user_response>...</user_response>, and every tool call in "
+ "<tool_call>...</tool_call>.",
raw,
),
),
recovered: state.recovered + 1,
dropped: state.dropped.appending({text: raw, reason: "salvaged_as_prose"}),
}
}
return state
+ {
violations: state.violations.appending(
protocol_violation(
"unknown_tag",
"Unknown top-level tag "
+ json_stringify(raw)
+ ". Use <tool_call>, <assistant_prose>, <user_response>, or <done> "
+ "— no other tags are accepted at the top level.",
raw,
),
),
dropped: state.dropped.appending({text: raw, reason: "salvaged_as_prose"}),
}
}
fn __tool_parse_markup_unit(unit, tools, state: dict) -> dict {
const parsed = __tool_parse_markup(to_string(unit?.text ?? ""), tools)
if parsed?.ok ?? false {
const name = to_string(parsed.call?.name ?? "")
return state
+ {
calls: state.calls.appending(parsed.call),
canonical: state.canonical.appending(call_block(parsed.call)),
violations: state.violations.appending(
protocol_violation(
"recovered_dialect",
"Tool call `"
+ name
+ "` was emitted as chat-template function markup instead of "
+ "`<tool_call>"
+ name
+ "({ ... })</tool_call>`. Recovered this turn so work moves forward; "
+ "emit the canonical form on subsequent turns.",
),
),
}
}
if parsed?.matched ?? false {
return state
+ {
errors: state.errors.appending(to_string(parsed?.error ?? "function-markup parse failed")),
}
}
return __tool_parse_consume_stray(unit, tools, state)
}
fn __tool_parse_tagged_unit(unit, tools, state: dict, candidates = nil) -> dict {
let next = state
if state.pending_response != nil {
const pending = __tool_parse_pending_unit(unit, state)
if pending.consumed {
return pending.state
}
next = pending.state
}
const kind = to_string(unit?.kind ?? "")
if TEXT_UNIT_KINDS.contains(kind) {
return __tool_parse_consume_stray(unit, tools, next, candidates?.bare)
}
if kind == "harmony_skip" || kind == "wrapper" || kind == "stray_close" {
return next
}
if kind == "angle_call" {
const call = {
id: "tc_angle_" + to_string(unit.name),
name: unit.name,
arguments: unit?.arguments ?? {},
}
return next
+ {
calls: next.calls.appending(call),
canonical: next.canonical.appending(call_block(call)),
violations: next.violations.appending(
protocol_violation(
"recovered_dialect",
"Recovered angle-wrapped tool call; emit it inside <tool_call>...</tool_call>.",
),
),
}
}
if kind == "markup" {
return __tool_parse_markup_unit(unit, tools, next)
}
if kind == "block" || kind == "unclosed_block" {
return __tool_parse_block_unit(unit, tools, next, candidates?.direct)
}
if kind == "tag" {
return __tool_parse_tag_unit(unit, next)
}
return __tool_parse_consume_stray(unit, tools, next)
}
fn __tool_parse_tagged_result(state: dict, response_tags: list<string>) -> dict {
const prose = state.prose.join("\n\n")
const answer = if len(state.answers) > 0 {
state.answers.join("\n")
} else if response_tags.contains(to_string(state.pending_response ?? "")) {
prose
} else {
nil
}
const visible_prose = if len(state.violations) > 0
&& len(state.calls) == 0
&& answer == nil
&& !state.unclosed_response_prose {
""
} else {
answer ?? prose
}
const recovered_count = if state.recovered > len(state.calls) {
len(state.calls)
} else {
state.recovered
}
return {
calls: state.calls,
tool_calls: state.calls,
tool_parse_errors: state.errors,
protocol_violations: state.violations,
recovered_from_stray_count: recovered_count,
prose: visible_prose,
user_response: answer,
done_marker: state.done,
canonical_text: __host_tool_render_parts(state.canonical),
dropped: state.dropped,
}
}
/**
* True when `text` opens a fence that names itself a tool block.
*
* Deliberately narrower than the fenced-JSON parser's own accept set: only the
* `"canonical"` intent counts, never the drift spellings. See
* `tool_fence_info_intent` — under a tagged-text pin the model was never asked
* for a fence, so a ```json block is documentation until proven otherwise, and
* dispatching it would run an example.
*
* This is a gate, not a parse. What the fence contains is still decided by
* `__tool_parse_json`.
*/
fn __tool_parse_has_canonical_tool_fence(text: string) -> bool {
for line in text.split("\n") {
const trimmed = trim(line)
const marker = if starts_with(trimmed, BACKTICK_FENCE) {
BACKTICK_FENCE
} else if starts_with(trimmed, TILDE_FENCE) {
TILDE_FENCE
} else {
""
}
if marker != ""
&& tool_fence_info_intent(trimmed.slice(len(marker), len(trimmed))) == "canonical" {
return true
}
}
return false
}
/**
* The tagged parser's fast path for a text unit outside any pending response.
*
* It differs from `__tool_parse_consume_stray` on purpose: no response-tag
* salvage (a pending response has already been ruled out here) and a `bare`
* canonical part that replays the sniffed call together with its prose.
*
* Fence recovery lives here and not in `__tool_parse_consume_stray` because
* that helper also runs while a response block is still open, where a fence is
* as likely to be quoted material as an attempted call. Recovery is only
* offered where the fence is unambiguously the model's own top-level output.
*
* It is offered only after the bare sniff comes up empty, so no text that
* already produced a call parses differently than before. Fenced text is
* accumulated on `state.fenced` rather than decided here: whether it dispatches
* depends on what the rest of the turn produced, which is not known until the
* unit loop ends.
*/
fn __tool_parse_stray_fast(unit, candidates, state: dict) -> dict {
const kind = to_string(unit?.kind ?? "")
const stray_text = trim(to_string(unit?.text ?? ""))
if stray_text == "" || stray_text == "<tool_call>" {
return state
}
const drop_reason = kind == "fenced_line" ? "fenced_narration" : "salvaged_as_prose"
const sniffed = candidates?.bare ?? {calls: [], errors: [], prose: stray_text}
if len(sniffed?.calls ?? []) > 0 {
const sniffed_prose = trim(to_string(sniffed?.prose ?? ""))
const prose = if sniffed_prose == "" {
state.prose
} else {
state.prose.appending(sniffed_prose)
}
return state
+ {
calls: state.calls + sniffed.calls,
canonical: state.canonical.appending(
{kind: "bare", calls: sniffed.calls, prose: sniffed_prose},
),
prose: prose,
// A unit that produced a call can still have produced a diagnostic
// beside it — an unknown name, or arguments that did not decode. Reading
// only the calls made the rest of the unit vanish, which is the same
// silence one recovered call used to hide the next behind.
errors: state.errors + (sniffed?.errors ?? []),
recovered: state.recovered + len(sniffed.calls),
}
}
// No call in the taught grammar, but a fence that names itself a tool block.
if __tool_parse_has_canonical_tool_fence(stray_text) {
return state + {fenced: state.fenced.appending(stray_text)}
}
if len(sniffed?.errors ?? []) > 0 {
return state
+ {
errors: state.errors + sniffed.errors,
dropped: state.dropped.appending({text: stray_text, reason: drop_reason}),
}
}
if has_response_protocol_fragment(stray_text) {
return state
+ {
violations: state.violations.appending(
protocol_violation(
"stray_text",
"Stray text outside response tags: "
+ json_stringify(stray_text)
+ ". Wrap all prose in <assistant_prose>...</assistant_prose> or "
+ "<user_response>...</user_response>, and every tool call in "
+ "<tool_call>...</tool_call>.",
stray_text,
),
),
recovered: state.recovered + 1,
dropped: state.dropped.appending({text: stray_text, reason: drop_reason}),
}
}
return state
+ {
prose: state.prose.appending(stray_text),
canonical: state.canonical.appending({kind: "prose", text: stray_text}),
dropped: state.dropped.appending({text: stray_text, reason: drop_reason}),
}
}
/**
* Decide what a turn's ```tool fences meant, once the taught grammar has had
* its say.
*
* A fence that names itself a tool block is an explicit statement of intent, so
* its body is parsed as a call and accepted — but only when the turn produced
* no call in the taught grammar. A turn that already called correctly is not
* improved by also mining its fences, and mixing the two would make the accept
* set depend on unit ordering.
*
* Either way the fence is REPORTED. Declining to dispatch is not licence to go
* quiet: a silent drop is the defect this exists to remove (burin-code#6388),
* and a conservative gate that reintroduced one would be worse than no gate.
*/
fn __tool_parse_recover_fenced_calls(blocks: list<string>, tools, state: dict) -> dict {
const fenced_text = blocks.join("\n")
const already_called = len(state.calls) > 0
const fenced_calls = if already_called {
[]
} else {
__tool_parse_json(fenced_text, tools, false)?.calls ?? []
}
if len(fenced_calls) > 0 {
let canonical = state.canonical
for call in fenced_calls {
canonical = canonical.appending(call_block(call))
}
return state
+ {
calls: state.calls + fenced_calls,
canonical: canonical,
recovered: state.recovered + len(fenced_calls),
violations: state.violations.appending(
protocol_violation(
"recovered_dialect",
"Recovered a tool call from a ```tool fence while `tool_format` is `text`; "
+ "accepted this turn, but emit it inside <tool_call>...</tool_call> "
+ "next turn.",
),
),
}
}
const reason = if already_called {
" because this turn already made a call in the taught grammar"
} else {
" because its body did not parse as a tool call"
}
return state
+ {
prose: state.prose.appending(fenced_text),
canonical: state.canonical.appending({kind: "prose", text: fenced_text}),
dropped: state.dropped.appending({text: fenced_text, reason: "fenced_narration"}),
violations: state.violations.appending(
protocol_violation(
"wrong_tool_format",
"A ```tool fence was present while `tool_format` is `text` but was not dispatched"
+ reason
+ "; emit calls inside <tool_call>...</tool_call>.",
),
),
}
}
fn __tool_parse_tagged(text: string, tools) -> dict {
const provider_dialect = parse_provider_dialect(text, tools)
if provider_dialect != nil {
return provider_dialect
}
// Which opener spellings wrap a JSON call body is not a property of the route
// pin. This lane owns `<tool_call>` and reads it far better than the envelope
// could, so that spelling stays here; the alternates it has no reading for —
// `<tool>`, `<tool_use>`, `[[tool]]` — went to the generic stray scan, which
// recovered the first object of a multi-call body and dropped the rest with
// nothing said. Ask the one owner instead, so a turn parses the same way on
// either pin.
const tagged_at = text.index_of("<tool_call>")
const alternate_at = tool_envelope_alternate_opener_at(text)
// A tagged block first and an alternate opener after it: each span owns only
// itself. Reading the whole message as one or the other loses whichever came
// second, and losing it here was silent, because an alternate opener is not
// tag-shaped and the stray scan has nothing to say about it.
if alternate_at > 0 && tagged_at >= 0 && tagged_at < alternate_at {
return __tool_parse_join(
__tool_parse_tagged(text.slice(0, alternate_at), tools),
__tool_parse_tagged(text.slice(alternate_at, len(text)), tools),
)
}
if tagged_at < 0 || (alternate_at >= 0 && alternate_at < tagged_at) {
const envelope = tool_parse_envelope(
text,
fn(body) { return __tool_parse_call_from_body(body, tools, "", "") },
)
if (envelope?.matched ?? false) && envelope?.result != nil
&& len(envelope.result?.calls ?? [])
> 0 {
const recovered = __tool_parse_envelope_remainder(
envelope,
fn(part) { return __tool_parse_tagged(part, tools) },
)
// The envelope phrases its violation for the fenced grammar because that
// is the pin it was written for. Say it in this lane's own vocabulary.
return recovered
+ {
protocol_violations: [
protocol_violation(
"recovered_dialect",
"Recovered a tool call from a chat-template envelope. Emit each call as "
+ "`<tool_call>name({ ... })</tool_call>` next turn.",
),
],
}
}
}
const scanned = __host_tool_scan_units(text, tool_parse_scan_spec(tools))
const bare_candidates = __host_tool_scan_bare_units(scanned.units, tools)
const known = registry_names(tools)
const assistant_tags = __tool_parse_tag_spellings("assistant_prose")
const response_tags = __tool_parse_tag_spellings("user_response")
let calls: list<dict> = []
let errors: list<string> = []
let violations: list = []
let prose_parts: list<string> = []
let answers: list<string> = []
let done = nil
let canonical: list<string> = []
let dropped: list<dict> = []
let recovered = 0
let pending_response = nil
let unclosed_response_prose = false
// Stray text carrying a ```tool fence. Held aside rather than dropped as
// narration: if the turn produces no call in the taught grammar, a call
// written in the fenced-JSON grammar is still a call the model meant to make,
// and discarding it silently is the burin-code#6388 failure. The fence label
// is what makes this unambiguous — prose *about* a tool is not written inside
// a fence that announces itself as a tool block.
let fenced_call_blocks: list<string> = []
for index in range(0, len(scanned.units)) {
const unit = scanned.units[index]
const candidates = bare_candidates[index]
const kind = unit?.kind ?? ""
const tag = unit?.tag ?? ""
if pending_response == nil && TEXT_UNIT_KINDS.contains(kind) {
const stray = __tool_parse_stray_fast(
unit,
candidates,
{
calls: calls,
errors: errors,
violations: violations,
prose: prose_parts,
canonical: canonical,
dropped: dropped,
recovered: recovered,
fenced: fenced_call_blocks,
},
)
calls = stray.calls
errors = stray.errors
violations = stray.violations
prose_parts = stray.prose
canonical = stray.canonical
dropped = stray.dropped
recovered = stray.recovered
fenced_call_blocks = stray.fenced
continue
}
if pending_response == nil && kind == "block" {
const body = trim(unit?.body ?? "")
if assistant_tags.contains(tag) {
if body != "" {
prose_parts = prose_parts.appending(body)
canonical = canonical.appending({kind: "prose", text: body})
}
continue
}
if response_tags.contains(tag) {
if body != "" {
answers = answers.appending(body)
canonical = canonical.appending({kind: "answer", text: body})
}
continue
}
const head_name = unit?.head_name ?? ""
const direct = candidates?.direct
if head_name != "" && known.contains(head_name) && (direct?.ok ?? false) {
const trailing = trim(body.slice(direct.consumed, len(body)))
if trailing == "" {
const call = {
id: "tc_0",
name: head_name,
arguments: direct.arguments ?? direct.value ?? {},
}
calls = calls.appending(call)
canonical = canonical.appending(
{kind: "call", name: call.name, arguments: call.arguments ?? {}},
)
continue
}
}
}
const state = __tool_parse_tagged_unit(
unit,
tools,
{
calls: calls,
errors: errors,
violations: violations,
prose: prose_parts,
answers: answers,
done: done,
canonical: canonical,
dropped: dropped,
recovered: recovered,
pending_response: pending_response,
unclosed_response_prose: unclosed_response_prose,
},
candidates,
)
calls = state.calls
errors = state.errors
violations = state.violations
prose_parts = state.prose
answers = state.answers
done = state.done
canonical = state.canonical
dropped = state.dropped
recovered = state.recovered
pending_response = state.pending_response
unclosed_response_prose = state.unclosed_response_prose
}
if len(fenced_call_blocks) > 0 {
const recovered_state = __tool_parse_recover_fenced_calls(
fenced_call_blocks,
tools,
{
calls: calls,
canonical: canonical,
prose: prose_parts,
dropped: dropped,
violations: violations,
recovered: recovered,
},
)
calls = recovered_state.calls
canonical = recovered_state.canonical
prose_parts = recovered_state.prose
dropped = recovered_state.dropped
violations = recovered_state.violations
recovered = recovered_state.recovered
}
return __tool_parse_tagged_result(
{
calls: calls,
errors: errors,
violations: violations,
prose: prose_parts,
answers: answers,
done: done,
canonical: canonical,
dropped: dropped,
recovered: recovered,
pending_response: pending_response,
unclosed_response_prose: unclosed_response_prose,
},
response_tags,
)
}
/**
* `recover_heredoc` breaks the one cycle between the two grammars.
*
* The recovery paths call each other: this function hands `<tool_call>` markup
* to `__tool_parse_tagged`, and `__tool_parse_recover_fenced_calls` hands a
* ```tool fence back here. Left open that is a loop, and its entry test is
* looser than what could stop it — `contains(text, "<tool_call>")` fires on the
* literal substring, while only the SCANNER decides whether that substring is a
* tag or narration. Termination must not depend on the two agreeing.
*
* So the fence path passes `false`: text arriving from
* `__tool_parse_recover_fenced_calls` has already been through the tagged
* parser once, and a second heredoc pass cannot find a call the first did not.
* Nothing is lost and the cycle cannot close.
*
* The guard is deliberately NOT symmetric. Disabling fence recovery in the
* tagged parser called from here would silently drop the fence in a mixed turn
* on a json pin — this branch delegates the WHOLE text to tagged, so tagged's
* fence recovery is the only thing that sees it. That would reintroduce exactly
* the silent drop this change exists to remove. One direction is enough.
*/
/**
* Fold the text an envelope did not consume back into the turn.
*
* An envelope owns its own span, not the message. An unclosed one runs to end
* of output, so the text after its objects routinely holds the rest of the
* turn — and models mix grammars within a turn precisely when they are unsure
* which one is real. Parsing the leading and trailing remainders in the
* ordinary way, in document order, is what stops one malformed envelope from
* taking a well-formed call elsewhere in the same message down with it.
*
* Recursion terminates because the envelope always consumes its opener, so
* each remainder is strictly shorter than the text it came from.
*/
/**
* Join two parses of adjacent halves of one message.
*
* Used when a message holds spans belonging to different owners, so each half
* is read by whoever owns it and the turn is the concatenation. Same shape as
* the envelope remainder merge, which does this for a span's leading and
* trailing text.
*/
fn __tool_parse_join(head: dict, tail: dict) -> dict {
const calls = (head?.calls ?? []) + (tail?.calls ?? [])
const prose = [to_string(head?.prose ?? ""), to_string(tail?.prose ?? "")].filter(
fn(part) { return trim(part) != "" },
)
.join("\n")
return head
+ {
calls: calls,
tool_calls: calls,
tool_parse_errors: (head?.tool_parse_errors ?? []) + (tail?.tool_parse_errors ?? []),
protocol_violations: (head?.protocol_violations ?? [])
+ (tail?.protocol_violations ?? []),
dropped: (head?.dropped ?? []) + (tail?.dropped ?? []),
prose: prose,
canonical_text: calls.map(fn(call) { return call_block(call) }).join("\n"),
}
}
fn __tool_parse_envelope_remainder(envelope: dict, parse_remainder) -> dict {
const result = envelope.result
const leading = to_string(envelope?.prose ?? "")
const trailing = to_string(envelope?.trailing ?? "")
if trim(leading) == "" && trim(trailing) == "" {
return result
}
// The remainder belongs to the lane that owns the turn, not to the envelope:
// the same leftover text means a fenced call on one pin and a heredoc call on
// the other, so the caller supplies the reading.
const before = if trim(leading) == "" {
empty_parse()
} else {
parse_remainder(leading)
}
const after = if trim(trailing) == "" {
empty_parse()
} else {
parse_remainder(trailing)
}
const calls = (before?.calls ?? []) + (result?.calls ?? []) + (after?.calls ?? [])
// The tail is not re-reported as a malformed envelope. Reading it as one was
// only ever a guess about text the envelope had no claim on, and it fired on
// ordinary narration after a call. Its real diagnostic comes from the same
// place every other unconsumed span's does: the lane records what it consumed
// without producing a call, and the shape detector speaks up when that text
// is shaped like one and stays quiet when it is prose.
const errors = (before?.tool_parse_errors ?? [])
+ (result?.tool_parse_errors ?? [])
+ (after?.tool_parse_errors ?? [])
const prose = [to_string(before?.prose ?? ""), to_string(after?.prose ?? "")].filter(
fn(part) { return trim(part) != "" },
)
.join("\n")
return result
+ {
calls: calls,
tool_calls: calls,
tool_parse_errors: errors,
protocol_violations: (result?.protocol_violations ?? [])
+ (before?.protocol_violations ?? [])
+ (after?.protocol_violations ?? []),
dropped: (before?.dropped ?? []) + (after?.dropped ?? []),
prose: prose,
canonical_text: calls.map(fn(call) { return call_block(call) }).join("\n"),
}
}
fn __tool_parse_json(text: string, tools = nil, recover_heredoc = true) -> dict {
const envelope = tool_parse_envelope(
text,
fn(body) { return __tool_parse_call_from_body(body, tools, "", "") },
)
if envelope?.matched ?? false {
if envelope?.result != nil {
if len(envelope.result?.tool_parse_errors ?? []) > 0 {
return envelope.result + {prose: trim(text)}
}
return __tool_parse_envelope_remainder(
envelope,
fn(part) { return __tool_parse_json(part, tools, false) },
)
}
return empty_parse()
+ {tool_parse_errors: ["chat-template envelope parse failed"], prose: text}
}
if recover_heredoc && contains(text, "<tool_call>") {
// Accept the heredoc grammar here rather than discarding it. A whole class
// of routes is exposed to this: any route whose capability row pairs
// `preferred = json` with `parity = text_only` teaches the fenced grammar
// while the model reaches for the tagged one. Probed on such a route, the
// turn parsed 1 fenced call, 0 heredoc calls, and 0 errors — a silent total
// drop of exactly the grammar most open-weight models reach for. The two
// grammars are mutually unambiguous, so forgiving one costs no soundness.
const recovered = __tool_parse_tagged(text, tools)
const violation = protocol_violation(
"recovered_dialect",
"Recovered `<tool_call>...</tool_call>` markup while `tool_format` is `json`; "
+ "accepted this turn, but wrap each call as a strict JSON object inside a "
+ "```tool fence next turn.",
)
if len(recovered?.calls ?? []) > 0 {
return recovered
+ {
protocol_violations: (recovered?.protocol_violations ?? []).appending(violation),
}
}
return empty_parse()
+ {
protocol_violations: [
protocol_violation(
"wrong_tool_format",
"protocol_violation: `<tool_call>...</tool_call>` markup was emitted while "
+ "`tool_format` is `json`; wrap each call as a strict JSON object inside "
+ "a ```tool fence instead.",
),
],
prose: trim(text),
}
}
const chunked = json_chunks(text)
let calls: list<dict> = []
let errors: list<string> = []
for body in chunked.bodies {
const source = trim(body)
const stream = __host_tool_json_stream(source)
let body_calls: list<dict> = []
for row in stream?.values ?? [] {
const normalized = json_call(row?.value, false)
if normalized?.ok ?? false {
body_calls = body_calls.appending(normalized.call)
} else {
errors = errors.appending(normalized.error)
}
}
const trailing = trim(source.slice(stream?.end ?? 0, len(source)))
if len(body_calls) > 0
&& (has_counted_verbatim(body_calls)
|| starts_with(trailing, "<<")) {
const bound = bind_verbatim(body_calls, trailing)
if bound?.ok ?? false {
body_calls = bound.calls
} else {
body_calls = []
errors = errors.appending(bound.error)
}
}
calls = calls + body_calls
if len(stream?.values ?? []) == 0 {
const missing_error = if stream?.eof ?? false {
"Unterminated ```tool fence opened on line 1 (1-based): the block reached "
+ "end-of-output with an incomplete JSON object. Re-emit the whole call inside "
+ "a ```tool ... ``` block and close it with a line that is exactly ```."
} else {
__tool_parse_json_error(
to_string(stream?.error ?? "Tool JSON block did not contain an object.")
+ " (near `"
+ source.slice(0, 80)
+ "`)",
)
}
errors = errors.appending(missing_error)
} else if stream?.error != nil && !starts_with(trailing, "<<") {
errors = errors.appending(
"The ```tool block is not valid JSON: "
+ to_string(stream.error)
+ " (near `"
+ source.slice(0, 80)
+ "`)"
+ ". Pass multi-line or code-bearing fields as ordinary JSON string values "
+ "(escape newlines as \\n, quotes as \\\", backslashes as \\\\); "
+ "backticks need no escaping.",
)
} else if stream?.eof ?? false {
errors = errors.appending(
"Unterminated ```tool fence opened on line 1 (1-based): the block reached "
+ "end-of-output with an incomplete JSON object. Re-emit the whole call inside "
+ "a ```tool ... ``` block and close it with a line that is exactly ```.",
)
}
}
const canonical = calls.map(fn(call) { return call_block(call) }).join("\n")
// This module's contract is that every consumed non-call is recorded as
// dropped, and the shape detector reads that ledger. The fenced lane used to
// keep no ledger at all, so a turn whose only tool-call-shaped span this
// grammar cannot read — a bare `name({ … })`, say — ended as ordinary prose
// with nothing for the detector to find and nothing for the model to act on.
// Record it only when the turn produced neither a call nor an error, so a
// turn that already carries a diagnostic is not double-reported and prose
// alongside a successful call stays prose.
const unparsed_turn = len(calls) == 0 && len(errors) == 0 && trim(chunked.prose) != ""
const dropped = if unparsed_turn {
[{text: chunked.prose, reason: "unparsed_fenced_turn"}]
} else {
[]
}
// The other grammar's bare call expression carries no structure this lane can
// key on, so the shape detector above cannot see it — but the registry can.
// A resolved, argument-bearing call to a DECLARED tool is not narration, and
// the model has to be told the turn produced nothing. It is reported rather
// than dispatched: what to accept on a json pin is the recovery ladder's
// decision, and silence is the only outcome that was never anyone's.
const stray = if unparsed_turn && tools != nil {
__host_tool_scan_bare_calls(chunked.prose, tools)
} else {
nil
}
const bare_violations = (stray?.calls ?? []).map(
fn(call) { return protocol_violation(
"wrong_tool_format",
"protocol_violation: `"
+ to_string(call?.name ?? "")
+ "({ ... })` was emitted as a bare call expression while `tool_format` is "
+ "`json`; it did NOT run. Re-emit it as a strict JSON object inside a "
+ "```tool fence.",
) },
)
return empty_parse()
+ {
calls: calls,
tool_calls: calls,
tool_parse_errors: errors,
protocol_violations: chunked.violations + bare_violations,
canonical_text: canonical,
prose: chunked.prose,
dropped: dropped,
}
}
fn __tool_parse_json_error(detail: string) -> string {
return "The ```tool block is not valid JSON: "
+ detail
+ ". Pass multi-line or code-bearing fields as ordinary JSON string values "
+ "(escape newlines as \\n, quotes as \\\", backslashes as \\\\); "
+ "backticks need no escaping."
}
/**
* Last resort on a json pin when the fenced grammar read no call: try the
* tagged one.
*
* Which grammar a route teaches is a steering decision, but which grammar a
* model actually emitted is a fact about the bytes, and the two disagree often
* enough that pinning the parser to the taught grammar loses real calls.
*
* The recovery runs in one direction only. Tagged framing says "this is a
* call" in the framing itself, so reading it under a json pin cannot mistake
* documentation for an action. The fenced grammar has no such guarantee: a
* fence info string is the only thing separating a call from an example ABOUT
* a call, so a ```json block explaining the shape must stay prose no matter
* which pin reads it. `tool_format_contract_parity` pins that floor.
*
* Only reached when the pinned lane recovered nothing, so no shape that
* already parses can change. The recovery always carries a violation:
* accepting the wrong grammar silently would teach the model that the pin does
* not mean anything.
*/
fn __tool_parse_sibling_recovery(text: string, tools, format: string, pinned: dict) -> dict {
if format != "json" || len(pinned?.calls ?? []) > 0 {
return pinned
}
const sibling = __tool_parse_tagged(text, tools)
if len(sibling?.calls ?? []) == 0 {
return pinned
}
return sibling
+ {
protocol_violations: (sibling?.protocol_violations ?? [])
+ [
protocol_violation(
"wrong_tool_format",
"Recovered a tool call written in the tagged grammar while `tool_format` "
+ "is `json`; accepted this turn, but emit each call as a strict JSON "
+ "object inside a ```tool fence next turn.",
),
],
}
}
/**
* Parse one model text response.
*
* @effects: []
* @errors: []
* @api_stability: experimental
*/
pub fn parse_tool_calls(payload: dict) -> dict {
const text = to_string(payload?.text ?? "")
const format = to_string(payload?.tool_format ?? "")
// `adaptive` was an opt-in union parser (#5272) deleted in the dialect
// cutover (#5700). Keep failing closed so an explicit pin cannot silently
// demote to tagged text and hide teach/accept drift (burin-code#4809).
if format == "adaptive" {
return empty_parse()
+ {
tool_parse_errors: [
"`tool_format=adaptive` is not implemented. Pin `json` (fenced JSON, "
+ "global default) or `text` (heredoc `<tool_call>`), or use `native` "
+ "when the provider tool channel is reliable.",
],
prose: trim(text),
}
}
const pinned = if format == "json" {
__tool_parse_json(text, payload?.tools)
} else {
__tool_parse_tagged(text, payload?.tools)
}
const parsed = __tool_parse_sibling_recovery(text, payload?.tools, format, pinned)
const unparsed = tool_shape_violations(tool_shape_findings(parsed?.dropped ?? []))
return parsed + {protocol_violations: (parsed?.protocol_violations ?? []) + unparsed}
}