/**
* JSON-fence normalization and counted-verbatim binding for tool parsing.
*
* Kept separate from std/llm/tool_parse so the public parser remains a small
* composition layer over the byte-oriented host scanners.
*/
import {
ARGUMENT_ALIASES,
FENCE_OPEN_INFO,
GENERIC_WRAPPER_NAMES,
NAME_ALIASES,
NAME_CHANNEL_MARKERS,
tool_fence_info_opens_call,
} from "std/llm/dialects"
import { protocol_violation } from "std/llm/tool_parse_result"
fn __tool_parse_json_name(value: dict) -> string {
for key in NAME_ALIASES {
const candidate = value[key]
if type_of(candidate) == "string" && trim(candidate) != "" {
return trim(candidate)
}
}
return ""
}
/**
* Arguments written as siblings of `name` rather than under an argument alias.
*
* Chat templates and several open-weight models inline the argument fields
* directly: `{"name": "edit", "action": "create", "path": …}`. Read only when
* no alias is present, so a canonical `{"name": …, "args": {…}}` object is
* untouched. Without it the object still carries a resolvable NAME, so the
* call dispatches with an empty argument dict instead of failing — a silent
* corruption strictly worse than a rejection.
*/
fn __tool_parse_json_flat_arguments(value: dict) -> dict {
let arguments: dict = {}
for key in keys(value) {
if !NAME_ALIASES.contains(key) && !ARGUMENT_ALIASES.contains(key) && key != "id" {
arguments[key] = value[key]
}
}
return arguments
}
fn __tool_parse_json_arguments(value: dict) {
for key in ARGUMENT_ALIASES {
if value[key] != nil {
return value[key]
}
}
return __tool_parse_json_flat_arguments(value)
}
/**
* The object that actually carries the call's name and arguments.
*
* Provider tool-call rows nest the pair under `function` and keep envelope
* metadata (`id`, `type`, `index`) outside it. Descending here means the flat
* reading above sees the call object, not the envelope around it.
*/
fn __tool_parse_json_source(value: dict) -> dict {
const nested = value?.function
if type_of(nested) == "dict" && __tool_parse_json_name(nested) != "" {
return nested
}
return value
}
/**
* Read canonical name and arguments fields from a decoded tool-call object.
*
* This is the one owner of "which key holds the name" and "which keys hold the
* arguments" for every JSON-bearing tool-call grammar. Grammars differ in what
* WRAPS the object; they must not differ in how the object reads.
*
* @effects: []
* @errors: []
*/
pub fn json_fields(value: dict) -> dict {
const source = __tool_parse_json_source(value)
return {name: __tool_parse_json_name(source), arguments: __tool_parse_json_arguments(source)}
}
/**
* Normalize one decoded JSON value into a canonical tool call.
* @effects: []
* @errors: []
*/
pub fn json_call(value, allow_flat_argument_string: bool) -> dict {
if type_of(value) != "dict" {
return {
ok: false,
error: "A ```tool block must contain one or more JSON objects `{ \"name\": ..., \"args\": { ... } }`, "
+ "one per tool call (several calls in a turn may share one block or use several blocks). "
+ "Arrays, scalars, and other non-object entries are rejected.",
}
}
const fields = json_fields(value)
let name = fields.name
for marker in NAME_CHANNEL_MARKERS {
const marker_at = name.index_of(marker)
if marker_at >= 0 {
name = name.slice(0, marker_at)
}
}
let arguments = fields.arguments
if GENERIC_WRAPPER_NAMES.contains(name) && type_of(arguments) == "dict" {
const nested_name = __tool_parse_json_name(arguments)
if nested_name != "" {
name = nested_name
arguments = __tool_parse_json_arguments(arguments)
}
}
if name == "" {
return {
ok: false,
error: "The ```tool JSON object is missing a non-empty string `name`. "
+ "Shape: `{ \"name\": \"edit\", \"args\": { ... } }`.",
}
}
if type_of(arguments) == "string"
&& (allow_flat_argument_string
|| type_of(value?.function) == "dict") {
arguments = try {
json_parse(arguments)
} catch (error) {
return {
ok: false,
error: "Tool `" + name + "` arguments string is invalid JSON: " + to_string(error),
}
}
}
if type_of(arguments) != "dict" {
return {
ok: false,
error: "The `args` field of a ```tool object must be a JSON object (`{ ... }`), "
+ "or omitted when the tool takes no arguments.",
}
}
return {ok: true, call: {id: "tc_json", name: name, arguments: arguments}}
}
fn __tool_parse_json_error(detail: string) -> string {
return "The ```tool block is not valid JSON: "
+ detail
+ ". For a multi-line or code-bearing field, do not hand-escape it: set the "
+ "value to \"<<BODY\" and put the raw text after the JSON object inside the "
+ "same ```tool fence, ending with a line that is exactly BODY. Short scalar "
+ "values stay ordinary JSON strings (escape newlines as \\n, quotes as \\\", "
+ "backslashes as \\\\); backticks need no escaping."
}
/**
* Parse one argument value as a verbatim body declaration, or return nil.
*
* Two forms, matching the text dialect's heredoc grammar (`scan_heredoc`):
*
* - `<<TAG` closes on the first trailing line that is exactly `TAG`. This is
* the everyday form: it asks the model for a delimiter it already wrote, not
* for an arithmetic fact about text it is still emitting.
* - `<<TAG:N` closes after exactly `N` body lines. Required only when the body
* itself contains a line that is exactly `TAG`, where the terminator alone
* would be ambiguous.
*
* The single owner of the declaration syntax: requests, both predicates, and
* the error messages all read the shape from here.
*
* `counted` records which OPENER was written, independently of whether its
* digits produced a usable number. A count too large to represent leaves
* `count` nil with `counted` true, so the binder rejects it instead of
* silently reading it as the terminator-closed form — a downgrade would hand a
* body different bytes than the one the model declared.
*/
fn __tool_parse_verbatim_declaration(value) {
if type_of(value) != "string" {
return nil
}
const counted = regex_captures("^<<([A-Za-z_][A-Za-z0-9_.-]*):(\\d+)$", value)
if len(counted) > 0 {
return {
tag: to_string(counted[0].groups[0]),
count: to_int(counted[0].groups[1]),
counted: true,
}
}
const tagged = regex_captures("^<<([A-Za-z_][A-Za-z0-9_.-]*)$", value)
if len(tagged) > 0 {
return {tag: to_string(tagged[0].groups[0]), count: nil, counted: false}
}
return nil
}
/**
* Return whether any call declares a counted-verbatim argument body.
* @effects: []
* @errors: []
*/
pub fn has_counted_verbatim(calls: list<dict>) -> bool {
for call in calls {
const arguments = call?.arguments ?? {}
for key in keys(arguments) {
const declaration = __tool_parse_verbatim_declaration(arguments[key])
if declaration != nil && declaration.counted {
return true
}
}
}
return false
}
/**
* Return whether any call declares a verbatim argument body in either form.
* @effects: []
* @errors: []
*/
pub fn has_verbatim_declaration(calls: list<dict>) -> bool {
for call in calls {
const arguments = call?.arguments ?? {}
for key in keys(arguments) {
if __tool_parse_verbatim_declaration(arguments[key]) != nil {
return true
}
}
}
return false
}
fn __tool_parse_verbatim_requests(calls: list<dict>) -> list<dict> {
let requests: list<dict> = []
let call_index = 0
while call_index < len(calls) {
for key in keys(calls[call_index].arguments) {
const value = calls[call_index].arguments[key]
const declaration = __tool_parse_verbatim_declaration(value)
if declaration == nil {
continue
}
requests = requests.appending(
{
call_index: call_index,
key: key,
opener: value,
tag: declaration.tag,
count: declaration.count,
counted: declaration.counted,
bound: false,
},
)
}
call_index = call_index + 1
}
return requests
}
fn __tool_parse_verbatim_count_error(requests: list<dict>, lines: list<string>) {
for request in requests {
let required = 0
for candidate in requests {
if candidate.opener == request.opener {
required = required + 1
}
}
let available = 0
for line in lines {
if trim(line) == request.opener {
available = available + 1
}
}
if available < required {
return __tool_parse_json_error(
"verbatim declaration `"
+ request.opener
+ "` has "
+ to_string(available)
+ " matching bodies, expected "
+ to_string(required),
)
}
}
return nil
}
/**
* Index of the line that closes an uncounted `<<TAG` body: the first line at or
* after `body_start` whose trimmed text is exactly `tag`. Returns -1 when the
* body never closes, which is a parse error rather than a recovery point.
*/
fn __tool_parse_verbatim_terminator(lines: list<string>, body_start: int, tag: string) -> int {
let index = body_start
while index < len(lines) {
if trim(lines[index]) == tag {
return index
}
index = index + 1
}
return -1
}
/**
* Bind trailing verbatim bodies to their declared call arguments.
* @effects: []
* @errors: []
*/
pub fn bind_verbatim(calls: list<dict>, trailing: string) -> dict {
let requests = __tool_parse_verbatim_requests(calls)
let next_calls = calls
if len(requests) == 0 {
const orphan = trim(trailing).split("\n")[0] ?? ""
return {
ok: false,
error: __tool_parse_json_error(
"a verbatim heredoc `"
+ orphan
+ "` trails the block but no argument's value declared it",
),
}
}
const lines = trailing.split("\n")
// Validate declaration/body cardinality before binding. This makes missing,
// mismatched, and duplicate bodies deterministic without guessing which
// stray body the model intended.
const count_error = __tool_parse_verbatim_count_error(requests, lines)
if count_error != nil {
return {ok: false, error: count_error}
}
let cursor = 0
// Tag of the most recent uncounted body that bound, so a later orphan can be
// diagnosed as that body's early close rather than as a nameless mismatch.
let last_uncounted_tag = nil
while cursor < len(lines) && trim(lines[cursor]) == "" {
cursor = cursor + 1
}
while cursor < len(lines) {
const opener = trim(lines[cursor])
let request_index = -1
let idx = 0
while idx < len(requests) {
if !requests[idx].bound && requests[idx].opener == opener {
request_index = idx
break
}
idx = idx + 1
}
if request_index < 0 {
// Leftover text that is not itself an opener, after an uncounted body has
// already bound, has exactly one ordinary cause: that body contained a
// line equal to its own tag, so it closed early and the rest of it is
// sitting here. Say that and name the remedy, because "no matching
// declaration" describes the symptom and leaves the model to guess. The
// diagnosis is read off state, not inferred: an uncounted body bound, and
// this line does not parse as a declaration.
if __tool_parse_verbatim_declaration(opener) == nil && last_uncounted_tag != nil {
return {
ok: false,
error: __tool_parse_json_error(
"verbatim body `<<"
+ last_uncounted_tag
+ "` closed early on a line that is exactly `"
+ last_uncounted_tag
+ "`, leaving `"
+ opener
+ "` unclaimed. When the body itself contains that line, declare `<<"
+ last_uncounted_tag
+ ":N` with N body lines, which closes on the count instead",
),
}
}
return {
ok: false,
error: "The ```tool block is not valid JSON: trailing verbatim body `"
+ opener
+ "` had no matching argument declaration.",
}
}
const request = requests[request_index] ?? {}
const body_start = cursor + 1
if request.counted && request.count == nil {
return {
ok: false,
error: __tool_parse_json_error(
"verbatim declaration `"
+ request.opener
+ "` states a line count that is not a usable number; re-emit the "
+ "call with `<<"
+ request.tag
+ "` and close the body with a line that is exactly `"
+ request.tag
+ "`",
),
}
}
// `<<TAG:N` is anchored by the count, so a body line that is exactly `TAG`
// stays body. `<<TAG` closes on the first such line. Neither form guesses:
// when the declared close is not where it must be, the call is rejected.
const close_at = if request.counted {
body_start + request.count
} else {
__tool_parse_verbatim_terminator(lines, body_start, request.tag)
}
if close_at < 0 {
return {
ok: false,
error: __tool_parse_json_error(
"verbatim body `"
+ request.opener
+ "` never closed; end it with a line that is exactly `"
+ request.tag
+ "`, or declare `<<"
+ request.tag
+ ":N` with N body lines when the body itself contains that line",
),
}
}
if close_at >= len(lines) || trim(lines[close_at]) != request.tag {
return {
ok: false,
error: __tool_parse_json_error(
"verbatim body `"
+ request.opener
+ "` did not close on the line its `:N` count declared; "
+ "recount the body lines, or drop the `:N` and close with a line "
+ "that is exactly `"
+ request.tag
+ "`",
),
}
}
// The two openers carry the two established newline contracts, and the
// opener is what says which: `<<TAG:N` is count-anchored, so each of its N
// lines keeps its terminator (a 1-line body is "x\n"); `<<TAG` matches the
// text dialect's heredoc, where the newline before the close tag is the
// delimiter rather than content (a 1-line body is "x", and a trailing blank
// line is how a body asks for a final newline). Keeping `<<TAG` identical
// across the two dialects is what lets one taught grammar mean one thing.
let content = lines.slice(body_start, close_at).join("\n")
if request.counted && close_at > body_start {
content = content + "\n"
}
next_calls[request.call_index].arguments[request.key] = content
requests[request_index] = request + {bound: true}
if !request.counted {
last_uncounted_tag = request.tag
}
cursor = close_at + 1
while cursor < len(lines) && trim(lines[cursor]) == "" {
cursor = cursor + 1
}
}
for request in requests {
if !request.bound {
return {
ok: false,
error: __tool_parse_json_error(
"verbatim declaration `"
+ request.opener
+ "` has 0 matching bodies, expected 1",
),
}
}
}
return {ok: true, calls: next_calls}
}
fn __tool_parse_tool_fence_info(marker: string, info: string) -> dict {
const normalized = lowercase(trim(info))
if marker == "```" && normalized == FENCE_OPEN_INFO {
return {tool: true, warning: nil}
}
if tool_fence_info_opens_call(normalized) {
return {
tool: true,
warning: protocol_violation(
"fence_dialect",
"protocol_violation: a tool call was emitted in a "
+ marker
+ normalized
+ " fence; the contract requires a bare ```tool fence. "
+ "Accepted this turn, but switch to ```tool.",
),
}
}
return {tool: false, warning: nil}
}
/**
* Split a response into JSON tool-fence bodies, visible prose, and drift warnings.
* @effects: []
* @errors: []
*/
pub fn json_chunks(text: string) -> dict {
let bodies: list<string> = []
let prose: list<string> = []
let violations: list = []
let active_marker = ""
let active_tool = false
let active_body: list<string> = []
let verbatim_remaining = 0
// Tag of an open `<<TAG` body. While set, every line rides through untouched
// until the line that is exactly `TAG`, so a body containing a ``` fence or a
// ~~~ fence cannot close the tool block that carries it. The counted form
// gets the same immunity from `verbatim_remaining`; both must agree with
// `bind_verbatim`, which re-reads the same declarations.
let verbatim_tag = ""
for line in text.split("\n") {
const trimmed = trim(line)
const marker = if starts_with(trimmed, "```") {
"```"
} else if starts_with(trimmed, "~~~") {
"~~~"
} else {
""
}
if active_marker != "" {
if active_tool && verbatim_remaining > 0 {
active_body = active_body.appending(line)
verbatim_remaining = verbatim_remaining - 1
continue
}
if active_tool && verbatim_tag != "" {
active_body = active_body.appending(line)
if trimmed == verbatim_tag {
verbatim_tag = ""
}
continue
}
if active_tool {
const counted = regex_captures("^<<[A-Za-z_][A-Za-z0-9_.-]*:(\\d+)$", trimmed)
if len(counted) > 0 {
const count = to_int(counted[0].groups[0])
if count != nil {
verbatim_remaining = count + 1
}
active_body = active_body.appending(line)
continue
}
const tagged = regex_captures("^<<([A-Za-z_][A-Za-z0-9_.-]*)$", trimmed)
if len(tagged) > 0 {
verbatim_tag = to_string(tagged[0].groups[0])
active_body = active_body.appending(line)
continue
}
}
if trimmed == active_marker {
if active_tool {
const candidate = active_body.join("\n")
const stream = __host_tool_json_stream(trim(candidate))
if stream?.eof ?? false {
active_body = active_body.appending(line)
continue
}
bodies = bodies.appending(candidate)
} else {
prose = prose + active_body + [line]
}
active_marker = ""
active_tool = false
active_body = []
verbatim_remaining = 0
verbatim_tag = ""
} else if active_tool && starts_with(trimmed, active_marker + FENCE_OPEN_INFO) {
bodies = bodies.appending(active_body.join("\n"))
const info = trim(trimmed.slice(3, len(trimmed)))
const classified = __tool_parse_tool_fence_info(active_marker, info)
active_tool = classified.tool
active_body = []
verbatim_tag = ""
if classified.warning != nil {
violations = violations.appending(classified.warning)
}
} else {
active_body = active_body.appending(line)
}
continue
}
if marker != "" {
const info = trim(trimmed.slice(3, len(trimmed)))
const classified = __tool_parse_tool_fence_info(marker, info)
active_marker = marker
active_tool = classified.tool
active_body = []
verbatim_remaining = 0
verbatim_tag = ""
if classified.warning != nil {
violations = violations.appending(classified.warning)
}
if !active_tool {
active_body = active_body.appending(line)
}
continue
}
prose = prose.appending(line)
}
if active_marker != "" {
if active_tool {
bodies = bodies.appending(active_body.join("\n"))
} else {
prose = prose + active_body
}
}
const outside = trim(prose.join("\n"))
if len(bodies) == 0 && (starts_with(outside, "{") || starts_with(outside, "[")) {
bodies = bodies.appending(outside)
prose = []
violations = violations.appending(
protocol_violation(
"bare_json",
"protocol_violation: a tool call was emitted as a bare JSON object; the contract "
+ "requires wrapping each `{ \"name\": ..., \"args\": { ... } }` object in a "
+ "```tool fence. Accepted this turn, but switch to ```tool.",
outside,
),
)
}
return {bodies: bodies, prose: trim(prose.join("\n")), violations: violations}
}