newt-core 0.8.0

//! Recover tool calls a weak model emitted in its CONTENT instead of the native
//! `tool_calls` field.
//!
//! The #1 weak-model failure observed in the dgx1 A/B (see
//! `docs/research/weak-model-plan-mode-findings.md`) is *tool-call format*:
//! small models emit the call as text — a fenced/bare JSON object,
//! `<function=NAME><parameter=K>V</parameter></function>`, or
//! `<read_file><path>…</path></read_file>`-style root tags — while the native
//! `tool_calls` array is empty, so the harness never executes it
//! (`tool_calls=0` → nothing runs). Even `qwen3-coder:30b` did this on 2/6 runs.
//!
//! This module recovers those calls into the **native Ollama shape**
//! (`{"function": {"name": …, "arguments": {…}}}`) so they flow into the existing
//! executor and the `is_hallucination` / dup-guard / caveat path unchanged.
//!
//! Authority note (see `docs/decisions/structural_parsing_over_regex.md`): the
//! JSON form is parsed with `serde_json` (structural); XML-ish forms are parsed
//! by bounded tag scans (no regex → no ReDoS) with charset-validated names. Root
//! tag recovery only accepts known built-in newt tool names, and recovery only
//! EXTRACTS `(name, args)`; the authority decision is still made downstream by
//! `resolve_tool_alias` + `is_hallucination` + the caveat check, so a spoofed
//! name cannot gain authority here.

use serde_json::{json, Value};

use content_addressable::{ContentId, MerkleNode};

use super::observability::{RecoveredIdentity, ToolCallDialect};

/// Outcome of attempting to recover tool calls from model content.
#[derive(Debug, Default, PartialEq)]
pub struct Recovery {
    /// Recovered calls in native shape (`{"function":{"name","arguments"}}`).
    /// Empty when nothing parseable was found.
    pub calls: Vec<Value>,
    /// The content *looked* like a tool-call attempt (a `<function=`, a known
    /// tool root tag, or a `"name"`+`"arguments"` JSON object) even if it did
    /// not fully parse — a format-hallucination signal the caller counts and
    /// steers on.
    pub tool_shaped: bool,
    /// Which dialect produced `calls` (W0 #1511: `recovered_tool_call{dialect}`
    /// provenance for the trace). `None` when nothing was recovered.
    pub dialect: Option<ToolCallDialect>,
}

/// What a recovered call IS, and so what its identity is minted from: the
/// dialect that matched, the call's position among the calls recovered from one
/// reply, and the call itself. `arguments` enters as the `Value`, encoded
/// canonically (dag-cbor, sorted keys) by the crate, so the id does not depend
/// on a serde_json feature flag. The causal parent is the inference attempt that
/// produced the reply.
#[derive(serde::Serialize)]
struct RecoveredCall<'a> {
    dialect: ToolCallDialect,
    ordinal: u32,
    name: &'a str,
    arguments: &'a Value,
}

/// Give each recovered call a harness-DERIVED id (identity is derived, not
/// assigned): the [`MerkleNode`] id of its [`RecoveredCall`] over `parent`, and
/// a wire locator `nwt-rc-` + 32 hex chars of that id's DIGEST (39 characters,
/// inside every documented `tool_call_id` charset and length cap). Two identical
/// calls in one reply differ by ordinal; the same call in a later round differs
/// by parent, so a looping model never repeats an id. A call whose identity
/// cannot be canonically encoded (e.g. a float dag-cbor forbids) is left id-less
/// and the caller's bounded re-ask handles it.
///
/// Only calls the harness itself recovered are touched: a provider call that
/// arrived without an id never passes through here.
pub fn assign_ids(recovery: &mut Recovery, parent: Option<ContentId>) -> Vec<RecoveredIdentity> {
    let Some(dialect) = recovery.dialect else {
        return Vec::new();
    };
    let mut identities = Vec::new();
    for (ordinal, call) in recovery.calls.iter_mut().enumerate() {
        let payload = RecoveredCall {
            dialect,
            ordinal: u32::try_from(ordinal).unwrap_or(u32::MAX),
            name: call["function"]["name"].as_str().unwrap_or_default(),
            arguments: &call["function"]["arguments"],
        };
        let Ok(id) = MerkleNode::new(payload, parent).id() else {
            continue;
        };
        let locator = format!("nwt-rc-{}", &id.digest_hex()[..32]);
        call["id"] = json!(locator);
        identities.push(RecoveredIdentity {
            locator,
            cid: id.to_string(),
        });
    }
    identities
}

/// The Chat Completions wire shape of recovered calls, for the replayed
/// assistant turn: `type:"function"` and STRINGIFIED arguments (recovery holds
/// them as an object; providers send and expect a string).
#[must_use]
pub fn wire_tool_calls(calls: &[Value]) -> Vec<Value> {
    calls
        .iter()
        .map(|call| {
            let args = &call["function"]["arguments"];
            let arguments = args
                .as_str()
                .map_or_else(|| args.to_string(), str::to_string);
            json!({
                "id": call["id"],
                "type": "function",
                "function": {"name": call["function"]["name"], "arguments": arguments},
            })
        })
        .collect()
}

/// Charset-validate a recovered tool name (downstream still validates authority).
fn valid_name(s: &str) -> bool {
    !s.is_empty()
        && s.len() <= 64
        && s.chars()
            .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
}

/// Parse a parameter value: a JSON literal (array/object/number/bool/null) if it
/// parses as one, otherwise the trimmed string.
fn param_value(raw: &str) -> Value {
    let t = raw.trim();
    if let Ok(v) = serde_json::from_str::<Value>(t) {
        if v.is_array() || v.is_object() || v.is_number() || v.is_boolean() || v.is_null() {
            return v;
        }
    }
    Value::String(t.to_string())
}

fn native(name: &str, args: Value) -> Value {
    json!({ "function": { "name": name, "arguments": args } })
}

fn close_tag(name: &str) -> String {
    format!("</{name}>")
}

/// Recover the `<function=NAME> <parameter=K>V</parameter> … </function>` form.
/// Bounded substring scan — no regex. Multiple blocks are recovered in order.
fn recover_function_tag(content: &str, out: &mut Vec<Value>) -> bool {
    let mut shaped = false;
    let mut rest = content;
    while let Some(open) = rest.find("<function=") {
        shaped = true;
        let after = &rest[open + "<function=".len()..];
        let Some(gt) = after.find('>') else { break };
        let name = after[..gt].trim();
        // Body runs to the matching </function> (or end of string).
        let body_start = open + "<function=".len() + gt + 1;
        let body_end = rest[body_start..]
            .find("</function>")
            .map(|e| body_start + e)
            .unwrap_or(rest.len());
        let body = &rest[body_start..body_end];
        if valid_name(name) {
            let mut args = serde_json::Map::new();
            let mut p = body;
            while let Some(po) = p.find("<parameter=") {
                let pa = &p[po + "<parameter=".len()..];
                let Some(pgt) = pa.find('>') else { break };
                let key = pa[..pgt].trim().to_string();
                let val_start = po + "<parameter=".len() + pgt + 1;
                let Some(pe) = p[val_start..].find("</parameter>") else {
                    break;
                };
                let val = &p[val_start..val_start + pe];
                if !key.is_empty() {
                    args.insert(key, param_value(val));
                }
                p = &p[val_start + pe + "</parameter>".len()..];
            }
            out.push(native(name, Value::Object(args)));
        }
        // Advance past this block.
        rest = &rest[body_end..];
        rest = rest
            .strip_prefix("</function>")
            .or(Some(rest))
            .unwrap_or(rest);
        if rest.is_empty() {
            break;
        }
    }
    shaped
}

/// Recover `<TOOL><arg>value</arg>…</TOOL>` where `TOOL` is a known built-in
/// newt tool name. This is the XML-ish root-tag dialect local models sometimes
/// learn from other harnesses. Unknown root tags are ignored so prose snippets,
/// HTML/XML examples, and arbitrary `<note>…</note>` blocks do not become tool
/// calls.
fn recover_root_tool_tag(content: &str, out: &mut Vec<Value>) -> bool {
    let mut shaped = false;
    let mut rest = content;
    while let Some(open) = rest.find('<') {
        let after = &rest[open + 1..];
        let Some(gt) = after.find('>') else { break };
        let tag = after[..gt].trim();
        let name = tag.split_ascii_whitespace().next().unwrap_or("");
        let body_start = open + 1 + gt + 1;

        if valid_name(name) && super::tools::known_builtin_tool_name(name) {
            shaped = true;
            let close = close_tag(name);
            let Some(body_len) = rest[body_start..].find(&close) else {
                break;
            };
            let body = &rest[body_start..body_start + body_len];
            out.push(native(name, Value::Object(xml_child_args(body))));
            rest = &rest[body_start + body_len + close.len()..];
        } else {
            rest = &after[gt + 1..];
        }
    }
    shaped
}

fn xml_child_args(body: &str) -> serde_json::Map<String, Value> {
    let mut args = serde_json::Map::new();
    let mut rest = body;
    while let Some(open) = rest.find('<') {
        let after = &rest[open + 1..];
        let Some(gt) = after.find('>') else { break };
        let tag = after[..gt].trim();
        let key = tag.split_ascii_whitespace().next().unwrap_or("");
        let val_start = open + 1 + gt + 1;
        if valid_name(key) {
            let close = close_tag(key);
            let Some(val_len) = rest[val_start..].find(&close) else {
                break;
            };
            let val = &rest[val_start..val_start + val_len];
            args.insert(key.to_string(), param_value(val));
            rest = &rest[val_start + val_len + close.len()..];
        } else {
            rest = &after[gt + 1..];
        }
    }
    args
}

/// Recover the JSON-object form: `{"name": "…", "arguments": {…}}`, whether bare
/// or fenced in a ``` block. Scans for balanced `{…}` spans and parses each.
fn recover_json(content: &str, out: &mut Vec<Value>) -> bool {
    let mut shaped = false;
    let bytes = content.as_bytes();
    let mut i = 0;
    while i < bytes.len() {
        if bytes[i] == b'{' {
            // Find the matching close brace (string-aware).
            let mut depth = 0i32;
            let mut j = i;
            let mut in_str = false;
            let mut esc = false;
            while j < bytes.len() {
                let c = bytes[j];
                if in_str {
                    if esc {
                        esc = false;
                    } else if c == b'\\' {
                        esc = true;
                    } else if c == b'"' {
                        in_str = false;
                    }
                } else if c == b'"' {
                    in_str = true;
                } else if c == b'{' {
                    depth += 1;
                } else if c == b'}' {
                    depth -= 1;
                    if depth == 0 {
                        break;
                    }
                }
                j += 1;
            }
            if depth == 0 && j < bytes.len() {
                let span = &content[i..=j];
                if let Ok(v) = serde_json::from_str::<Value>(span) {
                    if let (Some(name), args) =
                        (v.get("name").and_then(|n| n.as_str()), v.get("arguments"))
                    {
                        shaped = true;
                        if valid_name(name) {
                            let a = args.cloned().unwrap_or_else(|| json!({}));
                            // arguments may itself be a JSON string.
                            let a = match a {
                                Value::String(s) => serde_json::from_str(&s)
                                    .unwrap_or(Value::Object(Default::default())),
                                other => other,
                            };
                            out.push(native(name, a));
                        }
                    }
                }
                i = j + 1;
                continue;
            }
        }
        i += 1;
    }
    shaped
}

/// Recover tool calls from model content. Tries the `<function=>` tag form first
/// (it embeds JSON params), then the JSON-object form. Returns native-shaped
/// calls + whether the content looked tool-shaped at all + which dialect the
/// calls came from (trace provenance, W0 #1511).
#[must_use]
pub fn recover_tool_calls(content: &str) -> Recovery {
    let mut calls = Vec::new();
    let mut shaped = recover_function_tag(content, &mut calls);
    let mut dialect = (!calls.is_empty()).then_some(ToolCallDialect::FunctionTag);
    if calls.is_empty() {
        shaped |= recover_root_tool_tag(content, &mut calls);
        dialect = (!calls.is_empty()).then_some(ToolCallDialect::RootTag);
    }
    if calls.is_empty() {
        shaped |= recover_json(content, &mut calls);
        dialect = (!calls.is_empty()).then_some(ToolCallDialect::BareJson);
    }
    Recovery {
        calls,
        tool_shaped: shaped,
        dialect,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn fixed_parent() -> ContentId {
        use content_addressable::ContentAddressable;
        crate::attempts::AttemptKey {
            turn: "golden-turn".into(),
            role: "primary".into(),
            request: content_addressable::RawContentId::from_content(b"golden request body"),
            ordinal: 0,
        }
        .content_id()
        .expect("encodes")
    }

    fn golden_recovery() -> Recovery {
        Recovery {
            calls: vec![
                json!({"function": {"name": "read_file", "arguments": {"path": "a.rs", "limit": 5}}}),
                json!({"function": {"name": "read_file", "arguments": {"path": "a.rs", "limit": 5}}}),
            ],
            tool_shaped: true,
            dialect: Some(ToolCallDialect::BareJson),
        }
    }

    /// (locator, CID) for the two golden calls. The locator is `nwt-rc-` plus the
    /// first 32 hex chars of the CID's digest (checked independently by decoding
    /// the base32 CID: version, codec, hash code, length, then the digest).
    const GOLDEN: &[(&str, &str)] = &[
        (
            "nwt-rc-4cfe734efb8fb492009d376e086da619",
            "bafyr4icm7zzu564pwsjabhjxnyeg3jqzf5reyimo6ocgucaw2wxj3jdv4y",
        ),
        (
            "nwt-rc-2e2dc0f5868362b8e8ebc2af18e5fb7b",
            "bafyr4ibofxaplbudmk4or26cv4mol633kghspb7ke2qcvseukoedo3jrsy",
        ),
    ];

    /// GOLDEN VECTOR: a fixed recovered call and a fixed parent must derive
    /// exactly these ids. Any change to the derivation — the payload shape, the
    /// dialect label, the encoder, the locator rule — changes them and fails
    /// here, loudly. The two calls are textually identical and differ only by
    /// ordinal.
    #[test]
    fn golden_vector_fixed_call_and_parent_derive_fixed_ids() {
        let mut r = golden_recovery();
        let ids = assign_ids(&mut r, Some(fixed_parent()));
        let got: Vec<(&str, &str)> = ids
            .iter()
            .map(|i| (i.locator.as_str(), i.cid.as_str()))
            .collect();
        assert_eq!(got, GOLDEN, "the derivation changed: {got:?}");
        assert_eq!(r.calls[0]["id"], ids[0].locator);
        assert!(ids
            .iter()
            .all(|i| i.locator.len() == 39 && i.locator.starts_with("nwt-rc-")));
    }

    /// Recovery is a function of (call, parent): no parent is a different
    /// identity, and the same inputs derive the same ids.
    #[test]
    fn derivation_is_deterministic_and_parent_sensitive() {
        let mut a = golden_recovery();
        let mut b = golden_recovery();
        let mut none = golden_recovery();
        let ia = assign_ids(&mut a, Some(fixed_parent()));
        let ib = assign_ids(&mut b, Some(fixed_parent()));
        let inone = assign_ids(&mut none, None);
        assert_eq!(ia, ib);
        assert_ne!(ia[0].locator, inone[0].locator);
        assert_ne!(
            ia[0].locator, ia[1].locator,
            "ordinal separates identical calls"
        );
    }

    /// A finite float in a recovered argument encodes canonically (measured: the
    /// crate's encoder carries it), so it derives an id like any other argument.
    /// A value the encoder cannot carry would leave the call id-less and fall to
    /// the bounded re-ask; JSON cannot produce one (no NaN or infinity).
    #[test]
    fn a_finite_float_argument_still_derives_an_id() {
        let mut r = Recovery {
            calls: vec![json!({"function": {"name": "read_file", "arguments": {"x": 1.5}}})],
            tool_shaped: true,
            dialect: Some(ToolCallDialect::BareJson),
        };
        let ids = assign_ids(&mut r, None);
        assert_eq!(ids.len(), 1, "a float must not defeat identity derivation");
        assert!(r.calls[0]["id"]
            .as_str()
            .is_some_and(|id| id.starts_with("nwt-rc-")));
    }

    #[test]
    fn recovers_function_tag_form_qwen3_coder_observed() {
        // The exact shape qwen3-coder:30b emitted (and the harness dropped).
        let content = "I'll create a plan.\n\n<function=plan_set>\n<parameter=steps>\n[\"a\", \"b\"]\n</parameter>\n</function>\n</tool_call>";
        let r = recover_tool_calls(content);
        assert!(r.tool_shaped);
        assert_eq!(r.calls.len(), 1);
        assert_eq!(r.calls[0]["function"]["name"], "plan_set");
        assert_eq!(
            r.calls[0]["function"]["arguments"]["steps"],
            json!(["a", "b"])
        );
        assert_eq!(r.dialect, Some(ToolCallDialect::FunctionTag));
    }

    #[test]
    fn recovers_fenced_json_form_qwen25_coder_observed() {
        let content = "```json\n{\n  \"name\": \"edit_file\",\n  \"arguments\": {\"path\": \"service.py\", \"old_string\": \"x\", \"new_string\": \"y\"}\n}\n```";
        let r = recover_tool_calls(content);
        assert!(r.tool_shaped);
        assert_eq!(r.calls.len(), 1);
        assert_eq!(r.calls[0]["function"]["name"], "edit_file");
        assert_eq!(r.calls[0]["function"]["arguments"]["path"], "service.py");
        assert_eq!(r.dialect, Some(ToolCallDialect::BareJson));
    }

    #[test]
    fn recovers_bare_json_object() {
        let content = "{\"name\": \"read_file\", \"arguments\": {\"path\": \"a.rs\"}}";
        let r = recover_tool_calls(content);
        assert_eq!(r.calls.len(), 1);
        assert_eq!(r.calls[0]["function"]["name"], "read_file");
    }

    #[test]
    fn arguments_as_json_string_is_reparsed() {
        let content = "{\"name\": \"read_file\", \"arguments\": \"{\\\"path\\\": \\\"a.rs\\\"}\"}";
        let r = recover_tool_calls(content);
        assert_eq!(r.calls.len(), 1);
        assert_eq!(r.calls[0]["function"]["arguments"]["path"], "a.rs");
    }

    #[test]
    fn multiple_function_tags_recovered_in_order() {
        let content = "<function=plan_advance></function>\n<function=read_file><parameter=path>x.rs</parameter></function>";
        let r = recover_tool_calls(content);
        assert_eq!(r.calls.len(), 2);
        assert_eq!(r.calls[0]["function"]["name"], "plan_advance");
        assert_eq!(r.calls[1]["function"]["name"], "read_file");
        assert_eq!(r.calls[1]["function"]["arguments"]["path"], "x.rs");
    }

    #[test]
    fn recovers_known_tool_root_tag_form_ornith_observed() {
        let content = "Let me read that now.\n\n<read_file> <path>/Users/shawnhartsock/workspaces/newt-agent/newt-tui/src/help_sections.rs</path> </read_file>";
        let r = recover_tool_calls(content);
        assert!(r.tool_shaped);
        assert_eq!(r.calls.len(), 1);
        assert_eq!(r.calls[0]["function"]["name"], "read_file");
        assert_eq!(
            r.calls[0]["function"]["arguments"]["path"],
            "/Users/shawnhartsock/workspaces/newt-agent/newt-tui/src/help_sections.rs"
        );
        assert_eq!(r.dialect, Some(ToolCallDialect::RootTag));
    }

    #[test]
    fn recovers_multiple_known_tool_root_tags_in_order() {
        let content = "<plan_get></plan_get>\n<read_file><path>x.rs</path></read_file>";
        let r = recover_tool_calls(content);
        assert!(r.tool_shaped);
        assert_eq!(r.calls.len(), 2);
        assert_eq!(r.calls[0]["function"]["name"], "plan_get");
        assert_eq!(r.calls[0]["function"]["arguments"], json!({}));
        assert_eq!(r.calls[1]["function"]["name"], "read_file");
        assert_eq!(r.calls[1]["function"]["arguments"]["path"], "x.rs");
    }

    #[test]
    fn arbitrary_xml_root_tags_are_not_tool_calls() {
        let r = recover_tool_calls("<note><path>x.rs</path></note>");
        assert!(r.calls.is_empty());
        assert!(!r.tool_shaped);
    }

    #[test]
    fn plain_prose_is_not_tool_shaped() {
        let r = recover_tool_calls("I have finished editing the file and the tests pass.");
        assert!(r.calls.is_empty());
        assert!(!r.tool_shaped);
        assert_eq!(r.dialect, None, "no recovery → no dialect provenance");
    }

    #[test]
    fn narrated_intent_to_act_is_not_a_dropped_call() {
        // Real weak-model (ornith:35b) narrations that ended a turn with no tool
        // call: a plan announcement, not a mis-formatted call. Recovery finds
        // nothing AND does not flag tool_shaped, so the loop's format-
        // hallucination path stays quiet — proving the "narrate-then-stop" stall
        // is a turn-termination gap, not a parse drop. The fix belongs in the
        // loop's `!has_tools` branch (see `narration_action_nudge`), not here.
        for prose in [
            "Now I have everything I need. Let me make the two edits: \
             1. newt-cli/src/lib.rs ... 2. newt-core/src/config.rs ... \
             Let me make both edits now.",
            "Now I'll add the --home flag to the Cli struct. \
             I'll place it near the other directory-related options:",
        ] {
            let r = recover_tool_calls(prose);
            assert!(
                r.calls.is_empty(),
                "narration must yield no call: {prose:?}"
            );
            assert!(
                !r.tool_shaped,
                "narration must not look tool-shaped: {prose:?}"
            );
        }
    }

    #[test]
    fn json_without_name_arguments_is_not_a_call() {
        // A JSON blob that isn't a tool call (e.g. a config snippet) is ignored.
        let r = recover_tool_calls("Here is config: {\"debug\": true, \"level\": 3}");
        assert!(r.calls.is_empty());
    }

    #[test]
    fn malformed_function_tag_is_shaped_but_yields_no_call() {
        // Names the attempt (tool_shaped) but the bad name yields nothing — the
        // caller counts this as a format-hallucination and steers.
        let r = recover_tool_calls("<function=not a valid name!>oops");
        assert!(r.tool_shaped);
        assert!(r.calls.is_empty());
        assert_eq!(r.dialect, None, "an unrecovered attempt names no dialect");
    }

    #[test]
    fn name_charset_is_validated() {
        assert!(valid_name("edit_file"));
        assert!(valid_name("plan-advance"));
        assert!(!valid_name(""));
        assert!(!valid_name("rm -rf /"));
        assert!(!valid_name("a; drop table"));
    }
}