supercode-harness 0.4.13

//! B6 — deferred tool loading targeting the MCP surface: `tool_search` meta-tool.
//!
//! Follows the ScriptedProvider/Recorder patterns of `tests/agent_loop.rs`.
//! `body["tools"]`'s wire shape (`type: "function"`, `function: {name,
//! description, parameters}`) mirrors `provider::WireTool`/`build_request_body`
//! exactly (`provider.rs:116-150`, `447-472`) — reproduced here (rather than
//! reaching into `pub(crate)` internals from this external test crate) so byte
//! measurements match what actually goes over the wire.

use std::collections::{BTreeSet, HashSet};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};

use async_trait::async_trait;
use supercode_harness::tools::{Tool, ToolContext, ToolRegistry};
use supercode_harness::{
    Agent, ChatMessage, ChatRequest, Config, FunctionCall, Provider, Role, ToolAdvertising,
    ToolCall, ToolSchema, Usage,
};

fn temp_dir(tag: &str) -> std::path::PathBuf {
    static N: AtomicUsize = AtomicUsize::new(0);
    let dir = std::env::temp_dir().join(format!(
        "supercode-{tag}-{}-{}",
        std::process::id(),
        N.fetch_add(1, Ordering::SeqCst)
    ));
    std::fs::create_dir_all(&dir).unwrap();
    dir
}

/// The 10 built-ins registered by `ToolRegistry::with_builtins()` (`tools/mod.rs:176-191`).
fn builtin_names() -> Vec<String> {
    [
        "read_file",
        "write_file",
        "edit_file",
        "list_dir",
        "glob",
        "search",
        "apply_patch",
        "bash",
        "shell",
        "update_plan",
    ]
    .into_iter()
    .map(String::from)
    .collect()
}

/// The wire byte-length of `body["tools"]` for a set of advertised schemas —
/// reproduces `WireTool`'s exact shape (`provider.rs:447-472`).
fn tools_wire_bytes(tools: &[ToolSchema]) -> usize {
    let wire: Vec<serde_json::Value> = tools
        .iter()
        .map(|t| {
            serde_json::json!({
                "type": "function",
                "function": {
                    "name": t.name,
                    "description": t.description,
                    "parameters": t.parameters,
                }
            })
        })
        .collect();
    serde_json::to_vec(&wire).unwrap().len()
}

/// A fat, MCP-shaped tool: named `mcp__<server>__tool_<n>` (the `McpTool`
/// naming convention, `mcp.rs:175-201`) with a deliberately bloated
/// description + parameters schema, matching the #65 real-world bloat regime
/// (`McpTool::from_client` eagerly wraps every remote tool's full `input_schema`).
struct FatMcpTool {
    name: String,
    description: String,
    parameters: serde_json::Value,
}

#[async_trait]
impl Tool for FatMcpTool {
    fn name(&self) -> &str {
        &self.name
    }
    fn description(&self) -> &str {
        &self.description
    }
    fn parameters(&self) -> serde_json::Value {
        self.parameters.clone()
    }
    async fn execute(
        &self,
        _args: serde_json::Value,
        _ctx: &ToolContext,
    ) -> supercode_harness::Result<String> {
        Ok(format!("{} executed", self.name))
    }
}

fn fat_mcp_tool(server: &str, n: usize) -> FatMcpTool {
    let name = format!("mcp__{server}__tool_{n}");
    let description = format!(
        "Verbose remote-MCP tool description #{n} on server `{server}`. {}",
        "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod. ".repeat(30)
    );
    let mut props = serde_json::Map::new();
    for i in 0..25 {
        props.insert(
            format!("field_{i}"),
            serde_json::json!({
                "type": "string",
                "description": "A verbose per-field description, realistically bloating the \
                    input_schema the way real MCP servers do.".repeat(2)
            }),
        );
    }
    let parameters = serde_json::json!({
        "type": "object",
        "properties": serde_json::Value::Object(props),
        "required": [],
        "additionalProperties": false
    });
    FatMcpTool {
        name,
        description,
        parameters,
    }
}

/// Registry: 10 builtins + 12 fat MCP-shaped tools across two fake servers.
fn build_fat_registry() -> ToolRegistry {
    let mut registry = ToolRegistry::with_builtins();
    for n in 0..6 {
        registry.register(fat_mcp_tool("serverA", n));
        registry.register(fat_mcp_tool("serverB", n));
    }
    registry
}

/// Records the `req.tools` seen on the first `complete()` call, then answers
/// with plain text (no tool call) so the loop ends after one turn.
struct RecordFirstRequest {
    seen: Arc<Mutex<Option<Vec<ToolSchema>>>>,
}
#[async_trait]
impl Provider for RecordFirstRequest {
    async fn complete(
        &self,
        req: &ChatRequest,
        _on_delta: &(dyn for<'a> Fn(&'a str) + Send + Sync),
    ) -> supercode_harness::Result<(ChatMessage, Usage)> {
        let mut seen = self.seen.lock().unwrap();
        if seen.is_none() {
            *seen = Some(req.tools.clone());
        }
        Ok((ChatMessage::assistant("ok"), Usage::default()))
    }
}

#[tokio::test]
async fn ac1_mcp_schemas_deferred_headline_measurement() {
    // ---- Full: baseline, every schema advertised (the #65 bloat regime). ----
    let full_seen = Arc::new(Mutex::new(None));
    let full_config = Config::builder().build(); // ToolAdvertising::Full (default)
    let mut full_agent = Agent::with_parts(
        full_config,
        Box::new(RecordFirstRequest {
            seen: full_seen.clone(),
        }),
        build_fat_registry(),
    );
    full_agent.send("hi").await.unwrap();
    let full_tools = full_seen.lock().unwrap().clone().unwrap();
    let full_bytes = tools_wire_bytes(&full_tools);
    assert!(
        full_bytes > 60_000,
        "Full serialization of body[\"tools\"] should exceed 60 KB (the #65 regime), got {full_bytes} bytes"
    );

    // ---- Deferred: builtins core, all mcp__* deferred behind tool_search. ----
    let deferred_seen = Arc::new(Mutex::new(None));
    let deferred_config = Config::builder()
        .tool_advertising(ToolAdvertising::Deferred {
            core: builtin_names(),
        })
        .build();
    let mut deferred_agent = Agent::with_parts(
        deferred_config,
        Box::new(RecordFirstRequest {
            seen: deferred_seen.clone(),
        }),
        build_fat_registry(),
    );
    deferred_agent.send("hi").await.unwrap();
    let deferred_tools = deferred_seen.lock().unwrap().clone().unwrap();

    assert!(
        !deferred_tools.iter().any(|t| t.name.starts_with("mcp__")),
        "request 1 must contain no mcp__ name under Deferred, got: {:?}",
        deferred_tools.iter().map(|t| &t.name).collect::<Vec<_>>()
    );
    let deferred_bytes = tools_wire_bytes(&deferred_tools);
    let ratio = deferred_bytes as f64 / full_bytes as f64;
    assert!(
        ratio < 0.20,
        "deferred body[\"tools\"] bytes ({deferred_bytes}) should be < 20% of Full ({full_bytes}), ratio={ratio:.4}"
    );

    // Test-log measurement: bytes → ~tokens (bytes/4) → $/request at the D15
    // reference input price.
    let deferred_tokens = deferred_bytes as f64 / 4.0;
    let dollars_per_request =
        (deferred_tokens / 1_000_000.0) * supercode_harness::pricing_ref::REF_INPUT_PER_MTOK;
    eprintln!(
        "AC1: full={full_bytes}B deferred={deferred_bytes}B ratio={ratio:.4} \
         ~tokens={deferred_tokens:.0} $/request=${dollars_per_request:.6} \
         (REF_INPUT_PER_MTOK=${})",
        supercode_harness::pricing_ref::REF_INPUT_PER_MTOK
    );
}

#[tokio::test]
async fn ac2_builtins_stay_eager_under_the_default() {
    let seen = Arc::new(Mutex::new(None));
    let config = Config::builder()
        .tool_advertising(ToolAdvertising::Deferred {
            core: builtin_names(),
        })
        .build();
    let mut agent =
        Agent::with_provider(config, Box::new(RecordFirstRequest { seen: seen.clone() }));
    agent.send("hi").await.unwrap();
    let names: BTreeSet<String> = seen
        .lock()
        .unwrap()
        .clone()
        .unwrap()
        .into_iter()
        .map(|t| t.name)
        .collect();

    let mut expected: BTreeSet<String> = builtin_names().into_iter().collect();
    expected.insert("tool_search".to_string());
    assert_eq!(names, expected);
}

#[tokio::test]
async fn ac3_explicit_core_narrows_further_regression_guard_full() {
    // Explicit core list narrows to exactly {read_file, bash, tool_search}.
    let seen = Arc::new(Mutex::new(None));
    let config = Config::builder()
        .tool_advertising(ToolAdvertising::Deferred {
            core: vec!["read_file".to_string(), "bash".to_string()],
        })
        .build();
    let mut agent =
        Agent::with_provider(config, Box::new(RecordFirstRequest { seen: seen.clone() }));
    agent.send("hi").await.unwrap();
    let names: BTreeSet<String> = seen
        .lock()
        .unwrap()
        .clone()
        .unwrap()
        .into_iter()
        .map(|t| t.name)
        .collect();
    let expected: BTreeSet<String> = ["read_file", "bash", "tool_search"]
        .into_iter()
        .map(String::from)
        .collect();
    assert_eq!(names, expected);

    // Regression guard: under Full (no explicit core), all 10 builtins and
    // nothing else are advertised (no synthetic tool_search).
    let full_seen = Arc::new(Mutex::new(None));
    let full_config = Config::builder().build();
    let mut full_agent = Agent::with_provider(
        full_config,
        Box::new(RecordFirstRequest {
            seen: full_seen.clone(),
        }),
    );
    full_agent.send("hi").await.unwrap();
    let full_names: HashSet<String> = full_seen
        .lock()
        .unwrap()
        .clone()
        .unwrap()
        .into_iter()
        .map(|t| t.name)
        .collect();
    assert_eq!(
        full_names.len(),
        10,
        "Full should advertise exactly the 10 builtins, got {full_names:?}"
    );
    assert!(!full_names.contains("tool_search"));
}

/// Turn 1: `tool_search {"query":"patch"}`. Turn 2: calls `apply_patch` (now
/// advertised with a non-empty schema) against a temp-dir fixture. Turn 3:
/// answers.
struct PatchSearchThenApply {
    calls: AtomicUsize,
}
#[async_trait]
impl Provider for PatchSearchThenApply {
    async fn complete(
        &self,
        req: &ChatRequest,
        _on_delta: &(dyn for<'a> Fn(&'a str) + Send + Sync),
    ) -> supercode_harness::Result<(ChatMessage, Usage)> {
        let n = self.calls.fetch_add(1, Ordering::SeqCst);
        let has = |name: &str| req.tools.iter().any(|t| t.name == name);
        if n == 0 {
            // Not yet activated: only the deferred core + tool_search are advertised.
            assert!(
                !has("apply_patch"),
                "apply_patch must not be advertised before search"
            );
            assert!(
                has("tool_search"),
                "tool_search must always be advertised under Deferred"
            );
            let call = ChatMessage {
                role: Role::Assistant,
                content: None,
                content_parts: None,
                tool_calls: Some(vec![ToolCall {
                    id: "s1".into(),
                    kind: "function".into(),
                    function: FunctionCall {
                        name: "tool_search".into(),
                        arguments: serde_json::json!({"query": "patch"}).to_string(),
                    },
                }]),
                tool_call_id: None,
                name: None,
                metadata: Default::default(),
            };
            return Ok((call, Usage::default()));
        }
        if n == 1 {
            // apply_patch is now advertised with a non-empty parameters schema.
            let schema = req.tools.iter().find(|t| t.name == "apply_patch").unwrap();
            let props = schema
                .parameters
                .get("properties")
                .and_then(|p| p.as_object());
            assert!(
                props.is_some() && !props.unwrap().is_empty(),
                "apply_patch schema should carry a non-empty parameters object, got: {}",
                schema.parameters
            );
            let patch =
                "*** Begin Patch\n*** Add File: added.txt\n+hello from patch\n*** End Patch\n";
            let call = ChatMessage {
                role: Role::Assistant,
                content: None,
                content_parts: None,
                tool_calls: Some(vec![ToolCall {
                    id: "c2".into(),
                    kind: "function".into(),
                    function: FunctionCall {
                        name: "apply_patch".into(),
                        arguments: serde_json::json!({"patch": patch}).to_string(),
                    },
                }]),
                tool_call_id: None,
                name: None,
                metadata: Default::default(),
            };
            return Ok((call, Usage::default()));
        }
        // n == 2: the previous message must be the apply_patch tool result.
        assert_eq!(req.messages.last().map(|m| m.role), Some(Role::Tool));
        Ok((ChatMessage::assistant("patched"), Usage::default()))
    }
}

#[tokio::test]
async fn ac4_ac5_tool_search_activates_advertises_and_executes_apply_patch() {
    let dir = temp_dir("tool-search-patch");
    let config = Config::builder()
        .cwd(dir.clone())
        .tool_advertising(ToolAdvertising::Deferred {
            core: vec!["bash".to_string()],
        })
        .build();
    let mut agent = Agent::with_provider(
        config,
        Box::new(PatchSearchThenApply {
            calls: AtomicUsize::new(0),
        }),
    );

    let reply = agent.send("apply the patch").await.unwrap();
    assert_eq!(reply, "patched");

    // The patch actually applied on disk.
    let added = dir.join("added.txt");
    assert!(added.exists(), "apply_patch should have written added.txt");
    assert_eq!(
        std::fs::read_to_string(&added).unwrap().trim(),
        "hello from patch"
    );

    // AC5: history contains an assistant `tool_search` ToolCall and a matching
    // tool result whose content parses as a JSON array containing the
    // activated tool's name.
    let history = agent.history();
    let search_call = history
        .iter()
        .find_map(|m| {
            (m.role == Role::Assistant)
                .then(|| {
                    m.tool_calls()
                        .iter()
                        .find(|c| c.function.name == "tool_search")
                })
                .flatten()
        })
        .expect("a tool_search ToolCall must be in history");
    let result = history
        .iter()
        .find(|m| {
            m.role == Role::Tool && m.tool_call_id.as_deref() == Some(search_call.id.as_str())
        })
        .expect("a matching tool result for the tool_search call must be in history");
    let parsed: serde_json::Value =
        serde_json::from_str(result.content.as_deref().unwrap_or_default())
            .expect("tool_search result content should parse as JSON");
    let arr = parsed
        .as_array()
        .expect("tool_search result should be a JSON array");
    assert!(
        arr.iter()
            .any(|v| v.get("name").and_then(|n| n.as_str()) == Some("apply_patch")),
        "tool_search result should contain the activated apply_patch schema: {arr:?}"
    );

    std::fs::remove_dir_all(&dir).ok();
}

/// A fake MCP tool the model discovers via `tool_search` (the primary B6
/// target). `execute` returns text proving it actually ran.
struct McpWidgetTool;
#[async_trait]
impl Tool for McpWidgetTool {
    fn name(&self) -> &str {
        "mcp__serverA__widget"
    }
    fn description(&self) -> &str {
        "Control a remote widget device over MCP."
    }
    fn parameters(&self) -> serde_json::Value {
        serde_json::json!({
            "type": "object",
            "properties": {"value": {"type": "string", "description": "widget setting"}},
            "required": ["value"],
            "additionalProperties": false
        })
    }
    async fn execute(
        &self,
        args: serde_json::Value,
        _ctx: &ToolContext,
    ) -> supercode_harness::Result<String> {
        let value = args.get("value").and_then(|v| v.as_str()).unwrap_or("");
        Ok(format!("widget-executed:{value}"))
    }
}

/// Same shape as [`PatchSearchThenApply`], for an `mcp__`-named tool.
struct McpSearchThenCall {
    calls: AtomicUsize,
}
#[async_trait]
impl Provider for McpSearchThenCall {
    async fn complete(
        &self,
        req: &ChatRequest,
        _on_delta: &(dyn for<'a> Fn(&'a str) + Send + Sync),
    ) -> supercode_harness::Result<(ChatMessage, Usage)> {
        let n = self.calls.fetch_add(1, Ordering::SeqCst);
        let has = |name: &str| req.tools.iter().any(|t| t.name == name);
        if n == 0 {
            assert!(!has("mcp__serverA__widget"));
            assert!(has("tool_search"));
            let call = ChatMessage {
                role: Role::Assistant,
                content: None,
                content_parts: None,
                tool_calls: Some(vec![ToolCall {
                    id: "s1".into(),
                    kind: "function".into(),
                    function: FunctionCall {
                        name: "tool_search".into(),
                        arguments: serde_json::json!({"query": "widget"}).to_string(),
                    },
                }]),
                tool_call_id: None,
                name: None,
                metadata: Default::default(),
            };
            return Ok((call, Usage::default()));
        }
        if n == 1 {
            let schema = req
                .tools
                .iter()
                .find(|t| t.name == "mcp__serverA__widget")
                .unwrap();
            let props = schema
                .parameters
                .get("properties")
                .and_then(|p| p.as_object());
            assert!(props.is_some() && !props.unwrap().is_empty());
            let call = ChatMessage {
                role: Role::Assistant,
                content: None,
                content_parts: None,
                tool_calls: Some(vec![ToolCall {
                    id: "c2".into(),
                    kind: "function".into(),
                    function: FunctionCall {
                        name: "mcp__serverA__widget".into(),
                        arguments: serde_json::json!({"value": "on"}).to_string(),
                    },
                }]),
                tool_call_id: None,
                name: None,
                metadata: Default::default(),
            };
            return Ok((call, Usage::default()));
        }
        // n == 2: echo back the tool result to prove execution happened.
        let last = req.messages.last().unwrap();
        assert_eq!(last.role, Role::Tool);
        Ok((
            ChatMessage::assistant(last.content.clone().unwrap_or_default()),
            Usage::default(),
        ))
    }
}

#[tokio::test]
async fn ac4_tool_search_activates_advertises_and_executes_mcp_tool() {
    let mut registry = ToolRegistry::with_builtins();
    registry.register(McpWidgetTool);
    let config = Config::builder()
        .tool_advertising(ToolAdvertising::Deferred {
            core: vec!["bash".to_string()],
        })
        .build();
    let mut agent = Agent::with_parts(
        config,
        Box::new(McpSearchThenCall {
            calls: AtomicUsize::new(0),
        }),
        registry,
    );

    let reply = agent.send("turn on the widget").await.unwrap();
    assert!(reply.contains("widget-executed:on"), "reply: {reply}");
}