car-engine 0.47.0

// `Runtime::execute` is a deeply nested async fn; composing several of them in
// one test future overflows rustc's default layout-query depth.
#![recursion_limit = "256"]

//! The feedback loop, end to end: execute → persist trajectory → derive rates.
//!
//! Each half existed before and neither was connected. `persist_trajectory`
//! returned early on `self.trajectory_store.as_ref()?` because nothing called
//! `Runtime::with_trajectory_store`, so `ToolFeedback` had no data to read.
//! These tests execute real proposals through a real store and assert the rates
//! that come back out, so a regression that silently unhooks either end fails
//! here rather than surfacing as a Monte Carlo result quietly built on defaults.

use car_engine::Runtime;
use car_ir::ActionProposal;
use serde_json::Value;
use std::sync::Arc;
use tempfile::TempDir;

/// Succeeds for tools named `good*`, fails for `bad*`. Deterministic, so the
/// derived rates are exact rather than statistical.
struct ScriptedExecutor;

#[async_trait::async_trait]
impl car_engine::ToolExecutor for ScriptedExecutor {
    async fn execute(&self, tool: &str, _params: &Value) -> Result<Value, String> {
        if tool.starts_with("bad") {
            Err(format!("{tool} is broken"))
        } else {
            Ok(serde_json::json!({ "ok": true }))
        }
    }
}

fn proposal(actions: Vec<Value>) -> ActionProposal {
    serde_json::from_value(serde_json::json!({ "source": "test", "actions": actions }))
        .expect("proposal deserializes")
}

fn call(id: &str, tool: &str) -> Value {
    serde_json::json!({ "id": id, "type": "tool_call", "tool": tool, "parameters": {} })
}

async fn runtime_with_store(dir: &TempDir) -> Runtime {
    let store = Arc::new(car_memgine::TrajectoryStore::new(dir.path()));
    let rt = Runtime::new().with_trajectory_store(store);
    rt.set_executor(Arc::new(ScriptedExecutor)).await;
    for t in ["good_a", "good_b", "bad_a"] {
        rt.register_tool(t).await;
    }
    rt
}

#[tokio::test]
async fn executing_a_proposal_persists_a_trajectory() {
    let dir = TempDir::new().unwrap();
    let rt = runtime_with_store(&dir).await;

    rt.execute(&proposal(vec![call("a0", "good_a")])).await;

    let stored = rt.trajectory_store().expect("store attached").load_all();
    assert_eq!(
        stored.len(),
        1,
        "execution must persist exactly one trajectory"
    );
    assert!(stored[0]
        .events
        .iter()
        .any(|e| e.tool.as_deref() == Some("good_a") && e.kind == "action_succeeded"));
}

#[tokio::test]
async fn derived_rates_reflect_what_actually_happened() {
    let dir = TempDir::new().unwrap();
    let rt = runtime_with_store(&dir).await;

    // good_a runs 4×, always succeeding. bad_a runs 4×, always failing.
    for i in 0..4 {
        rt.execute(&proposal(vec![call(&format!("g{i}"), "good_a")]))
            .await;
        rt.execute(&proposal(vec![call(&format!("b{i}"), "bad_a")]))
            .await;
    }

    let fb = rt.tool_feedback(30).expect("store attached");

    assert_eq!(fb.tool_dispatch_counts["good_a"], (4, 4));
    assert_eq!(fb.tool_dispatch_counts["bad_a"], (0, 4));

    // Laplace-smoothed: 5/6 and 1/6. Directionally right, and not the
    // categorical 1.0/0.0 that four observations don't justify.
    assert!(
        (fb.rate("good_a") - 5.0 / 6.0).abs() < 1e-9,
        "{}",
        fb.rate("good_a")
    );
    assert!(
        (fb.rate("bad_a") - 1.0 / 6.0).abs() < 1e-9,
        "{}",
        fb.rate("bad_a")
    );
    assert!(fb.rate("good_a") > fb.rate("bad_a"));
}

#[tokio::test]
async fn a_tool_never_executed_gets_no_derived_rate() {
    let dir = TempDir::new().unwrap();
    let rt = runtime_with_store(&dir).await;
    rt.execute(&proposal(vec![call("a0", "good_a")])).await;

    let fb = rt.tool_feedback(30).unwrap();
    // `good_b` is registered but was never called — absent, so a consumer
    // applies its own default rather than reading an invented observation.
    assert!(!fb.tool_success_rates.contains_key("good_b"));
    assert!(fb.tool_success_rates.contains_key("good_a"));
}

#[tokio::test]
async fn rates_feed_monte_carlo_and_move_the_verdict() {
    let dir = TempDir::new().unwrap();
    let rt = runtime_with_store(&dir).await;

    // Build a history in which `bad_a` is observably unreliable.
    for i in 0..10 {
        rt.execute(&proposal(vec![call(&format!("g{i}"), "good_a")]))
            .await;
        rt.execute(&proposal(vec![call(&format!("b{i}"), "bad_a")]))
            .await;
    }
    let fb = rt.tool_feedback(30).unwrap();

    // A plan whose second step depends on the first, using the flaky tool.
    let plan: ActionProposal = serde_json::from_value(serde_json::json!({
        "source": "test",
        "actions": [
            {"id": "fetch", "type": "tool_call", "tool": "bad_a", "parameters": {},
             "expected_effects": {"data": 1}},
            {"id": "use", "type": "tool_call", "tool": "good_a", "parameters": {},
             "state_dependencies": ["data"], "expected_effects": {"done": true}},
        ],
    }))
    .unwrap();

    // `simulate` is unconditional — it reports the plan as fully landing.
    assert_eq!(
        car_verify::simulate(&plan, None).get("done"),
        Some(&Value::from(true))
    );

    let mc = car_verify::simulate_monte_carlo(
        &plan,
        None,
        &fb.tool_success_rates,
        None,
        &car_verify::MonteCarloConfig {
            trials: 20_000,
            ..Default::default()
        },
    );

    // Real history says bad_a succeeds ~1/12 of the time, so the plan almost
    // never completes — and the second step is starved rather than failing.
    assert!(
        mc.p_all_effects_landed < 0.15,
        "expected a low completion probability from real history, got {}",
        mc.p_all_effects_landed
    );
    let fetch = &mc.action_outcomes[0];
    let use_ = &mc.action_outcomes[1];
    assert!(
        fetch.p_failed > 0.8,
        "fetch should mostly fail: {}",
        fetch.p_failed
    );
    assert!(
        use_.p_rejected > 0.8,
        "the dependent should be starved, not failing on its own: {}",
        use_.p_rejected
    );
    assert!(use_.p_failed < 0.15);
}

#[tokio::test]
async fn no_store_means_no_feedback_rather_than_empty_feedback() {
    let rt = Runtime::new();
    rt.set_executor(Arc::new(ScriptedExecutor)).await;
    rt.register_tool("good_a").await;
    rt.execute(&proposal(vec![call("a0", "good_a")])).await;

    // `None` — distinguishable from "a store with nothing in it". A caller
    // must be able to tell "we have no history mechanism" from "we have one
    // and it is empty", because only the first is a misconfiguration.
    assert!(rt.trajectory_store().is_none());
    assert!(rt.tool_feedback(30).is_none());
}