use crate::{apply_action_effects, StaticState};
use car_ir::{build_dag, Action, ActionProposal, ActionType};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;
pub type State = HashMap<String, Value>;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Transition {
pub state_before: State,
pub action: Value,
pub state_after: State,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Failure {
pub index: usize,
pub action: Value,
pub expected: State,
#[serde(skip_serializing_if = "Option::is_none")]
pub predicted: Option<State>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScoreReport {
pub total: usize,
pub correct: usize,
pub errored: usize,
pub accuracy: f64,
pub failures: Vec<Failure>,
}
impl ScoreReport {
pub fn is_perfect(&self) -> bool {
self.total > 0 && self.correct == self.total
}
}
fn values_equal(a: &Value, b: &Value) -> bool {
match (a, b) {
(Value::Number(x), Value::Number(y)) => match (x.as_f64(), y.as_f64()) {
(Some(fx), Some(fy)) => fx == fy,
_ => x == y,
},
(Value::Array(xs), Value::Array(ys)) => {
xs.len() == ys.len() && xs.iter().zip(ys).all(|(x, y)| values_equal(x, y))
}
(Value::Object(xs), Value::Object(ys)) => {
xs.len() == ys.len()
&& xs
.iter()
.all(|(k, x)| ys.get(k).is_some_and(|y| values_equal(x, y)))
}
_ => a == b,
}
}
fn states_equal(a: &State, b: &State) -> bool {
a.len() == b.len()
&& a.iter()
.all(|(k, av)| b.get(k).is_some_and(|bv| values_equal(av, bv)))
}
pub fn score<P>(transitions: &[Transition], mut predict: P) -> ScoreReport
where
P: FnMut(&State, &Value) -> Result<State, String>,
{
let mut correct = 0;
let mut errored = 0;
let mut failures = Vec::new();
for (index, t) in transitions.iter().enumerate() {
match predict(&t.state_before, &t.action) {
Ok(predicted) => {
if states_equal(&predicted, &t.state_after) {
correct += 1;
} else {
failures.push(Failure {
index,
action: t.action.clone(),
expected: t.state_after.clone(),
predicted: Some(predicted),
error: None,
});
}
}
Err(e) => {
errored += 1;
failures.push(Failure {
index,
action: t.action.clone(),
expected: t.state_after.clone(),
predicted: None,
error: Some(e),
});
}
}
}
let total = transitions.len();
ScoreReport {
total,
correct,
errored,
accuracy: if total == 0 {
0.0
} else {
correct as f64 / total as f64
},
failures,
}
}
pub fn score_predictions(
transitions: &[Transition],
predictions: &[Result<State, String>],
) -> Result<ScoreReport, String> {
if transitions.len() != predictions.len() {
return Err(format!(
"transitions ({}) and predictions ({}) differ in length",
transitions.len(),
predictions.len()
));
}
let mut iter = predictions.iter();
Ok(score(transitions, |_, _| {
iter.next().expect("length checked above").clone()
}))
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CwmResult {
pub code: Option<String>,
pub perfect: bool,
pub train_accuracy: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub test_accuracy: Option<f64>,
pub attempts: u32,
pub failures: Vec<Failure>,
}
pub struct CwmRequest {
pub spec: String,
pub train: Vec<Transition>,
pub test: Vec<Transition>,
pub max_attempts: u32,
}
fn build_prompt(spec: &str, train: &[Transition], prior: &[Failure]) -> String {
let mut p = String::new();
p.push_str(
"Write a deterministic world model as a single function `apply(state, action)` \
returning the next state. Return ONLY the code.\n\n",
);
p.push_str("# Rules / tool schema\n");
p.push_str(spec);
p.push_str("\n\n# Transitions to reproduce exactly\n");
for t in train.iter().take(8) {
if let Ok(line) = serde_json::to_string(t) {
p.push_str(&line);
p.push('\n');
}
}
if !prior.is_empty() {
p.push_str("\n# Your previous model failed these cases — fix them:\n");
for f in prior.iter().take(8) {
if let Ok(line) = serde_json::to_string(f) {
p.push_str(&line);
p.push('\n');
}
}
}
p
}
pub async fn synthesize_cwm<G, GFut, R>(generate: G, mut run: R, req: &CwmRequest) -> CwmResult
where
G: Fn(String) -> GFut,
GFut: std::future::Future<Output = Result<String, String>>,
R: FnMut(&str, &State, &Value) -> Result<State, String>,
{
let max = req.max_attempts.max(1);
let mut prior: Vec<Failure> = Vec::new();
let mut best: Option<(String, ScoreReport)> = None;
let mut attempts = 0;
for attempt in 1..=max {
attempts = attempt;
let prompt = build_prompt(&req.spec, &req.train, &prior);
let code = match generate(prompt).await {
Ok(c) => c,
Err(_) => continue,
};
let report = score(&req.train, |s, a| run(&code, s, a));
if report.is_perfect() {
let test = score(&req.test, |s, a| run(&code, s, a));
return CwmResult {
code: Some(code),
perfect: true,
train_accuracy: report.accuracy,
test_accuracy: Some(test.accuracy),
attempts,
failures: Vec::new(),
};
}
prior = report.failures.clone();
let better = best
.as_ref()
.map(|(_, b)| report.accuracy > b.accuracy)
.unwrap_or(true);
if better {
best = Some((code, report));
}
}
match best {
Some((code, report)) => CwmResult {
code: Some(code),
perfect: false,
train_accuracy: report.accuracy,
test_accuracy: None,
attempts,
failures: report.failures,
},
None => CwmResult {
code: None,
perfect: false,
train_accuracy: 0.0,
test_accuracy: None,
attempts,
failures: Vec::new(),
},
}
}
pub trait EffectModel {
fn predict(&self, action: &Action, state_before: &State) -> Option<State>;
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GatedPrediction {
pub effects: State,
pub accuracy: f64,
}
pub struct GatedEffectModel {
pub predictions: HashMap<String, GatedPrediction>,
pub min_accuracy: f64,
}
impl EffectModel for GatedEffectModel {
fn predict(&self, action: &Action, _state_before: &State) -> Option<State> {
self.predictions
.get(&action.id)
.filter(|p| p.accuracy >= self.min_accuracy)
.map(|p| p.effects.clone())
}
}
pub fn simulate_with_model(
proposal: &ActionProposal,
initial_state: Option<&State>,
model: &dyn EffectModel,
) -> State {
let mut state = match initial_state {
Some(s) => StaticState::from_map(s.clone()),
None => StaticState::new(),
};
for level in build_dag(&proposal.actions) {
for idx in level {
let action = &proposal.actions[idx];
let predicted = model.predict(action, &state.known);
if action.action_type == ActionType::StateWrite {
if let Some(key) = action.parameters.get("key").and_then(|v| v.as_str()) {
let value = action
.parameters
.get("value")
.cloned()
.unwrap_or(Value::Null);
state.set(key, value);
}
}
match predicted {
Some(delta) => {
for (k, v) in delta {
state.set(&k, v);
}
}
None => apply_action_effects(action, &mut state),
}
}
}
state.known
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn st(pairs: &[(&str, Value)]) -> State {
pairs
.iter()
.map(|(k, v)| (k.to_string(), v.clone()))
.collect()
}
fn counter_transitions() -> Vec<Transition> {
vec![
Transition {
state_before: st(&[("count", json!(0))]),
action: json!({ "inc": 1 }),
state_after: st(&[("count", json!(1))]),
},
Transition {
state_before: st(&[("count", json!(1))]),
action: json!({ "inc": 2 }),
state_after: st(&[("count", json!(3))]),
},
]
}
fn good_model(s: &State, a: &Value) -> Result<State, String> {
let cur = s.get("count").and_then(|v| v.as_i64()).unwrap_or(0);
let inc = a.get("inc").and_then(|v| v.as_i64()).unwrap_or(0);
Ok(st(&[("count", json!(cur + inc))]))
}
#[test]
fn perfect_model_scores_1() {
let r = score(&counter_transitions(), good_model);
assert_eq!(r.total, 2);
assert_eq!(r.correct, 2);
assert!(r.is_perfect());
assert_eq!(r.accuracy, 1.0);
assert!(r.failures.is_empty());
}
#[test]
fn wrong_model_reports_failures() {
let r = score(&counter_transitions(), |s, _| {
let cur = s.get("count").and_then(|v| v.as_i64()).unwrap_or(0);
Ok(st(&[("count", json!(cur + 1))]))
});
assert_eq!(r.correct, 1); assert_eq!(r.failures.len(), 1);
assert_eq!(r.failures[0].index, 1);
assert!(r.failures[0].error.is_none());
assert_eq!(r.failures[0].predicted, Some(st(&[("count", json!(2))])));
}
#[test]
fn throwing_model_is_counted_as_errored() {
let r = score(&counter_transitions(), |_, _| Err("boom".to_string()));
assert_eq!(r.errored, 2);
assert_eq!(r.correct, 0);
assert_eq!(r.failures[0].error.as_deref(), Some("boom"));
}
#[test]
fn numeric_equality_int_vs_float() {
let trs = vec![Transition {
state_before: st(&[]),
action: json!({}),
state_after: st(&[("x", json!(2))]),
}];
let r = score(&trs, |_, _| Ok(st(&[("x", json!(2.0))])));
assert!(r.is_perfect());
}
#[test]
fn nested_numeric_equality() {
let trs = vec![Transition {
state_before: st(&[]),
action: json!({}),
state_after: st(&[("v", json!({ "items": [1, 2] }))]),
}];
let r = score(&trs, |_, _| {
Ok(st(&[("v", json!({ "items": [1.0, 2.0] }))]))
});
assert!(r.is_perfect());
}
#[test]
fn score_predictions_length_mismatch_errs() {
let trs = counter_transitions();
let preds = vec![Ok(st(&[("count", json!(1))]))]; assert!(score_predictions(&trs, &preds).is_err());
}
#[test]
fn score_predictions_matches_closure_path() {
let trs = counter_transitions();
let preds: Vec<Result<State, String>> =
trs.iter().map(|t| Ok(t.state_after.clone())).collect();
let r = score_predictions(&trs, &preds).unwrap();
assert!(r.is_perfect());
}
#[test]
fn empty_set_is_not_perfect() {
let r = score(&[], good_model);
assert!(!r.is_perfect());
assert_eq!(r.accuracy, 0.0);
}
#[tokio::test]
async fn synthesize_repairs_then_succeeds() {
use std::cell::Cell;
let attempt = Cell::new(0u32);
let generate = |_prompt: String| {
let n = attempt.get();
attempt.set(n + 1);
async move { Ok(format!("model-v{n}")) }
};
let run = |code: &str, s: &State, a: &Value| -> Result<State, String> {
if code == "model-v0" {
return Ok(st(&[("count", json!(999))])); }
good_model(s, a)
};
let req = CwmRequest {
spec: "counter world".to_string(),
train: counter_transitions(),
test: counter_transitions(),
max_attempts: 5,
};
let res = synthesize_cwm(generate, run, &req).await;
assert!(res.perfect, "should repair to a perfect model");
assert_eq!(res.code.as_deref(), Some("model-v1"));
assert_eq!(res.attempts, 2);
assert_eq!(res.test_accuracy, Some(1.0));
}
#[tokio::test]
async fn synthesize_returns_best_when_never_perfect() {
let generate = |_p: String| async { Ok("stuck".to_string()) };
let run = |_c: &str, s: &State, _a: &Value| -> Result<State, String> {
Ok(s.clone())
};
let req = CwmRequest {
spec: "counter".to_string(),
train: counter_transitions(),
test: counter_transitions(),
max_attempts: 3,
};
let res = synthesize_cwm(generate, run, &req).await;
assert!(!res.perfect);
assert_eq!(res.attempts, 3);
assert!(res.test_accuracy.is_none());
assert!(!res.failures.is_empty());
}
fn tool_action(id: &str, effects: &[(&str, Value)]) -> Action {
let eff: serde_json::Map<String, Value> = effects
.iter()
.map(|(k, v)| (k.to_string(), v.clone()))
.collect();
serde_json::from_value(json!({
"type": "tool_call", "id": id, "tool": "t", "expected_effects": eff
}))
.unwrap()
}
fn proposal(actions: Vec<Action>) -> ActionProposal {
serde_json::from_value(json!({ "actions": actions })).unwrap()
}
struct Abstain;
impl EffectModel for Abstain {
fn predict(&self, _a: &Action, _s: &State) -> Option<State> {
None
}
}
#[test]
fn abstaining_model_matches_static_simulate() {
let p = proposal(vec![
tool_action("a1", &[("x", json!(1))]),
tool_action("a2", &[("y", json!(2))]),
]);
let with_model = simulate_with_model(&p, None, &Abstain);
let static_sim = crate::simulate(&p, None);
assert_eq!(with_model, static_sim);
assert_eq!(with_model.get("x"), Some(&json!(1)));
assert_eq!(with_model.get("y"), Some(&json!(2)));
}
#[test]
fn gated_model_overrides_declared_effect_when_accurate() {
let p = proposal(vec![tool_action("a1", &[("x", json!(1))])]);
let mut predictions = HashMap::new();
predictions.insert(
"a1".to_string(),
GatedPrediction {
effects: st(&[("x", json!(42))]),
accuracy: 0.99,
},
);
let model = GatedEffectModel {
predictions,
min_accuracy: 0.9,
};
let out = simulate_with_model(&p, None, &model);
assert_eq!(out.get("x"), Some(&json!(42)), "prediction should win");
}
#[test]
fn gated_model_abstains_below_threshold() {
let p = proposal(vec![tool_action("a1", &[("x", json!(1))])]);
let mut predictions = HashMap::new();
predictions.insert(
"a1".to_string(),
GatedPrediction {
effects: st(&[("x", json!(42))]),
accuracy: 0.50, },
);
let model = GatedEffectModel {
predictions,
min_accuracy: 0.9,
};
let out = simulate_with_model(&p, None, &model);
assert_eq!(out.get("x"), Some(&json!(1)));
}
#[test]
fn gated_model_abstains_for_unknown_action() {
let p = proposal(vec![tool_action("a1", &[("x", json!(1))])]);
let model = GatedEffectModel {
predictions: HashMap::new(), min_accuracy: 0.9,
};
let out = simulate_with_model(&p, None, &model);
assert_eq!(out.get("x"), Some(&json!(1)));
}
#[test]
fn statewrite_deterministic_value_always_applies() {
let sw: Action = serde_json::from_value(json!({
"type": "state_write", "id": "w",
"parameters": { "key": "k", "value": "v" }
}))
.unwrap();
let out = simulate_with_model(&proposal(vec![sw]), None, &Abstain);
assert_eq!(out.get("k"), Some(&json!("v")));
}
}