mod common;
use agent_block_testkit::server::MockLlm;
use agent_block_testkit::shapes::{anthropic, openai};
use predicates::prelude::*;
use serde_json::json;
use std::sync::atomic::Ordering;
use tempfile::tempdir;
fn two_paths(paths: &[String]) -> (String, String) {
match paths.len() {
0 => ("file_a.lua".to_string(), "file_b.lua".to_string()),
1 => (paths[0].clone(), "file_b.lua".to_string()),
_ => (paths[0].clone(), paths[1].clone()),
}
}
fn sr_text_both(path_a: &str, path_b: &str) -> String {
format!(
"<<< path={path_a} >>>\n<<<<<<< SEARCH\nprint(\"a-old\")\n=======\nprint(\"a-new\")\n>>>>>>> REPLACE\n\n<<< path={path_b} >>>\n<<<<<<< SEARCH\nprint(\"b-old\")\n=======\nprint(\"b-new\")\n>>>>>>> REPLACE"
)
}
#[tokio::test]
async fn compile_loop_diff_anthropic_mock_iterates_until_pass() {
let (base_url, call_count, ct) =
common::compile_loop_diff_anthropic_mock::spawn_compile_loop_diff_anthropic_mock_server()
.await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let url_clone = base_url.clone();
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let target_file = tmp.path().join("target.lua");
common::agent_block_cmd()
.args([
"-s",
&common::fixture("compile_loop_diff_anthropic_mock.lua"),
])
.env("ANTHROPIC_BASE_URL_TEST", &url_clone)
.env(
"COMPILE_LOOP_TARGET",
target_file.to_str().expect("utf8 path"),
)
.env("AGENT_BLOCK_HOME", tmp.path())
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains("COMPILE_LOOP_DIFF_MOCK_PASS"));
})
.await
.expect("subprocess assertion task should not panic");
assert_eq!(
call_count.load(Ordering::SeqCst),
2,
"expected exactly 2 HTTP calls to the diff anthropic mock (iter1: apply-fail, iter2: pass)"
);
ct.cancel();
}
#[tokio::test]
async fn compile_loop_openai_mock_iterates_until_pass() {
let (base_url, call_count, ct) =
common::compile_loop_openai_mock::spawn_compile_loop_openai_mock_server().await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let url_clone = base_url.clone();
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let target_file = tmp.path().join("target.lua");
common::agent_block_cmd()
.args(["-s", &common::fixture("compile_loop_openai_mock.lua")])
.env("OPENAI_BASE_URL_TEST", &url_clone)
.env(
"COMPILE_LOOP_TARGET",
target_file.to_str().expect("utf8 path"),
)
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains("COMPILE_LOOP_MOCK_PASS"));
})
.await
.expect("subprocess assertion task should not panic");
assert_eq!(
call_count.load(Ordering::SeqCst),
2,
"expected exactly 2 HTTP calls to the mock (turn 1: broken, turn 2: fixed)"
);
ct.cancel();
}
#[tokio::test]
async fn compile_loop_anthropic_mock_iterates_until_pass() {
let (base_url, call_count, ct) =
common::compile_loop_anthropic_mock::spawn_compile_loop_anthropic_mock_server().await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let url_clone = base_url.clone();
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let target_file = tmp.path().join("target.lua");
common::agent_block_cmd()
.args(["-s", &common::fixture("compile_loop_anthropic_mock.lua")])
.env("ANTHROPIC_BASE_URL_TEST", &url_clone)
.env(
"COMPILE_LOOP_TARGET",
target_file.to_str().expect("utf8 path"),
)
.env("AGENT_BLOCK_HOME", tmp.path())
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains("COMPILE_LOOP_MOCK_PASS"));
})
.await
.expect("subprocess assertion task should not panic");
assert_eq!(
call_count.load(Ordering::SeqCst),
2,
"expected exactly 2 HTTP calls to the anthropic mock"
);
ct.cancel();
}
#[tokio::test]
async fn compile_loop_diff_multi_anthropic_mock_iterates_until_pass() {
let handle = MockLlm::anthropic(|req| {
let (path_a, path_b) = two_paths(&req.paths);
anthropic::text_response(&sr_text_both(&path_a, &path_b))
})
.spawn()
.await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let url_clone = handle.base_url.clone();
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let file_a = tmp.path().join("file_a.lua");
let file_b = tmp.path().join("file_b.lua");
common::agent_block_cmd()
.args([
"-s",
&common::fixture("compile_loop_diff_multi_anthropic_mock.lua"),
])
.env("ANTHROPIC_BASE_URL_TEST", &url_clone)
.env(
"COMPILE_LOOP_TARGET_FILES",
format!(
"{}:{}",
file_a.to_str().expect("utf8 path"),
file_b.to_str().expect("utf8 path")
),
)
.env("AGENT_BLOCK_HOME", tmp.path())
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains(
"COMPILE_LOOP_DIFF_MULTI_MOCK_PASS",
));
})
.await
.expect("subprocess assertion task should not panic");
assert_eq!(
handle.state.call_count(),
1,
"expected exactly 1 HTTP call to the multi diff mock (happy path: 2 files in 1 turn)"
);
handle.ct.cancel();
}
#[tokio::test]
async fn compile_loop_apply_search_replace_tool_converges() {
let handle = MockLlm::anthropic(|req| {
if !req.has_tool_results {
let (path_a, path_b) = two_paths(&req.paths);
anthropic::tool_use_response(vec![
anthropic::tool_use(
"toolu_asr_1",
"apply_search_replace",
json!({"path": path_a, "search": "print(\"a-old\")", "replace": "print(\"a-new\")"}),
),
anthropic::tool_use(
"toolu_asr_2",
"apply_search_replace",
json!({"path": path_b, "search": "print(\"b-old\")", "replace": "print(\"b-new\")"}),
),
])
} else {
anthropic::text_response("DONE")
}
})
.spawn()
.await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let url_clone = handle.base_url.clone();
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let file_a = tmp.path().join("file_a.lua");
let file_b = tmp.path().join("file_b.lua");
common::agent_block_cmd()
.args([
"-s",
&common::fixture("compile_loop_asr_anthropic_mock.lua"),
])
.env("ANTHROPIC_BASE_URL_TEST", &url_clone)
.env(
"COMPILE_LOOP_TARGET_FILES",
format!(
"{}:{}",
file_a.to_str().expect("utf8 path"),
file_b.to_str().expect("utf8 path")
),
)
.env("AGENT_BLOCK_HOME", tmp.path())
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains("COMPILE_LOOP_ASR_MOCK_PASS"));
})
.await
.expect("subprocess assertion task should not panic");
assert_eq!(
handle.state.call_count(),
2,
"expected exactly 2 HTTP calls (tool_use turn + DONE turn)"
);
assert!(
handle.state.declared_count_of("apply_search_replace") >= 1,
"tool_mode=auto must declare the apply_search_replace tool in the request"
);
handle.ct.cancel();
}
#[tokio::test]
async fn compile_loop_broken_openai_tool_calls_shape_converges() {
let handle = MockLlm::openai(|req| {
if !req.has_tool_results {
let (path_a, path_b) = two_paths(&req.paths);
openai::tool_calls_response(vec![
openai::tool_call_object_args_no_id(
"apply_search_replace",
json!({"path": path_a, "search": "print(\"a-old\")", "replace": "print(\"a-new\")"}),
),
openai::tool_call_object_args_no_id(
"apply_search_replace",
json!({"path": path_b, "search": "print(\"b-old\")", "replace": "print(\"b-new\")"}),
),
])
} else {
openai::text_response("DONE")
}
})
.spawn()
.await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let url_clone = handle.base_url.clone();
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let file_a = tmp.path().join("file_a.lua");
let file_b = tmp.path().join("file_b.lua");
common::agent_block_cmd()
.args([
"-s",
&common::fixture("compile_loop_broken_openai_mock.lua"),
])
.env("OPENAI_BASE_URL_TEST", &url_clone)
.env(
"COMPILE_LOOP_TARGET_FILES",
format!(
"{}:{}",
file_a.to_str().expect("utf8 path"),
file_b.to_str().expect("utf8 path")
),
)
.env("AGENT_BLOCK_HOME", tmp.path())
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains(
"COMPILE_LOOP_BROKEN_OPENAI_MOCK_PASS",
));
})
.await
.expect("subprocess assertion task should not panic");
assert_eq!(
handle.state.call_count(),
2,
"expected exactly 2 HTTP calls (broken tool_calls turn + DONE turn)"
);
let ids = handle.state.tool_result_ids();
assert!(
ids.len() >= 2,
"expected >=2 role=tool messages carrying tool results, got {ids:?}"
);
assert!(
ids.iter().all(|id| id.starts_with("call_synth_")),
"every role=tool message must carry a synthesized call_synth_* id, got {ids:?}"
);
handle.ct.cancel();
}
#[tokio::test]
async fn compile_loop_tool_mode_none_declares_no_tools() {
let handle = MockLlm::anthropic(|req| {
let (path_a, path_b) = two_paths(&req.paths);
anthropic::text_response(&sr_text_both(&path_a, &path_b))
})
.spawn()
.await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let url_clone = handle.base_url.clone();
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let file_a = tmp.path().join("file_a.lua");
let file_b = tmp.path().join("file_b.lua");
common::agent_block_cmd()
.args([
"-s",
&common::fixture("compile_loop_tool_mode_none_mock.lua"),
])
.env("ANTHROPIC_BASE_URL_TEST", &url_clone)
.env(
"COMPILE_LOOP_TARGET_FILES",
format!(
"{}:{}",
file_a.to_str().expect("utf8 path"),
file_b.to_str().expect("utf8 path")
),
)
.env("AGENT_BLOCK_HOME", tmp.path())
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains(
"COMPILE_LOOP_TOOL_MODE_NONE_MOCK_PASS",
));
})
.await
.expect("subprocess assertion task should not panic");
assert_eq!(
handle.state.call_count(),
1,
"expected exactly 1 HTTP call (single-turn SR text)"
);
assert_eq!(
handle.state.tools_declared_count(),
0,
"tool_mode=none must not declare any tools in the request"
);
handle.ct.cancel();
}
#[tokio::test]
async fn compile_loop_diff_multi_anthropic_mock_two_iter_converges() {
let handle = MockLlm::anthropic(|req| {
let (path_a, path_b) = two_paths(&req.paths);
if req.call_index == 0 {
anthropic::text_response(&format!(
"<<< path={path_a} >>>\n<<<<<<< SEARCH\nprint(\"WRONG\")\n=======\nprint(\"a-new\")\n>>>>>>> REPLACE"
))
} else {
anthropic::text_response(&sr_text_both(&path_a, &path_b))
}
})
.spawn()
.await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let url_clone = handle.base_url.clone();
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let file_a = tmp.path().join("file_a.lua");
let file_b = tmp.path().join("file_b.lua");
common::agent_block_cmd()
.args([
"-s",
&common::fixture("compile_loop_diff_multi_anthropic_mock_two_iter.lua"),
])
.env("ANTHROPIC_BASE_URL_TEST", &url_clone)
.env(
"COMPILE_LOOP_TARGET_FILES",
format!(
"{}:{}",
file_a.to_str().expect("utf8 path"),
file_b.to_str().expect("utf8 path")
),
)
.env("AGENT_BLOCK_HOME", tmp.path())
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains(
"COMPILE_LOOP_DIFF_MULTI_MOCK_TWO_ITER_PASS",
));
})
.await
.expect("subprocess assertion task should not panic");
assert_eq!(
handle.state.call_count(),
2,
"expected exactly 2 HTTP calls to the multi diff mock (iter1: apply-fail, iter2: pass)"
);
handle.ct.cancel();
}
#[tokio::test]
async fn compile_loop_distill_openai_mock_iterates_until_pass() {
let (addr, state) = common::compile_loop_distill_mock::spawn_distill_mock("openai").await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let base_url = format!("http://{addr}");
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let target_file = tmp.path().join("distill_target.lua");
common::agent_block_cmd()
.args(["-s", &common::fixture("compile_loop_distill_mock.lua")])
.env("OPENAI_BASE_URL_TEST", &base_url)
.env("DISTILL_MOCK_PROVIDER", "openai")
.env(
"COMPILE_LOOP_DISTILL_TARGET",
target_file.to_str().expect("utf8 path"),
)
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains("COMPILE_LOOP_DISTILL_MOCK_PASS"));
})
.await
.expect("subprocess assertion task should not panic");
assert!(
state.distill_call_count.load(Ordering::SeqCst) > 0,
"distill_call_count must be > 0: distill subloop was not triggered"
);
let body_guard = state.received_distill_body.lock().unwrap();
let distill_body = body_guard
.as_ref()
.expect("received_distill_body must be set after distill call");
assert!(
distill_body.get("tools").is_none(),
"BC5: distill LLM call must not include `tools` field in request body"
);
}
#[tokio::test]
async fn compile_loop_distill_anthropic_mock_iterates_until_pass() {
let (addr, state) = common::compile_loop_distill_mock::spawn_distill_mock("anthropic").await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let base_url = format!("http://{addr}");
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let target_file = tmp.path().join("distill_target.lua");
common::agent_block_cmd()
.args(["-s", &common::fixture("compile_loop_distill_mock.lua")])
.env("ANTHROPIC_BASE_URL_TEST", &base_url)
.env("DISTILL_MOCK_PROVIDER", "anthropic")
.env(
"COMPILE_LOOP_DISTILL_TARGET",
target_file.to_str().expect("utf8 path"),
)
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains("COMPILE_LOOP_DISTILL_MOCK_PASS"));
})
.await
.expect("subprocess assertion task should not panic");
assert!(
state.distill_call_count.load(Ordering::SeqCst) > 0,
"distill_call_count must be > 0: distill subloop was not triggered"
);
let body_guard = state.received_distill_body.lock().unwrap();
let distill_body = body_guard
.as_ref()
.expect("received_distill_body must be set after distill call");
assert!(
distill_body.get("tools").is_none(),
"BC5: distill LLM call must not include `tools` field in request body"
);
}
#[tokio::test]
async fn compile_loop_read_file_range_verbatim() {
let (addr, state) = common::compile_loop_distill_mock::spawn_range_mock().await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let base_url = format!("http://{addr}");
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let target_file = tmp.path().join("range_target.lua");
common::agent_block_cmd()
.args([
"-s",
&common::fixture("compile_loop_distill_range_mock.lua"),
])
.env("ANTHROPIC_BASE_URL_TEST", &base_url)
.env(
"COMPILE_LOOP_RANGE_TARGET",
target_file.to_str().expect("utf8 path"),
)
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains("READ_FILE_RANGE_VERBATIM_PASS"));
})
.await
.expect("subprocess assertion task should not panic");
assert_eq!(
state.call_count.load(Ordering::SeqCst),
2,
"expected exactly 2 HTTP calls to the range mock (turn 0: read_file_range, turn 1: SR pass)"
);
assert_eq!(
state.distill_call_count.load(Ordering::SeqCst),
0,
"range mock must not trigger distill subloop (read_file_range bypasses distill)"
);
}
#[tokio::test]
async fn compile_loop_openai_mock_three_turn_converges() {
let (base_url, call_count, ct) = common::compile_loop_openai_mock_three_turn::spawn_compile_loop_openai_mock_three_turn_server().await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let url_clone = base_url.clone();
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let target_file = tmp.path().join("target.lua");
common::agent_block_cmd()
.args([
"-s",
&common::fixture("compile_loop_openai_mock_three_turn.lua"),
])
.env("OPENAI_BASE_URL_TEST", &url_clone)
.env(
"COMPILE_LOOP_TARGET",
target_file.to_str().expect("utf8 path"),
)
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains("COMPILE_LOOP_MOCK_PASS"));
})
.await
.expect("subprocess assertion task (run 1) should not panic");
assert_eq!(
call_count.load(Ordering::SeqCst),
3,
"run 1: expected exactly 3 HTTP calls to the 3-turn mock (broken1, broken2, fixed)"
);
call_count.store(0, Ordering::SeqCst);
let url_clone2 = base_url.clone();
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let target_file = tmp.path().join("target.lua");
common::agent_block_cmd()
.args([
"-s",
&common::fixture("compile_loop_openai_mock_three_turn.lua"),
])
.env("OPENAI_BASE_URL_TEST", &url_clone2)
.env(
"COMPILE_LOOP_TARGET",
target_file.to_str().expect("utf8 path"),
)
.env("RUST_LOG", "off")
.assert()
.success()
.stdout(predicate::str::contains("COMPILE_LOOP_MOCK_PASS"));
})
.await
.expect("subprocess assertion task (run 2) should not panic");
assert_eq!(
call_count.load(Ordering::SeqCst),
3,
"run 2: expected exactly 3 HTTP calls to the 3-turn mock (broken1, broken2, fixed)"
);
ct.cancel();
}
#[tokio::test]
async fn compile_loop_anthropic_mock_emits_obs_events() {
let (base_url, call_count, ct) =
common::compile_loop_anthropic_mock::spawn_compile_loop_anthropic_mock_server().await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let url_clone = base_url.clone();
tokio::task::spawn_blocking(move || {
let tmp = tempdir().expect("tempdir");
let target_file = tmp.path().join("target.lua");
common::agent_block_cmd()
.args(["-s", &common::fixture("compile_loop_anthropic_mock.lua")])
.env("ANTHROPIC_BASE_URL_TEST", &url_clone)
.env(
"COMPILE_LOOP_TARGET",
target_file.to_str().expect("utf8 path"),
)
.env("AGENT_BLOCK_HOME", tmp.path())
.env("RUST_LOG", "info")
.env("AGENT_BLOCK_LLM_DUMP", "meta")
.assert()
.success()
.stdout(predicate::str::contains("COMPILE_LOOP_MOCK_PASS"))
.stdout(predicate::str::contains(
"prefix=ab.obs event=iter_start component=compile_loop",
))
.stdout(predicate::str::contains(
"prefix=ab.obs event=iter_result component=compile_loop",
))
.stdout(predicate::str::contains(
"prefix=ab.obs event=converged component=compile_loop",
));
})
.await
.expect("subprocess assertion task should not panic");
assert_eq!(
call_count.load(Ordering::SeqCst),
2,
"expected exactly 2 HTTP calls to the anthropic mock"
);
ct.cancel();
}