use super::*;
use serde_json::json;
#[test]
fn incomplete_action_response_catches_forward_looking_narration() {
assert!(is_incomplete_action_response(
"Now let me check which module is actually registered/exported:"
));
assert!(is_incomplete_action_response("Let me read the file first."));
assert!(is_incomplete_action_response(
"I will inspect the registration next:"
));
assert!(!is_incomplete_action_response(
"The two files are duplicates. shell_exec is the registered tool; shell.rs is unused."
));
assert!(!is_incomplete_action_response(
"Let me summarize: parse_port now returns Result<u16, String> and main exits on error."
));
assert!(!is_incomplete_action_response(
"To summarize, the fix changes the return type and updates the caller."
));
assert!(!is_incomplete_action_response(
"I used file_read() to examine the module, found the off-by-one bug, and fixed it."
));
assert!(!is_incomplete_action_response(
"The shell_exec() call confirmed the tests pass; the change is complete."
));
assert!(is_incomplete_action_response(
"Next I'll call file_read( to inspect the registration before editing."
));
}
#[test]
fn explicit_visual_expectation_takes_priority() {
let args = json!({
"action": "click",
"expected_visual": "A confirmation dialog should be visible."
});
assert_eq!(
visual_verification_expectation("computer_mouse", &args).as_deref(),
Some("A confirmation dialog should be visible.")
);
}
#[test]
fn computer_window_launch_has_default_expectation() {
let args = json!({
"action": "launch",
"app_name": "Firefox"
});
let expectation = visual_verification_expectation("computer_window", &args).unwrap();
assert!(expectation.contains("Firefox"));
assert!(expectation.contains("visible"));
}
#[test]
fn non_window_actions_without_expectation_skip_visual_gate() {
let args = json!({
"action": "type",
"text": "hello"
});
assert!(visual_verification_expectation("computer_keyboard", &args).is_none());
}
#[cfg(test)]
mod completion_gate_tests {
use super::*;
use crate::checkpoint::{TaskCheckpoint, ToolCallLog};
use crate::config::Config;
fn test_config() -> Config {
let mut config = Config::default();
config.agent.min_completion_steps = 0;
config.agent.require_verification_before_completion = true;
config
}
async fn agent_with_checkpoint(tool_calls: Vec<ToolCallLog>) -> Agent {
let mut agent = Agent::new(test_config()).await.expect("agent should build");
let mut checkpoint = TaskCheckpoint::new("task_1".to_string(), "test task".to_string());
for tc in tool_calls {
checkpoint.log_tool_call(tc);
}
agent.current_checkpoint = Some(checkpoint);
agent.has_written_any_file = true;
agent
}
fn shell_exec(command: &str, success: bool) -> ToolCallLog {
ToolCallLog {
timestamp: chrono::Utc::now(),
tool_name: "shell_exec".to_string(),
arguments: serde_json::json!({"command": command}).to_string(),
result: Some(if success {
"ok".to_string()
} else {
"failed".to_string()
}),
success,
duration_ms: Some(100),
}
}
fn checkpoint_call(tool_name: &str, arguments: Value, success: bool) -> ToolCallLog {
ToolCallLog {
timestamp: chrono::Utc::now(),
tool_name: tool_name.to_string(),
arguments: arguments.to_string(),
result: Some(if success {
"ok".to_string()
} else {
"failed".to_string()
}),
success,
duration_ms: Some(10),
}
}
async fn artifact_agent(task: &str, tool_calls: Vec<ToolCallLog>) -> Agent {
artifact_agent_with_config(test_config(), task, tool_calls).await
}
async fn artifact_agent_with_config(
config: Config,
task: &str,
tool_calls: Vec<ToolCallLog>,
) -> Agent {
let mut agent = Agent::new(config).await.expect("agent should build");
let mut checkpoint = TaskCheckpoint::new("artifact_task".to_string(), task.to_string());
for tc in tool_calls {
checkpoint.log_tool_call(tc);
}
agent.current_task_context = task.to_string();
agent.current_checkpoint = Some(checkpoint);
agent.has_written_any_file = true;
agent.last_assistant_response = "Done.".to_string();
agent
}
fn git(dir: &Path, args: &[&str]) {
let status = std::process::Command::new("git")
.args(args)
.current_dir(dir)
.env("GIT_AUTHOR_NAME", "gate-test")
.env("GIT_AUTHOR_EMAIL", "gate-test@example.com")
.env("GIT_COMMITTER_NAME", "gate-test")
.env("GIT_COMMITTER_EMAIL", "gate-test@example.com")
.status()
.expect("git should be available for gate tests");
assert!(
status.success(),
"git {:?} failed in {}",
args,
dir.display()
);
}
fn git_repo(files: &[(&str, &str)]) -> (tempfile::TempDir, crate::test_support::CwdGuard) {
let dir = tempfile::tempdir().unwrap();
git(dir.path(), &["init", "-q"]);
for (path, content) in files {
let full = dir.path().join(path);
if let Some(parent) = full.parent() {
std::fs::create_dir_all(parent).unwrap();
}
std::fs::write(full, content).unwrap();
}
git(dir.path(), &["add", "-A"]);
let status = std::process::Command::new("git")
.args(["commit", "-q", "-m", "base"])
.current_dir(dir.path())
.env("GIT_AUTHOR_NAME", "gate-test")
.env("GIT_AUTHOR_EMAIL", "gate-test@example.com")
.env("GIT_COMMITTER_NAME", "gate-test")
.env("GIT_COMMITTER_EMAIL", "gate-test@example.com")
.env("GIT_AUTHOR_DATE", "2000-01-01T00:00:00Z")
.env("GIT_COMMITTER_DATE", "2000-01-01T00:00:00Z")
.status()
.expect("git should be available for gate tests");
assert!(
status.success(),
"base commit failed in {}",
dir.path().display()
);
let guard = crate::test_support::CwdGuard::enter(dir.path());
(dir, guard)
}
async fn mutation_task_agent(task: &str) -> Agent {
let mut agent = Agent::new(test_config()).await.expect("agent should build");
agent.current_task_context = task.to_string();
agent.current_checkpoint = Some(TaskCheckpoint::new(
"mutation_task".to_string(),
task.to_string(),
));
agent
}
#[tokio::test]
async fn non_code_live_flow_txt_accepts_normalized_fresh_readback() {
let agent = artifact_agent(
"Create user-check_1+2=3.txt using file_write. Do not run shell commands or use pty_shell.",
vec![
checkpoint_call(
"file_write",
json!({"path": "./user-check_1+2=3.txt", "content": "hello\n"}),
true,
),
checkpoint_call(
"file_read",
json!({"path": "user-check_1+2=3.txt"}),
true,
),
],
)
.await;
let readback = agent
.non_code_artifact_readback()
.expect("the task-owned text artifact should be recognized from tool logs");
assert!(readback.artifact_only);
assert!(readback.missing_paths.is_empty());
assert!(
agent.check_completion_gate().await.is_none(),
"a successful same-path readback should complete without build/test evidence"
);
}
#[tokio::test]
async fn non_code_edit_verified_via_shell_cat_completes() {
let agent = artifact_agent(
"Edit only notes.md: add a line, then verify with shell.",
vec![
checkpoint_call(
"file_write",
json!({"path": "./notes.md", "content": "hi\n"}),
true,
),
checkpoint_call("shell_exec", json!({"command": "cat notes.md"}), true),
],
)
.await;
let readback = agent
.non_code_artifact_readback()
.expect("the task-owned markdown artifact should be recognized");
assert!(
readback.artifact_only,
"markdown-only edit should be artifact_only"
);
assert!(
readback.missing_paths.is_empty(),
"a `cat <file>` shell read should count as read-back"
);
assert!(
agent.check_completion_gate().await.is_none(),
"a shell-verified non-code edit should complete without build/test evidence"
);
}
#[tokio::test]
async fn non_code_artifact_without_readback_guides_to_file_read_only() {
let agent = artifact_agent(
"Create notes.txt containing hello.",
vec![checkpoint_call(
"file_write",
json!({"path": "notes.txt", "content": "hello\n"}),
true,
)],
)
.await;
let message = agent
.check_completion_gate()
.await
.expect("missing readback must block completion");
assert!(message.contains("file_read"));
assert!(message.contains("notes.txt"));
assert!(!message.to_ascii_lowercase().contains("shell"));
assert!(!message.contains("cargo"));
assert!(!message.contains("pytest"));
}
#[tokio::test]
async fn non_code_readback_before_latest_write_is_stale() {
let agent = artifact_agent(
"Update notes.txt.",
vec![
checkpoint_call(
"file_write",
json!({"path": "notes.txt", "content": "first\n"}),
true,
),
checkpoint_call("file_read", json!({"path": "./notes.txt"}), true),
checkpoint_call(
"file_edit",
json!({"path": "notes.txt", "old_string": "first", "new_string": "final"}),
true,
),
],
)
.await;
let readback = agent
.non_code_artifact_readback()
.expect("the text artifact should be tracked");
assert_eq!(readback.missing_paths, vec!["notes.txt"]);
}
#[tokio::test]
async fn non_code_partial_or_failed_readback_does_not_count() {
for read_call in [
checkpoint_call(
"file_read",
json!({"path": "notes.txt", "line_range": [1, 1]}),
true,
),
checkpoint_call("file_read", json!({"path": "notes.txt"}), false),
] {
let agent = artifact_agent(
"Create notes.txt.",
vec![
checkpoint_call(
"file_write",
json!({"path": "notes.txt", "content": "hello\n"}),
true,
),
read_call,
],
)
.await;
let readback = agent
.non_code_artifact_readback()
.expect("the text artifact should be tracked");
assert_eq!(readback.missing_paths, vec!["notes.txt"]);
}
}
#[tokio::test]
async fn non_code_patch_target_uses_the_same_readback_policy() {
let agent = artifact_agent(
"Update CHANGELOG.md.",
vec![
checkpoint_call(
"patch_apply",
json!({
"diff": "--- a/CHANGELOG.md\n+++ b/CHANGELOG.md\n@@ -1 +1 @@\n-old\n+new\n"
}),
true,
),
checkpoint_call("file_read", json!({"path": "./CHANGELOG.md"}), true),
],
)
.await;
let readback = agent
.non_code_artifact_readback()
.expect("patch target should be recognized as a written artifact");
assert!(readback.artifact_only);
assert!(readback.missing_paths.is_empty());
}
#[tokio::test]
async fn non_code_readback_does_not_bypass_source_task_gates() {
let agent = artifact_agent(
"Fix the bug in src/lib.rs and create notes.txt.",
vec![
checkpoint_call(
"file_write",
json!({"path": "notes.txt", "content": "hello\n"}),
true,
),
checkpoint_call("file_read", json!({"path": "notes.txt"}), true),
],
)
.await;
let readback = agent
.non_code_artifact_readback()
.expect("the named text artifact should still be recognized");
assert!(!readback.artifact_only);
assert!(
agent.check_completion_gate().await.is_some(),
"artifact evidence must not loosen completion for a source repair task"
);
}
#[tokio::test]
async fn accepts_completion_after_successful_pytest() {
let agent = agent_with_checkpoint(vec![shell_exec("pytest tests/", true)]).await;
assert!(
agent.check_completion_gate().await.is_none(),
"completion should be accepted after a successful pytest shell_exec"
);
}
#[tokio::test]
async fn accepts_completion_after_successful_direct_python_test_run() {
let agent = agent_with_checkpoint(vec![shell_exec("python3 test_calc.py", true)]).await;
assert!(
agent.check_completion_gate().await.is_none(),
"completion should be accepted after a successful direct python test run"
);
}
#[tokio::test]
async fn rejects_completion_when_no_verification_tool_call_succeeded() {
let agent = agent_with_checkpoint(vec![]).await;
let result = agent.check_completion_gate().await;
assert!(
result.is_some(),
"completion should be rejected when no verification tool succeeded"
);
let msg = result.unwrap();
assert!(
msg.contains("pytest") || msg.contains("cargo_test") || msg.contains("npm test"),
"rejection message should mention verification examples: {}",
msg
);
}
#[tokio::test]
async fn rejects_completion_when_only_failing_pytest_exists() {
let agent = agent_with_checkpoint(vec![shell_exec("pytest tests/", false)]).await;
let result = agent.check_completion_gate().await;
assert!(
result.is_some(),
"completion should be rejected when the only verification attempt failed"
);
}
#[tokio::test]
async fn read_only_review_completes_even_when_answer_quotes_code() {
let mut agent = Agent::new(test_config()).await.expect("agent should build");
agent.current_task_context = "Review this module and report any bugs you find".to_string();
agent.has_written_any_file = false;
agent.last_assistant_response = "Review complete. One real bug in `foo`:\n\
```rust\nfn foo() { let x: i32 = parse(); use_it(x); }\n```\n\
`parse()` can fail and the error is ignored. That is my full assessment."
.to_string();
assert!(
agent.check_completion_gate().await.is_none(),
"a read-only review that quotes code must complete, not livelock on a file_write demand"
);
}
#[tokio::test]
async fn mutation_task_still_rejects_unwritten_code() {
let mut agent = Agent::new(test_config()).await.expect("agent should build");
agent.current_task_context =
"Fix the bug in foo() and implement the missing error handling".to_string();
agent.has_written_any_file = false;
agent.last_assistant_response = "Here is the fix:\n\
```rust\nfn foo() { let x = parse().unwrap_or(0); use_it(x); }\n```"
.to_string();
assert!(
agent.check_completion_gate().await.is_some(),
"a mutation task that only pastes code as text must still be rejected"
);
}
#[tokio::test]
async fn tool_requirement_appendix_does_not_flip_readonly_to_mutation() {
let mut agent = Agent::new(test_config()).await.expect("agent should build");
agent.current_task_context = "Summarize the auth module\n\n\
This task explicitly requires these tools before answering:\n\
- `file_edit`\n\
Do not answer until each required tool has been called successfully."
.to_string();
assert_eq!(
agent.task_context_for_classification(),
"Summarize the auth module",
"the tool-requirement appendix must be stripped before classification"
);
assert!(
!crate::agent::tool_dispatch::task_requires_mutation(
agent.task_context_for_classification()
),
"a read-only task must stay read-only despite a file_edit tool appendix"
);
}
#[tokio::test]
async fn no_source_edit_gate_skips_when_task_names_the_deliverable() {
let (_dir, _cwd) = git_repo(&[("deploy.sh", "#!/bin/sh\nexit 0\n")]);
std::fs::write("deploy.sh", "#!/bin/sh\necho deployed\nexit 0\n").unwrap();
let agent = mutation_task_agent("Update deploy.sh to print deployed").await;
assert!(
agent.mutation_completion_gate().await.is_none(),
"a task-named artifact deliverable must not be rejected as NoSourceEdit"
);
}
#[tokio::test]
async fn no_source_edit_gate_still_rejects_unnamed_non_source_diffs() {
let (_dir, _cwd) = git_repo(&[("deploy.sh", "#!/bin/sh\nexit 0\n")]);
std::fs::write("deploy.sh", "#!/bin/sh\necho deployed\nexit 0\n").unwrap();
let agent = mutation_task_agent("Fix the deployment automation").await;
let message = agent
.mutation_completion_gate()
.await
.expect("an unnamed non-source diff must still be rejected");
assert!(
message.contains("NoSourceEdit"),
"expected NoSourceEdit, got: {}",
message
);
}
#[tokio::test]
async fn test_only_patch_accepted_when_task_is_writing_tests() {
let (_dir, _cwd) = git_repo(&[("tests/test_calc.py", "def test_div():\n pass\n")]);
std::fs::write(
"tests/test_calc.py",
"def test_div():\n assert 6 / 2 == 3\n",
)
.unwrap();
let agent = mutation_task_agent("Write tests for the calc module").await;
assert!(
agent.mutation_completion_gate().await.is_none(),
"a test-only patch must be accepted for a test-writing task"
);
}
#[tokio::test]
async fn test_only_patch_still_rejected_for_source_repair_task() {
let (_dir, _cwd) = git_repo(&[("tests/test_calc.py", "def test_div():\n pass\n")]);
std::fs::write(
"tests/test_calc.py",
"def test_div():\n assert 6 / 2 == 3\n",
)
.unwrap();
let agent = mutation_task_agent("Fix the divide-by-zero bug in the calc module").await;
let message = agent
.mutation_completion_gate()
.await
.expect("a test-only patch must still be rejected for a repair task");
assert!(
message.contains("TestOnlyPatch"),
"expected TestOnlyPatch, got: {}",
message
);
}
#[tokio::test]
async fn workflow_validator_keeps_test_writing_exemption() {
let mut agent = mutation_task_agent("Write tests for the calc module").await;
agent.messages.push(crate::api::types::Message {
role: "assistant".to_string(),
content: crate::api::types::MessageContent::Text(String::new()),
reasoning_content: None,
tool_calls: Some(vec![crate::api::types::ToolCall {
id: "tc_test".to_string(),
call_type: "function".to_string(),
function: crate::api::types::ToolFunction {
name: "file_write".to_string(),
arguments: r#"{"path":"tests/test_calc.py","content":"x"}"#.to_string(),
},
}]),
tool_call_id: None,
name: None,
});
assert!(
agent.validate_workflow_edits().is_none(),
"a test-writing task must be allowed to edit only test files"
);
}
#[tokio::test]
async fn empty_diff_gate_counts_work_committed_during_the_run() {
let (_dir, _cwd) = git_repo(&[("README.md", "base\n")]);
let agent = mutation_task_agent("Fix the divide-by-zero bug in calc.py").await;
std::fs::write("calc.py", "def div(a, b):\n return a / b\n").unwrap();
git(Path::new("."), &["add", "calc.py"]);
git(
Path::new("."),
&["commit", "-q", "-m", "fix divide-by-zero"],
);
assert!(
agent.mutation_completion_gate().await.is_none(),
"work committed during the run must satisfy the EmptyDiff gate"
);
}
#[tokio::test]
async fn empty_diff_gate_still_rejects_when_nothing_changed() {
let (_dir, _cwd) = git_repo(&[("README.md", "base\n")]);
let agent = mutation_task_agent("Fix the divide-by-zero bug in calc.py").await;
let message = agent
.mutation_completion_gate()
.await
.expect("a clean tree with no run commits must still be EmptyDiff");
assert!(
message.contains("EmptyDiff"),
"expected EmptyDiff, got: {}",
message
);
}
#[tokio::test]
async fn untracked_file_creation_satisfies_empty_diff_gate() {
let (_dir, _cwd) = git_repo(&[("README.md", "base\n")]);
let agent = mutation_task_agent("Create hello.py that prints hello").await;
std::fs::write("hello.py", "print('hello')\n").unwrap();
assert!(
agent.mutation_completion_gate().await.is_none(),
"a newly created untracked source file must satisfy the EmptyDiff gate"
);
}
#[tokio::test]
async fn gitignored_untracked_file_does_not_satisfy_empty_diff_gate() {
let (_dir, _cwd) = git_repo(&[("README.md", "base\n"), (".gitignore", "scratch.py\n")]);
let agent = mutation_task_agent("Fix the divide-by-zero bug in calc.py").await;
std::fs::write("scratch.py", "x = 1\n").unwrap();
let message = agent
.mutation_completion_gate()
.await
.expect("an ignored untracked file must not count as a change");
assert!(
message.contains("EmptyDiff"),
"expected EmptyDiff, got: {}",
message
);
}
#[tokio::test]
async fn non_rust_direct_test_run_satisfies_stale_verification_gate() {
let (_dir, _cwd) = git_repo(&[("calc.py", "def div(a, b):\n return a // b\n")]);
let mut agent = mutation_task_agent("Fix the divide-by-zero bug in calc.py").await;
std::fs::write("calc.py", "def div(a, b):\n return a / b\n").unwrap();
agent.note_mutating_tool_call();
let args = serde_json::json!({"command": "python3 test_calc.py"});
if crate::agent::tool_dispatch::tool_call_is_mutating("shell_exec", &args) {
agent.note_mutating_tool_call();
}
agent.note_verification_outcome("shell_exec", &args.to_string(), true, "1 passed");
assert!(
agent.mutation_completion_gate().await.is_none(),
"a direct run of the project's own passing test must satisfy the gate"
);
}
#[tokio::test]
async fn verified_trivial_artifact_task_completes_before_min_steps() {
let mut config = test_config();
config.agent.min_completion_steps = 3;
let agent = artifact_agent_with_config(
config,
"Create notes.txt containing hello.",
vec![
checkpoint_call(
"file_write",
json!({"path": "notes.txt", "content": "hello\n"}),
true,
),
checkpoint_call("file_read", json!({"path": "notes.txt"}), true),
],
)
.await;
assert_eq!(
agent.loop_control.current_step(),
0,
"the test agent must be below min_completion_steps"
);
assert!(
agent.check_completion_gate().await.is_none(),
"a verified-complete trivial task must not be taxed up to min_completion_steps"
);
}
#[tokio::test]
async fn unverified_artifact_task_is_still_blocked_with_min_steps() {
let mut config = test_config();
config.agent.min_completion_steps = 3;
let agent = artifact_agent_with_config(
config,
"Create notes.txt containing hello.",
vec![checkpoint_call(
"file_write",
json!({"path": "notes.txt", "content": "hello\n"}),
true,
)],
)
.await;
let message = agent
.check_completion_gate()
.await
.expect("an unverified artifact must not complete");
assert!(
message.contains("file_read"),
"expected read-back guidance, got: {}",
message
);
}
}