use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use io_harness::provider::{CompletionRequest, CompletionResponse, ToolCall, Usage};
use io_harness::{
resume_tree, resume_with_decision, ApproveAll, Containment, Decision, Policy, Provider,
RunOutcome, Store, TaskContract, Verification, CHECKPOINT_FORMAT,
};
use serde_json::{json, Value};
fn fixtures() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/store-0.22.0")
}
fn sidecar(name: &str) -> Value {
let path = fixtures().join(format!("{name}.json"));
let text =
std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read sidecar {path:?}: {e}"));
serde_json::from_str(&text).unwrap_or_else(|e| panic!("parse sidecar {path:?}: {e}"))
}
fn working_copy(name: &str, workspace: bool) -> (tempfile::TempDir, PathBuf, PathBuf) {
let dir = tempfile::tempdir().unwrap();
let db = dir.path().join(format!("{name}.sqlite3"));
let from = fixtures().join(format!("{name}.sqlite3"));
std::fs::copy(&from, &db).unwrap_or_else(|e| panic!("copy {from:?}: {e}"));
let ws = dir.path().join(format!("{name}-workspace"));
if workspace {
copy_dir(&fixtures().join(format!("{name}-workspace")), &ws);
}
(dir, db, ws)
}
fn copy_dir(from: &Path, to: &Path) {
std::fs::create_dir_all(to).unwrap();
for entry in std::fs::read_dir(from).unwrap_or_else(|e| panic!("read dir {from:?}: {e}")) {
let entry = entry.unwrap();
let dst = to.join(entry.file_name());
if entry.file_type().unwrap().is_dir() {
copy_dir(&entry.path(), &dst);
} else {
std::fs::copy(entry.path(), &dst).unwrap();
}
}
}
fn sqlite(db: &Path) -> rusqlite::Connection {
rusqlite::Connection::open(db).expect("open the store as a plain SQLite file")
}
fn schema(db: &Path) -> Vec<String> {
let conn = sqlite(db);
let mut stmt = conn
.prepare(
"SELECT sql FROM sqlite_master
WHERE sql IS NOT NULL AND type IN ('table', 'index') AND name NOT LIKE 'sqlite_%'",
)
.unwrap();
let mut sql: Vec<String> = stmt
.query_map([], |r| r.get::<_, String>(0))
.unwrap()
.map(|s| s.unwrap().split_whitespace().collect::<Vec<_>>().join(" "))
.collect();
sql.sort();
sql
}
fn assert_same_schema(left: &[String], right: &[String], what: &str) {
let only_left: Vec<&String> = left.iter().filter(|s| !right.contains(s)).collect();
let only_right: Vec<&String> = right.iter().filter(|s| !left.contains(s)).collect();
assert!(
only_left.is_empty() && only_right.is_empty(),
"{what}\n present only on the left: {only_left:#?}\n \
present only on the right: {only_right:#?}"
);
}
fn user_version(db: &Path) -> i64 {
sqlite(db)
.query_row("PRAGMA user_version", [], |r| r.get(0))
.unwrap()
}
fn call(name: &str, args: Value) -> ToolCall {
ToolCall {
name: name.into(),
arguments: args,
}
}
const RESUME_TOKENS: u64 = 25;
fn resume_usage() -> Usage {
Usage {
prompt_tokens: 20,
completion_tokens: 5,
total_tokens: RESUME_TOKENS,
..Default::default()
}
}
struct Finisher {
calls: AtomicUsize,
}
impl Provider for Finisher {
async fn complete(&self, req: CompletionRequest) -> io_harness::Result<CompletionResponse> {
self.calls.fetch_add(1, Ordering::SeqCst);
let tool = if req.user.contains("COORDINATOR") {
call(
"spawn_agent",
json!({
"goal": FIXUP_GOAL,
"verify_file": "b.txt",
"verify_contains": "BETA",
"max_steps": 2,
}),
)
} else {
call(
"write_file",
json!({ "path": "b.txt", "content": "BETA\n" }),
)
};
Ok(CompletionResponse {
tool_calls: vec![tool],
usage: Some(resume_usage()),
..Default::default()
})
}
}
const FIXUP_GOAL: &str = "finish b.txt with BETA";
struct WriteOnce {
path: &'static str,
content: &'static str,
}
impl Provider for WriteOnce {
async fn complete(&self, _req: CompletionRequest) -> io_harness::Result<CompletionResponse> {
Ok(CompletionResponse {
tool_calls: vec![call(
"write_file",
json!({ "path": self.path, "content": self.content }),
)],
usage: Some(resume_usage()),
..Default::default()
})
}
}
fn tree_contract(root: &Path, max_steps: u32) -> TaskContract {
TaskContract::workspace(
"COORDINATOR: delegate to sub-agents; do not write files yourself.",
root,
Verification::WorkspaceFileContains {
file: "b.txt".into(),
needle: "BETA".into(),
},
)
.with_max_steps(max_steps)
}
fn out_contract(root: &Path, needle: &str, max_steps: u32) -> TaskContract {
TaskContract::workspace(
"write out.txt",
root,
Verification::WorkspaceFileContains {
file: "out.txt".into(),
needle: needle.into(),
},
)
.with_max_steps(max_steps)
}
fn containment() -> Containment {
Containment::new(10, 4, 3, 1_000_000)
}
const NOT_PUBLICLY_READABLE: [&str; 2] = ["goal", "workspace"];
#[test]
fn a_0_22_0_store_reads_back_row_for_row_through_the_public_api() {
let (_dir, db, _) = working_copy("populated", false);
let expected = sidecar("populated");
let store = Store::open(&db).expect("a 0.22.0 store opens");
let run_id = store
.last_run()
.unwrap()
.expect("the 0.22.0 store holds a run");
let request_id = expected["pending"]["request_id"]
.as_i64()
.expect("the sidecar names the pending request");
let pending = store
.pending(request_id)
.unwrap()
.unwrap_or_else(|| panic!("pending approval {request_id} is gone from the 0.22.0 store"));
let actual = json!({
"run_id": run_id,
"outcome": store.outcome(run_id).unwrap(),
"status": store.status(run_id).unwrap(),
"step_count": store.steps(run_id).unwrap().len(),
"last_step": store.last_step(run_id).unwrap(),
"spent_tokens": store.spent_tokens(run_id).unwrap(),
"checkpoint_event_count": store.checkpoint_events(run_id).unwrap().len(),
"steps": store.steps(run_id).unwrap().iter()
.map(|s| json!({ "step": s.step, "tokens": s.tokens, "decision": s.decision }))
.collect::<Vec<_>>(),
"provider_calls": store.provider_calls(run_id).unwrap().iter()
.map(|c| json!({
"step": c.step,
"attempt": c.attempt,
"provider": c.provider,
"model": c.model,
"finish_reason": c.finish_reason,
"failure": c.failure,
"usage": c.usage,
}))
.collect::<Vec<_>>(),
"observations": store.observations(run_id).unwrap().iter()
.map(|o| json!({ "step": o.step, "kind": o.kind, "target": o.target }))
.collect::<Vec<_>>(),
"edits": store.edits(run_id).unwrap().iter()
.map(|e| json!({
"step": e.step,
"tool": e.tool,
"path": e.path,
"lines_added": e.lines_added,
"lines_removed": e.lines_removed,
}))
.collect::<Vec<_>>(),
"policy_events": store.events(run_id).unwrap().iter()
.map(|e| json!({
"step": e.step,
"kind": e.kind,
"act": e.act,
"target": e.target,
"rule": e.rule,
"layer": e.layer,
"decision": e.decision,
"source": e.source,
"performed": e.performed,
}))
.collect::<Vec<_>>(),
"pending": {
"request_id": pending.id,
"run_id": pending.run_id,
"step": pending.step,
"act": pending.act,
"target": pending.target,
"content": pending.content,
"resolved": pending.resolved,
},
"citations": store.citations(run_id).unwrap().iter()
.map(|c| json!({ "url": c.url, "title": c.title, "cited_text": c.cited_text }))
.collect::<Vec<_>>(),
"server_tool_calls": store.server_tool_calls(run_id).unwrap().iter()
.map(|c| json!({ "provider": c.provider, "tool": c.tool, "error": c.error }))
.collect::<Vec<_>>(),
});
let want = expected.as_object().unwrap();
let got = actual.as_object().unwrap();
for key in NOT_PUBLICLY_READABLE {
assert!(
want.contains_key(key),
"`{key}` is excused as unreadable but the sidecar does not record it — \
the exception list has drifted from the fixture"
);
}
for (key, expect) in want {
if NOT_PUBLICLY_READABLE.contains(&key.as_str()) {
continue;
}
let read_back = got.get(key).unwrap_or_else(|| {
panic!("the sidecar records `{key}` and this test reads nothing back for it")
});
assert_eq!(
read_back, expect,
"`{key}` read back by 0.23.0 is not what 0.22.0 stored — the rusqlite \
upgrade changed how this table round-trips"
);
}
assert_eq!(
got.len() + NOT_PUBLICLY_READABLE.len(),
want.len(),
"every key in populated.json is either asserted or named unreadable; \
read back {:?}, sidecar holds {:?}",
got.keys().collect::<Vec<_>>(),
want.keys().collect::<Vec<_>>()
);
}
#[test]
fn the_schema_this_release_creates_is_identical_to_the_one_0_22_0_created() {
let dir = tempfile::tempdir().unwrap();
let fresh = dir.path().join("fresh.sqlite3");
drop(Store::open(&fresh).expect("0.23.0 creates a store"));
let (_fixture_dir, old, _) = working_copy("populated", false);
let (new_schema, old_schema) = (schema(&fresh), schema(&old));
assert_same_schema(
&new_schema,
&old_schema,
"0.23.0 creates a different schema than 0.22.0 did — this release is \
documented as changing no table, and a divergence here means databases \
written from now on are not the ones the previous release can read",
);
assert!(
new_schema.len() > 20,
"the comparison is vacuous unless it actually found the schema (got {} statements)",
new_schema.len()
);
}
#[tokio::test]
async fn a_0_22_0_interrupted_tree_resumes_without_re_running_or_double_charging() {
let (_dir, db, ws) = working_copy("interrupted", true);
let expected = sidecar("interrupted");
let store = Store::open(&db).expect("a 0.22.0 tree store opens");
let root = expected["root_run_id"].as_i64().unwrap();
assert_eq!(
json!(store.tree_run_ids(root).unwrap()),
expected["tree_run_ids"],
"the tree 0.23.0 walks is not the tree 0.22.0 wrote"
);
assert_eq!(
json!(store.agent_count_tree(root).unwrap()),
expected["agent_count_tree"],
"the agent count differs from what 0.22.0 recorded"
);
assert_eq!(
json!(store.status(root).unwrap()),
expected["root_status"],
"the root's status differs from what 0.22.0 recorded"
);
assert_eq!(
json!(store.outcome(root).unwrap()),
expected["root_outcome"],
"the root's outcome differs — `status` alone would call a step-capped run \
'completed', so this is the one that says it stopped short"
);
assert_eq!(
json!(store.last_step(root).unwrap()),
expected["root_last_step"],
"the root's last committed step differs, so a resume would start in the \
wrong place"
);
let before_tree_tokens = store.spent_tokens_tree(root).unwrap();
assert_eq!(
json!(before_tree_tokens),
expected["spent_tokens_tree"],
"the partly-drawn tree budget differs from what 0.22.0 drew"
);
let children = expected["children"].as_array().unwrap();
for child in children {
let id = child["run_id"].as_i64().unwrap();
for (key, read_back) in [
("status", json!(store.status(id).unwrap())),
("outcome", json!(store.outcome(id).unwrap())),
("depth", json!(store.depth(id).unwrap())),
("last_step", json!(store.last_step(id).unwrap())),
("spent_tokens", json!(store.spent_tokens(id).unwrap())),
(
"wrote",
json!(store
.edits(id)
.unwrap()
.iter()
.map(|e| e.path.clone())
.collect::<Vec<_>>()),
),
] {
assert_eq!(
read_back, child[key],
"child {id}'s `{key}` differs from what 0.22.0 recorded"
);
}
}
assert_eq!(
json!({
"a.txt": std::fs::read_to_string(ws.join("a.txt")).ok(),
"b.txt": std::fs::read_to_string(ws.join("b.txt")).ok(),
}),
expected["workspace_files"],
"the interrupted workspace is not the one the fixture was committed with"
);
let done_child = children
.iter()
.find(|c| c["outcome"] == json!("success"))
.expect("the fixture has one child that finished");
let done_id = done_child["run_id"].as_i64().unwrap();
let done_steps = store.steps(done_id).unwrap().len();
let done_tokens = store.spent_tokens(done_id).unwrap();
let finisher = Finisher {
calls: AtomicUsize::new(0),
};
let result = resume_tree(
&tree_contract(&ws, 4),
&finisher,
&store,
root,
&Policy::permissive(),
&ApproveAll,
&containment(),
)
.await
.expect("a 0.22.0 checkpoint resumes under 0.23.0");
assert!(
matches!(result.outcome, RunOutcome::Success { .. }),
"the tree 0.22.0 interrupted must reach verified success here: {:?}",
result.outcome
);
assert_eq!(
store.outcome(root).unwrap().as_deref(),
Some("success"),
"the durable outcome agrees with the returned one"
);
for id in store.tree_run_ids(root).unwrap() {
let steps: Vec<u32> = store.steps(id).unwrap().iter().map(|s| s.step).collect();
let mut sorted = steps.clone();
sorted.sort_unstable();
sorted.dedup();
assert_eq!(
sorted.len(),
steps.len(),
"run {id} has a duplicate step number, so a step 0.22.0 had already \
committed was re-run across the boundary: {steps:?}"
);
}
assert_eq!(
store.steps(done_id).unwrap().len(),
done_steps,
"the child that had already finished gained a step on resume"
);
assert_eq!(
store.spent_tokens(done_id).unwrap(),
done_tokens,
"the child that had already finished was charged again"
);
let served = finisher.calls.load(Ordering::SeqCst) as u64;
assert_eq!(
served, 2,
"the resume half is expected to serve one coordinator completion and one \
child completion; a different number means the loop took a different \
path and the budget assertion below would be measuring something else"
);
assert_eq!(
store.spent_tokens_tree(root).unwrap(),
before_tree_tokens + served * RESUME_TOKENS,
"the tree's total must be what 0.22.0 drew plus exactly what this release \
drew — anything higher is a step charged twice, anything lower is a \
ledger that reset across the boundary"
);
let a = std::fs::read_to_string(ws.join("a.txt")).unwrap();
assert_eq!(
a, "ALPHA\n",
"the completed child's file was rewritten by the resume"
);
assert_eq!(
a.matches("ALPHA").count(),
1,
"the completed child's write was applied a second time (appended, not \
overwritten): {a:?}"
);
assert_eq!(
std::fs::read_to_string(ws.join("b.txt")).unwrap(),
"BETA\n",
"the half of the fan-out that 0.22.0 could not finish was finished here"
);
assert!(
store
.checkpoint_events(root)
.unwrap()
.iter()
.any(|e| e.kind == "resume"),
"the crossing is recorded in the trace as a resume"
);
}
#[tokio::test]
async fn a_0_22_0_deferred_approval_is_resolved_here_and_the_run_continues() {
let (_dir, db, ws) = working_copy("deferred-approval", true);
let expected = sidecar("deferred-approval");
let store = Store::open(&db).expect("a 0.22.0 paused store opens");
let run_id = expected["run_id"].as_i64().unwrap();
let request_id = expected["request_id"].as_i64().unwrap();
let pending = store
.pending(request_id)
.unwrap()
.unwrap_or_else(|| panic!("request {request_id} is unreadable by this release"));
assert_eq!(
json!({
"request_id": pending.id,
"run_id": pending.run_id,
"step": pending.step,
"act": pending.act,
"target": pending.target,
"content": pending.content,
"resolved": pending.resolved,
}),
expected["pending"],
"the deferred request does not read back as 0.22.0 stored it"
);
assert!(
pending.resolved.is_none(),
"an approval nobody has decided must read back undecided, not as an empty \
string or a default"
);
assert_eq!(
json!(store.status(run_id).unwrap()),
expected["status"],
"the run's status differs from what 0.22.0 recorded"
);
assert_eq!(
json!(store.outcome(run_id).unwrap()),
expected["outcome"],
"the run's outcome differs from what 0.22.0 recorded"
);
assert_eq!(
json!(store.last_step(run_id).unwrap()),
expected["last_step"],
"the paused run's last committed step differs"
);
assert!(
!ws.join("out.txt").exists(),
"a run that already performed its deferred action was never paused"
);
let policy = Policy::default()
.layer("base")
.allow_read("*")
.ask_write("out.txt");
let result = resume_with_decision(
&out_contract(&ws, "DONE", 8),
&WriteOnce {
path: "out.txt",
content: "DONE\n",
},
&store,
run_id,
request_id,
Decision::Approve {
modified: None,
remember: vec![],
},
&policy,
&ApproveAll,
)
.await
.expect("a decision delivered across the release boundary resumes the run");
assert!(
matches!(result.outcome, RunOutcome::Success { .. }),
"the run must continue to a terminal outcome once the approval is \
resolved: {:?}",
result.outcome
);
assert_eq!(
store
.pending(request_id)
.unwrap()
.unwrap()
.resolved
.as_deref(),
Some("approve"),
"the pending row is resolved, so a second resume cannot re-ask and \
re-perform the same action"
);
assert_eq!(
std::fs::read_to_string(ws.join("out.txt")).unwrap(),
"DONE\n",
"the approved action is the one that was pending, performed once"
);
assert_eq!(
store.status(run_id).unwrap().as_deref(),
Some("completed"),
"the run is no longer paused"
);
}
#[test]
fn the_checkpoint_format_is_still_7_and_opening_a_0_22_0_store_migrates_nothing() {
assert_eq!(
CHECKPOINT_FORMAT, 7,
"0.23.0 upgrades a driver and changes no layout; bumping the format would \
make check_resumable refuse every store 0.22.0 wrote"
);
for name in ["populated", "interrupted", "deferred-approval"] {
let (_dir, db, _) = working_copy(name, false);
let before_version = user_version(&db);
let before_schema = schema(&db);
assert_eq!(
before_version, CHECKPOINT_FORMAT,
"{name}.sqlite3 was stamped at {before_version} by 0.22.0, so it is not \
the fixture this test thinks it is"
);
drop(Store::open(&db).expect("0.23.0 opens the 0.22.0 store"));
assert_eq!(
user_version(&db),
before_version,
"opening {name}.sqlite3 changed its checkpoint format — a migration ran"
);
assert_same_schema(
&schema(&db),
&before_schema,
&format!(
"opening {name}.sqlite3 changed its schema — a migration ran, and a \
0.22.0 binary may no longer read this file"
),
);
}
}