use crate::harness::{Harness, interpretation_tests::stream};
use serde_json::{Value, json};
#[test]
fn review_claude_permission_denial_metadata_overrides_prior_tool_error() {
use serde_json::json;
let mut decoder =
crate::harness::interpretation_tests::stream(crate::harness::Harness::Claude, "");
let mut events = Vec::new();
for record in [
json!({"type":"assistant","message":{"id":"m1","content":[{"type":"tool_use","id":"deny","name":"Bash","input":{"command":"git push"}}]}}),
json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"deny","content":"Tool use rejected by policy.","is_error":true}]}}),
json!({"type":"result","subtype":"success","is_error":false,"permission_denials":[{"tool_use_id":"deny","tool_name":"Bash","tool_input":{"command":"git push"}}]}),
] {
events.extend(decoder.decode(&record.to_string()).events);
}
let events = serde_json::to_value(events).unwrap();
let events = events.as_array().unwrap();
let last_end = events
.iter()
.rfind(|event| event["type"] == "tool_call_ended" && event["id"] == "deny")
.unwrap();
assert_eq!(last_end["outcome"], "denied", "{events:?}");
}
#[test]
fn late_denial_metadata_amends_the_same_call_once() {
let denial = json!({"type":"result","subtype":"success","permission_denials":[{"tool_use_id":"deny","tool_name":"Bash","tool_input":{"command":"git push"}}]});
let events = decoded(
Harness::Claude,
[
json!({"type":"assistant","message":{"id":"m1","content":[{"type":"tool_use","id":"deny","name":"Bash","input":{"command":"git push"}}]}}),
json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"deny","content":"opaque failure","is_error":true}]}}),
denial.clone(),
denial,
],
);
assert_eq!(
events
.iter()
.filter(|event| event["type"] == "tool_call_started")
.count(),
1
);
let ends: Vec<_> = events
.iter()
.filter(|event| event["type"] == "tool_call_ended")
.collect();
assert_eq!(ends.len(), 2);
assert_eq!(ends[0]["outcome"], "failed");
assert_eq!(ends[0]["amended"], false);
assert_eq!(ends[1]["id"], "deny");
assert_eq!(ends[1]["outcome"], "denied");
assert_eq!(ends[1]["amended"], true);
}
#[test]
#[ignore = "Codex exec has no per-request boundaries; Spec decision: https://github.com/JacobStephens2/thirdshift/issues/1047"]
fn review_codex_fixture_contains_model_call_lifecycles() {
let mut decoder =
crate::harness::interpretation_tests::stream(crate::harness::Harness::Codex, "");
let mut events = Vec::new();
for raw in include_str!("../../../tests/fixtures/normalized-codex.jsonl").lines() {
events.extend(decoder.decode(raw).events);
}
let events = serde_json::to_value(events).unwrap();
let events = events.as_array().unwrap();
let starts: Vec<_> = events
.iter()
.filter(|event| event["type"] == "model_call_started")
.collect();
assert!(
!starts.is_empty(),
"The successful recorded Codex turn reports usage, but no model calls: {events:?}"
);
for start in starts {
assert!(
events
.iter()
.any(|event| { event["type"] == "model_call_ended" && event["id"] == start["id"] })
);
}
}
#[test]
fn api_error_envelopes_do_not_invent_model_calls_and_fail_a_started_request() {
let events = decoded(
Harness::Claude,
[
json!({"type":"assistant","error":"rate_limit","message":{"model":"<synthetic>","content":[{"type":"text","text":"limited"}]}}),
json!({"type":"stream_event","event":{"type":"message_start","message":{"id":"real","model":"claude-opus-4-8","content":[],"usage":{"input_tokens":8}}}}),
json!({"type":"assistant","api_error":"usage_limit_reached","message":{"content":[{"type":"text","text":"usage exhausted"}]}}),
json!({"type":"result","subtype":"success"}),
],
);
let calls: Vec<_> = events
.iter()
.filter(|event| {
event["type"] == "model_call_started" || event["type"] == "model_call_ended"
})
.collect();
assert_eq!(calls.len(), 2);
assert_eq!(calls[0]["id"], "real");
assert_eq!(calls[1]["id"], "real");
assert_eq!(calls[1]["outcome"], "failed");
}
#[test]
fn stopping_an_agent_closes_descendants_and_preserves_other_agents() {
let events = decoded(
Harness::Claude,
[
json!({"type":"assistant","message":{"id":"root","content":[{"type":"tool_use","id":"a1","name":"Agent","input":{"run_in_background":true}},{"type":"tool_use","id":"a2","name":"Agent","input":{"run_in_background":true}}]}}),
json!({"type":"system","subtype":"task_started","task_id":"task1","tool_use_id":"a1","task_type":"local_agent"}),
json!({"type":"system","subtype":"task_started","task_id":"task2","tool_use_id":"a2","task_type":"local_agent"}),
json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"a1","content":"launched"},{"type":"tool_result","tool_use_id":"a2","content":"launched"}]}}),
json!({"type":"assistant","parent_tool_use_id":"a1","message":{"id":"child1","content":[{"type":"tool_use","id":"g1","name":"Agent","input":{"run_in_background":true}},{"type":"tool_use","id":"t1","name":"Bash","input":{"command":"sleep 100"}}]}}),
json!({"type":"assistant","parent_tool_use_id":"g1","message":{"id":"grandchild","content":[{"type":"tool_use","id":"tg","name":"Bash","input":{"command":"sleep 200"}}]}}),
json!({"type":"assistant","parent_tool_use_id":"a2","message":{"id":"child2","content":[{"type":"tool_use","id":"t2","name":"Bash","input":{"command":"sleep 300"}}]}}),
json!({"type":"system","subtype":"task_notification","task_id":"task1","status":"stopped","summary":"Terminated"}),
json!({"type":"result","subtype":"success"}),
],
);
for id in ["t1", "tg", "g1"] {
assert!(events.iter().any(|event| event["type"] == "tool_call_ended"
&& event["id"] == id
&& event["outcome"] == "killed"));
}
for id in ["child1", "grandchild"] {
assert!(
events
.iter()
.any(|event| event["type"] == "model_call_ended"
&& event["id"] == id
&& event["outcome"] == "killed")
);
}
assert!(
!events
.iter()
.any(|event| event["type"] == "tool_call_ended" && event["id"] == "t2")
);
assert!(
!events
.iter()
.any(|event| event["type"] == "model_call_ended" && event["id"] == "child2")
);
assert!(
!events
.iter()
.any(|event| event["type"] == "subagent_ended" && event["id"] == "a2")
);
}
#[test]
fn claude_result_metadata_records_denials_and_interrupted_work() {
let events = decoded(
Harness::Claude,
[
json!({"type":"assistant","message":{"id":"m1","content":[{"type":"tool_use","id":"deny","name":"Bash","input":{"command":"git push"}},{"type":"tool_use","id":"active","name":"Bash","input":{"command":"sleep 100"}}],"usage":{"input_tokens":8,"output_tokens":4}}}),
json!({"type":"result","subtype":"success","is_error":false,"terminal_reason":"aborted_tools","permission_denials":[{"tool_use_id":"deny","tool_name":"Bash","tool_input":{"command":"git push"}}]}),
],
);
assert!(events.iter().any(|event| event["type"] == "tool_call_ended"
&& event["id"] == "deny"
&& event["outcome"] == "denied"));
assert!(events.iter().any(|event| event["type"] == "tool_call_ended"
&& event["id"] == "active"
&& event["outcome"] == "interrupted"));
assert!(
events
.iter()
.any(|event| event["type"] == "model_call_ended"
&& event["id"] == "m1"
&& event["outcome"] == "interrupted")
);
assert!(events.contains(&json!({"type":"work_ended","id":"active","outcome":"interrupted"})));
}
#[test]
fn review_claude_terminated_subagent_closes_its_calls() {
use serde_json::json;
for (status, outcome) in [("stopped", "killed"), ("interrupted", "interrupted")] {
let mut decoder =
crate::harness::interpretation_tests::stream(crate::harness::Harness::Claude, "");
let mut events = Vec::new();
for record in [
json!({"type":"assistant","message":{"id":"root","content":[{"type":"tool_use","id":"a1","name":"Agent","input":{"description":"Review","run_in_background":true}}]}}),
json!({"type":"system","subtype":"task_started","task_id":"agent-task","tool_use_id":"a1","task_type":"local_agent"}),
json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"a1","content":"Async agent launched"}]}}),
json!({"type":"assistant","parent_tool_use_id":"a1","message":{"id":"child","model":"claude-opus-4-8","content":[{"type":"tool_use","id":"t1","name":"Bash","input":{"command":"sleep 100"}}],"usage":{"input_tokens":8,"output_tokens":4}}}),
json!({"type":"system","subtype":"task_notification","task_id":"agent-task","status":status,"summary":"Terminated"}),
json!({"type":"result","subtype":"success","result":"Done"}),
] {
events.extend(decoder.decode(&record.to_string()).events);
}
let events = serde_json::to_value(events).unwrap();
let events = events.as_array().unwrap();
assert!(events.iter().any(|event| {
event["type"] == "subagent_ended" && event["id"] == "a1" && event["outcome"] == outcome
}));
for (event_type, id) in [("tool_call_ended", "t1"), ("model_call_ended", "child")] {
assert!(
events.iter().any(|event| {
event["type"] == event_type && event["id"] == id && event["outcome"] == outcome
}),
"Missing {event_type} for {id} after {status}: {events:?}"
);
}
}
}
fn decoded(harness: Harness, records: impl IntoIterator<Item = Value>) -> Vec<Value> {
let mut decoder = stream(harness, "");
records
.into_iter()
.flat_map(|record| {
decoder
.decode(&record.to_string())
.events
.into_iter()
.map(|event| serde_json::to_value(event).unwrap())
.collect::<Vec<_>>()
})
.collect()
}
#[test]
fn codex_final_snapshots_keep_inputs_and_turn_end_stops_open_work_and_subagents() {
let events = decoded(
Harness::Codex,
include_str!("../../../tests/fixtures/normalized-codex.jsonl")
.lines()
.map(|line| serde_json::from_str(line).unwrap()),
);
let web = events
.iter()
.find(|event| event["type"] == "tool_call_ended" && event["id"] == "web")
.unwrap();
assert_eq!(web["input"]["query"], "rust docs");
let build = events
.iter()
.find(|event| event["type"] == "tool_call_ended" && event["id"] == "build")
.unwrap();
assert_eq!(build["outcome"], "killed");
assert!(
events.contains(
&json!({"type":"subagent_ended","id":"child","outcome":"killed","result":null})
)
);
assert!(
events.contains(&json!({"type":"work_ended","id":"interrupted","outcome":"interrupted"}))
);
assert!(
events
.iter()
.any(|event| event["type"] == "error" && event["kind"] == "rate_limit")
);
assert!(events.iter().any(|event| event["type"] == "turn_ended"
&& event["id"] == "turn-2"
&& event["outcome"] == "interrupted"));
}
#[test]
fn claude_partial_frames_close_one_model_call_with_the_final_usage() {
let events = decoded(
Harness::Claude,
include_str!("../../../tests/fixtures/normalized-claude.jsonl")
.lines()
.map(|line| serde_json::from_str(line).unwrap()),
);
assert_eq!(
events
.iter()
.map(|event| event["type"].as_str().unwrap())
.collect::<Vec<_>>(),
["model_call_started", "message", "model_call_ended", "usage"]
);
assert_eq!(
events[2]["usage"],
json!({"input":8,"output":4,"cache_read":2,"cache_write":3})
);
assert_eq!(events[3]["cost_usd"].as_f64(), Some(0.0));
}
#[test]
fn grok_messages_keep_usage_but_zero_cost_stays_unknown_and_result_text_is_not_lost() {
let mut decoder = stream(Harness::Grok, "");
let mut events = Vec::new();
let mut progress = Vec::new();
for raw in include_str!("../../../tests/fixtures/normalized-grok.jsonl").lines() {
let update = decoder.decode(raw);
progress.extend(update.progress);
events.extend(
update
.events
.into_iter()
.map(|event| serde_json::to_value(event).unwrap()),
);
}
assert_eq!(progress, ["session started", "$ cargo check"]);
assert_eq!(
events
.iter()
.filter(|event| event["type"] == "model_call_started")
.count(),
2
);
assert_eq!(
events
.iter()
.filter(|event| event["type"] == "model_call_ended")
.count(),
2
);
assert!(events.contains(&json!({"type":"message","role":"assistant","text":"Final reply from Grok.","parent_id":null})));
let usage = events
.iter()
.find(|event| event["type"] == "usage")
.unwrap();
assert_eq!(usage["cost_usd"], Value::Null);
assert_eq!(
usage["usage"],
json!({"input":20,"output":6,"cache_read":2,"cache_write":3})
);
let completion = crate::harness::interpretation_tests::finish(decoder);
assert_eq!(
completion.outcome.unwrap().final_message.as_deref(),
Some("Final reply from Grok.")
);
}
#[test]
fn codex_normalizes_item_lifecycles_and_keeps_turn_totals_distinct_from_model_calls() {
let events = decoded(
Harness::Codex,
[
json!({"type":"thread.started","thread_id":"session"}),
json!({"type":"turn.started"}),
json!({"type":"item.started","item":{"id":"c1","type":"command_execution","command":"cargo test","status":"in_progress"}}),
json!({"type":"item.updated","item":{"id":"c1","type":"command_execution","command":"cargo test","status":"in_progress","aggregated_output":"running"}}),
json!({"type":"item.completed","item":{"id":"c1","type":"command_execution","command":"cargo test","status":"failed","exit_code":1,"aggregated_output":"assertion failed"}}),
json!({"type":"item.completed","item":{"id":"patch","type":"file_change","changes":[{"path":"src/main.rs","kind":"update"}],"status":"completed"}}),
json!({"type":"item.completed","item":{"id":"deny","type":"command_execution","command":"git push","status":"declined"}}),
json!({"type":"item.started","item":{"id":"mcp","type":"mcp_tool_call","server":"files","tool":"read","arguments":{"path":"README.md"},"status":"in_progress"}}),
json!({"type":"item.completed","item":{"id":"mcp","type":"mcp_tool_call","server":"files","tool":"read","arguments":{"path":"README.md"},"error":{"message":"unavailable"},"status":"failed"}}),
json!({"type":"item.started","item":{"id":"web","type":"web_search","query":"rust docs"}}),
json!({"type":"item.completed","item":{"id":"web","type":"web_search","query":"rust docs","results":[{"url":"https://doc.rust-lang.org"}]}}),
json!({"type":"item.started","item":{"id":"spawn","type":"collab_tool_call","tool":"spawn_agent","sender_thread_id":"session","receiver_thread_ids":[],"status":"in_progress"}}),
json!({"type":"item.completed","item":{"id":"spawn","type":"collab_tool_call","tool":"spawn_agent","sender_thread_id":"session","receiver_thread_ids":["child"],"agents_states":{"child":{"status":"running"}},"status":"completed"}}),
json!({"type":"item.completed","item":{"id":"wait","type":"collab_tool_call","tool":"wait","receiver_thread_ids":["child"],"agents_states":{"child":{"status":"completed","message":"Reviewed."}},"status":"completed"}}),
json!({"type":"item.started","item":{"id":"background","type":"command_execution","command":"cargo build","status":"in_progress"}}),
json!({"type":"item.completed","item":{"id":"reply","type":"agent_message","text":"Done."}}),
json!({"type":"turn.completed","usage":{"input_tokens":100,"output_tokens":7,"cached_input_tokens":20,"cache_write_input_tokens":3}}),
],
);
let starts: Vec<_> = events
.iter()
.filter(|event| event["type"] == "tool_call_started")
.map(|event| event["id"].as_str().unwrap())
.collect();
let ends: Vec<_> = events
.iter()
.filter(|event| event["type"] == "tool_call_ended")
.map(|event| event["id"].as_str().unwrap())
.collect();
assert_eq!(
starts,
[
"c1",
"patch",
"deny",
"mcp",
"web",
"spawn",
"wait",
"background"
]
);
assert_eq!(ends, starts);
assert_eq!(
events
.iter()
.find(|event| event["type"] == "tool_call_ended" && event["id"] == "c1")
.unwrap()["outcome"],
"failed"
);
assert_eq!(
events
.iter()
.find(|event| event["type"] == "tool_call_ended" && event["id"] == "deny")
.unwrap()["outcome"],
"denied"
);
assert_eq!(
events
.iter()
.find(|event| event["type"] == "tool_call_ended" && event["id"] == "mcp")
.unwrap()["result"],
json!({"message":"unavailable"})
);
assert_eq!(
events
.iter()
.filter(|event| event["type"] == "subagent_started")
.count(),
1
);
assert!(events.contains(
&json!({"type":"subagent_ended","id":"child","outcome":"succeeded","result":"Reviewed."})
));
assert!(events.contains(&json!({"type":"work_ended","id":"background","outcome":"killed"})));
assert!(
events.contains(
&json!({"type":"message","role":"assistant","text":"Done.","parent_id":null})
)
);
assert!(events.contains(&json!({"type":"turn_ended","id":"turn-1","usage":{"input":100,"output":7,"cache_read":20,"cache_write":3},"outcome":"succeeded"})));
assert!(
!events
.iter()
.any(|event| event["type"] == "model_call_started")
);
}
#[test]
fn claude_keeps_denials_limits_killed_tasks_and_background_subagent_lifecycles() {
let events = decoded(
Harness::Claude,
[
json!({"type":"assistant","message":{"id":"m1","content":[{"type":"tool_use","id":"a1","name":"Agent","input":{"description":"Review","run_in_background":true}}]}}),
json!({"type":"system","subtype":"task_started","task_id":"agent-task","tool_use_id":"a1","task_type":"local_agent","description":"Review"}),
json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"a1","content":"Async agent launched"}]}}),
json!({"type":"assistant","message":{"id":"m2","content":[{"type":"tool_use","id":"t1","name":"Bash","input":{"command":"git push"}}]}}),
json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1","content":"Permission to use Bash has been denied.","is_error":true}]}}),
json!({"type":"result","subtype":"success"}),
json!({"type":"system","subtype":"task_notification","task_id":"agent-task","status":"completed","summary":"Reviewed."}),
json!({"type":"system","subtype":"task_started","task_id":"build","task_type":"local_bash","description":"cargo test"}),
json!({"type":"system","subtype":"task_updated","task_id":"build","patch":{"status":"killed"}}),
json!({"type":"system","subtype":"task_notification","task_id":"build","status":"stopped"}),
json!({"type":"assistant","is_api_error_message":true,"api_error":"usage_limit_reached","api_error_status":429,"api_error_params":{"rate_limit_info":{"resetsAt":1791597600}},"message":{"content":[{"type":"text","text":"You've hit your limit"}]}}),
json!({"type":"rate_limit_event","rate_limit_info":{"status":"rejected","resetsAt":1791597600}}),
],
);
assert!(events.iter().any(|event| event["type"] == "tool_call_ended"
&& event["id"] == "t1"
&& event["outcome"] == "denied"));
assert_eq!(
events
.iter()
.filter(|event| event["type"] == "subagent_started")
.count(),
1
);
let ended: Vec<_> = events
.iter()
.filter(|event| event["type"] == "subagent_ended")
.collect();
assert_eq!(
ended,
[&json!({"type":"subagent_ended","id":"a1","outcome":"succeeded","result":"Reviewed."})]
);
assert_eq!(
events
.iter()
.filter(|event| event["type"] == "work_ended")
.collect::<Vec<_>>(),
[&json!({"type":"work_ended","id":"build","outcome":"killed"})]
);
let errors: Vec<_> = events
.iter()
.filter(|event| event["type"] == "error")
.collect();
assert_eq!(errors.len(), 2);
assert_eq!(errors[0]["kind"], "usage_limit");
assert_eq!(errors[0]["details"]["api_error_status"], 429);
assert_eq!(errors[1]["kind"], "rate_limit");
assert_eq!(
errors[1]["details"]["rate_limit_info"]["resetsAt"],
1791597600_u64
);
}
#[test]
fn claude_normalizes_calls_messages_and_reported_usage_without_changing_progress() {
let mut decoder = stream(Harness::Claude, "");
let mut events = Vec::new();
let mut progress = Vec::new();
for raw in [
json!({"type":"assistant","message":{"id":"m1","model":"claude-opus-4-8","role":"assistant","content":[{"type":"text","text":"Checking."}],"usage":{"input_tokens":8,"output_tokens":1,"cache_read_input_tokens":2}}}),
json!({"type":"assistant","message":{"id":"m1","model":"claude-opus-4-8","role":"assistant","content":[{"type":"tool_use","id":"t1","name":"Bash","input":{"command":"cargo check"}}],"usage":{"input_tokens":8,"output_tokens":4,"cache_read_input_tokens":2,"cache_creation_input_tokens":3}}}),
json!({"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","content":"ok","is_error":false}]}}),
json!({"type":"assistant","message":{"id":"m2","model":"claude-opus-4-8","role":"assistant","content":[{"type":"text","text":"Done."}],"usage":{"input_tokens":12,"output_tokens":2}}}),
json!({"type":"result","subtype":"success","result":"Done.","total_cost_usd":0.25,"usage":{"input_tokens":20,"output_tokens":6}}),
] {
let update = decoder.decode(&raw.to_string());
progress.extend(update.progress);
events.extend(update.events);
}
assert_eq!(progress, ["$ cargo check"]);
let events: Value = serde_json::to_value(events).unwrap();
assert_eq!(
events,
json!([
{"type":"model_call_started","id":"m1","parent_id":null,"model":"claude-opus-4-8"},
{"type":"message","role":"assistant","text":"Checking.","parent_id":null},
{"type":"tool_call_started","id":"t1","parent_id":null,"tool":"Bash","input":{"command":"cargo check"}},
{"type":"model_call_ended","id":"m1","parent_id":null,"model":"claude-opus-4-8","usage":{"input":8,"output":4,"cache_read":2,"cache_write":3},"cost_usd":null,"outcome":"succeeded"},
{"type":"tool_call_ended","id":"t1","parent_id":null,"tool":"Bash","input":{"command":"cargo check"},"result":"ok","outcome":"succeeded","amended":false},
{"type":"model_call_started","id":"m2","parent_id":null,"model":"claude-opus-4-8"},
{"type":"message","role":"assistant","text":"Done.","parent_id":null},
{"type":"model_call_ended","id":"m2","parent_id":null,"model":"claude-opus-4-8","usage":{"input":12,"output":2,"cache_read":null,"cache_write":null},"cost_usd":null,"outcome":"succeeded"},
{"type":"usage","scope":"session","model":null,"usage":{"input":20,"output":6,"cache_read":null,"cache_write":null},"cost_usd":0.25}
])
);
assert_eq!(decoder.observed_models(), ["claude-opus-4-8"]);
}