use std::{
path::Path,
time::{Instant, SystemTime, UNIX_EPOCH},
};
use murmur_artifact::{ContainmentClass, TraceCapture};
use serde::Serialize;
use serde_json::Value;
use tokio::{
fs::{File, OpenOptions},
io::{AsyncWriteExt, BufWriter},
};
use crate::{
agent::DriverUsage,
containment::ScopeReport,
inference_credential::CredentialChange,
lanes::TaskLane,
origin::{TaskProvenance, TrustClass},
trace_blobs::BlobStore,
};
pub(crate) struct TraceWriter {
writer: BufWriter<File>,
session_id: String,
capsule_name: String,
capsule_version: String,
model: String,
capabilities: Vec<String>,
effective_grants: ScopeReport,
capture: TraceCapture,
blobs: BlobStore,
system_prompt_source: &'static str,
system_prompt_sha256: Option<String>,
credential_source: &'static str,
resumed_from: Option<String>,
context_id: Option<String>,
spawned_by: Option<String>,
spawn_delegation_id: Option<String>,
session_start_time: Instant,
session_event_id: String,
session_started: bool,
session_ended: bool,
task_event_id: Option<String>,
turn_event_id: Option<String>,
total_turns: u32,
total_input_tokens: u64,
total_output_tokens: u64,
total_tool_calls: u32,
total_shell_calls: u32,
task_turns: u32,
task_input_tokens: u64,
task_output_tokens: u64,
task_tool_calls: u32,
task_shell_calls: u32,
task_start_instant: Option<Instant>,
pub(crate) active_task_id: Option<String>,
max_session_tokens: Option<u64>,
machine_tokens_per_day: Option<u64>,
}
#[derive(Debug, Clone)]
pub(crate) struct InferenceOrigin {
pub(crate) source: String,
pub(crate) model: String,
}
pub(crate) const COMPACTION_DECLINED_NO_HOOK_REPLACEMENT: &str = "no_hook_replacement";
pub(crate) const COMPACTION_DECLINED_UNRESOLVED_TOOL_CALL: &str = "unresolved_tool_call";
pub(crate) const SEED_OUTCOME_SEEDED: &str = "seeded";
pub(crate) const SEED_OUTCOME_TRIMMED: &str = "trimmed";
pub(crate) const SEED_OUTCOME_COMPACTED: &str = "compacted";
pub(crate) const SEED_OUTCOME_REJECTED: &str = "rejected";
pub(crate) const SEED_REJECTED_MESSAGE_OVER_BUDGET: &str = "message_over_budget";
pub(crate) const SEED_REJECTED_OVERFLOW_OVER_LIMIT: &str = "overflow_over_limit";
pub(crate) const SEED_REJECTED_NO_BUDGET: &str = "no_budget";
pub(crate) const SEED_REJECTED_UNSUPPORTED_TRANSPORT: &str = "unsupported_transport";
pub(crate) const RETENTION_STORE_SESSIONS: &str = "sessions";
pub(crate) const RETENTION_STORE_RECORDS: &str = "records";
pub(crate) const RETENTION_REASON_MAX_SESSIONS: &str = "max_sessions";
pub(crate) const RETENTION_REASON_MAX_AGE: &str = "max_age";
pub(crate) const RETENTION_REASON_MAX_MESSAGES: &str = "max_messages";
fn new_event_id() -> String {
format!("evt_{}", uuid::Uuid::now_v7().simple())
}
#[derive(Serialize)]
struct SessionStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
capsule_name: String,
capsule_version: String,
model: String,
max_turns: u32,
max_session_tokens: Option<u64>,
machine_tokens_per_day: Option<u64>,
capabilities: Vec<String>,
tools_declared: Vec<String>,
containment_declared: ContainmentClass,
containment_achieved: ContainmentClass,
workdir_exec: bool,
userns_grant: Option<crate::sealed::UsernsGrant>,
effective_grants: ScopeReport,
system_prompt_source: &'static str,
system_prompt_sha256: Option<String>,
credential_source: &'static str,
resumed_from: Option<String>,
context_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
spawned_by: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
delegation_id: Option<String>,
}
#[derive(Serialize)]
struct InferenceEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
input_tokens: u64,
output_tokens: u64,
decision: String,
#[serde(skip_serializing_if = "Option::is_none")]
stop_reason: Option<String>,
tool_name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
input_tokens_actual: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
output_tokens_actual: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
cached_tokens: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
cache_write_tokens: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
origin: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
model: Option<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
message_ids: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
system_sha: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
tools_sha: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
response_sha: Option<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
message_shas: Vec<String>,
}
pub(crate) struct WireCapture {
system: Vec<u8>,
tools: Vec<u8>,
messages: Vec<Vec<u8>>,
response: Vec<u8>,
}
impl WireCapture {
pub(crate) fn from_driver_payload(payload: &Value, response: &str) -> Self {
Self {
system: payload
.get("system")
.and_then(Value::as_str)
.unwrap_or_default()
.as_bytes()
.to_vec(),
tools: payload
.get("tools")
.map(|tools| serde_json::to_vec(tools).unwrap_or_default())
.unwrap_or_default(),
messages: payload
.get("messages")
.and_then(Value::as_array)
.map(|messages| {
messages
.iter()
.map(|message| serde_json::to_vec(message).unwrap_or_default())
.collect()
})
.unwrap_or_default(),
response: response.as_bytes().to_vec(),
}
}
}
#[derive(Serialize)]
struct ToolCallEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
tool_name: String,
tool_call_id: Option<String>,
input: Value,
input_bytes: u64,
#[serde(skip_serializing_if = "Option::is_none")]
output: Option<String>,
output_bytes: u64,
duration_ms: u64,
status: String,
#[serde(skip_serializing_if = "Option::is_none")]
state_effect: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
resource_id: Option<String>,
}
#[derive(Serialize)]
struct SkillCallEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
skill_name: String,
output_bytes: u64,
duration_ms: u64,
status: String,
}
#[derive(Serialize)]
struct ShellEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
binary: String,
command: String,
exit_code: i32,
stdout_bytes: u64,
stderr_bytes: u64,
duration_ms: u64,
#[serde(skip_serializing_if = "Option::is_none")]
resource_limit: Option<String>,
}
#[derive(Serialize)]
struct ShellDetachedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
work_id: String,
binary: String,
command: String,
grace_ms: u64,
}
#[derive(Serialize)]
struct ShellCompletedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
work_id: String,
binary: String,
command: String,
exit_code: i32,
duration_ms: u64,
output_path: String,
output_bytes: u64,
#[serde(skip_serializing_if = "Option::is_none")]
resource_limit: Option<String>,
status: String,
completion_task_id: String,
}
#[derive(Serialize)]
struct ShellAbandonedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
work_id: String,
binary: String,
command: String,
running_ms: u64,
#[serde(skip_serializing_if = "Option::is_none")]
exit_code: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
output_path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
output_bytes: Option<u64>,
}
#[derive(Serialize)]
struct ShellLostEvent {
event_type: &'static str,
event_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
parent_id: Option<String>,
session_id: String,
timestamp: u64,
work_id: String,
binary: String,
command: String,
detached_at_ms: u64,
reconciled_by_session: String,
reconciled_task_id: String,
}
#[derive(Serialize)]
struct ShellDetachUnrecordedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
work_id: String,
binary: String,
reason: String,
}
#[derive(Serialize)]
struct CompactionEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
tokens_before: u64,
tokens_after: u64,
}
#[derive(Serialize)]
struct CompactionDeclinedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
tokens: u64,
reason: String,
}
#[derive(Serialize)]
struct ContextSeedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: Option<String>,
hook_name: String,
tokens: u64,
proposed_tokens: u64,
budget_tokens: u64,
outcome: String,
#[serde(skip_serializing_if = "Option::is_none")]
reason: Option<String>,
message_ids: Vec<String>,
}
#[derive(Serialize)]
struct RetentionEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
store: String,
reason: String,
removed: u32,
targets: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
messages_dropped: Option<u64>,
}
#[derive(Serialize)]
struct A2aTaskReceivedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
context_id: String,
message_id: String,
traceparent_from_caller: Option<String>,
}
#[derive(Serialize)]
struct A2aSendEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
peer_url: String,
message_id: String,
task_id: String,
context_id: String,
traceparent: Option<String>,
trust: String,
}
#[derive(Serialize)]
struct SessionEndEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
total_turns: u32,
total_input_tokens: u64,
total_output_tokens: u64,
total_tool_calls: u32,
total_shell_calls: u32,
duration_ms: u64,
exit_status: String,
}
#[derive(Serialize)]
struct TaskStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
context_id: String,
source: String,
origin: String,
trust: String,
lane: String,
#[serde(skip_serializing_if = "Option::is_none")]
delegation_id: Option<String>,
message_parts_bytes: u64,
}
#[derive(Serialize)]
struct TaskEndEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
exit_status: String,
duration_ms: u64,
turns: u32,
input_tokens: u64,
output_tokens: u64,
tool_calls: u32,
shell_calls: u32,
reopen_count: u32,
}
#[derive(Serialize)]
struct TaskCanceledEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
turn: Option<u32>,
phase: String,
detached_work_ids: Vec<String>,
delegation_ids: Vec<String>,
}
#[derive(Serialize)]
struct TaskReopenedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
hook_name: String,
reason: String,
reopen_number: u32,
}
#[derive(Serialize)]
struct SpendCeilingReachedEvent<'a> {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
#[serde(flatten)]
refusal: &'a crate::spend::SpendRefusal,
#[serde(skip_serializing_if = "Option::is_none")]
origin: Option<&'a str>,
}
#[derive(Serialize)]
struct CallDeniedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
event: String,
hook_name: String,
target: String,
reason: String,
}
#[derive(Serialize)]
struct ProtectedPathDeniedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
call: String,
target: String,
path: String,
rule: String,
signal: String,
reason: String,
}
#[derive(Serialize)]
struct HookDispatchErrorEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
hook_name: String,
event: String,
arm: String,
}
#[derive(Serialize)]
struct ResourceListEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
root: String,
entry_count: usize,
total_bytes: u64,
generation: u64,
containment_achieved: ContainmentClass,
outcome: String,
reason: Option<String>,
}
#[derive(Serialize)]
struct ResourceReadEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
path: String,
outcome: String,
bytes: Option<u64>,
sha256: Option<String>,
generation: u64,
containment_achieved: ContainmentClass,
reason: Option<String>,
}
#[derive(Serialize)]
struct PeerHandleMintEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
handle_id: Option<String>,
path: String,
audience: String,
expires_at_ms: Option<u64>,
outcome: String,
reason: Option<String>,
}
#[derive(Serialize)]
struct PeerHandleRedeemEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
handle_id: String,
path: Option<String>,
generation: u64,
audience_asserted: Option<String>,
bytes: Option<u64>,
sha256: Option<String>,
outcome: String,
reason: Option<String>,
}
#[derive(Serialize)]
struct PeerFileFetchEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
peer: String,
handle_id: String,
stored_path: Option<String>,
bytes: Option<u64>,
sha256: Option<String>,
outcome: String,
reason: Option<String>,
}
#[derive(Serialize)]
struct DelegationStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
delegation_id: String,
capsule: String,
version: String,
child_session_id: String,
child_workdir: String,
}
impl DelegationStartEvent {
fn new(
session_id: &str,
session_event_id: &str,
delegation_id: &str,
capsule: &str,
version: &str,
child_session_id: &str,
child_workdir: &str,
) -> Self {
Self {
event_type: "delegation_start",
event_id: new_event_id(),
parent_id: Some(session_event_id.to_string()),
session_id: session_id.to_string(),
timestamp: timestamp_ms(),
delegation_id: delegation_id.to_string(),
capsule: capsule.to_string(),
version: version.to_string(),
child_session_id: child_session_id.to_string(),
child_workdir: child_workdir.to_string(),
}
}
}
#[derive(Serialize)]
struct DelegationEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
capsule: String,
version: String,
delegation_id: Option<String>,
child_session_id: Option<String>,
duration_ms: u64,
outcome: String,
reason: Option<String>,
}
impl DelegationEvent {
#[allow(clippy::too_many_arguments)]
fn new(
session_id: &str,
session_event_id: &str,
capsule: &str,
version: &str,
delegation_id: Option<String>,
child_session_id: Option<String>,
duration_ms: u64,
outcome: &str,
reason: Option<String>,
) -> Self {
Self {
event_type: "delegation",
event_id: new_event_id(),
parent_id: Some(session_event_id.to_string()),
session_id: session_id.to_string(),
timestamp: timestamp_ms(),
capsule: capsule.to_string(),
version: version.to_string(),
delegation_id,
child_session_id,
duration_ms,
outcome: outcome.to_string(),
reason,
}
}
}
impl TraceWriter {
#[allow(clippy::too_many_arguments)]
pub(crate) async fn open(
workdir: &Path,
session_id: String,
capsule_name: String,
capsule_version: String,
model: String,
capabilities: Vec<String>,
effective_grants: ScopeReport,
capture: TraceCapture,
system_prompt: Option<String>,
system_prompt_overridden: bool,
resumed_from: Option<String>,
context_id: Option<String>,
spawned_by: Option<String>,
spawn_delegation_id: Option<String>,
) -> std::io::Result<Self> {
let file = OpenOptions::new()
.create(true)
.append(true)
.open(workdir.join("trace.jsonl"))
.await?;
let system_prompt_source = match (system_prompt_overridden, system_prompt.is_some()) {
(true, _) => "cli",
(false, true) => "manifest",
(false, false) => "none",
};
let system_prompt_sha256 = system_prompt
.as_ref()
.map(|prompt| murmur_artifact::sha256_hex(prompt.as_bytes()));
let blobs = BlobStore::new(workdir);
if let (true, Some(prompt), Some(sha)) = (
capture.captures_content(),
system_prompt.as_ref(),
system_prompt_sha256.as_deref(),
) {
blobs.put(sha, prompt.as_bytes()).await?;
}
Ok(Self {
writer: BufWriter::new(file),
session_id,
capsule_name,
capsule_version,
model,
capabilities,
effective_grants,
capture,
blobs,
system_prompt_source,
system_prompt_sha256,
credential_source: "none",
resumed_from,
context_id,
spawned_by,
spawn_delegation_id,
session_start_time: Instant::now(),
session_event_id: new_event_id(),
session_started: false,
session_ended: false,
task_event_id: None,
turn_event_id: None,
total_turns: 0,
total_input_tokens: 0,
total_output_tokens: 0,
total_tool_calls: 0,
total_shell_calls: 0,
task_turns: 0,
task_input_tokens: 0,
task_output_tokens: 0,
task_tool_calls: 0,
task_shell_calls: 0,
task_start_instant: None,
active_task_id: None,
max_session_tokens: None,
machine_tokens_per_day: None,
})
}
pub(crate) fn set_spend_ceilings(
&mut self,
max_session_tokens: Option<u64>,
machine_tokens_per_day: Option<u64>,
) {
self.max_session_tokens = max_session_tokens;
self.machine_tokens_per_day = machine_tokens_per_day;
}
pub(crate) fn session_event_id(&self) -> &str {
&self.session_event_id
}
pub(crate) fn captures_wire(&self) -> bool {
self.capture.captures_hashes()
}
fn session_parent(&self) -> Option<String> {
self.session_started.then(|| self.session_event_id.clone())
}
fn task_parent(&self) -> Option<String> {
self.task_event_id.clone().or_else(|| self.session_parent())
}
fn turn_parent(&self) -> Option<String> {
self.turn_event_id.clone().or_else(|| self.task_parent())
}
pub(crate) fn set_credential_source(&mut self, source: &'static str) {
self.credential_source = source;
}
pub(crate) async fn write_session_start(
&mut self,
max_turns: u32,
tools_declared: Vec<String>,
) -> std::io::Result<()> {
let event = SessionStartEvent {
event_type: "session_start",
event_id: self.session_event_id.clone(),
parent_id: None,
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
capsule_name: self.capsule_name.clone(),
capsule_version: self.capsule_version.clone(),
model: self.model.clone(),
max_turns,
max_session_tokens: self.max_session_tokens,
machine_tokens_per_day: self.machine_tokens_per_day,
capabilities: self.capabilities.clone(),
tools_declared,
containment_declared: self.effective_grants.declared_containment,
containment_achieved: self.effective_grants.achieved_containment,
workdir_exec: self.effective_grants.workdir_exec,
userns_grant: self.effective_grants.userns_grant,
effective_grants: self.effective_grants.clone(),
system_prompt_source: self.system_prompt_source,
system_prompt_sha256: self.system_prompt_sha256.clone(),
credential_source: self.credential_source,
resumed_from: self.resumed_from.clone(),
context_id: self.context_id.clone(),
spawned_by: self.spawned_by.clone(),
delegation_id: self.spawn_delegation_id.clone(),
};
self.write_event(&event).await?;
self.session_started = true;
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_inference(
&mut self,
turn: u32,
input_tokens: u64,
output_tokens: u64,
decision: String,
stop_reason: Option<&str>,
tool_name: Option<String>,
origin: Option<&InferenceOrigin>,
usage: Option<&DriverUsage>,
message_ids: Vec<String>,
wire: Option<&WireCapture>,
) -> std::io::Result<()> {
let event_id = new_event_id();
let parent_id = if origin.is_none() {
let parent = self.task_parent();
self.turn_event_id = Some(event_id.clone());
parent
} else {
self.turn_parent()
};
let (system_sha, tools_sha, response_sha, message_shas) =
match wire.filter(|_| self.capture.captures_hashes()) {
Some(wire) => self.record_wire(wire).await?,
None => (None, None, None, Vec::new()),
};
let event = InferenceEvent {
event_type: "inference",
event_id,
parent_id,
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
input_tokens,
output_tokens,
decision,
stop_reason: stop_reason.map(str::to_string),
tool_name,
input_tokens_actual: usage.and_then(|u| u.input_tokens),
output_tokens_actual: usage.and_then(|u| u.output_tokens),
cached_tokens: usage.and_then(|u| u.cached_tokens),
cache_write_tokens: usage.and_then(|u| u.cache_write_tokens),
origin: origin.map(|o| o.source.clone()),
model: origin.map(|o| o.model.clone()),
message_ids,
system_sha,
tools_sha,
response_sha,
message_shas,
};
self.write_event(&event).await?;
self.total_turns = self.total_turns.saturating_add(1);
self.total_input_tokens = self.total_input_tokens.saturating_add(input_tokens);
self.total_output_tokens = self.total_output_tokens.saturating_add(output_tokens);
self.task_turns = self.task_turns.saturating_add(1);
self.task_input_tokens = self.task_input_tokens.saturating_add(input_tokens);
self.task_output_tokens = self.task_output_tokens.saturating_add(output_tokens);
Ok(())
}
async fn record_wire(
&mut self,
wire: &WireCapture,
) -> std::io::Result<(Option<String>, Option<String>, Option<String>, Vec<String>)> {
let system_sha = murmur_artifact::sha256_hex(&wire.system);
let tools_sha = murmur_artifact::sha256_hex(&wire.tools);
let response_sha = murmur_artifact::sha256_hex(&wire.response);
let message_shas: Vec<String> = wire
.messages
.iter()
.map(|message| murmur_artifact::sha256_hex(message))
.collect();
if self.capture.captures_content() {
self.blobs.put(&system_sha, &wire.system).await?;
self.blobs.put(&tools_sha, &wire.tools).await?;
self.blobs.put(&response_sha, &wire.response).await?;
for (sha, body) in message_shas.iter().zip(&wire.messages) {
self.blobs.put(sha, body).await?;
}
}
Ok((
Some(system_sha),
Some(tools_sha),
Some(response_sha),
message_shas,
))
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_tool_call(
&mut self,
turn: u32,
tool_name: String,
tool_call_id: Option<String>,
mut input: Value,
input_bytes: u64,
output: &str,
output_bytes: u64,
duration_ms: u64,
status: String,
state_effect: Option<String>,
resource_id: Option<String>,
) -> std::io::Result<()> {
crate::peer_handoff::redact_handles_in_json(&mut input);
let event = ToolCallEvent {
event_type: "tool_call",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
tool_name,
tool_call_id: tool_call_id.filter(|id| !id.is_empty()),
input,
input_bytes,
output: self
.capture
.captures_content()
.then(|| crate::peer_handoff::redact_handle_tokens(output).into_owned()),
output_bytes,
duration_ms,
status,
state_effect,
resource_id,
};
self.write_event(&event).await?;
self.total_tool_calls = self.total_tool_calls.saturating_add(1);
self.task_tool_calls = self.task_tool_calls.saturating_add(1);
Ok(())
}
pub(crate) async fn write_skill_call(
&mut self,
turn: u32,
skill_name: String,
output_bytes: u64,
duration_ms: u64,
status: String,
) -> std::io::Result<()> {
let event = SkillCallEvent {
event_type: "skill_call",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
skill_name,
output_bytes,
duration_ms,
status,
};
self.write_event(&event).await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_shell(
&mut self,
turn: u32,
binary: String,
command: String,
exit_code: i32,
stdout_bytes: u64,
stderr_bytes: u64,
duration_ms: u64,
resource_limit: Option<String>,
) -> std::io::Result<()> {
let event = ShellEvent {
event_type: "shell",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
binary,
command,
exit_code,
stdout_bytes,
stderr_bytes,
duration_ms,
resource_limit,
};
self.write_event(&event).await?;
self.total_shell_calls = self.total_shell_calls.saturating_add(1);
self.task_shell_calls = self.task_shell_calls.saturating_add(1);
Ok(())
}
pub(crate) async fn write_shell_detached(
&mut self,
turn: u32,
work_id: &str,
binary: &str,
command: &str,
grace_ms: u64,
) -> std::io::Result<()> {
let event = ShellDetachedEvent {
event_type: "shell_detached",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
work_id: work_id.to_string(),
binary: binary.to_string(),
command: command.to_string(),
grace_ms,
};
self.write_event(&event).await?;
self.total_shell_calls = self.total_shell_calls.saturating_add(1);
self.task_shell_calls = self.task_shell_calls.saturating_add(1);
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_shell_completed(
&mut self,
work_id: &str,
binary: &str,
command: &str,
exit_code: i32,
duration_ms: u64,
output_path: &str,
output_bytes: u64,
resource_limit: Option<String>,
status: &str,
completion_task_id: &str,
) -> std::io::Result<()> {
let event = ShellCompletedEvent {
event_type: "shell_completed",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
work_id: work_id.to_string(),
binary: binary.to_string(),
command: command.to_string(),
exit_code,
duration_ms,
output_path: output_path.to_string(),
output_bytes,
resource_limit,
status: status.to_string(),
completion_task_id: completion_task_id.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_shell_abandoned(
&mut self,
work_id: &str,
binary: &str,
command: &str,
running_ms: u64,
recovered: Option<(i32, &str, u64)>,
) -> std::io::Result<()> {
let event = ShellAbandonedEvent {
event_type: "shell_abandoned",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
work_id: work_id.to_string(),
binary: binary.to_string(),
command: command.to_string(),
running_ms,
exit_code: recovered.map(|(exit_code, _, _)| exit_code),
output_path: recovered.map(|(_, path, _)| path.to_string()),
output_bytes: recovered.map(|(_, _, bytes)| bytes),
};
self.write_event(&event).await
}
pub(crate) async fn write_shell_detach_unrecorded(
&mut self,
turn: u32,
work_id: &str,
binary: &str,
reason: &str,
) -> std::io::Result<()> {
let event = ShellDetachUnrecordedEvent {
event_type: "shell_detach_unrecorded",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
work_id: work_id.to_string(),
binary: binary.to_string(),
reason: reason.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_a2a_task_received(
&mut self,
task_id: &str,
context_id: &str,
message_id: &str,
traceparent_from_caller: Option<&str>,
) -> std::io::Result<()> {
let event = A2aTaskReceivedEvent {
event_type: "a2a_task_received",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
context_id: context_id.to_string(),
message_id: message_id.to_string(),
traceparent_from_caller: traceparent_from_caller.map(str::to_string),
};
self.write_event(&event).await
}
pub(crate) async fn write_a2a_send(
&mut self,
peer_url: &str,
message_id: &str,
task_id: &str,
context_id: &str,
traceparent: Option<&str>,
trust: TrustClass,
) -> std::io::Result<()> {
let event = A2aSendEvent {
event_type: "a2a_send",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
peer_url: peer_url.to_string(),
message_id: message_id.to_string(),
task_id: task_id.to_string(),
context_id: context_id.to_string(),
traceparent: traceparent.map(str::to_string),
trust: trust.as_str().to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_task_start(
&mut self,
task_id: &str,
context_id: &str,
source: &str,
provenance: TaskProvenance,
delegation_id: Option<&str>,
message_parts_bytes: u64,
) -> std::io::Result<()> {
self.task_turns = 0;
self.task_input_tokens = 0;
self.task_output_tokens = 0;
self.task_tool_calls = 0;
self.task_shell_calls = 0;
self.task_start_instant = Some(Instant::now());
self.active_task_id = Some(task_id.to_string());
let event_id = new_event_id();
self.task_event_id = Some(event_id.clone());
self.turn_event_id = None;
let event = TaskStartEvent {
event_type: "task_start",
event_id: event_id.clone(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
context_id: context_id.to_string(),
source: source.to_string(),
origin: provenance.origin().as_str().to_string(),
trust: provenance.trust().as_str().to_string(),
lane: TaskLane::for_origin(provenance.origin())
.as_str()
.to_string(),
delegation_id: delegation_id.map(str::to_string),
message_parts_bytes,
};
self.write_event(&event).await
}
pub(crate) async fn write_task_end(
&mut self,
task_id: &str,
exit_status: &str,
reopen_count: u32,
) -> std::io::Result<()> {
let duration_ms = self
.task_start_instant
.take()
.map(|t| t.elapsed().as_millis().try_into().unwrap_or(u64::MAX))
.unwrap_or(0);
let event = TaskEndEvent {
event_type: "task_end",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
exit_status: exit_status.to_string(),
duration_ms,
turns: self.task_turns,
input_tokens: self.task_input_tokens,
output_tokens: self.task_output_tokens,
tool_calls: self.task_tool_calls,
shell_calls: self.task_shell_calls,
reopen_count,
};
self.active_task_id = None;
self.task_event_id = None;
self.turn_event_id = None;
self.write_event(&event).await
}
pub(crate) async fn write_task_canceled(
&mut self,
task_id: &str,
turn: Option<u32>,
phase: &str,
detached_work_ids: Vec<String>,
delegation_ids: Vec<String>,
) -> std::io::Result<()> {
let event = TaskCanceledEvent {
event_type: "task_canceled",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
turn,
phase: phase.to_string(),
detached_work_ids,
delegation_ids,
};
self.write_event(&event).await
}
pub(crate) async fn write_task_reopened(
&mut self,
task_id: &str,
hook_name: &str,
reason: &str,
reopen_number: u32,
) -> std::io::Result<()> {
let event = TaskReopenedEvent {
event_type: "task_reopened",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
hook_name: hook_name.to_string(),
reason: reason.to_string(),
reopen_number,
};
self.write_event(&event).await
}
pub(crate) fn task_turns(&self) -> u32 {
self.task_turns
}
pub(crate) async fn write_compaction(
&mut self,
turn: u32,
tokens_before: u64,
tokens_after: u64,
) -> std::io::Result<()> {
let event = CompactionEvent {
event_type: "compaction",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
tokens_before,
tokens_after,
};
self.write_event(&event).await
}
pub(crate) async fn write_compaction_declined(
&mut self,
turn: u32,
tokens: u64,
reason: &str,
) -> std::io::Result<()> {
let event = CompactionDeclinedEvent {
event_type: "compaction_declined",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
tokens,
reason: reason.to_string(),
};
self.write_event(&event).await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_context_seed(
&mut self,
hook_name: &str,
tokens: u64,
proposed_tokens: u64,
budget_tokens: u64,
outcome: &str,
reason: Option<&str>,
message_ids: Vec<String>,
) -> std::io::Result<()> {
let event = ContextSeedEvent {
event_type: "context_seed",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
hook_name: hook_name.to_string(),
tokens,
proposed_tokens,
budget_tokens,
outcome: outcome.to_string(),
reason: reason.map(str::to_string),
message_ids,
};
self.write_event(&event).await
}
pub(crate) async fn write_retention(
&mut self,
store: &str,
reason: &str,
targets: Vec<String>,
messages_dropped: Option<u64>,
) -> std::io::Result<()> {
let event = RetentionEvent {
event_type: "retention",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
store: store.to_string(),
reason: reason.to_string(),
removed: u32::try_from(targets.len()).unwrap_or(u32::MAX),
targets,
messages_dropped,
};
self.write_event(&event).await
}
pub(crate) async fn write_call_denied(
&mut self,
turn: u32,
event: &str,
hook_name: &str,
target: &str,
reason: &str,
) -> std::io::Result<()> {
let event = CallDeniedEvent {
event_type: "call_denied",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
event: event.to_string(),
hook_name: hook_name.to_string(),
target: target.to_string(),
reason: reason.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_spend_ceiling_reached(
&mut self,
turn: u32,
refusal: &crate::spend::SpendRefusal,
origin: Option<&str>,
) -> std::io::Result<()> {
let event = SpendCeilingReachedEvent {
event_type: "spend_ceiling_reached",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
refusal,
origin,
};
self.write_event(&event).await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_protected_path_denied(
&mut self,
turn: u32,
call: &str,
target: &str,
path: &str,
rule: &str,
signal: &str,
reason: &str,
) -> std::io::Result<()> {
let event = ProtectedPathDeniedEvent {
event_type: "protected_path_denied",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
call: call.to_string(),
target: target.to_string(),
path: path.to_string(),
rule: rule.to_string(),
signal: signal.to_string(),
reason: reason.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_hook_dispatch_error(
&mut self,
hook_name: &str,
event: &str,
arm: &str,
) -> std::io::Result<()> {
let event = HookDispatchErrorEvent {
event_type: "hook_dispatch_error",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
hook_name: hook_name.to_string(),
event: event.to_string(),
arm: arm.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_session_end(&mut self, exit_status: &str) -> std::io::Result<()> {
let duration_ms = self
.session_start_time
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX);
let event = SessionEndEvent {
event_type: "session_end",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
total_turns: self.total_turns,
total_input_tokens: self.total_input_tokens,
total_output_tokens: self.total_output_tokens,
total_tool_calls: self.total_tool_calls,
total_shell_calls: self.total_shell_calls,
duration_ms,
exit_status: exit_status.to_string(),
};
self.write_event(&event).await?;
self.session_ended = true;
Ok(())
}
pub(crate) async fn write_session_end_if_not_ended(
&mut self,
exit_status: &str,
) -> std::io::Result<()> {
if self.session_started && !self.session_ended {
self.write_session_end(exit_status).await?;
}
Ok(())
}
pub(crate) async fn flush(&mut self) -> std::io::Result<()> {
self.writer.flush().await
}
async fn write_event(&mut self, event: &impl Serialize) -> std::io::Result<()> {
let mut line = serde_json::to_string(event).map_err(std::io::Error::other)?;
line.push('\n');
self.writer.write_all(line.as_bytes()).await?;
self.writer.flush().await
}
}
#[derive(Serialize)]
struct InferenceCredentialEvent<'a> {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
source: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
credential: Option<&'a str>,
change: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
trigger: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
status: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
retried: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
reason: Option<&'static str>,
}
pub struct ResourceTraceAppender {
file: tokio::sync::Mutex<File>,
session_id: String,
session_event_id: String,
}
impl ResourceTraceAppender {
pub async fn open(
workdir: &Path,
session_id: String,
session_event_id: String,
) -> std::io::Result<Self> {
let file = OpenOptions::new()
.create(true)
.append(true)
.open(workdir.join("trace.jsonl"))
.await?;
Ok(Self {
file: tokio::sync::Mutex::new(file),
session_id,
session_event_id,
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_resource_list(
&self,
root: &str,
entry_count: usize,
total_bytes: u64,
generation: u64,
containment_achieved: ContainmentClass,
outcome: &str,
reason: Option<String>,
) {
let event = ResourceListEvent {
event_type: "resource_list",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
root: root.to_string(),
entry_count,
total_bytes,
generation,
containment_achieved,
outcome: outcome.to_string(),
reason,
};
self.append(&event).await;
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_resource_read(
&self,
path: &str,
outcome: &str,
bytes: Option<u64>,
sha256: Option<String>,
generation: u64,
containment_achieved: ContainmentClass,
reason: Option<String>,
) {
let event = ResourceReadEvent {
event_type: "resource_read",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
path: path.to_string(),
outcome: outcome.to_string(),
bytes,
sha256,
generation,
containment_achieved,
reason,
};
self.append(&event).await;
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_peer_handle_mint(
&self,
handle_id: Option<String>,
path: &str,
audience: &str,
expires_at_ms: Option<u64>,
outcome: &str,
reason: Option<String>,
) {
let event = PeerHandleMintEvent {
event_type: "peer_handle_mint",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
handle_id,
path: path.to_string(),
audience: audience.to_string(),
expires_at_ms,
outcome: outcome.to_string(),
reason,
};
self.append(&event).await;
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_peer_handle_redeem(
&self,
handle_id: &str,
path: Option<String>,
generation: u64,
audience_asserted: Option<String>,
bytes: Option<u64>,
sha256: Option<String>,
outcome: &str,
reason: Option<String>,
) {
let event = PeerHandleRedeemEvent {
event_type: "peer_handle_redeem",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
handle_id: handle_id.to_string(),
path,
generation,
audience_asserted,
bytes,
sha256,
outcome: outcome.to_string(),
reason,
};
self.append(&event).await;
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_peer_file_fetch(
&self,
peer: &str,
handle_id: &str,
stored_path: Option<String>,
bytes: Option<u64>,
sha256: Option<String>,
outcome: &str,
reason: Option<String>,
) {
let event = PeerFileFetchEvent {
event_type: "peer_file_fetch",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
peer: peer.to_string(),
handle_id: handle_id.to_string(),
stored_path,
bytes,
sha256,
outcome: outcome.to_string(),
reason,
};
self.append(&event).await;
}
pub(crate) async fn write_delegation_start(
&self,
delegation_id: &str,
capsule: &str,
version: &str,
child_session_id: &str,
child_workdir: &str,
) {
let event = DelegationStartEvent::new(
&self.session_id,
&self.session_event_id,
delegation_id,
capsule,
version,
child_session_id,
child_workdir,
);
self.append(&event).await;
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_delegation(
&self,
capsule: &str,
version: &str,
delegation_id: Option<String>,
child_session_id: Option<String>,
duration_ms: u64,
outcome: &str,
reason: Option<String>,
) {
let event = DelegationEvent::new(
&self.session_id,
&self.session_event_id,
capsule,
version,
delegation_id,
child_session_id,
duration_ms,
outcome,
reason,
);
self.append(&event).await;
}
pub(crate) async fn write_inference_credential(
&self,
source: &'static str,
credential: Option<&str>,
change: CredentialChange,
) {
let (change_name, trigger, status, retried, reason) = match change {
CredentialChange::Rotated { trigger } => ("rotated", Some(trigger), None, None, None),
CredentialChange::Rejected { status, retried } => {
("rejected", None, Some(status), Some(retried), None)
}
CredentialChange::Unreadable { reason } => {
("unreadable", None, None, None, Some(reason))
}
};
let event = InferenceCredentialEvent {
event_type: "inference_credential",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
source,
credential,
change: change_name,
trigger,
status,
retried,
reason,
};
self.append(&event).await;
}
async fn append(&self, event: &impl Serialize) {
let Ok(mut line) = serde_json::to_string(event) else {
return;
};
line.push('\n');
let mut file = self.file.lock().await;
if file.write_all(line.as_bytes()).await.is_ok() {
let _ = file.flush().await;
}
}
}
pub(crate) struct PriorSessionTraceAppender {
file: File,
session_id: String,
session_event_id: Option<String>,
terminate_torn_tail: bool,
}
impl PriorSessionTraceAppender {
pub(crate) async fn open(
trace_path: &Path,
session_id: String,
session_event_id: Option<String>,
) -> std::io::Result<Self> {
let terminate_torn_tail = !ends_with_newline(trace_path)?;
let file = OpenOptions::new().append(true).open(trace_path).await?;
Ok(Self {
file,
session_id,
session_event_id,
terminate_torn_tail,
})
}
pub(crate) async fn write_shell_lost(
&mut self,
work: &crate::detached::LostWork,
reconciled_by_session: &str,
reconciled_task_id: &str,
) -> std::io::Result<()> {
let event = ShellLostEvent {
event_type: "shell_lost",
event_id: new_event_id(),
parent_id: self.session_event_id.clone(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
work_id: work.work_id.clone(),
binary: work.binary.clone(),
command: work.command.clone(),
detached_at_ms: work.detached_at_ms,
reconciled_by_session: reconciled_by_session.to_string(),
reconciled_task_id: reconciled_task_id.to_string(),
};
let mut line = if self.terminate_torn_tail {
"\n".to_string()
} else {
String::new()
};
line.push_str(&serde_json::to_string(&event).map_err(std::io::Error::other)?);
line.push('\n');
self.file.write_all(line.as_bytes()).await?;
self.file.flush().await?;
self.terminate_torn_tail = false;
Ok(())
}
}
fn ends_with_newline(path: &Path) -> std::io::Result<bool> {
use std::io::{Read, Seek, SeekFrom};
let mut file = std::fs::File::open(path)?;
let length = file.metadata()?.len();
if length == 0 {
return Ok(true);
}
file.seek(SeekFrom::End(-1))?;
let mut last = [0u8; 1];
file.read_exact(&mut last)?;
Ok(last[0] == b'\n')
}
pub(crate) fn timestamp_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64
}
#[derive(Serialize, Clone)]
pub(crate) struct PlanStepShape {
pub(crate) step_id: String,
pub(crate) kind: &'static str,
pub(crate) depends_on: Vec<String>,
pub(crate) has_condition: bool,
}
#[derive(Serialize)]
struct PlanStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
plan_id: String,
step_count: usize,
steps: Vec<PlanStepShape>,
}
#[derive(Serialize)]
struct PlanStepStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
plan_id: String,
step_id: String,
kind: &'static str,
depends_on: Vec<String>,
}
#[derive(Serialize)]
struct PlanStepEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
plan_id: String,
step_id: String,
kind: &'static str,
status: &'static str,
attempts: u32,
duration_ms: u64,
#[serde(skip_serializing_if = "Option::is_none")]
error: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
input: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
state_effect: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
resource_id: Option<String>,
}
#[derive(Serialize)]
struct PlanEndEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
plan_id: String,
outcome: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
failed_step: Option<String>,
steps_total: usize,
steps_succeeded: usize,
steps_failed: usize,
steps_skipped: usize,
duration_ms: u64,
#[serde(skip_serializing_if = "Option::is_none")]
reason: Option<String>,
}
pub(crate) struct PlanStepRecord {
pub(crate) plan_id: String,
pub(crate) step_id: String,
pub(crate) kind: &'static str,
pub(crate) status: &'static str,
pub(crate) attempts: u32,
pub(crate) duration_ms: u64,
pub(crate) error: Option<String>,
pub(crate) input: Option<Value>,
pub(crate) state_effect: Option<String>,
pub(crate) resource_id: Option<String>,
}
pub(crate) struct PlanEndRecord {
pub(crate) plan_id: String,
pub(crate) outcome: &'static str,
pub(crate) failed_step: Option<String>,
pub(crate) steps_total: usize,
pub(crate) steps_succeeded: usize,
pub(crate) steps_failed: usize,
pub(crate) steps_skipped: usize,
pub(crate) duration_ms: u64,
pub(crate) reason: Option<String>,
}
pub struct PlanTraceAppender {
file: std::sync::Mutex<std::fs::File>,
session_id: String,
session_event_id: String,
}
impl PlanTraceAppender {
pub fn open(
workdir: &Path,
session_id: String,
session_event_id: String,
) -> std::io::Result<Self> {
let file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(workdir.join("trace.jsonl"))?;
Ok(Self {
file: std::sync::Mutex::new(file),
session_id,
session_event_id,
})
}
pub(crate) fn write_plan_start(&self, plan_id: &str, steps: Vec<PlanStepShape>) -> String {
let event_id = new_event_id();
let event = PlanStartEvent {
event_type: "plan_start",
event_id: event_id.clone(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
plan_id: plan_id.to_string(),
step_count: steps.len(),
steps,
};
self.append(&event);
event_id
}
pub(crate) fn write_plan_step_start(
&self,
plan_node: &str,
plan_id: &str,
step_id: &str,
kind: &'static str,
depends_on: Vec<String>,
) {
let event = PlanStepStartEvent {
event_type: "plan_step_start",
event_id: new_event_id(),
parent_id: Some(plan_node.to_string()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
plan_id: plan_id.to_string(),
step_id: step_id.to_string(),
kind,
depends_on,
};
self.append(&event);
}
pub(crate) fn write_plan_step(&self, plan_node: &str, record: PlanStepRecord) {
let input = record.input.map(|mut value| {
crate::peer_handoff::redact_handles_in_json(&mut value);
value
});
let event = PlanStepEvent {
event_type: "plan_step",
event_id: new_event_id(),
parent_id: Some(plan_node.to_string()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
plan_id: record.plan_id,
step_id: record.step_id,
kind: record.kind,
status: record.status,
attempts: record.attempts,
duration_ms: record.duration_ms,
error: record.error,
input,
state_effect: record.state_effect,
resource_id: record.resource_id,
};
self.append(&event);
}
pub(crate) fn write_plan_end(&self, plan_node: Option<&str>, record: PlanEndRecord) {
let event = PlanEndEvent {
event_type: "plan_end",
event_id: new_event_id(),
parent_id: Some(
plan_node
.map(str::to_string)
.unwrap_or_else(|| self.session_event_id.clone()),
),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
plan_id: record.plan_id,
outcome: record.outcome,
failed_step: record.failed_step,
steps_total: record.steps_total,
steps_succeeded: record.steps_succeeded,
steps_failed: record.steps_failed,
steps_skipped: record.steps_skipped,
duration_ms: record.duration_ms,
reason: record.reason,
};
self.append(&event);
}
pub(crate) fn write_delegation_start(
&self,
delegation_id: &str,
capsule: &str,
version: &str,
child_session_id: &str,
child_workdir: &str,
) {
let event = DelegationStartEvent::new(
&self.session_id,
&self.session_event_id,
delegation_id,
capsule,
version,
child_session_id,
child_workdir,
);
self.append(&event);
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn write_delegation(
&self,
capsule: &str,
version: &str,
delegation_id: Option<String>,
child_session_id: Option<String>,
duration_ms: u64,
outcome: &str,
reason: Option<String>,
) {
let event = DelegationEvent::new(
&self.session_id,
&self.session_event_id,
capsule,
version,
delegation_id,
child_session_id,
duration_ms,
outcome,
reason,
);
self.append(&event);
}
fn append(&self, event: &impl Serialize) {
use std::io::Write as _;
let Ok(mut line) = serde_json::to_string(event) else {
return;
};
line.push('\n');
let Ok(mut file) = self.file.lock() else {
return;
};
if file.write_all(line.as_bytes()).is_ok() {
let _ = file.flush();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
containment::scope_report_for_tier, sandbox::EnforcementTier, sealed::UsernsGrant,
trace_blobs::BLOB_DIR_NAME, types::CapabilityPolicy,
};
use murmur_artifact::{InterpreterRuntimeDir, InterpreterRuntimeGrant};
use serde_json::Value;
use crate::origin::TaskOrigin;
fn event_provenance() -> TaskProvenance {
TaskProvenance::derive(TaskOrigin::Event, None)
}
fn report_for(
declared: ContainmentClass,
tier: EnforcementTier,
workdir_exec: bool,
) -> ScopeReport {
scope_report_for_tier(
&CapabilityPolicy {
workdir_exec_allowed: workdir_exec,
..CapabilityPolicy::default()
},
declared,
tier,
None,
None,
None,
Vec::new(),
Vec::new(),
Vec::new(),
crate::cgroup::IoMaxReport::default(),
)
}
fn demotion_info() -> crate::detached::DetachedDispatchInfo {
crate::detached::DetachedDispatchInfo {
work_id: "wrk_0a1b2c3d4e5f6a7b".to_string(),
binary: "/usr/bin/bash".to_string(),
command: "make -j8".to_string(),
grace_ms: 1_000,
}
}
#[tokio::test]
async fn a_demotion_marker_that_cannot_be_written_does_not_fail_the_demotion() {
let dir = tempfile::tempdir().unwrap();
std::os::unix::fs::symlink("/dev/full", dir.path().join("trace.jsonl")).unwrap();
let mut writer = make_writer(dir.path()).await;
let info = demotion_info();
assert!(
writer
.write_shell_detached(1, &info.work_id, &info.binary, &info.command, info.grace_ms)
.await
.is_err(),
"the sink under this writer must be unwritable for the claim below to mean anything"
);
crate::agent::record_demotion(&mut writer, 1, &info).await;
}
#[tokio::test]
async fn a_recorded_demotion_writes_only_its_own_marker() {
let dir = tempfile::tempdir().unwrap();
let mut writer = make_writer(dir.path()).await;
crate::agent::record_demotion(&mut writer, 3, &demotion_info()).await;
let events = read_events(dir.path());
assert_eq!(events.len(), 1);
assert_eq!(events[0]["event_type"], "shell_detached");
assert_eq!(events[0]["work_id"], "wrk_0a1b2c3d4e5f6a7b");
assert_eq!(events[0]["turn"], 3);
}
#[tokio::test]
async fn an_unrecorded_demotion_names_the_work_id_and_the_write_error() {
let dir = tempfile::tempdir().unwrap();
let mut writer = make_writer(dir.path()).await;
writer
.write_shell_detach_unrecorded(2, "wrk_0a1b", "/usr/bin/bash", "No space left")
.await
.unwrap();
let events = read_events(dir.path());
assert_eq!(events.len(), 1);
assert_eq!(events[0]["event_type"], "shell_detach_unrecorded");
assert_eq!(events[0]["turn"], 2);
assert_eq!(events[0]["work_id"], "wrk_0a1b");
assert_eq!(events[0]["binary"], "/usr/bin/bash");
assert_eq!(events[0]["reason"], "No space left");
}
#[tokio::test]
async fn an_abandoned_command_still_running_omits_the_fields_it_cannot_know() {
let dir = tempfile::tempdir().unwrap();
let mut writer = make_writer(dir.path()).await;
writer
.write_shell_abandoned(
"wrk_0a1b2c3d4e5f6a7b",
"/usr/bin/bash",
"sleep 45; echo done",
45_012,
None,
)
.await
.unwrap();
let events = read_events(dir.path());
assert_eq!(events.len(), 1);
let event = &events[0];
assert_eq!(event["event_type"], "shell_abandoned");
assert_eq!(event["work_id"], "wrk_0a1b2c3d4e5f6a7b");
assert_eq!(event["binary"], "/usr/bin/bash");
assert_eq!(event["command"], "sleep 45; echo done");
assert_eq!(event["running_ms"], 45_012);
for absent in ["exit_code", "output_path", "output_bytes"] {
assert!(
event.get(absent).is_none(),
"{absent} must be omitted, not written as null: {event}"
);
}
}
#[tokio::test]
async fn an_abandoned_command_that_finished_carries_its_result() {
let dir = tempfile::tempdir().unwrap();
let mut writer = make_writer(dir.path()).await;
writer
.write_shell_abandoned(
"wrk_0a1b2c3d4e5f6a7b",
"/usr/bin/bash",
"sleep 2; echo done",
2_014,
Some((0, "logs/wrk_0a1b2c3d4e5f6a7b.log", 91)),
)
.await
.unwrap();
let event = &read_events(dir.path())[0];
assert_eq!(event["event_type"], "shell_abandoned");
assert_eq!(event["running_ms"], 2_014);
assert_eq!(event["exit_code"], 0);
assert_eq!(event["output_path"], "logs/wrk_0a1b2c3d4e5f6a7b.log");
assert_eq!(event["output_bytes"], 91);
}
async fn make_writer(dir: &std::path::Path) -> TraceWriter {
make_writer_with_opts(dir, TraceCapture::Meta).await
}
async fn make_writer_with_opts(dir: &std::path::Path, capture: TraceCapture) -> TraceWriter {
make_writer_with_prompt(dir, capture, None, false).await
}
async fn make_writer_with_prompt(
dir: &std::path::Path,
capture: TraceCapture,
system_prompt: Option<&str>,
system_prompt_overridden: bool,
) -> TraceWriter {
TraceWriter::open(
dir,
"test-session-id".to_string(),
"test-capsule".to_string(),
"1.0.0".to_string(),
"claude-test".to_string(),
vec!["shell".to_string()],
report_for(
ContainmentClass::Advisory,
EnforcementTier::EnvironmentOnly,
false,
),
capture,
system_prompt.map(str::to_string),
system_prompt_overridden,
None,
None,
None,
None,
)
.await
.unwrap()
}
fn read_events(dir: &std::path::Path) -> Vec<Value> {
let content = std::fs::read_to_string(dir.join("trace.jsonl")).unwrap();
content
.lines()
.filter(|l| !l.is_empty())
.map(|l| serde_json::from_str(l).unwrap())
.collect()
}
#[tokio::test]
async fn session_start_fields() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec!["bash".to_string()])
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events.len(), 1);
let e = &events[0];
assert_eq!(e["event_type"], "session_start");
assert_eq!(e["session_id"], "test-session-id");
assert_eq!(e["capsule_name"], "test-capsule");
assert_eq!(e["capsule_version"], "1.0.0");
assert_eq!(e["model"], "claude-test");
assert_eq!(e["max_turns"], 10);
assert_eq!(e["capabilities"], serde_json::json!(["shell"]));
assert_eq!(e["tools_declared"], serde_json::json!(["bash"]));
assert_eq!(e["containment_declared"], "advisory");
assert_eq!(e["containment_achieved"], "advisory");
assert_eq!(e["workdir_exec"], false);
assert!(e["timestamp"].as_u64().unwrap() > 0);
}
#[tokio::test]
async fn session_start_records_system_prompt_source_and_hash() {
let expected_sha = murmur_artifact::sha256_hex(b"Be terse.");
for (prompt, overridden, source, sha) in [
(Some("Be terse."), false, "manifest", Some(&expected_sha)),
(Some("Be terse."), true, "cli", Some(&expected_sha)),
(None, false, "none", None),
(None, true, "cli", None),
] {
let dir = tempfile::tempdir().unwrap();
let mut w =
make_writer_with_prompt(dir.path(), TraceCapture::Meta, prompt, overridden).await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
assert_eq!(e["system_prompt_source"], source, "prompt={prompt:?}");
match sha {
Some(sha) => assert_eq!(e["system_prompt_sha256"], sha.as_str()),
None => assert!(
e["system_prompt_sha256"].is_null(),
"no prompt in effect must hash to null, got {}",
e["system_prompt_sha256"]
),
}
}
}
#[tokio::test]
async fn session_start_records_the_prompt_by_hash_and_stores_it_only_under_content() {
let sha = murmur_artifact::sha256_hex(b"Be terse.");
let dir = tempfile::tempdir().unwrap();
let mut w =
make_writer_with_prompt(dir.path(), TraceCapture::Meta, Some("Be terse."), false).await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
assert!(
e.get("system_prompt").is_none(),
"the verbatim prompt is not a session_start field, got {e}"
);
assert_eq!(e["system_prompt_sha256"], sha.as_str());
assert!(
!dir.path().join(BLOB_DIR_NAME).exists(),
"meta stores no bodies"
);
let opted_in = tempfile::tempdir().unwrap();
let mut w = make_writer_with_prompt(
opted_in.path(),
TraceCapture::Content,
Some("Be terse."),
false,
)
.await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let event = &read_events(opted_in.path())[0];
assert!(event.get("system_prompt").is_none());
assert_eq!(event["system_prompt_sha256"], sha.as_str());
let blob = opted_in.path().join(BLOB_DIR_NAME).join(&sha);
assert_eq!(std::fs::read(&blob).unwrap(), b"Be terse.");
}
#[tokio::test]
async fn content_capture_stores_no_prompt_blob_when_there_is_no_prompt() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_prompt(dir.path(), TraceCapture::Content, None, false).await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
assert!(e.get("system_prompt").is_none());
assert!(e["system_prompt_sha256"].is_null());
assert!(!dir.path().join(BLOB_DIR_NAME).exists());
}
#[tokio::test]
async fn every_task_session_start_repeats_the_same_system_prompt_record() {
let dir = tempfile::tempdir().unwrap();
let mut w =
make_writer_with_prompt(dir.path(), TraceCapture::Meta, Some("Be terse."), true).await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events.len(), 2);
assert_eq!(
events[0]["system_prompt_source"],
events[1]["system_prompt_source"]
);
assert_eq!(
events[0]["system_prompt_sha256"],
events[1]["system_prompt_sha256"]
);
assert_eq!(events[1]["system_prompt_source"], "cli");
}
#[tokio::test]
async fn session_start_records_workdir_exec_next_to_the_class_it_forced() {
let dir = tempfile::tempdir().unwrap();
let mut w = TraceWriter::open(
dir.path(),
"test-session-id".to_string(),
"test-capsule".to_string(),
"1.0.0".to_string(),
"claude-test".to_string(),
Vec::new(),
report_for(
ContainmentClass::Advisory,
EnforcementTier::KernelFull,
true,
),
TraceCapture::Meta,
None,
false,
None,
None,
None,
None,
)
.await
.unwrap();
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events[0]["workdir_exec"], true);
assert_eq!(events[0]["containment_achieved"], "advisory");
}
#[tokio::test]
async fn session_start_records_declared_and_achieved_containment_separately() {
let dir = tempfile::tempdir().unwrap();
let mut w = TraceWriter::open(
dir.path(),
"test-session-id".to_string(),
"test-capsule".to_string(),
"1.0.0".to_string(),
"claude-test".to_string(),
Vec::new(),
report_for(ContainmentClass::Sealed, EnforcementTier::KernelFull, false),
TraceCapture::Meta,
None,
false,
None,
None,
None,
None,
)
.await
.unwrap();
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events[0]["containment_declared"], "sealed");
assert_eq!(events[0]["containment_achieved"], "scoped");
}
#[tokio::test]
async fn session_start_distinguishes_two_hosts_with_the_same_achieved_class() {
async fn event_for(grant: UsernsGrant) -> Value {
let report = scope_report_for_tier(
&CapabilityPolicy::default(),
ContainmentClass::Sealed,
EnforcementTier::KernelSealed,
None,
Some(grant),
None,
Vec::new(),
Vec::new(),
Vec::new(),
crate::cgroup::IoMaxReport::default(),
);
let dir = tempfile::tempdir().unwrap();
let mut w = TraceWriter::open(
dir.path(),
"test-session-id".to_string(),
"test-capsule".to_string(),
"1.0.0".to_string(),
"claude-test".to_string(),
Vec::new(),
report,
TraceCapture::Meta,
None,
false,
None,
None,
None,
None,
)
.await
.unwrap();
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let mut event = read_events(dir.path()).remove(0);
event.as_object_mut().unwrap().remove("timestamp");
event
}
let through_profile = event_for(UsernsGrant::ProfileConfining).await;
let host_wide = event_for(UsernsGrant::RestrictionDisabledHostWide).await;
assert_eq!(through_profile["containment_achieved"], "sealed");
assert_eq!(host_wide["containment_achieved"], "sealed");
assert_eq!(through_profile["userns_grant"], "profile_confining");
assert_eq!(host_wide["userns_grant"], "restriction_disabled_host_wide");
assert_ne!(
through_profile, host_wide,
"two hosts granting the user namespace by different mechanisms must not write the \
same session_start record"
);
let unprobed = scope_report_for_tier(
&CapabilityPolicy::default(),
ContainmentClass::Advisory,
EnforcementTier::EnvironmentOnly,
None,
None,
None,
Vec::new(),
Vec::new(),
Vec::new(),
crate::cgroup::IoMaxReport::default(),
);
assert!(
serde_json::to_value(&unprobed).unwrap()["userns_grant"].is_null(),
"off Linux the key is present and null, not absent"
);
}
#[tokio::test]
async fn session_start_records_the_whole_scope_report_as_effective_grants() {
let policy = CapabilityPolicy {
network_allow: vec!["https://api.example.com".to_string()],
unix_sockets_allowed: false,
filesystem_scope: Some("workdir".to_string()),
shell_allow: vec!["python3".to_string()],
spawn_allow: vec!["helper".to_string()],
env_allow: vec!["TZ".to_string()],
shell_interpreter_runtime: vec![InterpreterRuntimeGrant {
binary: "python3".to_string(),
dirs: vec![InterpreterRuntimeDir {
path: "/usr/lib/python3.11".to_string(),
list_dir: true,
}],
}],
..CapabilityPolicy::default()
};
let expected = scope_report_for_tier(
&policy,
ContainmentClass::Scoped,
EnforcementTier::KernelFull,
None,
None,
None,
vec![crate::containment::StateStoreReport {
artifact: "notes-tool".to_string(),
store: "shey".to_string(),
host_path: "/home/dev/.murmur/state/shey".to_string(),
}],
vec!["config-echo".to_string()],
vec![crate::containment::PreopenReport {
artifact: "notes-tool".to_string(),
role: "tool".to_string(),
scope: None,
surface: crate::containment::PreopenSurface::WholeWorkdir,
}],
crate::cgroup::IoMaxReport::default(),
);
let dir = tempfile::tempdir().unwrap();
let mut w = TraceWriter::open(
dir.path(),
"test-session-id".to_string(),
"test-capsule".to_string(),
"1.0.0".to_string(),
"claude-test".to_string(),
vec!["network".to_string(), "shell".to_string()],
expected.clone(),
TraceCapture::Meta,
None,
false,
None,
None,
None,
None,
)
.await
.unwrap();
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(
events[0]["effective_grants"],
serde_json::to_value(&expected).unwrap()
);
assert_eq!(
events[0]["capabilities"],
serde_json::json!(["network", "shell"])
);
assert_eq!(
events[0]["effective_grants"]["state_stores"],
serde_json::json!([{
"artifact": "notes-tool",
"store": "shey",
"host_path": "/home/dev/.murmur/state/shey",
}])
);
assert_eq!(
events[0]["effective_grants"]["configured_artifacts"],
serde_json::json!(["config-echo"])
);
assert_eq!(
events[0]["effective_grants"]["preopens"],
serde_json::json!([{
"artifact": "notes-tool",
"role": "tool",
"scope": null,
"surface": "whole-workdir",
}])
);
assert_eq!(events[0]["containment_declared"], "scoped");
assert_eq!(events[0]["containment_achieved"], "scoped");
assert_eq!(events[0]["workdir_exec"], false);
assert_eq!(
events[0]["effective_grants"]["network_allow"],
serde_json::json!(["https://api.example.com"])
);
assert_eq!(
events[0]["effective_grants"]["interpreter_runtime_grants"],
serde_json::json!(["python3: /usr/lib/python3.11 (list_dir)"])
);
}
#[tokio::test]
async fn inference_fields_and_snake_case() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_inference(
0,
100,
50,
"tool_call".to_string(),
Some("tool_call"),
Some("bash".to_string()),
None,
None,
vec!["msg_0000000000000000000000000000000a".to_string()],
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "inference");
assert_eq!(
e["message_ids"],
serde_json::json!(["msg_0000000000000000000000000000000a"]),
"an agent-loop turn names the messages its request embedded"
);
assert_eq!(e["turn"], 0);
assert_eq!(e["input_tokens"], 100);
assert_eq!(e["output_tokens"], 50);
assert_eq!(e["decision"], "tool_call");
assert_eq!(e["tool_name"], "bash");
assert!(e.get("inputTokens").is_none(), "must use snake_case");
assert!(e.get("toolName").is_none(), "must use snake_case");
for key in [
"input_tokens_actual",
"output_tokens_actual",
"cached_tokens",
"cache_write_tokens",
] {
assert!(e.get(key).is_none(), "{key} must be omitted, not null or 0");
}
}
#[tokio::test]
async fn inference_writes_only_the_reported_usage_members() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
let usage = DriverUsage {
input_tokens: Some(12043),
output_tokens: None,
cached_tokens: Some(0),
cache_write_tokens: None,
};
w.write_inference(
0,
100,
50,
"end_turn".to_string(),
Some("end_turn"),
None,
None,
Some(&usage),
Vec::new(),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["input_tokens"], 100, "the estimate is untouched");
assert_eq!(e["input_tokens_actual"], 12043);
assert_eq!(e["cached_tokens"], 0, "a reported zero is written");
assert!(e.get("output_tokens_actual").is_none());
assert!(e.get("cache_write_tokens").is_none());
}
#[tokio::test]
async fn tool_call_fields() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_tool_call(
1,
"bash".to_string(),
None,
serde_json::json!({"command": "echo hi"}),
42,
"hi\n",
100,
55,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "tool_call");
assert_eq!(e["turn"], 1);
assert_eq!(e["tool_name"], "bash");
assert_eq!(e["input_bytes"], 42);
assert_eq!(e["output_bytes"], 100);
assert_eq!(e["duration_ms"], 55);
assert_eq!(e["status"], "ok");
assert!(
e.get("state_effect").is_none(),
"state_effect must be omitted when the tool declared nothing"
);
}
#[tokio::test]
async fn tool_call_records_declared_state_effect() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_tool_call(
0,
"murmur-tool-editor".to_string(),
None,
serde_json::json!({"operation": "read_file", "path": "a.txt"}),
10,
"hi\n",
3,
5,
"ok".to_string(),
Some("read".to_string()),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(
events[0]["state_effect"], "read",
"declared state_effect must be recorded verbatim on the tool_call event"
);
}
#[tokio::test]
async fn tool_call_records_declared_resource_id() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_tool_call(
0,
"murmur-tool-code-graph".to_string(),
None,
serde_json::json!({"symbol": "Foo::bar"}),
10,
"hi\n",
3,
5,
"ok".to_string(),
Some("read".to_string()),
Some("sym:Foo".to_string()),
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(
events[0]["resource_id"], "sym:Foo",
"declared resource_id must reach trace.jsonl verbatim, unparsed"
);
}
#[tokio::test]
async fn tool_call_omits_undeclared_resource_id() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "echo hi"}),
10,
"hi\n",
3,
5,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert!(
events[0].get("resource_id").is_none(),
"resource_id must be omitted entirely (not null) when the tool declared nothing"
);
}
#[tokio::test]
async fn shell_fields() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_shell(
0,
"/usr/bin/echo".to_string(),
"echo hi".to_string(),
0,
7,
0,
10,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "shell");
assert_eq!(e["binary"], "/usr/bin/echo");
assert_eq!(e["command"], "echo hi");
assert_eq!(e["exit_code"], 0);
assert_eq!(e["stdout_bytes"], 7);
assert_eq!(e["stderr_bytes"], 0);
assert_eq!(e["duration_ms"], 10);
}
#[tokio::test]
async fn compaction_fields() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_compaction(3, 80000, 20000).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "compaction");
assert_eq!(e["turn"], 3);
assert_eq!(e["tokens_before"], 80000);
assert_eq!(e["tokens_after"], 20000);
}
#[tokio::test]
async fn hook_dispatch_error_fields() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_hook_dispatch_error("my-hook", "on-tool-call", "write-manifests")
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "hook_dispatch_error");
assert_eq!(e["session_id"], "test-session-id");
assert_eq!(e["hook_name"], "my-hook");
assert_eq!(e["event"], "on-tool-call");
assert_eq!(e["arm"], "write-manifests");
assert!(e["timestamp"].as_u64().unwrap() > 0);
assert!(e.get("hookName").is_none(), "must use snake_case");
}
#[tokio::test]
async fn session_end_fields_and_snake_case() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_inference(
0,
100,
50,
"tool_call".to_string(),
Some("tool_call"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "ls"}),
10,
"file.txt\n",
20,
5,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.write_shell(0, "/bin/ls".to_string(), "ls".to_string(), 0, 3, 0, 2, None)
.await
.unwrap();
w.write_session_end("ok").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let se = events.last().unwrap();
assert_eq!(se["event_type"], "session_end");
assert_eq!(se["total_turns"], 1);
assert_eq!(se["total_input_tokens"], 100);
assert_eq!(se["total_output_tokens"], 50);
assert_eq!(se["total_tool_calls"], 1);
assert_eq!(se["total_shell_calls"], 1);
assert_eq!(se["exit_status"], "ok");
assert!(se["duration_ms"].as_u64().is_some());
assert!(se.get("exitStatus").is_none(), "must use snake_case");
assert!(se.get("totalTurns").is_none(), "must use snake_case");
}
#[tokio::test]
async fn write_session_end_if_not_ended_is_noop_when_already_ended() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec![]).await.unwrap();
w.write_session_end("ok").await.unwrap();
w.write_session_end_if_not_ended("failed").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(
events
.iter()
.filter(|e| e["event_type"] == "session_end")
.count(),
1,
"session_end should appear exactly once"
);
}
#[tokio::test]
async fn write_session_end_if_not_ended_writes_when_started_not_ended() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec![]).await.unwrap();
w.write_session_end_if_not_ended("failed").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let se = events
.iter()
.find(|e| e["event_type"] == "session_end")
.unwrap();
assert_eq!(se["exit_status"], "failed");
}
#[tokio::test]
async fn no_session_end_written_when_session_never_started() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_end_if_not_ended("failed").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert!(
events.iter().all(|e| e["event_type"] != "session_end"),
"session_end must not appear when session_start was never written"
);
}
#[tokio::test]
async fn session_id_on_every_event() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec!["bash".to_string()])
.await
.unwrap();
w.write_inference(
0,
10,
5,
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_session_end("ok").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
for e in &events {
assert_eq!(e["session_id"], "test-session-id");
}
}
#[tokio::test]
async fn counts_match_per_event_sum() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec![]).await.unwrap();
w.write_inference(
0,
50,
25,
"tool_call".to_string(),
Some("tool_call"),
Some("bash".to_string()),
None,
None,
vec!["msg_0000000000000000000000000000000a".to_string()],
None,
)
.await
.unwrap();
w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "echo"}),
5,
"ok",
10,
3,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.write_shell(
0,
"/bin/echo".to_string(),
"echo".to_string(),
0,
4,
0,
1,
None,
)
.await
.unwrap();
w.write_inference(
1,
60,
30,
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_session_end("ok").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let inference_count = events
.iter()
.filter(|e| e["event_type"] == "inference")
.count();
let tool_count = events
.iter()
.filter(|e| e["event_type"] == "tool_call")
.count();
let shell_count = events.iter().filter(|e| e["event_type"] == "shell").count();
let se = events
.iter()
.find(|e| e["event_type"] == "session_end")
.unwrap();
assert_eq!(
se["total_turns"].as_u64().unwrap() as usize,
inference_count
);
assert_eq!(
se["total_tool_calls"].as_u64().unwrap() as usize,
tool_count
);
assert_eq!(
se["total_shell_calls"].as_u64().unwrap() as usize,
shell_count
);
}
#[tokio::test]
async fn tool_call_input_always_present() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await; w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "ls -la"}),
20,
"total 8\n",
80,
12,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "tool_call");
assert_eq!(e["input"]["command"], "ls -la");
assert!(e.get("input").is_some(), "input must always be present");
}
#[tokio::test]
async fn tool_call_output_absent_by_default() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await; w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "ls"}),
10,
"file.txt\n",
90,
8,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert!(
e.get("output").is_none(),
"output must be absent under a capture mode that stores no bodies"
);
assert_eq!(e["output_bytes"], 90, "output_bytes must still be recorded");
}
#[tokio::test]
async fn skill_call_writes_skill_call_event() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_skill_call(2, "my-skill".to_string(), 1024, 8, "ok".to_string())
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "skill_call");
assert_eq!(e["turn"], 2);
assert_eq!(e["skill_name"], "my-skill");
assert_eq!(e["output_bytes"], 1024);
assert_eq!(e["duration_ms"], 8);
assert_eq!(e["status"], "ok");
}
#[tokio::test]
async fn skill_call_does_not_increment_total_tool_calls() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec![]).await.unwrap();
w.write_tool_call(
0,
"real-tool".to_string(),
None,
serde_json::json!({}),
2,
"data",
4,
5,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.write_skill_call(1, "my-skill".to_string(), 512, 3, "ok".to_string())
.await
.unwrap();
w.write_session_end("ok").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let se = events
.iter()
.find(|e| e["event_type"] == "session_end")
.unwrap();
assert_eq!(
se["total_tool_calls"], 1,
"skill_call must not inflate total_tool_calls"
);
}
#[tokio::test]
async fn tool_call_output_present_when_opted_in() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "echo hello"}),
22,
"hello\n",
60,
5,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(
e["output"].as_str().unwrap(),
"hello\n",
"output must be present under capture: content"
);
assert_eq!(e["output_bytes"], 60);
}
#[tokio::test]
async fn task_end_carries_reopen_count() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), None, 3)
.await
.unwrap();
w.write_task_end("tsk_1", "ok", 2).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let end = events
.iter()
.find(|e| e["event_type"] == "task_end")
.unwrap();
assert_eq!(end["task_id"], "tsk_1");
assert_eq!(end["exit_status"], "ok");
assert_eq!(end["reopen_count"], 2);
}
#[tokio::test]
async fn task_reopened_names_hook_reason_and_ordinal() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), None, 3)
.await
.unwrap();
w.write_task_reopened("tsk_1", "gatekeeper", "tests still fail", 1)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let re = events
.iter()
.find(|e| e["event_type"] == "task_reopened")
.unwrap();
assert_eq!(re["task_id"], "tsk_1");
assert_eq!(re["hook_name"], "gatekeeper");
assert_eq!(re["reason"], "tests still fail");
assert_eq!(re["reopen_number"], 1);
assert!(re["timestamp"].as_u64().unwrap() > 0);
}
#[tokio::test]
async fn task_turns_accessor_tracks_and_resets() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), None, 3)
.await
.unwrap();
assert_eq!(w.task_turns(), 0);
w.write_inference(
0,
10,
5,
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_inference(
1,
10,
5,
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
assert_eq!(w.task_turns(), 2);
w.write_task_start("tsk_2", "ctx_2", "a2a", event_provenance(), None, 3)
.await
.unwrap();
assert_eq!(w.task_turns(), 0);
}
#[tokio::test]
async fn every_event_carries_a_unique_id_and_parents_to_an_earlier_event() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec!["bash".to_string()])
.await
.unwrap();
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), None, 3)
.await
.unwrap();
w.write_inference(
0,
10,
5,
"tool_call".to_string(),
Some("tool_call"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_tool_call(
0,
"bash".to_string(),
Some("toolu_1".to_string()),
serde_json::json!({"command": "echo hi"}),
10,
"hi\n",
3,
5,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.write_shell(
0,
"/bin/bash".to_string(),
"echo hi".to_string(),
0,
3,
0,
2,
None,
)
.await
.unwrap();
w.write_task_end("tsk_1", "ok", 0).await.unwrap();
w.write_session_end("ok").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
for (i, event) in events.iter().enumerate() {
let id = event["event_id"].as_str().unwrap();
assert!(id.starts_with("evt_") && id.len() == 36, "event {i}: {id}");
assert!(id[4..].chars().all(|c| c.is_ascii_hexdigit()), "event {i}");
assert!(seen.insert(id.to_string()), "event {i} reuses {id}");
assert!(
event.get("parent_id").is_some(),
"event {i} has no parent_id"
);
if let Some(parent) = event["parent_id"].as_str() {
assert!(seen.contains(parent), "event {i} names dangling {parent}");
}
}
let by_type = |ty: &str| {
events
.iter()
.find(|e| e["event_type"] == ty)
.unwrap_or_else(|| panic!("no {ty}"))
.clone()
};
let session = by_type("session_start");
assert!(session["parent_id"].is_null());
assert_eq!(session["event_id"], w.session_event_id());
let task = by_type("task_start");
assert_eq!(task["parent_id"], session["event_id"]);
let turn = by_type("inference");
assert_eq!(turn["parent_id"], task["event_id"]);
assert_eq!(by_type("tool_call")["parent_id"], turn["event_id"]);
assert_eq!(by_type("shell")["parent_id"], turn["event_id"]);
assert_eq!(by_type("task_end")["parent_id"], task["event_id"]);
assert_eq!(by_type("session_end")["parent_id"], session["event_id"]);
}
#[tokio::test]
async fn hook_origin_inference_hangs_off_the_turn_rather_than_becoming_one() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, Vec::new()).await.unwrap();
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), None, 3)
.await
.unwrap();
w.write_inference(
0,
10,
5,
"tool_call".to_string(),
Some("tool_call"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
let origin = InferenceOrigin {
source: "hook:gatekeeper".to_string(),
model: "claude-test".to_string(),
};
w.write_inference(
0,
1,
1,
"end_turn".to_string(),
None,
None,
Some(&origin),
None,
Vec::new(),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let turn = &events[2];
let hook = &events[3];
assert!(turn["origin"].is_null());
assert_eq!(hook["origin"], "hook:gatekeeper");
assert_eq!(hook["parent_id"], turn["event_id"]);
assert!(hook.get("message_ids").is_none(), "{hook}");
}
#[tokio::test]
async fn a2a_send_parents_to_null_without_a_session_frame_and_to_it_with_one() {
let bare = tempfile::tempdir().unwrap();
let mut w = make_writer(bare.path()).await;
w.write_a2a_send(
"http://peer",
"msg_1",
"tsk_1",
"ctx_1",
None,
TrustClass::Untrusted,
)
.await
.unwrap();
w.flush().await.unwrap();
assert!(read_events(bare.path())[0]["parent_id"].is_null());
let framed = tempfile::tempdir().unwrap();
let mut w = make_writer(framed.path()).await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.write_a2a_send(
"http://peer",
"msg_1",
"tsk_1",
"ctx_1",
None,
TrustClass::Untrusted,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(framed.path());
assert_eq!(events[1]["parent_id"], events[0]["event_id"]);
}
#[tokio::test]
async fn turn_level_events_carry_the_active_task_id() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, Vec::new()).await.unwrap();
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), None, 3)
.await
.unwrap();
w.write_inference(
0,
10,
5,
"tool_call".to_string(),
Some("tool_call"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_skill_call(0, "house-style".to_string(), 12, 1, "ok".to_string())
.await
.unwrap();
w.write_compaction(0, 100, 40).await.unwrap();
w.write_task_end("tsk_1", "ok", 0).await.unwrap();
w.write_inference(
1,
1,
1,
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
for event in events
.iter()
.filter(|e| {
matches!(
e["event_type"].as_str(),
Some("inference" | "skill_call" | "compaction")
)
})
.take(3)
{
assert_eq!(event["task_id"], "tsk_1", "{event}");
}
let after = events.last().unwrap();
assert_eq!(after["event_type"], "inference");
assert!(
after["task_id"].is_null(),
"an inference outside any task carries a null task_id, not the last task's"
);
assert_eq!(after["parent_id"], events[0]["event_id"]);
}
#[tokio::test]
async fn tool_call_records_the_provider_id_verbatim_and_empty_as_null() {
for (given, expected) in [
(Some("toolu_1".to_string()), Some("toolu_1")),
(Some(String::new()), None),
(None, None),
] {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_tool_call(
0,
"bash".to_string(),
given.clone(),
serde_json::json!({}),
2,
"",
0,
1,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
match expected {
Some(id) => assert_eq!(e["tool_call_id"], id, "given={given:?}"),
None => assert!(e["tool_call_id"].is_null(), "given={given:?}"),
}
}
}
#[tokio::test]
async fn compaction_declined_records_turn_task_tokens_and_reason() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, Vec::new()).await.unwrap();
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), None, 3)
.await
.unwrap();
w.write_inference(
2,
10,
5,
"tool_call".to_string(),
Some("tool_call"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_compaction_declined(2, 4321, COMPACTION_DECLINED_UNRESOLVED_TOOL_CALL)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let declined: Vec<&Value> = events
.iter()
.filter(|e| e["event_type"] == "compaction_declined")
.collect();
assert_eq!(declined.len(), 1);
let d = declined[0];
assert_eq!(d["reason"], "unresolved_tool_call");
assert_eq!(d["turn"], 2);
assert_eq!(d["task_id"], "tsk_1");
assert_eq!(d["tokens"], 4321);
assert!(d["event_id"].as_str().unwrap().starts_with("evt_"));
assert_eq!(
d["parent_id"],
events
.iter()
.find(|e| e["event_type"] == "inference")
.unwrap()["event_id"]
);
}
#[tokio::test]
async fn event_ids_sort_by_mint_time() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
for turn in 0..5 {
w.write_inference(
turn,
1,
1,
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
}
w.flush().await.unwrap();
let ids: Vec<String> = read_events(dir.path())
.iter()
.map(|e| e["event_id"].as_str().unwrap().to_string())
.collect();
let mut sorted = ids.clone();
sorted.sort();
assert_eq!(ids, sorted, "evt_ ids must sort into mint order");
}
fn wire_payload(system: &str, messages: usize) -> Value {
let messages: Vec<Value> = (0..messages)
.map(|i| serde_json::json!({"role": "user", "content": format!("m{i}")}))
.collect();
serde_json::json!({
"model": "claude-test",
"max_tokens": 1024,
"messages": messages,
"tools": [{"name": "bash"}],
"params": {},
"system": system,
})
}
async fn write_turn(w: &mut TraceWriter, turn: u32, payload: &Value, response: &str) {
let wire = WireCapture::from_driver_payload(payload, response);
w.write_inference(
turn,
10,
5,
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
vec!["msg_0000000000000000000000000000000a".to_string()],
Some(&wire),
)
.await
.unwrap();
}
fn blob_path(dir: &std::path::Path, sha: &str) -> std::path::PathBuf {
dir.join(BLOB_DIR_NAME).join(sha)
}
#[tokio::test]
async fn content_capture_names_blobs_that_exist_and_rehash_to_their_names() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let payload = wire_payload("You are a capsule.", 3);
write_turn(&mut w, 0, &payload, r#"{"stop_reason":"end_turn"}"#).await;
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
let mut named: Vec<String> = ["system_sha", "tools_sha", "response_sha"]
.iter()
.map(|key| {
e[*key]
.as_str()
.unwrap_or_else(|| panic!("{key} missing from {e}"))
.to_string()
})
.collect();
let shas = e["message_shas"].as_array().expect("message_shas");
assert_eq!(shas.len(), 3);
named.extend(shas.iter().map(|s| s.as_str().unwrap().to_string()));
for sha in &named {
let path = blob_path(dir.path(), sha);
let bytes =
std::fs::read(&path).unwrap_or_else(|err| panic!("blob {sha} must exist: {err}"));
assert_eq!(murmur_artifact::sha256_hex(&bytes), *sha);
assert_eq!(sha.len(), 64, "{sha}");
assert!(
sha.chars()
.all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()),
"{sha}"
);
}
}
#[tokio::test]
async fn the_system_blob_holds_the_prompt_as_readable_text() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let payload = wire_payload("[Capsule] you are cap.\nBe terse.", 1);
write_turn(&mut w, 0, &payload, "{}").await;
w.flush().await.unwrap();
let sha = read_events(dir.path())[0]["system_sha"]
.as_str()
.unwrap()
.to_string();
assert_eq!(
std::fs::read_to_string(blob_path(dir.path(), &sha)).unwrap(),
"[Capsule] you are cap.\nBe terse."
);
}
#[tokio::test]
async fn meta_writes_the_same_hashes_and_no_bodies() {
let payload = wire_payload("You are a capsule.", 2);
let response = r#"{"stop_reason":"end_turn"}"#;
let meta = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(meta.path(), TraceCapture::Meta).await;
write_turn(&mut w, 0, &payload, response).await;
w.flush().await.unwrap();
let content = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(content.path(), TraceCapture::Content).await;
write_turn(&mut w, 0, &payload, response).await;
w.flush().await.unwrap();
let meta_event = read_events(meta.path()).remove(0);
let content_event = read_events(content.path()).remove(0);
for key in ["system_sha", "tools_sha", "response_sha", "message_shas"] {
assert_eq!(meta_event[key], content_event[key], "{key}");
}
assert!(
!meta.path().join(BLOB_DIR_NAME).exists(),
"meta must not create the blob directory"
);
assert!(content.path().join(BLOB_DIR_NAME).is_dir());
}
#[tokio::test]
async fn the_default_capture_mode_is_meta() {
assert_eq!(TraceCapture::default(), TraceCapture::Meta);
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_prompt(dir.path(), TraceCapture::default(), None, false).await;
write_turn(&mut w, 0, &wire_payload("s", 1), "{}").await;
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
assert!(e.get("system_sha").is_some());
assert!(!dir.path().join(BLOB_DIR_NAME).exists());
}
#[tokio::test]
async fn none_omits_every_hash_and_leaves_the_rest_of_the_event_intact() {
let payload = wire_payload("You are a capsule.", 2);
let off = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(off.path(), TraceCapture::None).await;
write_turn(&mut w, 3, &payload, "{}").await;
w.flush().await.unwrap();
let e = read_events(off.path()).remove(0);
for key in ["system_sha", "tools_sha", "response_sha", "message_shas"] {
assert!(
e.get(key).is_none(),
"{key} must be absent under none, got {e}"
);
}
assert!(!off.path().join(BLOB_DIR_NAME).exists());
for key in [
"event_type",
"event_id",
"parent_id",
"session_id",
"timestamp",
"turn",
"task_id",
"input_tokens",
"output_tokens",
"decision",
"tool_name",
"message_ids",
] {
assert!(
e.get(key).is_some(),
"{key} must survive under none, got {e}"
);
}
assert_eq!(e["turn"], 3);
assert_eq!(e["message_ids"].as_array().unwrap().len(), 1);
}
#[tokio::test]
async fn an_unchanged_system_prompt_is_stored_once() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let payload = wire_payload("You are a capsule.", 1);
write_turn(&mut w, 0, &payload, "{}").await;
let sha = {
w.flush().await.unwrap();
read_events(dir.path())[0]["system_sha"]
.as_str()
.unwrap()
.to_string()
};
let path = blob_path(dir.path(), &sha);
std::fs::write(&path, b"sentinel").unwrap();
for turn in 1..10 {
write_turn(&mut w, turn, &payload, "{}").await;
}
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events.len(), 10);
for e in &events {
assert_eq!(e["system_sha"], sha.as_str());
}
assert_eq!(
std::fs::read(&path).unwrap(),
b"sentinel",
"an existing blob is never rewritten"
);
let matching = std::fs::read_dir(dir.path().join(BLOB_DIR_NAME))
.unwrap()
.filter(|entry| entry.as_ref().unwrap().file_name() == sha.as_str())
.count();
assert_eq!(matching, 1, "exactly one file bears that name");
}
#[tokio::test]
async fn stop_reason_is_absent_when_no_driver_response_was_parsed() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_inference(
0,
10,
5,
"end_turn".to_string(),
None,
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let event = read_events(dir.path()).remove(0);
assert!(
event.get("stop_reason").is_none(),
"stop_reason must be absent, got {event}"
);
}
#[tokio::test]
async fn a_hook_origin_record_carries_no_hashes() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let origin = InferenceOrigin {
source: "hook:gatekeeper".to_string(),
model: "claude-test".to_string(),
};
w.write_inference(
0,
1,
1,
"end_turn".to_string(),
None,
None,
Some(&origin),
None,
Vec::new(),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let e = read_events(dir.path()).remove(0);
assert_eq!(e["origin"], "hook:gatekeeper");
for key in ["system_sha", "tools_sha", "response_sha", "message_shas"] {
assert!(e.get(key).is_none(), "{key} must be absent, got {e}");
}
assert!(!dir.path().join(BLOB_DIR_NAME).exists());
}
#[tokio::test]
async fn message_shas_locate_the_index_two_runs_diverge_at() {
async fn shas_for(payload: &Value) -> Vec<String> {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Meta).await;
write_turn(&mut w, 0, payload, "{}").await;
w.flush().await.unwrap();
read_events(dir.path())[0]["message_shas"]
.as_array()
.unwrap()
.iter()
.map(|s| s.as_str().unwrap().to_string())
.collect()
}
let first = wire_payload("You are a capsule.", 4);
let mut second = first.clone();
second["messages"][2]["content"] = serde_json::json!("diverged");
let a = shas_for(&first).await;
let b = shas_for(&second).await;
assert_eq!(a.len(), 4);
let divergence = a.iter().zip(&b).position(|(x, y)| x != y);
assert_eq!(divergence, Some(2));
assert_eq!(a[3], b[3], "only the changed message changes its own sha");
}
#[tokio::test]
async fn identical_messages_share_one_sha_and_one_blob() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let payload = serde_json::json!({
"messages": [{"role": "user", "content": "same"}, {"role": "user", "content": "same"}],
"tools": [],
"system": "s",
});
write_turn(&mut w, 0, &payload, "{}").await;
w.flush().await.unwrap();
let shas = read_events(dir.path())[0]["message_shas"]
.as_array()
.unwrap()
.iter()
.map(|s| s.as_str().unwrap().to_string())
.collect::<Vec<_>>();
assert_eq!(shas[0], shas[1]);
assert!(blob_path(dir.path(), &shas[0]).is_file());
}
#[tokio::test]
async fn a_message_blob_is_the_body_verbatim() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let message = serde_json::json!({"role": "user", "content": "hello"});
let payload = serde_json::json!({
"messages": [message.clone()],
"tools": [],
"system": "s",
});
write_turn(&mut w, 0, &payload, "{}").await;
w.flush().await.unwrap();
let sha = read_events(dir.path())[0]["message_shas"][0]
.as_str()
.unwrap()
.to_string();
assert_eq!(
std::fs::read(blob_path(dir.path(), &sha)).unwrap(),
serde_json::to_vec(&message).unwrap()
);
}
#[tokio::test]
async fn a_plan_written_delegation_record_matches_a_delegate_task_one() {
let plan_dir = tempfile::tempdir().unwrap();
let tool_dir = tempfile::tempdir().unwrap();
let session = "ses_0000000000004000800000000000plan".to_string();
let node = "evt_0000000000004000800000000000node".to_string();
let plan = PlanTraceAppender::open(plan_dir.path(), session.clone(), node.clone()).unwrap();
let tool = ResourceTraceAppender::open(tool_dir.path(), session.clone(), node.clone())
.await
.unwrap();
plan.write_delegation_start("dlg_abc123", "worker", "0.1.0", "ses_child", ".murmur/c");
tool.write_delegation_start("dlg_abc123", "worker", "0.1.0", "ses_child", ".murmur/c")
.await;
plan.write_delegation(
"worker",
"0.1.0",
Some("dlg_abc123".to_string()),
Some("ses_child".to_string()),
42,
"completed",
None,
);
tool.write_delegation(
"worker",
"0.1.0",
Some("dlg_abc123".to_string()),
Some("ses_child".to_string()),
42,
"completed",
None,
)
.await;
let read = |dir: &Path| -> Vec<serde_json::Map<String, Value>> {
std::fs::read_to_string(dir.join("trace.jsonl"))
.unwrap()
.lines()
.map(|line| serde_json::from_str::<serde_json::Map<_, _>>(line).unwrap())
.collect()
};
let from_plan = read(plan_dir.path());
let from_tool = read(tool_dir.path());
assert_eq!(from_plan.len(), 2);
assert_eq!(from_tool.len(), 2);
for (planned, called, event_type) in [
(&from_plan[0], &from_tool[0], "delegation_start"),
(&from_plan[1], &from_tool[1], "delegation"),
] {
assert_eq!(planned["event_type"], event_type);
assert_eq!(called["event_type"], event_type);
assert_eq!(
planned.keys().collect::<Vec<_>>(),
called.keys().collect::<Vec<_>>(),
"{planned:?} vs {called:?}"
);
assert_eq!(planned["parent_id"], node.as_str());
assert_eq!(called["parent_id"], node.as_str());
for (key, value) in planned {
if key == "event_id" || key == "timestamp" {
continue;
}
assert_eq!(value, &called[key], "{key}: {planned:?} vs {called:?}");
}
for key in ["plan_id", "step_id", "surface", "kind"] {
assert!(!planned.contains_key(key), "{key} is on {planned:?}");
assert!(!called.contains_key(key), "{key} is on {called:?}");
}
}
}
}