use std::{
path::Path,
time::{Instant, SystemTime, UNIX_EPOCH},
};
use murmur_artifact::{ContainmentClass, LockOrigin, TraceCapture};
use serde::Serialize;
use serde_json::Value;
use tokio::{
fs::{File, OpenOptions},
io::{AsyncWriteExt, BufWriter},
};
use crate::{
agent::DriverUsage,
containment::ScopeReport,
formation::FormationId,
gateway_credential::CredentialChange,
lanes::TaskLane,
origin::{TaskProvenance, TrustClass},
trace_blobs::BlobStore,
types::InstalledArtifactSummary,
};
pub(crate) struct TraceWriter {
writer: BufWriter<File>,
session_id: String,
capsule_name: String,
capsule_version: String,
model: String,
capabilities: Vec<String>,
effective_grants: ScopeReport,
capture: TraceCapture,
blobs: BlobStore,
system_prompt_source: &'static str,
system_prompt_sha256: Option<String>,
credential_source: &'static str,
gateways: Vec<SessionGateway>,
control: Option<SessionControl>,
inference_choices: Vec<SessionInferenceChoice>,
tool_refresh: Option<&'static str>,
runtime_artifacts: Vec<RuntimeArtifact>,
resumed_from: Option<String>,
context_id: Option<String>,
spawned_by: Option<String>,
spawn_delegation_id: Option<String>,
formation_id: Option<FormationId>,
formation_member: Option<(String, Vec<String>)>,
session_start_time: Instant,
session_event_id: String,
session_started: bool,
session_ended: bool,
task_event_id: Option<String>,
turn_event_id: Option<String>,
total_turns: u32,
total_input_tokens: u64,
total_output_tokens: u64,
total_tool_calls: u32,
total_shell_calls: u32,
task_turns: u32,
task_input_tokens: u64,
task_output_tokens: u64,
task_tool_calls: u32,
task_shell_calls: u32,
task_agent_turns: u32,
task_hook_inferences: u32,
task_start_instant: Option<Instant>,
pub(crate) active_task_id: Option<String>,
task_failures_written: u64,
last_task_failure: Option<String>,
max_session_tokens: Option<u64>,
machine_tokens_per_day: Option<u64>,
}
#[derive(Debug, Clone)]
pub(crate) struct InferenceOrigin {
pub(crate) source: String,
pub(crate) model: String,
}
pub(crate) const COMPACTION_DECLINED_NO_HOOK_REPLACEMENT: &str = "no_hook_replacement";
pub(crate) const COMPACTION_DECLINED_UNRESOLVED_TOOL_CALL: &str = "unresolved_tool_call";
pub(crate) const SEED_OUTCOME_SEEDED: &str = "seeded";
pub(crate) const SEED_OUTCOME_TRIMMED: &str = "trimmed";
pub(crate) const SEED_OUTCOME_COMPACTED: &str = "compacted";
pub(crate) const SEED_OUTCOME_REJECTED: &str = "rejected";
pub(crate) const SEED_REJECTED_MESSAGE_OVER_BUDGET: &str = "message_over_budget";
pub(crate) const SEED_REJECTED_OVERFLOW_OVER_LIMIT: &str = "overflow_over_limit";
pub(crate) const SEED_REJECTED_NO_BUDGET: &str = "no_budget";
pub(crate) const SEED_REJECTED_UNSUPPORTED_TRANSPORT: &str = "unsupported_transport";
pub(crate) const RETENTION_STORE_SESSIONS: &str = "sessions";
pub(crate) const RETENTION_STORE_RECORDS: &str = "records";
pub(crate) const RETENTION_REASON_MAX_SESSIONS: &str = "max_sessions";
pub(crate) const RETENTION_REASON_MAX_AGE: &str = "max_age";
pub(crate) const RETENTION_REASON_MAX_MESSAGES: &str = "max_messages";
pub(crate) fn new_event_id() -> String {
format!("evt_{}", uuid::Uuid::now_v7().simple())
}
#[derive(Serialize)]
struct SessionStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
capsule_name: String,
capsule_version: String,
model: String,
max_turns: u32,
max_session_tokens: Option<u64>,
machine_tokens_per_day: Option<u64>,
capabilities: Vec<String>,
tools_declared: Vec<String>,
containment_declared: ContainmentClass,
containment_achieved: ContainmentClass,
workdir_exec: bool,
userns_grant: Option<crate::sealed::UsernsGrant>,
effective_grants: ScopeReport,
system_prompt_source: &'static str,
system_prompt_sha256: Option<String>,
credential_source: &'static str,
gateways: Vec<SessionGateway>,
#[serde(skip_serializing_if = "Option::is_none")]
control: Option<SessionControl>,
#[serde(skip_serializing_if = "Vec::is_empty")]
inference_choices: Vec<SessionInferenceChoice>,
tool_refresh: Option<&'static str>,
runtime_artifacts: Vec<RuntimeArtifact>,
resumed_from: Option<String>,
context_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
spawned_by: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
delegation_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
formation_id: Option<FormationId>,
#[serde(skip_serializing_if = "Option::is_none")]
formation_member: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
formation_callees: Option<Vec<String>>,
}
#[derive(Serialize)]
struct InferenceEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
input_tokens: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
output_tokens: Option<u64>,
decision: String,
#[serde(skip_serializing_if = "Option::is_none")]
stop_reason: Option<String>,
tool_name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
input_tokens_actual: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
output_tokens_actual: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
cached_tokens: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
cache_write_tokens: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
thinking_tokens: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
origin: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
driver_choice: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
model: Option<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
message_ids: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
system_sha: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
tools_sha: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
response_sha: Option<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
message_shas: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
error_code: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
error: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
provider_status: Option<u16>,
}
pub(crate) struct WireCapture {
system: Vec<u8>,
tools: Vec<u8>,
messages: Vec<Vec<u8>>,
response: Vec<u8>,
}
impl WireCapture {
pub(crate) fn from_driver_payload(payload: &Value, response: &str) -> Self {
Self {
system: payload
.get("system")
.and_then(Value::as_str)
.unwrap_or_default()
.as_bytes()
.to_vec(),
tools: payload
.get("tools")
.map(|tools| serde_json::to_vec(tools).unwrap_or_default())
.unwrap_or_default(),
messages: payload
.get("messages")
.and_then(Value::as_array)
.map(|messages| {
messages
.iter()
.map(|message| serde_json::to_vec(message).unwrap_or_default())
.collect()
})
.unwrap_or_default(),
response: response.as_bytes().to_vec(),
}
}
}
#[derive(Serialize)]
struct ToolCallEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
tool_name: String,
tool_call_id: Option<String>,
input: Value,
input_bytes: u64,
#[serde(skip_serializing_if = "Option::is_none")]
output: Option<String>,
output_bytes: u64,
duration_ms: u64,
status: String,
#[serde(skip_serializing_if = "Option::is_none")]
state_effect: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
resource_id: Option<String>,
}
#[derive(Serialize)]
struct SkillCallEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
skill_name: String,
output_bytes: u64,
duration_ms: u64,
status: String,
origin: &'static str,
trust: &'static str,
}
#[derive(Serialize)]
struct ShellEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
binary: String,
command: String,
exit_code: i32,
stdout_bytes: u64,
stderr_bytes: u64,
duration_ms: u64,
#[serde(skip_serializing_if = "Option::is_none")]
resource_limit: Option<String>,
}
#[derive(Serialize)]
struct ShellDetachedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
work_id: String,
binary: String,
command: String,
grace_ms: u64,
}
#[derive(Serialize)]
struct ShellCompletedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
work_id: String,
binary: String,
command: String,
exit_code: i32,
duration_ms: u64,
output_path: String,
output_bytes: u64,
#[serde(skip_serializing_if = "Option::is_none")]
resource_limit: Option<String>,
status: String,
completion_task_id: String,
}
#[derive(Serialize)]
struct ShellAbandonedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
work_id: String,
binary: String,
command: String,
running_ms: u64,
#[serde(skip_serializing_if = "Option::is_none")]
exit_code: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
output_path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
output_bytes: Option<u64>,
}
#[derive(Serialize)]
struct ShellLostEvent {
event_type: &'static str,
event_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
parent_id: Option<String>,
session_id: String,
timestamp: u64,
work_id: String,
binary: String,
command: String,
detached_at_ms: u64,
reconciled_by_session: String,
reconciled_task_id: String,
}
#[derive(Serialize)]
struct ShellDetachUnrecordedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
work_id: String,
binary: String,
reason: String,
}
#[derive(Serialize)]
struct CompactionEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
tokens_before: u64,
tokens_after: u64,
}
#[derive(Serialize)]
struct CompactionDeclinedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
tokens: u64,
reason: String,
}
#[derive(Serialize)]
struct ContextSeedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: Option<String>,
hook_name: String,
tokens: u64,
proposed_tokens: u64,
budget_tokens: u64,
outcome: String,
#[serde(skip_serializing_if = "Option::is_none")]
reason: Option<String>,
message_ids: Vec<String>,
}
#[derive(Serialize)]
struct RetentionEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
store: String,
reason: String,
removed: u32,
targets: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
messages_dropped: Option<u64>,
}
#[derive(Serialize)]
struct A2aTaskReceivedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
context_id: String,
message_id: String,
traceparent_from_caller: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
caller_member: Option<String>,
}
#[derive(Serialize)]
struct A2aSendEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
peer_url: String,
message_id: String,
task_id: String,
context_id: String,
traceparent: Option<String>,
trust: String,
}
#[derive(Serialize)]
struct ArtifactPulledEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
name: String,
version: String,
runtime: &'static str,
origin: &'static str,
session: Option<String>,
trust: &'static str,
}
#[derive(Serialize)]
struct SessionEndEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
total_turns: u32,
total_input_tokens: u64,
total_output_tokens: u64,
total_tool_calls: u32,
total_shell_calls: u32,
duration_ms: u64,
exit_status: String,
}
#[derive(Serialize)]
struct TaskStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
context_id: String,
source: String,
origin: String,
trust: String,
lane: String,
message_parts_bytes: u64,
}
#[derive(Serialize)]
struct TaskEndEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
exit_status: String,
duration_ms: u64,
turns: u32,
input_tokens: u64,
output_tokens: u64,
tool_calls: u32,
shell_calls: u32,
reopen_count: u32,
}
#[derive(Serialize)]
struct TaskCanceledEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
turn: Option<u32>,
phase: String,
detached_work_ids: Vec<String>,
delegation_ids: Vec<String>,
}
pub(crate) const TASK_REJECTED_SESSION_ENDED: &str = "session_ended";
pub(crate) const TASK_REJECTED_SESSION_STOPPED: &str = "session_stopped";
#[derive(Serialize)]
struct TaskRejectedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
context_id: String,
source: String,
cause: String,
reason: String,
}
pub(crate) const TASK_FAILED_DRIVER_ERROR: &str = "driver_error";
pub(crate) const TASK_FAILED_CREDENTIAL_REJECTED: &str = "credential_rejected";
pub(crate) const TASK_FAILED_MALFORMED_RESPONSE: &str = "malformed_response";
pub(crate) const TASK_FAILED_COMPACTION_HOOK: &str = "compaction_hook";
pub(crate) const TASK_FAILED_INPUT_TIMEOUT: &str = "input_timeout";
pub(crate) const TASK_FAILED_REOPEN_BUDGET_EXHAUSTED: &str = "reopen_budget_exhausted";
pub(crate) const TASK_FAILED_RUNTIME_ERROR: &str = "runtime_error";
pub(crate) const INFERENCE_ERROR_CREDENTIAL_REJECTED: &str = TASK_FAILED_CREDENTIAL_REJECTED;
pub(crate) const INFERENCE_ERROR_DRIVER_ERROR: &str = TASK_FAILED_DRIVER_ERROR;
pub(crate) const INFERENCE_ERROR_DRIVER_FAILED: &str = "driver_failed";
pub(crate) const INFERENCE_ERROR_MALFORMED_RESPONSE: &str = TASK_FAILED_MALFORMED_RESPONSE;
pub(crate) const INFERENCE_ERROR_HARNESS_AUTH: &str = "harness_auth";
pub(crate) const INFERENCE_ERROR_HARNESS_QUOTA: &str = "harness_quota";
pub(crate) const INFERENCE_ERROR_HARNESS_ERROR: &str = "harness_error";
pub(crate) const INFERENCE_ERROR_HARNESS_OTHER: &str = "harness_other";
pub(crate) const INFERENCE_ERROR_HARNESS_INACTIVE: &str = "harness_inactive";
pub(crate) struct FailedInference<'a> {
pub(crate) turn: u32,
pub(crate) error_code: &'static str,
pub(crate) error: &'a str,
pub(crate) provider_status: Option<u16>,
pub(crate) usage: Option<&'a DriverUsage>,
pub(crate) origin: Option<&'a InferenceOrigin>,
pub(crate) choice: Option<&'a crate::driver_choice::DriverChoice>,
pub(crate) message_ids: Vec<String>,
pub(crate) tool_name: Option<String>,
pub(crate) spent: Option<(Option<u64>, Option<u64>)>,
}
struct InferenceLine<'a> {
turn: u32,
input_tokens: Option<u64>,
output_tokens: Option<u64>,
decision: String,
stop_reason: Option<&'a str>,
tool_name: Option<String>,
origin: Option<&'a InferenceOrigin>,
usage: Option<&'a DriverUsage>,
message_ids: Vec<String>,
wire: Option<&'a WireCapture>,
choice: Option<&'a crate::driver_choice::DriverChoice>,
failure: Option<(&'static str, String, Option<u16>)>,
counted: bool,
}
pub(crate) const MAX_TASK_FAILURE_REASON_BYTES: usize = 2_000;
#[derive(Serialize)]
struct TaskFailedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
#[serde(skip_serializing_if = "Option::is_none")]
task_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
turn: Option<u32>,
cause: String,
reason: String,
}
pub(crate) const REOPEN_CONTEXT_CONTINUED: &str = "continued";
pub(crate) const REOPEN_CONTEXT_RESTARTED: &str = "restarted";
#[derive(Serialize)]
struct TaskReopenedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
hook_name: String,
reason: String,
reopen_number: u32,
attempt_context: &'static str,
turns_remaining: u32,
}
#[derive(Serialize)]
struct TaskContinuedEvent<'a> {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
continuation_number: u32,
member_calls: &'a [String],
delegations: &'a [String],
waited_ms: u64,
turns_remaining: u32,
}
#[derive(Serialize)]
struct SpendCeilingReachedEvent<'a> {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
#[serde(flatten)]
refusal: &'a crate::spend::SpendRefusal,
#[serde(skip_serializing_if = "Option::is_none")]
origin: Option<&'a str>,
}
#[derive(Serialize)]
pub(crate) struct HarnessStart {
pub driver: String,
pub driver_version: String,
pub harness: String,
pub binary: String,
pub binary_source: String,
pub harness_version: Option<String>,
pub version_tested: bool,
pub harness_session_id: String,
pub session_mode: String,
pub args_count: usize,
pub env_names: Vec<String>,
pub files: Vec<String>,
pub bridge_tools: Vec<String>,
pub stdin_bytes: usize,
pub keep_stdin_open: bool,
}
#[derive(Serialize)]
struct HarnessStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: Option<String>,
#[serde(flatten)]
start: HarnessStart,
}
#[derive(Serialize)]
struct HarnessWarningEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: Option<String>,
code: String,
message: String,
}
#[derive(Serialize)]
struct HarnessSessionEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: Option<String>,
harness_session_id: String,
auth: String,
model: Option<String>,
}
#[derive(Serialize)]
struct HarnessSessionForgottenEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: Option<String>,
context_id: String,
harness_session_id: String,
requested_by: String,
}
#[derive(Serialize)]
struct HarnessRetryEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: Option<String>,
attempt: u32,
reason: String,
}
#[derive(Serialize)]
struct HarnessNoteEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: Option<String>,
text: String,
}
#[derive(Serialize)]
struct HarnessFailedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: Option<String>,
kind: String,
message: String,
source: String,
}
#[derive(Serialize)]
struct HarnessInterruptEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: Option<String>,
method: String,
delivered: bool,
grace_ms: u64,
}
#[derive(Serialize)]
pub(crate) struct HarnessExit {
pub code: Option<i32>,
pub signal: Option<i32>,
pub cause: String,
pub killed: bool,
pub duration_ms: u64,
}
#[derive(Serialize)]
struct HarnessExitEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: Option<String>,
#[serde(flatten)]
exit: HarnessExit,
}
#[derive(Serialize)]
struct CallDeniedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
event: String,
hook_name: String,
target: String,
reason: String,
}
#[derive(Serialize)]
struct ToolInputRefusedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
tool_name: String,
tool_call_id: Option<String>,
missing: Vec<String>,
reason: String,
}
#[derive(Serialize)]
struct ProtectedPathDeniedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
call: String,
target: String,
path: String,
rule: String,
signal: String,
reason: String,
}
#[derive(Serialize)]
struct HookDispatchErrorEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
hook_name: String,
event: String,
arm: String,
}
#[derive(Serialize)]
struct FormationEndedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
formation_id: String,
}
#[derive(Serialize)]
struct SpawnerEndedEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
#[serde(skip_serializing_if = "Option::is_none")]
spawned_by: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
delegation_id: Option<String>,
}
#[derive(Serialize)]
struct ResourceListEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
root: String,
entry_count: usize,
total_bytes: u64,
generation: u64,
containment_achieved: ContainmentClass,
outcome: String,
reason: Option<String>,
}
#[derive(Serialize)]
struct ResourceReadEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
path: String,
outcome: String,
bytes: Option<u64>,
sha256: Option<String>,
generation: u64,
containment_achieved: ContainmentClass,
reason: Option<String>,
}
#[derive(Serialize)]
struct PeerHandleMintEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
handle_id: Option<String>,
path: String,
audience: String,
expires_at_ms: Option<u64>,
outcome: String,
reason: Option<String>,
}
#[derive(Serialize)]
struct PeerHandleRedeemEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
handle_id: String,
path: Option<String>,
generation: u64,
audience_asserted: Option<String>,
bytes: Option<u64>,
sha256: Option<String>,
outcome: String,
reason: Option<String>,
}
#[derive(Serialize)]
struct PeerFileFetchEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
peer: String,
handle_id: String,
stored_path: Option<String>,
bytes: Option<u64>,
sha256: Option<String>,
outcome: String,
reason: Option<String>,
}
#[derive(Serialize)]
struct DelegationStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
delegation_id: String,
capsule: String,
version: String,
child_session_id: String,
child_workdir: String,
}
impl DelegationStartEvent {
fn new(
session_id: &str,
session_event_id: &str,
delegation_id: &str,
capsule: &str,
version: &str,
child_session_id: &str,
child_workdir: &str,
) -> Self {
Self {
event_type: "delegation_start",
event_id: new_event_id(),
parent_id: Some(session_event_id.to_string()),
session_id: session_id.to_string(),
timestamp: timestamp_ms(),
delegation_id: delegation_id.to_string(),
capsule: capsule.to_string(),
version: version.to_string(),
child_session_id: child_session_id.to_string(),
child_workdir: child_workdir.to_string(),
}
}
}
#[derive(Serialize)]
struct MemberCallStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
call_id: String,
member: String,
member_task_id: String,
}
#[derive(Serialize)]
struct MemberCallBusyEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
call_id: String,
member: String,
offer: u32,
waited_ms: u64,
message: String,
}
#[derive(Serialize)]
struct MemberCallEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
task_id: String,
call_id: String,
member: String,
#[serde(skip_serializing_if = "Option::is_none")]
member_task_id: Option<String>,
status: String,
duration_ms: u64,
output: String,
truncated: bool,
delivered: bool,
#[serde(skip_serializing_if = "Vec::is_empty")]
no_answer_below: Vec<NoAnswerBelow>,
}
#[derive(Serialize)]
struct NoAnswerBelow {
member: String,
status: String,
}
#[derive(Serialize)]
struct DelegationEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
capsule: String,
version: String,
delegation_id: Option<String>,
child_session_id: Option<String>,
duration_ms: u64,
outcome: String,
reason: Option<String>,
}
impl DelegationEvent {
#[allow(clippy::too_many_arguments)]
fn new(
session_id: &str,
session_event_id: &str,
capsule: &str,
version: &str,
delegation_id: Option<String>,
child_session_id: Option<String>,
duration_ms: u64,
outcome: &str,
reason: Option<String>,
) -> Self {
Self {
event_type: "delegation",
event_id: new_event_id(),
parent_id: Some(session_event_id.to_string()),
session_id: session_id.to_string(),
timestamp: timestamp_ms(),
capsule: capsule.to_string(),
version: version.to_string(),
delegation_id,
child_session_id,
duration_ms,
outcome: outcome.to_string(),
reason,
}
}
}
impl TraceWriter {
#[allow(clippy::too_many_arguments)]
pub(crate) async fn open(
workdir: &Path,
session_id: String,
capsule_name: String,
capsule_version: String,
model: String,
capabilities: Vec<String>,
effective_grants: ScopeReport,
capture: TraceCapture,
system_prompt: Option<String>,
system_prompt_overridden: bool,
resumed_from: Option<String>,
context_id: Option<String>,
spawned_by: Option<String>,
spawn_delegation_id: Option<String>,
) -> std::io::Result<Self> {
let file = OpenOptions::new()
.create(true)
.append(true)
.open(workdir.join("trace.jsonl"))
.await?;
let system_prompt_source = match (system_prompt_overridden, system_prompt.is_some()) {
(true, _) => "cli",
(false, true) => "manifest",
(false, false) => "none",
};
let system_prompt_sha256 = system_prompt
.as_ref()
.map(|prompt| murmur_artifact::sha256_hex(prompt.as_bytes()));
let blobs = BlobStore::new(workdir);
if let (true, Some(prompt), Some(sha)) = (
capture.captures_content(),
system_prompt.as_ref(),
system_prompt_sha256.as_deref(),
) {
blobs.put(sha, prompt.as_bytes()).await?;
}
Ok(Self {
writer: BufWriter::new(file),
session_id,
capsule_name,
capsule_version,
model,
capabilities,
effective_grants,
capture,
blobs,
system_prompt_source,
system_prompt_sha256,
credential_source: "none",
gateways: Vec::new(),
control: None,
inference_choices: Vec::new(),
tool_refresh: None,
runtime_artifacts: Vec::new(),
resumed_from,
context_id,
spawned_by,
spawn_delegation_id,
formation_id: None,
formation_member: None,
session_start_time: Instant::now(),
session_event_id: new_event_id(),
session_started: false,
session_ended: false,
task_event_id: None,
turn_event_id: None,
total_turns: 0,
total_input_tokens: 0,
total_output_tokens: 0,
total_tool_calls: 0,
total_shell_calls: 0,
task_turns: 0,
task_input_tokens: 0,
task_output_tokens: 0,
task_tool_calls: 0,
task_shell_calls: 0,
task_agent_turns: 0,
task_hook_inferences: 0,
task_start_instant: None,
active_task_id: None,
task_failures_written: 0,
last_task_failure: None,
max_session_tokens: None,
machine_tokens_per_day: None,
})
}
pub(crate) fn set_spend_ceilings(
&mut self,
max_session_tokens: Option<u64>,
machine_tokens_per_day: Option<u64>,
) {
self.max_session_tokens = max_session_tokens;
self.machine_tokens_per_day = machine_tokens_per_day;
}
pub(crate) fn session_event_id(&self) -> &str {
&self.session_event_id
}
pub(crate) fn captures_wire(&self) -> bool {
self.capture.captures_hashes()
}
fn session_parent(&self) -> Option<String> {
self.session_started.then(|| self.session_event_id.clone())
}
fn task_parent(&self) -> Option<String> {
self.task_event_id.clone().or_else(|| self.session_parent())
}
fn turn_parent(&self) -> Option<String> {
self.turn_event_id.clone().or_else(|| self.task_parent())
}
pub(crate) fn set_credential_source(&mut self, source: &'static str) {
self.credential_source = source;
}
pub(crate) fn set_gateways(&mut self, gateways: Vec<SessionGateway>) {
self.gateways = gateways;
}
pub(crate) fn set_control(&mut self, control: Option<SessionControl>) {
self.control = control;
}
pub(crate) fn set_inference_choices(&mut self, choices: Vec<SessionInferenceChoice>) {
self.inference_choices = choices;
}
pub(crate) fn set_runtime_artifacts(&mut self, installed: &[InstalledArtifactSummary]) {
self.runtime_artifacts = installed
.iter()
.filter_map(|artifact| match &artifact.origin {
LockOrigin::Runtime { session } => Some(RuntimeArtifact {
name: artifact.name.clone(),
version: artifact.version.clone(),
origin: artifact.origin.as_str(),
session: session.clone(),
trust: crate::origin::artifact_trust(&artifact.origin).as_str(),
}),
LockOrigin::Operator => None,
})
.collect();
}
pub(crate) async fn write_control_applied(
&mut self,
turn: u32,
name: &str,
value: Value,
change_id: &str,
) -> std::io::Result<()> {
let event = ControlAppliedEvent {
event_type: "control_applied",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
name,
value,
change_id,
};
self.write_event(&event).await
}
pub(crate) fn set_formation_id(&mut self, formation_id: Option<&FormationId>) {
self.formation_id = formation_id.cloned();
}
pub(crate) fn set_formation_member(
&mut self,
member: Option<&crate::formation_credentials::FormationMember>,
) {
self.formation_member = member.map(|member| {
(
member.name().to_string(),
member.callees().map(str::to_string).collect(),
)
});
}
pub(crate) fn set_tool_refresh(&mut self, tool_refresh: Option<&'static str>) {
self.tool_refresh = tool_refresh;
}
pub(crate) async fn write_tools_refreshed(
&mut self,
turn: u32,
trigger: &str,
added: &[String],
removed: &[String],
tools: &[String],
) -> std::io::Result<()> {
let event = ToolsRefreshedEvent {
event_type: "tools_refreshed",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
trigger,
added,
removed,
tools,
};
self.write_event(&event).await
}
pub(crate) async fn write_session_start(
&mut self,
max_turns: u32,
tools_declared: Vec<String>,
) -> std::io::Result<()> {
let event = SessionStartEvent {
event_type: "session_start",
event_id: self.session_event_id.clone(),
parent_id: None,
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
capsule_name: self.capsule_name.clone(),
capsule_version: self.capsule_version.clone(),
model: self.model.clone(),
max_turns,
max_session_tokens: self.max_session_tokens,
machine_tokens_per_day: self.machine_tokens_per_day,
capabilities: self.capabilities.clone(),
tools_declared,
containment_declared: self.effective_grants.declared_containment,
containment_achieved: self.effective_grants.achieved_containment,
workdir_exec: self.effective_grants.workdir_exec,
userns_grant: self.effective_grants.userns_grant,
effective_grants: self.effective_grants.clone(),
system_prompt_source: self.system_prompt_source,
system_prompt_sha256: self.system_prompt_sha256.clone(),
credential_source: self.credential_source,
gateways: self.gateways.clone(),
control: self.control.clone(),
inference_choices: self.inference_choices.clone(),
tool_refresh: self.tool_refresh,
runtime_artifacts: self.runtime_artifacts.clone(),
resumed_from: self.resumed_from.clone(),
context_id: self.context_id.clone(),
spawned_by: self.spawned_by.clone(),
delegation_id: self.spawn_delegation_id.clone(),
formation_id: self.formation_id.clone(),
formation_member: self.formation_member.as_ref().map(|(name, _)| name.clone()),
formation_callees: self
.formation_member
.as_ref()
.map(|(_, callees)| callees.clone()),
};
self.write_event(&event).await?;
self.session_started = true;
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_inference(
&mut self,
turn: u32,
input_tokens: Option<u64>,
output_tokens: Option<u64>,
decision: String,
stop_reason: Option<&str>,
tool_name: Option<String>,
origin: Option<&InferenceOrigin>,
usage: Option<&DriverUsage>,
message_ids: Vec<String>,
wire: Option<&WireCapture>,
) -> std::io::Result<()> {
self.write_inference_event(InferenceLine {
turn,
input_tokens,
output_tokens,
decision,
stop_reason,
tool_name,
origin,
usage,
message_ids,
wire,
choice: None,
failure: None,
counted: true,
})
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_turn_inference(
&mut self,
turn: u32,
input_tokens: Option<u64>,
output_tokens: Option<u64>,
decision: String,
stop_reason: Option<&str>,
tool_name: Option<String>,
usage: Option<&DriverUsage>,
message_ids: Vec<String>,
wire: Option<&WireCapture>,
choice: Option<&crate::driver_choice::DriverChoice>,
) -> std::io::Result<()> {
self.write_inference_event(InferenceLine {
turn,
input_tokens,
output_tokens,
decision,
stop_reason,
tool_name,
origin: None,
usage,
message_ids,
wire,
choice,
failure: None,
counted: true,
})
.await
}
pub(crate) async fn write_failed_inference(
&mut self,
failed: FailedInference<'_>,
) -> std::io::Result<()> {
let error =
crate::agent::truncate_on_char_boundary(failed.error, MAX_TASK_FAILURE_REASON_BYTES)
.to_string();
let (input_tokens, output_tokens) = failed.spent.unwrap_or((None, None));
self.write_inference_event(InferenceLine {
turn: failed.turn,
input_tokens,
output_tokens,
decision: "error".to_string(),
stop_reason: Some("error"),
tool_name: failed.tool_name,
origin: failed.origin,
usage: failed.usage,
message_ids: failed.message_ids,
wire: None,
choice: failed.choice,
failure: Some((failed.error_code, error, failed.provider_status)),
counted: failed.spent.is_some(),
})
.await
}
async fn write_inference_event(&mut self, line: InferenceLine<'_>) -> std::io::Result<()> {
let InferenceLine {
turn,
input_tokens,
output_tokens,
decision,
stop_reason,
tool_name,
origin,
usage,
message_ids,
wire,
choice,
failure,
counted,
} = line;
let (error_code, error, provider_status) = match failure {
Some((code, error, status)) => (Some(code.to_string()), Some(error), status),
None => (None, None, None),
};
let event_id = new_event_id();
let parent_id = if origin.is_none() {
let parent = self.task_parent();
self.turn_event_id = Some(event_id.clone());
parent
} else {
self.turn_parent()
};
let (system_sha, tools_sha, response_sha, message_shas) =
match wire.filter(|_| self.capture.captures_hashes()) {
Some(wire) => self.record_wire(wire).await?,
None => (None, None, None, Vec::new()),
};
let event = InferenceEvent {
event_type: "inference",
event_id,
parent_id,
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
input_tokens,
output_tokens,
decision,
stop_reason: stop_reason.map(str::to_string),
tool_name,
input_tokens_actual: usage.and_then(|u| u.input_tokens),
output_tokens_actual: usage.and_then(|u| u.output_tokens),
cached_tokens: usage.and_then(|u| u.cached_tokens),
cache_write_tokens: usage.and_then(|u| u.cache_write_tokens),
thinking_tokens: usage.and_then(|u| u.thinking_tokens),
origin: origin.map(|o| o.source.clone()),
driver_choice: choice.map(|choice| choice.name.clone()),
model: origin
.map(|o| o.model.clone())
.or_else(|| choice.map(|choice| choice.model.clone())),
message_ids,
system_sha,
tools_sha,
response_sha,
message_shas,
error_code,
error,
provider_status,
};
self.write_event(&event).await?;
if origin.is_none() {
self.task_agent_turns = self.task_agent_turns.saturating_add(1);
}
if !counted {
return Ok(());
}
let (input_tokens, output_tokens) = (input_tokens.unwrap_or(0), output_tokens.unwrap_or(0));
self.total_turns = self.total_turns.saturating_add(1);
self.total_input_tokens = self.total_input_tokens.saturating_add(input_tokens);
self.total_output_tokens = self.total_output_tokens.saturating_add(output_tokens);
self.task_turns = self.task_turns.saturating_add(1);
if origin.is_some() {
self.task_hook_inferences = self.task_hook_inferences.saturating_add(1);
}
self.task_input_tokens = self.task_input_tokens.saturating_add(input_tokens);
self.task_output_tokens = self.task_output_tokens.saturating_add(output_tokens);
Ok(())
}
async fn record_wire(
&mut self,
wire: &WireCapture,
) -> std::io::Result<(Option<String>, Option<String>, Option<String>, Vec<String>)> {
let system_sha = murmur_artifact::sha256_hex(&wire.system);
let tools_sha = murmur_artifact::sha256_hex(&wire.tools);
let response_sha = murmur_artifact::sha256_hex(&wire.response);
let message_shas: Vec<String> = wire
.messages
.iter()
.map(|message| murmur_artifact::sha256_hex(message))
.collect();
if self.capture.captures_content() {
self.blobs.put(&system_sha, &wire.system).await?;
self.blobs.put(&tools_sha, &wire.tools).await?;
self.blobs.put(&response_sha, &wire.response).await?;
for (sha, body) in message_shas.iter().zip(&wire.messages) {
self.blobs.put(sha, body).await?;
}
}
Ok((
Some(system_sha),
Some(tools_sha),
Some(response_sha),
message_shas,
))
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_tool_call(
&mut self,
turn: u32,
tool_name: String,
tool_call_id: Option<String>,
mut input: Value,
input_bytes: u64,
output: &str,
output_bytes: u64,
duration_ms: u64,
status: String,
state_effect: Option<String>,
resource_id: Option<String>,
) -> std::io::Result<()> {
crate::peer_handoff::redact_handles_in_json(&mut input);
let event = ToolCallEvent {
event_type: "tool_call",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
tool_name,
tool_call_id: tool_call_id.filter(|id| !id.is_empty()),
input,
input_bytes,
output: self
.capture
.captures_content()
.then(|| crate::peer_handoff::redact_handle_tokens(output).into_owned()),
output_bytes,
duration_ms,
status,
state_effect,
resource_id,
};
self.write_event(&event).await?;
self.total_tool_calls = self.total_tool_calls.saturating_add(1);
self.task_tool_calls = self.task_tool_calls.saturating_add(1);
Ok(())
}
pub(crate) async fn write_skill_call(
&mut self,
turn: u32,
skill_name: String,
origin: &LockOrigin,
output_bytes: u64,
duration_ms: u64,
status: String,
) -> std::io::Result<()> {
let event = SkillCallEvent {
event_type: "skill_call",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
skill_name,
output_bytes,
duration_ms,
status,
origin: origin.as_str(),
trust: crate::origin::artifact_trust(origin).as_str(),
};
self.write_event(&event).await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_shell(
&mut self,
turn: u32,
binary: String,
command: String,
exit_code: i32,
stdout_bytes: u64,
stderr_bytes: u64,
duration_ms: u64,
resource_limit: Option<String>,
) -> std::io::Result<()> {
let event = ShellEvent {
event_type: "shell",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
binary,
command,
exit_code,
stdout_bytes,
stderr_bytes,
duration_ms,
resource_limit,
};
self.write_event(&event).await?;
self.total_shell_calls = self.total_shell_calls.saturating_add(1);
self.task_shell_calls = self.task_shell_calls.saturating_add(1);
Ok(())
}
pub(crate) async fn write_shell_detached(
&mut self,
turn: u32,
work_id: &str,
binary: &str,
command: &str,
grace_ms: u64,
) -> std::io::Result<()> {
let event = ShellDetachedEvent {
event_type: "shell_detached",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
work_id: work_id.to_string(),
binary: binary.to_string(),
command: command.to_string(),
grace_ms,
};
self.write_event(&event).await?;
self.total_shell_calls = self.total_shell_calls.saturating_add(1);
self.task_shell_calls = self.task_shell_calls.saturating_add(1);
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_shell_completed(
&mut self,
work_id: &str,
binary: &str,
command: &str,
exit_code: i32,
duration_ms: u64,
output_path: &str,
output_bytes: u64,
resource_limit: Option<String>,
status: &str,
completion_task_id: &str,
) -> std::io::Result<()> {
let event = ShellCompletedEvent {
event_type: "shell_completed",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
work_id: work_id.to_string(),
binary: binary.to_string(),
command: command.to_string(),
exit_code,
duration_ms,
output_path: output_path.to_string(),
output_bytes,
resource_limit,
status: status.to_string(),
completion_task_id: completion_task_id.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_shell_abandoned(
&mut self,
work_id: &str,
binary: &str,
command: &str,
running_ms: u64,
recovered: Option<(i32, &str, u64)>,
) -> std::io::Result<()> {
let event = ShellAbandonedEvent {
event_type: "shell_abandoned",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
work_id: work_id.to_string(),
binary: binary.to_string(),
command: command.to_string(),
running_ms,
exit_code: recovered.map(|(exit_code, _, _)| exit_code),
output_path: recovered.map(|(_, path, _)| path.to_string()),
output_bytes: recovered.map(|(_, _, bytes)| bytes),
};
self.write_event(&event).await
}
pub(crate) async fn write_shell_detach_unrecorded(
&mut self,
turn: u32,
work_id: &str,
binary: &str,
reason: &str,
) -> std::io::Result<()> {
let event = ShellDetachUnrecordedEvent {
event_type: "shell_detach_unrecorded",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
work_id: work_id.to_string(),
binary: binary.to_string(),
reason: reason.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_a2a_task_received(
&mut self,
task_id: &str,
context_id: &str,
message_id: &str,
traceparent_from_caller: Option<&str>,
caller_member: Option<&str>,
) -> std::io::Result<()> {
let event = A2aTaskReceivedEvent {
event_type: "a2a_task_received",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
context_id: context_id.to_string(),
message_id: message_id.to_string(),
traceparent_from_caller: traceparent_from_caller.map(str::to_string),
caller_member: caller_member.map(str::to_string),
};
self.write_event(&event).await
}
pub(crate) async fn write_a2a_send(
&mut self,
peer_url: &str,
message_id: &str,
task_id: &str,
context_id: &str,
traceparent: Option<&str>,
trust: TrustClass,
) -> std::io::Result<()> {
let event = A2aSendEvent {
event_type: "a2a_send",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
peer_url: peer_url.to_string(),
message_id: message_id.to_string(),
task_id: task_id.to_string(),
context_id: context_id.to_string(),
traceparent: traceparent.map(str::to_string),
trust: trust.as_str().to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_artifact_pulled(
&mut self,
artifact: &InstalledArtifactSummary,
) -> std::io::Result<()> {
let session = match &artifact.origin {
LockOrigin::Runtime { session } => Some(session.clone()),
LockOrigin::Operator => None,
};
let event = ArtifactPulledEvent {
event_type: "artifact_pulled",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
name: artifact.name.clone(),
version: artifact.version.clone(),
runtime: artifact.runtime.as_str(),
origin: artifact.origin.as_str(),
session,
trust: crate::origin::artifact_trust(&artifact.origin).as_str(),
};
self.write_event(&event).await
}
pub(crate) async fn write_task_start(
&mut self,
task_id: &str,
context_id: &str,
source: &str,
provenance: TaskProvenance,
message_parts_bytes: u64,
) -> std::io::Result<()> {
self.task_turns = 0;
self.task_input_tokens = 0;
self.task_output_tokens = 0;
self.task_tool_calls = 0;
self.task_shell_calls = 0;
self.task_agent_turns = 0;
self.task_hook_inferences = 0;
self.task_start_instant = Some(Instant::now());
self.active_task_id = Some(task_id.to_string());
let event_id = new_event_id();
self.task_event_id = Some(event_id.clone());
self.turn_event_id = None;
let event = TaskStartEvent {
event_type: "task_start",
event_id: event_id.clone(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
context_id: context_id.to_string(),
source: source.to_string(),
origin: provenance.origin().as_str().to_string(),
trust: provenance.trust().as_str().to_string(),
lane: TaskLane::for_origin(provenance.origin())
.as_str()
.to_string(),
message_parts_bytes,
};
self.write_event(&event).await
}
pub(crate) async fn write_task_end(
&mut self,
task_id: &str,
exit_status: &str,
reopen_count: u32,
) -> std::io::Result<()> {
let duration_ms = self
.task_start_instant
.take()
.map(|t| t.elapsed().as_millis().try_into().unwrap_or(u64::MAX))
.unwrap_or(0);
let event = TaskEndEvent {
event_type: "task_end",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
exit_status: exit_status.to_string(),
duration_ms,
turns: self.task_turns,
input_tokens: self.task_input_tokens,
output_tokens: self.task_output_tokens,
tool_calls: self.task_tool_calls,
shell_calls: self.task_shell_calls,
reopen_count,
};
self.active_task_id = None;
self.task_event_id = None;
self.turn_event_id = None;
self.task_agent_turns = 0;
self.write_event(&event).await
}
pub(crate) async fn write_task_canceled(
&mut self,
task_id: &str,
turn: Option<u32>,
phase: &str,
detached_work_ids: Vec<String>,
delegation_ids: Vec<String>,
) -> std::io::Result<()> {
let event = TaskCanceledEvent {
event_type: "task_canceled",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
turn,
phase: phase.to_string(),
detached_work_ids,
delegation_ids,
};
self.write_event(&event).await
}
pub(crate) async fn write_task_rejected(
&mut self,
task_id: &str,
context_id: &str,
source: &str,
cause: &str,
reason: &str,
) -> std::io::Result<()> {
let event = TaskRejectedEvent {
event_type: "task_rejected",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
context_id: context_id.to_string(),
source: source.to_string(),
cause: cause.to_string(),
reason: reason.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_task_failed(
&mut self,
turn: Option<u32>,
cause: &str,
reason: &str,
) -> std::io::Result<()> {
let reason = crate::agent::truncate_on_char_boundary(reason, MAX_TASK_FAILURE_REASON_BYTES)
.to_string();
let event = TaskFailedEvent {
event_type: "task_failed",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
turn,
cause: cause.to_string(),
reason: reason.clone(),
};
self.task_failures_written += 1;
self.last_task_failure = Some(reason);
self.write_event(&event).await
}
pub(crate) fn last_task_failure(&self) -> Option<&str> {
self.last_task_failure.as_deref()
}
pub(crate) fn task_failures_written(&self) -> u64 {
self.task_failures_written
}
pub(crate) async fn write_task_reopened(
&mut self,
task_id: &str,
hook_name: &str,
reason: &str,
reopen_number: u32,
attempt_context: &'static str,
turns_remaining: u32,
) -> std::io::Result<()> {
let event = TaskReopenedEvent {
event_type: "task_reopened",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
hook_name: hook_name.to_string(),
reason: reason.to_string(),
reopen_number,
attempt_context,
turns_remaining,
};
self.write_event(&event).await
}
pub(crate) async fn write_task_continued(
&mut self,
task_id: &str,
continuation_number: u32,
member_calls: &[String],
delegations: &[String],
waited_ms: u64,
turns_remaining: u32,
) -> std::io::Result<()> {
let event = TaskContinuedEvent {
event_type: "task_continued",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
continuation_number,
member_calls,
delegations,
waited_ms,
turns_remaining,
};
self.write_event(&event).await
}
pub(crate) fn task_turns(&self) -> u32 {
self.task_turns
}
pub(crate) fn task_hook_inferences(&self) -> u32 {
self.task_hook_inferences
}
pub(crate) fn task_agent_turns(&self) -> u32 {
self.task_agent_turns
}
pub(crate) async fn write_compaction(
&mut self,
turn: u32,
tokens_before: u64,
tokens_after: u64,
) -> std::io::Result<()> {
let event = CompactionEvent {
event_type: "compaction",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
tokens_before,
tokens_after,
};
self.write_event(&event).await
}
pub(crate) async fn write_compaction_declined(
&mut self,
turn: u32,
tokens: u64,
reason: &str,
) -> std::io::Result<()> {
let event = CompactionDeclinedEvent {
event_type: "compaction_declined",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
tokens,
reason: reason.to_string(),
};
self.write_event(&event).await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_context_seed(
&mut self,
hook_name: &str,
tokens: u64,
proposed_tokens: u64,
budget_tokens: u64,
outcome: &str,
reason: Option<&str>,
message_ids: Vec<String>,
) -> std::io::Result<()> {
let event = ContextSeedEvent {
event_type: "context_seed",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
hook_name: hook_name.to_string(),
tokens,
proposed_tokens,
budget_tokens,
outcome: outcome.to_string(),
reason: reason.map(str::to_string),
message_ids,
};
self.write_event(&event).await
}
pub(crate) async fn write_retention(
&mut self,
store: &str,
reason: &str,
targets: Vec<String>,
messages_dropped: Option<u64>,
) -> std::io::Result<()> {
let event = RetentionEvent {
event_type: "retention",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
store: store.to_string(),
reason: reason.to_string(),
removed: u32::try_from(targets.len()).unwrap_or(u32::MAX),
targets,
messages_dropped,
};
self.write_event(&event).await
}
pub(crate) async fn write_call_denied(
&mut self,
turn: u32,
event: &str,
hook_name: &str,
target: &str,
reason: &str,
) -> std::io::Result<()> {
let event = CallDeniedEvent {
event_type: "call_denied",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
event: event.to_string(),
hook_name: hook_name.to_string(),
target: target.to_string(),
reason: reason.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_tool_input_refused(
&mut self,
turn: u32,
tool_name: &str,
tool_call_id: Option<&str>,
missing: &[String],
reason: &str,
) -> std::io::Result<()> {
let event = ToolInputRefusedEvent {
event_type: "tool_input_refused",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
tool_name: tool_name.to_string(),
tool_call_id: tool_call_id.filter(|id| !id.is_empty()).map(str::to_string),
missing: missing.to_vec(),
reason: reason.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_spend_ceiling_reached(
&mut self,
turn: u32,
refusal: &crate::spend::SpendRefusal,
origin: Option<&str>,
) -> std::io::Result<()> {
let event = SpendCeilingReachedEvent {
event_type: "spend_ceiling_reached",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
task_id: self.active_task_id.clone(),
refusal,
origin,
};
self.write_event(&event).await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_protected_path_denied(
&mut self,
turn: u32,
call: &str,
target: &str,
path: &str,
rule: &str,
signal: &str,
reason: &str,
) -> std::io::Result<()> {
let event = ProtectedPathDeniedEvent {
event_type: "protected_path_denied",
event_id: new_event_id(),
parent_id: self.turn_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
turn,
call: call.to_string(),
target: target.to_string(),
path: path.to_string(),
rule: rule.to_string(),
signal: signal.to_string(),
reason: reason.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_hook_dispatch_error(
&mut self,
hook_name: &str,
event: &str,
arm: &str,
) -> std::io::Result<()> {
let event = HookDispatchErrorEvent {
event_type: "hook_dispatch_error",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
hook_name: hook_name.to_string(),
event: event.to_string(),
arm: arm.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_harness_start(&mut self, start: HarnessStart) -> std::io::Result<()> {
let event = HarnessStartEvent {
event_type: "harness_start",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
start,
};
self.write_event(&event).await
}
pub(crate) async fn write_harness_warning(
&mut self,
code: &str,
message: &str,
) -> std::io::Result<()> {
let event = HarnessWarningEvent {
event_type: "harness_warning",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
code: code.to_string(),
message: message.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_harness_session(
&mut self,
harness_session_id: &str,
auth: &str,
model: Option<&str>,
) -> std::io::Result<()> {
let event = HarnessSessionEvent {
event_type: "harness_session",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
harness_session_id: harness_session_id.to_string(),
auth: auth.to_string(),
model: model.map(str::to_string),
};
self.write_event(&event).await
}
pub(crate) async fn write_harness_session_forgotten(
&mut self,
context_id: &str,
harness_session_id: &str,
requested_by: &str,
) -> std::io::Result<()> {
let event = HarnessSessionForgottenEvent {
event_type: "harness_session_forgotten",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
context_id: context_id.to_string(),
harness_session_id: harness_session_id.to_string(),
requested_by: requested_by.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_harness_retry(
&mut self,
attempt: u32,
reason: &str,
) -> std::io::Result<()> {
let event = HarnessRetryEvent {
event_type: "harness_retry",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
attempt,
reason: reason.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_harness_note(&mut self, text: &str) -> std::io::Result<()> {
let event = HarnessNoteEvent {
event_type: "harness_note",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
text: text.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_harness_failed(
&mut self,
kind: &str,
message: &str,
source: &str,
) -> std::io::Result<()> {
let event = HarnessFailedEvent {
event_type: "harness_failed",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
kind: kind.to_string(),
message: message.to_string(),
source: source.to_string(),
};
self.write_event(&event).await
}
pub(crate) async fn write_harness_interrupt(
&mut self,
method: &str,
delivered: bool,
grace_ms: u64,
) -> std::io::Result<()> {
let event = HarnessInterruptEvent {
event_type: "harness_interrupt",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
method: method.to_string(),
delivered,
grace_ms,
};
self.write_event(&event).await
}
pub(crate) async fn write_harness_exit(&mut self, exit: HarnessExit) -> std::io::Result<()> {
let event = HarnessExitEvent {
event_type: "harness_exit",
event_id: new_event_id(),
parent_id: self.task_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: self.active_task_id.clone(),
exit,
};
self.write_event(&event).await
}
pub(crate) async fn write_session_end(&mut self, exit_status: &str) -> std::io::Result<()> {
let duration_ms = self
.session_start_time
.elapsed()
.as_millis()
.try_into()
.unwrap_or(u64::MAX);
let event = SessionEndEvent {
event_type: "session_end",
event_id: new_event_id(),
parent_id: self.session_parent(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
total_turns: self.total_turns,
total_input_tokens: self.total_input_tokens,
total_output_tokens: self.total_output_tokens,
total_tool_calls: self.total_tool_calls,
total_shell_calls: self.total_shell_calls,
duration_ms,
exit_status: exit_status.to_string(),
};
self.write_event(&event).await?;
self.session_ended = true;
Ok(())
}
pub(crate) async fn write_session_end_if_not_ended(
&mut self,
exit_status: &str,
) -> std::io::Result<()> {
if self.session_started && !self.session_ended {
self.write_session_end(exit_status).await?;
}
Ok(())
}
pub(crate) async fn flush(&mut self) -> std::io::Result<()> {
self.writer.flush().await
}
async fn write_event(&mut self, event: &impl Serialize) -> std::io::Result<()> {
let mut line = serde_json::to_string(event).map_err(std::io::Error::other)?;
line.push('\n');
self.writer.write_all(line.as_bytes()).await?;
self.writer.flush().await
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct SessionGateway {
pub(crate) artifact: String,
pub(crate) host: String,
pub(crate) credential_source: &'static str,
pub(crate) metered: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct RuntimeArtifact {
name: String,
version: String,
origin: &'static str,
session: String,
trust: &'static str,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct SessionControl {
pub(crate) settings: Vec<&'static str>,
pub(crate) secrets: Vec<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub(crate) agent_settings: Vec<&'static str>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct SessionInferenceChoice {
pub(crate) name: String,
pub(crate) driver: String,
pub(crate) model: String,
pub(crate) credential_source: &'static str,
pub(crate) available: bool,
}
pub(crate) struct ControlChange<'a> {
pub(crate) event_id: String,
pub(crate) principal: crate::control_plane::Principal,
pub(crate) token_id: Option<&'a str>,
pub(crate) kind: &'static str,
pub(crate) name: &'a str,
pub(crate) action: &'static str,
pub(crate) previous: Option<Value>,
pub(crate) value: Option<Value>,
pub(crate) applies_from: Option<&'static str>,
pub(crate) replaced: Option<bool>,
}
#[derive(Serialize)]
struct ControlChangeEvent<'a> {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
principal: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
token_id: Option<&'a str>,
kind: &'static str,
name: &'a str,
action: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
previous: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
value: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
applies_from: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
replaced: Option<bool>,
}
#[derive(Serialize)]
struct ControlRefusedEvent<'a> {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
principal: &'static str,
status: u16,
reason: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
kind: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
name: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
token_id: Option<&'a str>,
}
#[derive(Serialize)]
struct ControlAppliedEvent<'a> {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
name: &'a str,
value: Value,
change_id: &'a str,
}
#[derive(Serialize)]
struct ToolsRefreshedEvent<'a> {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
turn: u32,
task_id: Option<String>,
trigger: &'a str,
added: &'a [String],
removed: &'a [String],
tools: &'a [String],
}
#[derive(Serialize)]
struct GatewayCredentialEvent<'a> {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
artifact: &'a str,
source: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
credential: Option<&'a str>,
change: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
trigger: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
status: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
retried: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
reason: Option<&'static str>,
}
#[derive(Serialize)]
struct InferenceCredentialEvent<'a> {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
source: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
credential: Option<&'a str>,
change: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
trigger: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
status: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
retried: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
reason: Option<&'static str>,
}
#[allow(clippy::type_complexity)]
fn credential_change_fields(
change: CredentialChange,
) -> (
&'static str,
Option<&'static str>,
Option<u16>,
Option<bool>,
Option<&'static str>,
) {
match change {
CredentialChange::Rotated { trigger } => ("rotated", Some(trigger), None, None, None),
CredentialChange::Rejected { status, retried } => {
("rejected", None, Some(status), Some(retried), None)
}
CredentialChange::Unreadable { reason } => ("unreadable", None, None, None, Some(reason)),
}
}
pub struct ResourceTraceAppender {
file: tokio::sync::Mutex<File>,
session_id: String,
session_event_id: String,
}
impl ResourceTraceAppender {
pub async fn open(
workdir: &Path,
session_id: String,
session_event_id: String,
) -> std::io::Result<Self> {
let file = OpenOptions::new()
.create(true)
.append(true)
.open(workdir.join("trace.jsonl"))
.await?;
Ok(Self {
file: tokio::sync::Mutex::new(file),
session_id,
session_event_id,
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_resource_list(
&self,
root: &str,
entry_count: usize,
total_bytes: u64,
generation: u64,
containment_achieved: ContainmentClass,
outcome: &str,
reason: Option<String>,
) {
let event = ResourceListEvent {
event_type: "resource_list",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
root: root.to_string(),
entry_count,
total_bytes,
generation,
containment_achieved,
outcome: outcome.to_string(),
reason,
};
self.append(&event).await;
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_resource_read(
&self,
path: &str,
outcome: &str,
bytes: Option<u64>,
sha256: Option<String>,
generation: u64,
containment_achieved: ContainmentClass,
reason: Option<String>,
) {
let event = ResourceReadEvent {
event_type: "resource_read",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
path: path.to_string(),
outcome: outcome.to_string(),
bytes,
sha256,
generation,
containment_achieved,
reason,
};
self.append(&event).await;
}
pub(crate) async fn write_formation_ended(&self, formation_id: &crate::FormationId) {
let event = FormationEndedEvent {
event_type: "formation_ended",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
formation_id: formation_id.as_str().to_string(),
};
self.append(&event).await;
}
pub(crate) async fn write_spawner_ended(
&self,
spawned_by: Option<&str>,
delegation_id: Option<&str>,
) {
let event = SpawnerEndedEvent {
event_type: "spawner_ended",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
spawned_by: spawned_by.map(str::to_string),
delegation_id: delegation_id.map(str::to_string),
};
self.append(&event).await;
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_peer_handle_mint(
&self,
handle_id: Option<String>,
path: &str,
audience: &str,
expires_at_ms: Option<u64>,
outcome: &str,
reason: Option<String>,
) {
let event = PeerHandleMintEvent {
event_type: "peer_handle_mint",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
handle_id,
path: path.to_string(),
audience: audience.to_string(),
expires_at_ms,
outcome: outcome.to_string(),
reason,
};
self.append(&event).await;
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_peer_handle_redeem(
&self,
handle_id: &str,
path: Option<String>,
generation: u64,
audience_asserted: Option<String>,
bytes: Option<u64>,
sha256: Option<String>,
outcome: &str,
reason: Option<String>,
) {
let event = PeerHandleRedeemEvent {
event_type: "peer_handle_redeem",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
handle_id: handle_id.to_string(),
path,
generation,
audience_asserted,
bytes,
sha256,
outcome: outcome.to_string(),
reason,
};
self.append(&event).await;
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_peer_file_fetch(
&self,
peer: &str,
handle_id: &str,
stored_path: Option<String>,
bytes: Option<u64>,
sha256: Option<String>,
outcome: &str,
reason: Option<String>,
) {
let event = PeerFileFetchEvent {
event_type: "peer_file_fetch",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
peer: peer.to_string(),
handle_id: handle_id.to_string(),
stored_path,
bytes,
sha256,
outcome: outcome.to_string(),
reason,
};
self.append(&event).await;
}
pub(crate) async fn write_delegation_start(
&self,
delegation_id: &str,
capsule: &str,
version: &str,
child_session_id: &str,
child_workdir: &str,
) {
let event = DelegationStartEvent::new(
&self.session_id,
&self.session_event_id,
delegation_id,
capsule,
version,
child_session_id,
child_workdir,
);
self.append(&event).await;
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn write_delegation(
&self,
capsule: &str,
version: &str,
delegation_id: Option<String>,
child_session_id: Option<String>,
duration_ms: u64,
outcome: &str,
reason: Option<String>,
) {
let event = DelegationEvent::new(
&self.session_id,
&self.session_event_id,
capsule,
version,
delegation_id,
child_session_id,
duration_ms,
outcome,
reason,
);
self.append(&event).await;
}
pub(crate) async fn write_member_call_start(
&self,
task_id: &str,
call_id: &str,
member: &str,
member_task_id: &str,
) {
let event = self.member_call_start(task_id, call_id, member, member_task_id);
self.append(&event).await;
}
pub(crate) async fn write_member_call_start_if(
&self,
task_id: &str,
call_id: &str,
member: &str,
member_task_id: &str,
admit: impl FnOnce() -> bool,
) -> bool {
let event = self.member_call_start(task_id, call_id, member, member_task_id);
self.append_if(&event, admit).await
}
fn member_call_start(
&self,
task_id: &str,
call_id: &str,
member: &str,
member_task_id: &str,
) -> MemberCallStartEvent {
MemberCallStartEvent {
event_type: "member_call_start",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
call_id: call_id.to_string(),
member: member.to_string(),
member_task_id: member_task_id.to_string(),
}
}
pub(crate) async fn write_member_call_busy(
&self,
task_id: &str,
call_id: &str,
member: &str,
offer: u32,
waited_ms: u64,
message: &str,
) {
let event = MemberCallBusyEvent {
event_type: "member_call_busy",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
call_id: call_id.to_string(),
member: member.to_string(),
offer,
waited_ms,
message: message.to_string(),
};
self.append(&event).await;
}
pub(crate) async fn write_member_call(
&self,
task_id: &str,
outcome: &crate::member_call::MemberCallOutcome,
delivered: bool,
) {
let event = MemberCallEvent {
event_type: "member_call",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
task_id: task_id.to_string(),
call_id: outcome.call_id.clone(),
member: outcome.member.clone(),
member_task_id: outcome.member_task_id.clone(),
status: outcome.status.as_str().to_string(),
duration_ms: outcome.duration_ms,
output: outcome.output.clone(),
truncated: outcome.truncated,
delivered,
no_answer_below: outcome
.below
.iter()
.map(|(member, status)| NoAnswerBelow {
member: member.clone(),
status: status.as_str().to_string(),
})
.collect(),
};
self.append(&event).await;
}
pub(crate) async fn write_inference_credential(
&self,
source: &'static str,
credential: Option<&str>,
change: CredentialChange,
) {
let (change_name, trigger, status, retried, reason) = credential_change_fields(change);
let event = InferenceCredentialEvent {
event_type: "inference_credential",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
source,
credential,
change: change_name,
trigger,
status,
retried,
reason,
};
self.append(&event).await;
}
pub(crate) async fn write_gateway_credential(
&self,
artifact: &str,
source: &'static str,
credential: Option<&str>,
change: CredentialChange,
) {
let (change_name, trigger, status, retried, reason) = credential_change_fields(change);
let event = GatewayCredentialEvent {
event_type: "gateway_credential",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
artifact,
source,
credential,
change: change_name,
trigger,
status,
retried,
reason,
};
self.append(&event).await;
}
pub(crate) async fn write_control_change(&self, change: ControlChange<'_>) {
let event = ControlChangeEvent {
event_type: "control_change",
event_id: change.event_id,
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
principal: change.principal.as_str(),
token_id: change.token_id,
kind: change.kind,
name: change.name,
action: change.action,
previous: change.previous,
value: change.value,
applies_from: change.applies_from,
replaced: change.replaced,
};
self.append(&event).await;
}
pub(crate) async fn write_control_refused(
&self,
principal: crate::control_plane::Principal,
status: u16,
reason: &'static str,
kind: Option<&'static str>,
name: Option<&str>,
token_id: Option<&str>,
) {
let event = ControlRefusedEvent {
event_type: "control_refused",
event_id: new_event_id(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
principal: principal.as_str(),
status,
reason,
kind,
name,
token_id,
};
self.append(&event).await;
}
async fn append(&self, event: &impl Serialize) {
self.append_if(event, || true).await;
}
async fn append_if(&self, event: &impl Serialize, admit: impl FnOnce() -> bool) -> bool {
let Ok(mut line) = serde_json::to_string(event) else {
return admit();
};
line.push('\n');
let mut file = self.file.lock().await;
if !admit() {
return false;
}
if file.write_all(line.as_bytes()).await.is_ok() {
let _ = file.flush().await;
}
true
}
}
pub(crate) struct PriorSessionTraceAppender {
file: File,
session_id: String,
session_event_id: Option<String>,
terminate_torn_tail: bool,
}
impl PriorSessionTraceAppender {
pub(crate) async fn open(
trace_path: &Path,
session_id: String,
session_event_id: Option<String>,
) -> std::io::Result<Self> {
let terminate_torn_tail = !ends_with_newline(trace_path)?;
let file = OpenOptions::new().append(true).open(trace_path).await?;
Ok(Self {
file,
session_id,
session_event_id,
terminate_torn_tail,
})
}
pub(crate) async fn write_shell_lost(
&mut self,
work: &crate::detached::LostWork,
reconciled_by_session: &str,
reconciled_task_id: &str,
) -> std::io::Result<()> {
let event = ShellLostEvent {
event_type: "shell_lost",
event_id: new_event_id(),
parent_id: self.session_event_id.clone(),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
work_id: work.work_id.clone(),
binary: work.binary.clone(),
command: work.command.clone(),
detached_at_ms: work.detached_at_ms,
reconciled_by_session: reconciled_by_session.to_string(),
reconciled_task_id: reconciled_task_id.to_string(),
};
let mut line = if self.terminate_torn_tail {
"\n".to_string()
} else {
String::new()
};
line.push_str(&serde_json::to_string(&event).map_err(std::io::Error::other)?);
line.push('\n');
self.file.write_all(line.as_bytes()).await?;
self.file.flush().await?;
self.terminate_torn_tail = false;
Ok(())
}
}
fn ends_with_newline(path: &Path) -> std::io::Result<bool> {
use std::io::{Read, Seek, SeekFrom};
let mut file = std::fs::File::open(path)?;
let length = file.metadata()?.len();
if length == 0 {
return Ok(true);
}
file.seek(SeekFrom::End(-1))?;
let mut last = [0u8; 1];
file.read_exact(&mut last)?;
Ok(last[0] == b'\n')
}
pub(crate) fn timestamp_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64
}
#[derive(Serialize, Clone)]
pub(crate) struct PlanStepShape {
pub(crate) step_id: String,
pub(crate) kind: &'static str,
pub(crate) depends_on: Vec<String>,
pub(crate) has_condition: bool,
}
#[derive(Serialize)]
struct PlanStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
plan_id: String,
step_count: usize,
steps: Vec<PlanStepShape>,
}
#[derive(Serialize)]
struct PlanStepStartEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
plan_id: String,
step_id: String,
kind: &'static str,
depends_on: Vec<String>,
}
#[derive(Serialize)]
struct PlanStepEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
plan_id: String,
step_id: String,
kind: &'static str,
status: &'static str,
attempts: u32,
duration_ms: u64,
#[serde(skip_serializing_if = "Option::is_none")]
error: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
input: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
state_effect: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
resource_id: Option<String>,
}
#[derive(Serialize)]
struct PlanEndEvent {
event_type: &'static str,
event_id: String,
parent_id: Option<String>,
session_id: String,
timestamp: u64,
plan_id: String,
outcome: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
failed_step: Option<String>,
steps_total: usize,
steps_succeeded: usize,
steps_failed: usize,
steps_skipped: usize,
duration_ms: u64,
#[serde(skip_serializing_if = "Option::is_none")]
reason: Option<String>,
}
pub(crate) struct PlanStepRecord {
pub(crate) plan_id: String,
pub(crate) step_id: String,
pub(crate) kind: &'static str,
pub(crate) status: &'static str,
pub(crate) attempts: u32,
pub(crate) duration_ms: u64,
pub(crate) error: Option<String>,
pub(crate) input: Option<Value>,
pub(crate) state_effect: Option<String>,
pub(crate) resource_id: Option<String>,
}
pub(crate) struct PlanEndRecord {
pub(crate) plan_id: String,
pub(crate) outcome: &'static str,
pub(crate) failed_step: Option<String>,
pub(crate) steps_total: usize,
pub(crate) steps_succeeded: usize,
pub(crate) steps_failed: usize,
pub(crate) steps_skipped: usize,
pub(crate) duration_ms: u64,
pub(crate) reason: Option<String>,
}
pub struct PlanTraceAppender {
file: std::sync::Mutex<std::fs::File>,
session_id: String,
session_event_id: String,
}
impl PlanTraceAppender {
pub fn open(
workdir: &Path,
session_id: String,
session_event_id: String,
) -> std::io::Result<Self> {
let file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(workdir.join("trace.jsonl"))?;
Ok(Self {
file: std::sync::Mutex::new(file),
session_id,
session_event_id,
})
}
pub(crate) fn write_plan_start(&self, plan_id: &str, steps: Vec<PlanStepShape>) -> String {
let event_id = new_event_id();
let event = PlanStartEvent {
event_type: "plan_start",
event_id: event_id.clone(),
parent_id: Some(self.session_event_id.clone()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
plan_id: plan_id.to_string(),
step_count: steps.len(),
steps,
};
self.append(&event);
event_id
}
pub(crate) fn write_plan_step_start(
&self,
plan_node: &str,
plan_id: &str,
step_id: &str,
kind: &'static str,
depends_on: Vec<String>,
) {
let event = PlanStepStartEvent {
event_type: "plan_step_start",
event_id: new_event_id(),
parent_id: Some(plan_node.to_string()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
plan_id: plan_id.to_string(),
step_id: step_id.to_string(),
kind,
depends_on,
};
self.append(&event);
}
pub(crate) fn write_plan_step(&self, plan_node: &str, record: PlanStepRecord) {
let input = record.input.map(|mut value| {
crate::peer_handoff::redact_handles_in_json(&mut value);
value
});
let event = PlanStepEvent {
event_type: "plan_step",
event_id: new_event_id(),
parent_id: Some(plan_node.to_string()),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
plan_id: record.plan_id,
step_id: record.step_id,
kind: record.kind,
status: record.status,
attempts: record.attempts,
duration_ms: record.duration_ms,
error: record.error,
input,
state_effect: record.state_effect,
resource_id: record.resource_id,
};
self.append(&event);
}
pub(crate) fn write_plan_end(&self, plan_node: Option<&str>, record: PlanEndRecord) {
let event = PlanEndEvent {
event_type: "plan_end",
event_id: new_event_id(),
parent_id: Some(
plan_node
.map(str::to_string)
.unwrap_or_else(|| self.session_event_id.clone()),
),
session_id: self.session_id.clone(),
timestamp: timestamp_ms(),
plan_id: record.plan_id,
outcome: record.outcome,
failed_step: record.failed_step,
steps_total: record.steps_total,
steps_succeeded: record.steps_succeeded,
steps_failed: record.steps_failed,
steps_skipped: record.steps_skipped,
duration_ms: record.duration_ms,
reason: record.reason,
};
self.append(&event);
}
pub(crate) fn write_delegation_start(
&self,
delegation_id: &str,
capsule: &str,
version: &str,
child_session_id: &str,
child_workdir: &str,
) {
let event = DelegationStartEvent::new(
&self.session_id,
&self.session_event_id,
delegation_id,
capsule,
version,
child_session_id,
child_workdir,
);
self.append(&event);
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn write_delegation(
&self,
capsule: &str,
version: &str,
delegation_id: Option<String>,
child_session_id: Option<String>,
duration_ms: u64,
outcome: &str,
reason: Option<String>,
) {
let event = DelegationEvent::new(
&self.session_id,
&self.session_event_id,
capsule,
version,
delegation_id,
child_session_id,
duration_ms,
outcome,
reason,
);
self.append(&event);
}
fn append(&self, event: &impl Serialize) {
use std::io::Write as _;
let Ok(mut line) = serde_json::to_string(event) else {
return;
};
line.push('\n');
let Ok(mut file) = self.file.lock() else {
return;
};
if file.write_all(line.as_bytes()).is_ok() {
let _ = file.flush();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
containment::scope_report_for_tier, sandbox::EnforcementTier, sealed::UsernsGrant,
trace_blobs::BLOB_DIR_NAME, types::CapabilityPolicy,
};
use murmur_artifact::{ArtifactRuntime, InterpreterRuntimeDir, InterpreterRuntimeGrant};
use serde_json::Value;
use crate::origin::TaskOrigin;
fn event_provenance() -> TaskProvenance {
TaskProvenance::derive(TaskOrigin::Event, None)
}
fn report_for(
declared: ContainmentClass,
tier: EnforcementTier,
workdir_exec: bool,
) -> ScopeReport {
scope_report_for_tier(
&CapabilityPolicy {
workdir_exec_allowed: workdir_exec,
..CapabilityPolicy::default()
},
declared,
tier,
None,
None,
None,
Vec::new(),
Vec::new(),
Vec::new(),
crate::cgroup::IoMaxReport::default(),
)
}
fn demotion_info() -> crate::detached::DetachedDispatchInfo {
crate::detached::DetachedDispatchInfo {
work_id: "wrk_0a1b2c3d4e5f6a7b".to_string(),
binary: "/usr/bin/bash".to_string(),
command: "make -j8".to_string(),
grace_ms: 1_000,
}
}
#[tokio::test]
async fn a_demotion_marker_that_cannot_be_written_does_not_fail_the_demotion() {
let dir = tempfile::tempdir().unwrap();
std::os::unix::fs::symlink("/dev/full", dir.path().join("trace.jsonl")).unwrap();
let mut writer = make_writer(dir.path()).await;
let info = demotion_info();
assert!(
writer
.write_shell_detached(1, &info.work_id, &info.binary, &info.command, info.grace_ms)
.await
.is_err(),
"the sink under this writer must be unwritable for the claim below to mean anything"
);
crate::agent::record_demotion(&mut writer, 1, &info).await;
}
#[tokio::test]
async fn a_recorded_demotion_writes_only_its_own_marker() {
let dir = tempfile::tempdir().unwrap();
let mut writer = make_writer(dir.path()).await;
crate::agent::record_demotion(&mut writer, 3, &demotion_info()).await;
let events = read_events(dir.path());
assert_eq!(events.len(), 1);
assert_eq!(events[0]["event_type"], "shell_detached");
assert_eq!(events[0]["work_id"], "wrk_0a1b2c3d4e5f6a7b");
assert_eq!(events[0]["turn"], 3);
}
#[tokio::test]
async fn an_unrecorded_demotion_names_the_work_id_and_the_write_error() {
let dir = tempfile::tempdir().unwrap();
let mut writer = make_writer(dir.path()).await;
writer
.write_shell_detach_unrecorded(2, "wrk_0a1b", "/usr/bin/bash", "No space left")
.await
.unwrap();
let events = read_events(dir.path());
assert_eq!(events.len(), 1);
assert_eq!(events[0]["event_type"], "shell_detach_unrecorded");
assert_eq!(events[0]["turn"], 2);
assert_eq!(events[0]["work_id"], "wrk_0a1b");
assert_eq!(events[0]["binary"], "/usr/bin/bash");
assert_eq!(events[0]["reason"], "No space left");
}
#[tokio::test]
async fn an_abandoned_command_still_running_omits_the_fields_it_cannot_know() {
let dir = tempfile::tempdir().unwrap();
let mut writer = make_writer(dir.path()).await;
writer
.write_shell_abandoned(
"wrk_0a1b2c3d4e5f6a7b",
"/usr/bin/bash",
"sleep 45; echo done",
45_012,
None,
)
.await
.unwrap();
let events = read_events(dir.path());
assert_eq!(events.len(), 1);
let event = &events[0];
assert_eq!(event["event_type"], "shell_abandoned");
assert_eq!(event["work_id"], "wrk_0a1b2c3d4e5f6a7b");
assert_eq!(event["binary"], "/usr/bin/bash");
assert_eq!(event["command"], "sleep 45; echo done");
assert_eq!(event["running_ms"], 45_012);
for absent in ["exit_code", "output_path", "output_bytes"] {
assert!(
event.get(absent).is_none(),
"{absent} must be omitted, not written as null: {event}"
);
}
}
#[tokio::test]
async fn an_abandoned_command_that_finished_carries_its_result() {
let dir = tempfile::tempdir().unwrap();
let mut writer = make_writer(dir.path()).await;
writer
.write_shell_abandoned(
"wrk_0a1b2c3d4e5f6a7b",
"/usr/bin/bash",
"sleep 2; echo done",
2_014,
Some((0, "logs/wrk_0a1b2c3d4e5f6a7b.log", 91)),
)
.await
.unwrap();
let event = &read_events(dir.path())[0];
assert_eq!(event["event_type"], "shell_abandoned");
assert_eq!(event["running_ms"], 2_014);
assert_eq!(event["exit_code"], 0);
assert_eq!(event["output_path"], "logs/wrk_0a1b2c3d4e5f6a7b.log");
assert_eq!(event["output_bytes"], 91);
}
async fn make_writer(dir: &std::path::Path) -> TraceWriter {
make_writer_with_opts(dir, TraceCapture::Meta).await
}
async fn make_writer_with_opts(dir: &std::path::Path, capture: TraceCapture) -> TraceWriter {
make_writer_with_prompt(dir, capture, None, false).await
}
async fn make_writer_with_prompt(
dir: &std::path::Path,
capture: TraceCapture,
system_prompt: Option<&str>,
system_prompt_overridden: bool,
) -> TraceWriter {
TraceWriter::open(
dir,
"test-session-id".to_string(),
"test-capsule".to_string(),
"1.0.0".to_string(),
"claude-test".to_string(),
vec!["shell".to_string()],
report_for(
ContainmentClass::Advisory,
EnforcementTier::EnvironmentOnly,
false,
),
capture,
system_prompt.map(str::to_string),
system_prompt_overridden,
None,
None,
None,
None,
)
.await
.unwrap()
}
fn read_events(dir: &std::path::Path) -> Vec<Value> {
let content = std::fs::read_to_string(dir.join("trace.jsonl")).unwrap();
content
.lines()
.filter(|l| !l.is_empty())
.map(|l| serde_json::from_str(l).unwrap())
.collect()
}
#[tokio::test]
async fn session_start_fields() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec!["bash".to_string()])
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events.len(), 1);
let e = &events[0];
assert_eq!(e["event_type"], "session_start");
assert_eq!(e["session_id"], "test-session-id");
assert_eq!(e["capsule_name"], "test-capsule");
assert_eq!(e["capsule_version"], "1.0.0");
assert_eq!(e["model"], "claude-test");
assert_eq!(e["max_turns"], 10);
assert_eq!(e["capabilities"], serde_json::json!(["shell"]));
assert_eq!(e["tools_declared"], serde_json::json!(["bash"]));
assert_eq!(e["containment_declared"], "advisory");
assert_eq!(e["containment_achieved"], "advisory");
assert_eq!(e["workdir_exec"], false);
assert!(e["timestamp"].as_u64().unwrap() > 0);
}
#[tokio::test]
async fn session_start_is_the_first_line_and_carries_a_members_formation() {
let formation = FormationId::mint();
for member in [Some(&formation), None] {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.set_formation_id(member);
w.write_session_start(10, Vec::new()).await.unwrap();
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
w.flush().await.unwrap();
let content = std::fs::read_to_string(dir.path().join("trace.jsonl")).unwrap();
let first_line = content.lines().next().unwrap();
let first: Value = serde_json::from_str(first_line).unwrap();
assert_eq!(first["event_type"], "session_start");
match member {
Some(id) => {
assert_eq!(first["formation_id"], id.as_str());
assert!(
first_line.contains(&format!("\"formation_id\":\"{id}\"")),
"{first_line}"
);
}
None => assert!(!first_line.contains("formation_id"), "{first_line}"),
}
assert_eq!(
content.matches("formation_id").count(),
usize::from(member.is_some())
);
}
}
#[tokio::test]
async fn a_members_trace_names_it_its_callees_and_its_callers() {
let authority = crate::formation_credentials::FormationAuthority::for_test();
let cases = [
(Some(("coder", vec!["reviewer"])), Some("planner")),
(Some(("reviewer", vec![])), None),
(None, None),
];
for (member, caller) in cases {
let member = member.map(|(name, callees)| {
crate::formation_credentials::FormationMember::from_bundle(
authority.member_bundle(name, &callees),
)
});
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.set_formation_member(member.as_ref());
w.write_session_start(10, Vec::new()).await.unwrap();
w.write_a2a_task_received("tsk_1", "ctx_1", "msg_1", None, caller)
.await
.unwrap();
w.flush().await.unwrap();
let content = std::fs::read_to_string(dir.path().join("trace.jsonl")).unwrap();
let lines: Vec<Value> = content
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
let start = &lines[0];
let received = lines
.iter()
.find(|line| line["event_type"] == "a2a_task_received")
.unwrap();
match &member {
Some(member) => {
assert_eq!(start["formation_member"], member.name());
assert_eq!(
start["formation_callees"],
serde_json::json!(member.callees().collect::<Vec<_>>())
);
}
None => {
assert!(start.get("formation_member").is_none(), "{start}");
assert!(start.get("formation_callees").is_none(), "{start}");
}
}
match caller {
Some(caller) => assert_eq!(received["caller_member"], caller),
None => assert!(received.get("caller_member").is_none(), "{received}"),
}
assert!(!content.contains("mft1."), "{content}");
}
}
#[tokio::test]
async fn session_start_records_system_prompt_source_and_hash() {
let expected_sha = murmur_artifact::sha256_hex(b"Be terse.");
for (prompt, overridden, source, sha) in [
(Some("Be terse."), false, "manifest", Some(&expected_sha)),
(Some("Be terse."), true, "cli", Some(&expected_sha)),
(None, false, "none", None),
(None, true, "cli", None),
] {
let dir = tempfile::tempdir().unwrap();
let mut w =
make_writer_with_prompt(dir.path(), TraceCapture::Meta, prompt, overridden).await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
assert_eq!(e["system_prompt_source"], source, "prompt={prompt:?}");
match sha {
Some(sha) => assert_eq!(e["system_prompt_sha256"], sha.as_str()),
None => assert!(
e["system_prompt_sha256"].is_null(),
"no prompt in effect must hash to null, got {}",
e["system_prompt_sha256"]
),
}
}
}
#[tokio::test]
async fn session_start_records_the_prompt_by_hash_and_stores_it_only_under_content() {
let sha = murmur_artifact::sha256_hex(b"Be terse.");
let dir = tempfile::tempdir().unwrap();
let mut w =
make_writer_with_prompt(dir.path(), TraceCapture::Meta, Some("Be terse."), false).await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
assert!(
e.get("system_prompt").is_none(),
"the verbatim prompt is not a session_start field, got {e}"
);
assert_eq!(e["system_prompt_sha256"], sha.as_str());
assert!(
!dir.path().join(BLOB_DIR_NAME).exists(),
"meta stores no bodies"
);
let opted_in = tempfile::tempdir().unwrap();
let mut w = make_writer_with_prompt(
opted_in.path(),
TraceCapture::Content,
Some("Be terse."),
false,
)
.await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let event = &read_events(opted_in.path())[0];
assert!(event.get("system_prompt").is_none());
assert_eq!(event["system_prompt_sha256"], sha.as_str());
let blob = opted_in.path().join(BLOB_DIR_NAME).join(&sha);
assert_eq!(std::fs::read(&blob).unwrap(), b"Be terse.");
}
#[tokio::test]
async fn content_capture_stores_no_prompt_blob_when_there_is_no_prompt() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_prompt(dir.path(), TraceCapture::Content, None, false).await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
assert!(e.get("system_prompt").is_none());
assert!(e["system_prompt_sha256"].is_null());
assert!(!dir.path().join(BLOB_DIR_NAME).exists());
}
#[tokio::test]
async fn every_task_session_start_repeats_the_same_system_prompt_record() {
let dir = tempfile::tempdir().unwrap();
let mut w =
make_writer_with_prompt(dir.path(), TraceCapture::Meta, Some("Be terse."), true).await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events.len(), 2);
assert_eq!(
events[0]["system_prompt_source"],
events[1]["system_prompt_source"]
);
assert_eq!(
events[0]["system_prompt_sha256"],
events[1]["system_prompt_sha256"]
);
assert_eq!(events[1]["system_prompt_source"], "cli");
}
#[tokio::test]
async fn session_start_records_workdir_exec_next_to_the_class_it_forced() {
let dir = tempfile::tempdir().unwrap();
let mut w = TraceWriter::open(
dir.path(),
"test-session-id".to_string(),
"test-capsule".to_string(),
"1.0.0".to_string(),
"claude-test".to_string(),
Vec::new(),
report_for(
ContainmentClass::Advisory,
EnforcementTier::KernelFull,
true,
),
TraceCapture::Meta,
None,
false,
None,
None,
None,
None,
)
.await
.unwrap();
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events[0]["workdir_exec"], true);
assert_eq!(events[0]["containment_achieved"], "advisory");
}
#[tokio::test]
async fn session_start_records_declared_and_achieved_containment_separately() {
let dir = tempfile::tempdir().unwrap();
let mut w = TraceWriter::open(
dir.path(),
"test-session-id".to_string(),
"test-capsule".to_string(),
"1.0.0".to_string(),
"claude-test".to_string(),
Vec::new(),
report_for(ContainmentClass::Sealed, EnforcementTier::KernelFull, false),
TraceCapture::Meta,
None,
false,
None,
None,
None,
None,
)
.await
.unwrap();
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events[0]["containment_declared"], "sealed");
assert_eq!(events[0]["containment_achieved"], "scoped");
}
#[tokio::test]
async fn session_start_distinguishes_two_hosts_with_the_same_achieved_class() {
async fn event_for(grant: UsernsGrant) -> Value {
let report = scope_report_for_tier(
&CapabilityPolicy::default(),
ContainmentClass::Sealed,
EnforcementTier::KernelSealed,
None,
Some(grant),
None,
Vec::new(),
Vec::new(),
Vec::new(),
crate::cgroup::IoMaxReport::default(),
);
let dir = tempfile::tempdir().unwrap();
let mut w = TraceWriter::open(
dir.path(),
"test-session-id".to_string(),
"test-capsule".to_string(),
"1.0.0".to_string(),
"claude-test".to_string(),
Vec::new(),
report,
TraceCapture::Meta,
None,
false,
None,
None,
None,
None,
)
.await
.unwrap();
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let mut event = read_events(dir.path()).remove(0);
event.as_object_mut().unwrap().remove("timestamp");
event
}
let through_profile = event_for(UsernsGrant::ProfileConfining).await;
let host_wide = event_for(UsernsGrant::RestrictionDisabledHostWide).await;
assert_eq!(through_profile["containment_achieved"], "sealed");
assert_eq!(host_wide["containment_achieved"], "sealed");
assert_eq!(through_profile["userns_grant"], "profile_confining");
assert_eq!(host_wide["userns_grant"], "restriction_disabled_host_wide");
assert_ne!(
through_profile, host_wide,
"two hosts granting the user namespace by different mechanisms must not write the \
same session_start record"
);
let unprobed = scope_report_for_tier(
&CapabilityPolicy::default(),
ContainmentClass::Advisory,
EnforcementTier::EnvironmentOnly,
None,
None,
None,
Vec::new(),
Vec::new(),
Vec::new(),
crate::cgroup::IoMaxReport::default(),
);
assert!(
serde_json::to_value(&unprobed).unwrap()["userns_grant"].is_null(),
"off Linux the key is present and null, not absent"
);
}
#[tokio::test]
async fn session_start_records_the_whole_scope_report_as_effective_grants() {
let policy = CapabilityPolicy {
network_allow: vec!["https://api.example.com".to_string()],
unix_sockets_allowed: false,
filesystem_scope: Some("workdir".to_string()),
shell_allow: vec!["python3".to_string()],
spawn_allow: vec!["helper".to_string()],
env_allow: vec!["TZ".to_string()],
shell_interpreter_runtime: vec![InterpreterRuntimeGrant {
binary: "python3".to_string(),
dirs: vec![InterpreterRuntimeDir {
path: "/usr/lib/python3.11".to_string(),
list_dir: true,
}],
}],
..CapabilityPolicy::default()
};
let expected = scope_report_for_tier(
&policy,
ContainmentClass::Scoped,
EnforcementTier::KernelFull,
None,
None,
None,
vec![crate::containment::StateStoreReport {
artifact: "notes-tool".to_string(),
store: "shey".to_string(),
host_path: "/home/dev/.murmur/state/shey".to_string(),
}],
vec!["config-echo".to_string()],
vec![crate::containment::PreopenReport {
artifact: "notes-tool".to_string(),
role: "tool".to_string(),
scope: None,
surface: crate::containment::PreopenSurface::WholeWorkdir,
}],
crate::cgroup::IoMaxReport::default(),
);
let dir = tempfile::tempdir().unwrap();
let mut w = TraceWriter::open(
dir.path(),
"test-session-id".to_string(),
"test-capsule".to_string(),
"1.0.0".to_string(),
"claude-test".to_string(),
vec!["network".to_string(), "shell".to_string()],
expected.clone(),
TraceCapture::Meta,
None,
false,
None,
None,
None,
None,
)
.await
.unwrap();
w.write_session_start(1, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(
events[0]["effective_grants"],
serde_json::to_value(&expected).unwrap()
);
assert_eq!(
events[0]["capabilities"],
serde_json::json!(["network", "shell"])
);
assert_eq!(
events[0]["effective_grants"]["state_stores"],
serde_json::json!([{
"artifact": "notes-tool",
"store": "shey",
"host_path": "/home/dev/.murmur/state/shey",
}])
);
assert_eq!(
events[0]["effective_grants"]["configured_artifacts"],
serde_json::json!(["config-echo"])
);
assert_eq!(
events[0]["effective_grants"]["preopens"],
serde_json::json!([{
"artifact": "notes-tool",
"role": "tool",
"scope": null,
"surface": "whole-workdir",
}])
);
assert_eq!(events[0]["containment_declared"], "scoped");
assert_eq!(events[0]["containment_achieved"], "scoped");
assert_eq!(events[0]["workdir_exec"], false);
assert_eq!(
events[0]["effective_grants"]["network_allow"],
serde_json::json!(["https://api.example.com"])
);
assert_eq!(
events[0]["effective_grants"]["interpreter_runtime_grants"],
serde_json::json!(["python3: /usr/lib/python3.11 (list_dir)"])
);
}
#[tokio::test]
async fn inference_fields_and_snake_case() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_inference(
0,
Some(100),
Some(50),
"tool_call".to_string(),
Some("tool_call"),
Some("bash".to_string()),
None,
None,
vec!["msg_0000000000000000000000000000000a".to_string()],
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "inference");
assert_eq!(
e["message_ids"],
serde_json::json!(["msg_0000000000000000000000000000000a"]),
"an agent-loop turn names the messages its request embedded"
);
assert_eq!(e["turn"], 0);
assert_eq!(e["input_tokens"], 100);
assert_eq!(e["output_tokens"], 50);
assert_eq!(e["decision"], "tool_call");
assert_eq!(e["tool_name"], "bash");
assert!(e.get("inputTokens").is_none(), "must use snake_case");
assert!(e.get("toolName").is_none(), "must use snake_case");
for key in [
"input_tokens_actual",
"output_tokens_actual",
"cached_tokens",
"cache_write_tokens",
] {
assert!(e.get(key).is_none(), "{key} must be omitted, not null or 0");
}
}
#[tokio::test]
async fn inference_writes_only_the_reported_usage_members() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
let usage = DriverUsage {
input_tokens: Some(12043),
output_tokens: None,
cached_tokens: Some(0),
cache_write_tokens: None,
thinking_tokens: None,
};
w.write_inference(
0,
Some(100),
Some(50),
"end_turn".to_string(),
Some("end_turn"),
None,
None,
Some(&usage),
Vec::new(),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["input_tokens"], 100, "the estimate is untouched");
assert_eq!(e["input_tokens_actual"], 12043);
assert_eq!(e["cached_tokens"], 0, "a reported zero is written");
assert!(e.get("output_tokens_actual").is_none());
assert!(e.get("cache_write_tokens").is_none());
}
#[tokio::test]
async fn failed_inference_writes_its_keys_and_counts_only_what_it_spent() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
let usage = DriverUsage {
input_tokens: Some(7),
..DriverUsage::default()
};
let long = "é".repeat(MAX_TASK_FAILURE_REASON_BYTES);
w.write_failed_inference(FailedInference {
turn: 0,
error_code: INFERENCE_ERROR_CREDENTIAL_REJECTED,
error: &long,
provider_status: Some(401),
usage: Some(&usage),
origin: None,
choice: None,
message_ids: vec!["msg_1".to_string()],
tool_name: None,
spent: None,
})
.await
.unwrap();
assert_eq!(w.task_turns(), 0, "an uncounted record moves no counter");
assert_eq!((w.total_turns, w.total_input_tokens), (0, 0));
assert_eq!((w.task_input_tokens, w.task_output_tokens), (0, 0));
w.write_failed_inference(FailedInference {
turn: 1,
error_code: INFERENCE_ERROR_HARNESS_AUTH,
error: "bad login",
provider_status: None,
usage: None,
origin: None,
choice: None,
message_ids: Vec::new(),
tool_name: None,
spent: Some((Some(10), Some(5))),
})
.await
.unwrap();
assert_eq!(w.task_turns(), 1);
assert_eq!(w.total_turns, 1);
assert_eq!((w.total_input_tokens, w.total_output_tokens), (10, 5));
assert_eq!((w.task_input_tokens, w.task_output_tokens), (10, 5));
w.flush().await.unwrap();
let events = read_events(dir.path());
let failed: Vec<&Value> = events
.iter()
.filter(|e| e["event_type"] == "inference")
.collect();
let keys = |e: &Value| {
let mut keys: Vec<String> = e.as_object().unwrap().keys().cloned().collect();
keys.sort();
keys
};
assert_eq!(
keys(failed[0]),
[
"decision",
"error",
"error_code",
"event_id",
"event_type",
"input_tokens_actual",
"message_ids",
"parent_id",
"provider_status",
"session_id",
"stop_reason",
"task_id",
"timestamp",
"tool_name",
"turn",
]
);
assert_eq!(failed[0]["decision"], "error");
assert_eq!(failed[0]["stop_reason"], "error");
assert_eq!(failed[0]["error_code"], "credential_rejected");
assert_eq!(failed[0]["provider_status"], 401);
assert_eq!(failed[0]["input_tokens_actual"], 7);
let error = failed[0]["error"].as_str().unwrap();
assert!(error.len() <= MAX_TASK_FAILURE_REASON_BYTES);
assert!(error.len() > MAX_TASK_FAILURE_REASON_BYTES - 2);
assert_eq!(failed[1]["error_code"], "harness_auth");
assert_eq!(failed[1]["error"], "bad login");
assert_eq!(failed[1]["input_tokens"], 10);
assert_eq!(failed[1]["output_tokens"], 5);
assert!(failed[1].get("provider_status").is_none());
assert!(
!dir.path().join("blobs").exists(),
"a failed record stores no blob, even under trace.capture: content"
);
let task_start = events
.iter()
.find(|e| e["event_type"] == "task_start")
.unwrap();
assert_eq!(failed[0]["parent_id"], task_start["event_id"]);
}
#[tokio::test]
async fn a_failed_hook_record_hangs_off_the_turn_and_keeps_hook_accounting() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_inference(
0,
Some(10),
Some(5),
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
let origin = InferenceOrigin {
source: "hook:compactor".to_string(),
model: "m".to_string(),
};
w.write_failed_inference(FailedInference {
turn: 0,
error_code: INFERENCE_ERROR_DRIVER_ERROR,
error: "HTTP 500: boom",
provider_status: Some(500),
usage: None,
origin: Some(&origin),
choice: None,
message_ids: Vec::new(),
tool_name: None,
spent: Some((Some(40), None)),
})
.await
.unwrap();
assert_eq!(w.total_turns, 2);
assert_eq!((w.total_input_tokens, w.total_output_tokens), (50, 5));
w.flush().await.unwrap();
let events = read_events(dir.path());
let turn = events.iter().find(|e| e["decision"] == "end_turn").unwrap();
let hook = events.iter().find(|e| e["decision"] == "error").unwrap();
assert_eq!(hook["parent_id"], turn["event_id"]);
assert_eq!(hook["origin"], "hook:compactor");
assert_eq!(hook["model"], "m");
assert_eq!(hook["input_tokens"], 40);
assert!(hook.get("output_tokens").is_none());
}
#[tokio::test]
async fn tool_call_fields() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_tool_call(
1,
"bash".to_string(),
None,
serde_json::json!({"command": "echo hi"}),
42,
"hi\n",
100,
55,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "tool_call");
assert_eq!(e["turn"], 1);
assert_eq!(e["tool_name"], "bash");
assert_eq!(e["input_bytes"], 42);
assert_eq!(e["output_bytes"], 100);
assert_eq!(e["duration_ms"], 55);
assert_eq!(e["status"], "ok");
assert!(
e.get("state_effect").is_none(),
"state_effect must be omitted when the tool declared nothing"
);
}
#[tokio::test]
async fn tool_call_records_declared_state_effect() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_tool_call(
0,
"murmur-tool-editor".to_string(),
None,
serde_json::json!({"operation": "read_file", "path": "a.txt"}),
10,
"hi\n",
3,
5,
"ok".to_string(),
Some("read".to_string()),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(
events[0]["state_effect"], "read",
"declared state_effect must be recorded verbatim on the tool_call event"
);
}
#[tokio::test]
async fn tool_call_records_declared_resource_id() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_tool_call(
0,
"murmur-tool-code-graph".to_string(),
None,
serde_json::json!({"symbol": "Foo::bar"}),
10,
"hi\n",
3,
5,
"ok".to_string(),
Some("read".to_string()),
Some("sym:Foo".to_string()),
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(
events[0]["resource_id"], "sym:Foo",
"declared resource_id must reach trace.jsonl verbatim, unparsed"
);
}
#[tokio::test]
async fn tool_call_omits_undeclared_resource_id() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "echo hi"}),
10,
"hi\n",
3,
5,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert!(
events[0].get("resource_id").is_none(),
"resource_id must be omitted entirely (not null) when the tool declared nothing"
);
}
#[tokio::test]
async fn shell_fields() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_shell(
0,
"/usr/bin/echo".to_string(),
"echo hi".to_string(),
0,
7,
0,
10,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "shell");
assert_eq!(e["binary"], "/usr/bin/echo");
assert_eq!(e["command"], "echo hi");
assert_eq!(e["exit_code"], 0);
assert_eq!(e["stdout_bytes"], 7);
assert_eq!(e["stderr_bytes"], 0);
assert_eq!(e["duration_ms"], 10);
}
#[tokio::test]
async fn compaction_fields() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_compaction(3, 80000, 20000).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "compaction");
assert_eq!(e["turn"], 3);
assert_eq!(e["tokens_before"], 80000);
assert_eq!(e["tokens_after"], 20000);
}
#[tokio::test]
async fn hook_dispatch_error_fields() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_hook_dispatch_error("my-hook", "on-tool-call", "write-manifests")
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "hook_dispatch_error");
assert_eq!(e["session_id"], "test-session-id");
assert_eq!(e["hook_name"], "my-hook");
assert_eq!(e["event"], "on-tool-call");
assert_eq!(e["arm"], "write-manifests");
assert!(e["timestamp"].as_u64().unwrap() > 0);
assert!(e.get("hookName").is_none(), "must use snake_case");
}
#[tokio::test]
async fn session_end_fields_and_snake_case() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_inference(
0,
Some(100),
Some(50),
"tool_call".to_string(),
Some("tool_call"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "ls"}),
10,
"file.txt\n",
20,
5,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.write_shell(0, "/bin/ls".to_string(), "ls".to_string(), 0, 3, 0, 2, None)
.await
.unwrap();
w.write_session_end("ok").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let se = events.last().unwrap();
assert_eq!(se["event_type"], "session_end");
assert_eq!(se["total_turns"], 1);
assert_eq!(se["total_input_tokens"], 100);
assert_eq!(se["total_output_tokens"], 50);
assert_eq!(se["total_tool_calls"], 1);
assert_eq!(se["total_shell_calls"], 1);
assert_eq!(se["exit_status"], "ok");
assert!(se["duration_ms"].as_u64().is_some());
assert!(se.get("exitStatus").is_none(), "must use snake_case");
assert!(se.get("totalTurns").is_none(), "must use snake_case");
}
#[tokio::test]
async fn write_session_end_if_not_ended_is_noop_when_already_ended() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec![]).await.unwrap();
w.write_session_end("ok").await.unwrap();
w.write_session_end_if_not_ended("failed").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(
events
.iter()
.filter(|e| e["event_type"] == "session_end")
.count(),
1,
"session_end should appear exactly once"
);
}
#[tokio::test]
async fn write_session_end_if_not_ended_writes_when_started_not_ended() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec![]).await.unwrap();
w.write_session_end_if_not_ended("failed").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let se = events
.iter()
.find(|e| e["event_type"] == "session_end")
.unwrap();
assert_eq!(se["exit_status"], "failed");
}
#[tokio::test]
async fn no_session_end_written_when_session_never_started() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_end_if_not_ended("failed").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert!(
events.iter().all(|e| e["event_type"] != "session_end"),
"session_end must not appear when session_start was never written"
);
}
#[tokio::test]
async fn session_id_on_every_event() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec!["bash".to_string()])
.await
.unwrap();
w.write_inference(
0,
Some(10),
Some(5),
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_session_end("ok").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
for e in &events {
assert_eq!(e["session_id"], "test-session-id");
}
}
#[tokio::test]
async fn counts_match_per_event_sum() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec![]).await.unwrap();
w.write_inference(
0,
Some(50),
Some(25),
"tool_call".to_string(),
Some("tool_call"),
Some("bash".to_string()),
None,
None,
vec!["msg_0000000000000000000000000000000a".to_string()],
None,
)
.await
.unwrap();
w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "echo"}),
5,
"ok",
10,
3,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.write_shell(
0,
"/bin/echo".to_string(),
"echo".to_string(),
0,
4,
0,
1,
None,
)
.await
.unwrap();
w.write_inference(
1,
Some(60),
Some(30),
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_session_end("ok").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let inference_count = events
.iter()
.filter(|e| e["event_type"] == "inference")
.count();
let tool_count = events
.iter()
.filter(|e| e["event_type"] == "tool_call")
.count();
let shell_count = events.iter().filter(|e| e["event_type"] == "shell").count();
let se = events
.iter()
.find(|e| e["event_type"] == "session_end")
.unwrap();
assert_eq!(
se["total_turns"].as_u64().unwrap() as usize,
inference_count
);
assert_eq!(
se["total_tool_calls"].as_u64().unwrap() as usize,
tool_count
);
assert_eq!(
se["total_shell_calls"].as_u64().unwrap() as usize,
shell_count
);
}
#[tokio::test]
async fn tool_call_input_always_present() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await; w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "ls -la"}),
20,
"total 8\n",
80,
12,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "tool_call");
assert_eq!(e["input"]["command"], "ls -la");
assert!(e.get("input").is_some(), "input must always be present");
}
#[tokio::test]
async fn tool_call_output_absent_by_default() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await; w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "ls"}),
10,
"file.txt\n",
90,
8,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert!(
e.get("output").is_none(),
"output must be absent under a capture mode that stores no bodies"
);
assert_eq!(e["output_bytes"], 90, "output_bytes must still be recorded");
}
#[tokio::test]
async fn skill_call_writes_skill_call_event() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_skill_call(
2,
"my-skill".to_string(),
&LockOrigin::Operator,
1024,
8,
"ok".to_string(),
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(e["event_type"], "skill_call");
assert_eq!(e["turn"], 2);
assert_eq!(e["skill_name"], "my-skill");
assert_eq!(e["output_bytes"], 1024);
assert_eq!(e["duration_ms"], 8);
assert_eq!(e["status"], "ok");
}
#[tokio::test]
async fn skill_call_does_not_increment_total_tool_calls() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec![]).await.unwrap();
w.write_tool_call(
0,
"real-tool".to_string(),
None,
serde_json::json!({}),
2,
"data",
4,
5,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.write_skill_call(
1,
"my-skill".to_string(),
&LockOrigin::Operator,
512,
3,
"ok".to_string(),
)
.await
.unwrap();
w.write_session_end("ok").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let se = events
.iter()
.find(|e| e["event_type"] == "session_end")
.unwrap();
assert_eq!(
se["total_tool_calls"], 1,
"skill_call must not inflate total_tool_calls"
);
}
#[tokio::test]
async fn tool_call_output_present_when_opted_in() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
w.write_tool_call(
0,
"bash".to_string(),
None,
serde_json::json!({"command": "echo hello"}),
22,
"hello\n",
60,
5,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let e = &events[0];
assert_eq!(
e["output"].as_str().unwrap(),
"hello\n",
"output must be present under capture: content"
);
assert_eq!(e["output_bytes"], 60);
}
#[tokio::test]
async fn task_end_carries_reopen_count() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
w.write_task_end("tsk_1", "ok", 2).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let end = events
.iter()
.find(|e| e["event_type"] == "task_end")
.unwrap();
assert_eq!(end["task_id"], "tsk_1");
assert_eq!(end["exit_status"], "ok");
assert_eq!(end["reopen_count"], 2);
}
#[tokio::test]
async fn task_reopened_names_hook_reason_and_ordinal() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
w.write_task_reopened(
"tsk_1",
"gatekeeper",
"tests still fail",
1,
REOPEN_CONTEXT_CONTINUED,
7,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let re = events
.iter()
.find(|e| e["event_type"] == "task_reopened")
.unwrap();
assert_eq!(re["task_id"], "tsk_1");
assert_eq!(re["hook_name"], "gatekeeper");
assert_eq!(re["reason"], "tests still fail");
assert_eq!(re["reopen_number"], 1);
assert_eq!(re["attempt_context"], "continued");
assert_eq!(re["turns_remaining"], 7);
assert!(re["timestamp"].as_u64().unwrap() > 0);
}
#[tokio::test]
async fn task_continued_names_the_round_under_the_task_node() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
w.write_task_continued(
"tsk_1",
1,
&["mcl_b".to_string(), "mcl_a".to_string()],
&["dlg_1".to_string()],
1234,
8,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let start = events
.iter()
.find(|e| e["event_type"] == "task_start")
.unwrap();
let continued = events
.iter()
.find(|e| e["event_type"] == "task_continued")
.unwrap();
assert_eq!(continued["parent_id"], start["event_id"]);
assert_eq!(continued["task_id"], "tsk_1");
assert_eq!(continued["continuation_number"], 1);
assert_eq!(
continued["member_calls"],
serde_json::json!(["mcl_b", "mcl_a"])
);
assert_eq!(continued["delegations"], serde_json::json!(["dlg_1"]));
assert_eq!(continued["waited_ms"], 1234);
assert_eq!(continued["turns_remaining"], 8);
}
#[tokio::test]
async fn task_turns_accessor_tracks_and_resets() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
assert_eq!(w.task_turns(), 0);
w.write_inference(
0,
Some(10),
Some(5),
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_inference(
1,
Some(10),
Some(5),
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
assert_eq!(w.task_turns(), 2);
w.write_task_start("tsk_2", "ctx_2", "a2a", event_provenance(), 3)
.await
.unwrap();
assert_eq!(w.task_turns(), 0);
}
#[tokio::test]
async fn task_agent_turns_counts_agent_loop_inference_only() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
assert_eq!(w.task_agent_turns(), 0);
let hook = InferenceOrigin {
source: "hook:x".into(),
model: "m".into(),
};
for (turn, origin) in [(0, None), (0, Some(&hook)), (1, None)] {
w.write_inference(
turn,
Some(10),
Some(5),
"end_turn".to_string(),
Some("end_turn"),
None,
origin,
None,
Vec::new(),
None,
)
.await
.unwrap();
}
assert_eq!(w.task_agent_turns(), 2);
assert_eq!(w.task_turns(), 3);
assert_eq!(w.task_hook_inferences(), 1);
w.write_task_end("tsk_1", "completed", 0).await.unwrap();
assert_eq!(
w.task_agent_turns(),
0,
"a run outside a task frame starts at 0"
);
w.write_inference(
0,
Some(10),
Some(5),
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_task_start("tsk_2", "ctx_2", "a2a", event_provenance(), 3)
.await
.unwrap();
assert_eq!(w.task_agent_turns(), 0);
}
#[tokio::test]
async fn every_event_carries_a_unique_id_and_parents_to_an_earlier_event() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, vec!["bash".to_string()])
.await
.unwrap();
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
w.write_inference(
0,
Some(10),
Some(5),
"tool_call".to_string(),
Some("tool_call"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_tool_call(
0,
"bash".to_string(),
Some("toolu_1".to_string()),
serde_json::json!({"command": "echo hi"}),
10,
"hi\n",
3,
5,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.write_shell(
0,
"/bin/bash".to_string(),
"echo hi".to_string(),
0,
3,
0,
2,
None,
)
.await
.unwrap();
w.write_artifact_pulled(&runtime_summary("pulled-style", ArtifactRuntime::Skill))
.await
.unwrap();
w.write_task_end("tsk_1", "ok", 0).await.unwrap();
w.write_session_end("ok").await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
for (i, event) in events.iter().enumerate() {
let id = event["event_id"].as_str().unwrap();
assert!(id.starts_with("evt_") && id.len() == 36, "event {i}: {id}");
assert!(id[4..].chars().all(|c| c.is_ascii_hexdigit()), "event {i}");
assert!(seen.insert(id.to_string()), "event {i} reuses {id}");
assert!(
event.get("parent_id").is_some(),
"event {i} has no parent_id"
);
if let Some(parent) = event["parent_id"].as_str() {
assert!(seen.contains(parent), "event {i} names dangling {parent}");
}
}
let by_type = |ty: &str| {
events
.iter()
.find(|e| e["event_type"] == ty)
.unwrap_or_else(|| panic!("no {ty}"))
.clone()
};
let session = by_type("session_start");
assert!(session["parent_id"].is_null());
assert_eq!(session["event_id"], w.session_event_id());
let task = by_type("task_start");
assert_eq!(task["parent_id"], session["event_id"]);
let turn = by_type("inference");
assert_eq!(turn["parent_id"], task["event_id"]);
assert_eq!(by_type("tool_call")["parent_id"], turn["event_id"]);
assert_eq!(by_type("shell")["parent_id"], turn["event_id"]);
assert_eq!(by_type("task_end")["parent_id"], task["event_id"]);
assert_eq!(by_type("session_end")["parent_id"], session["event_id"]);
assert_eq!(by_type("artifact_pulled")["parent_id"], session["event_id"]);
}
fn runtime_summary(name: &str, runtime: ArtifactRuntime) -> InstalledArtifactSummary {
InstalledArtifactSummary {
name: name.to_string(),
version: "1.0.0".to_string(),
runtime,
implementation: None,
origin: LockOrigin::Runtime {
session: "ses_puller".to_string(),
},
}
}
fn operator_summary(name: &str, runtime: ArtifactRuntime) -> InstalledArtifactSummary {
InstalledArtifactSummary {
origin: LockOrigin::Operator,
..runtime_summary(name, runtime)
}
}
#[tokio::test]
async fn runtime_origin_session_start_lists_nothing_for_an_operator_session() {
for set in [false, true] {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
if set {
w.set_runtime_artifacts(&[
operator_summary("house-style", ArtifactRuntime::Skill),
operator_summary("fetcher", ArtifactRuntime::Tool),
]);
}
w.write_session_start(10, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events[0]["runtime_artifacts"], serde_json::json!([]));
}
}
#[tokio::test]
async fn runtime_origin_session_start_lists_runtime_artifacts_in_order() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.set_runtime_artifacts(&[
runtime_summary("zeta-tool", ArtifactRuntime::Tool),
operator_summary("house-style", ArtifactRuntime::Skill),
InstalledArtifactSummary {
origin: LockOrigin::Runtime {
session: "ses_other".to_string(),
},
..runtime_summary("alpha-style", ArtifactRuntime::Skill)
},
]);
w.write_session_start(10, Vec::new()).await.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(
events[0]["runtime_artifacts"],
serde_json::json!([
{"name": "zeta-tool", "version": "1.0.0", "origin": "runtime",
"session": "ses_puller", "trust": "untrusted"},
{"name": "alpha-style", "version": "1.0.0", "origin": "runtime",
"session": "ses_other", "trust": "untrusted"},
])
);
}
#[tokio::test]
async fn runtime_origin_skill_call_carries_origin_and_trust() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
for origin in [
LockOrigin::Operator,
LockOrigin::Runtime {
session: "ses_puller".to_string(),
},
] {
w.write_skill_call(0, "a-skill".to_string(), &origin, 1, 1, "ok".to_string())
.await
.unwrap();
}
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events[0]["origin"], "operator");
assert_eq!(events[0]["trust"], "trusted");
assert_eq!(events[1]["origin"], "runtime");
assert_eq!(events[1]["trust"], "untrusted");
}
#[tokio::test]
async fn runtime_origin_artifact_pulled_writes_every_key() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_artifact_pulled(&runtime_summary("pulled-style", ArtifactRuntime::Skill))
.await
.unwrap();
w.write_artifact_pulled(&operator_summary("fetcher", ArtifactRuntime::Tool))
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let runtime = &events[0];
assert_eq!(runtime["event_type"], "artifact_pulled");
assert_eq!(runtime["name"], "pulled-style");
assert_eq!(runtime["version"], "1.0.0");
assert_eq!(runtime["runtime"], "skill");
assert_eq!(runtime["origin"], "runtime");
assert_eq!(runtime["session"], "ses_puller");
assert_eq!(runtime["trust"], "untrusted");
assert!(
runtime["parent_id"].is_null(),
"no session_start was written"
);
let operator = &events[1];
assert_eq!(operator["runtime"], "tool");
assert_eq!(operator["origin"], "operator");
assert!(operator.get("session").is_some_and(Value::is_null));
assert_eq!(operator["trust"], "trusted");
}
#[tokio::test]
async fn hook_origin_inference_hangs_off_the_turn_rather_than_becoming_one() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, Vec::new()).await.unwrap();
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
w.write_inference(
0,
Some(10),
Some(5),
"tool_call".to_string(),
Some("tool_call"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
let origin = InferenceOrigin {
source: "hook:gatekeeper".to_string(),
model: "claude-test".to_string(),
};
w.write_inference(
0,
Some(1),
Some(1),
"end_turn".to_string(),
None,
None,
Some(&origin),
None,
Vec::new(),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let turn = &events[2];
let hook = &events[3];
assert!(turn["origin"].is_null());
assert_eq!(hook["origin"], "hook:gatekeeper");
assert_eq!(hook["parent_id"], turn["event_id"]);
assert!(hook.get("message_ids").is_none(), "{hook}");
}
#[tokio::test]
async fn a2a_send_parents_to_null_without_a_session_frame_and_to_it_with_one() {
let bare = tempfile::tempdir().unwrap();
let mut w = make_writer(bare.path()).await;
w.write_a2a_send(
"http://peer",
"msg_1",
"tsk_1",
"ctx_1",
None,
TrustClass::Untrusted,
)
.await
.unwrap();
w.flush().await.unwrap();
assert!(read_events(bare.path())[0]["parent_id"].is_null());
let framed = tempfile::tempdir().unwrap();
let mut w = make_writer(framed.path()).await;
w.write_session_start(1, Vec::new()).await.unwrap();
w.write_a2a_send(
"http://peer",
"msg_1",
"tsk_1",
"ctx_1",
None,
TrustClass::Untrusted,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(framed.path());
assert_eq!(events[1]["parent_id"], events[0]["event_id"]);
}
#[tokio::test]
async fn task_rejected_names_the_task_source_cause_and_reason_under_the_session() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, Vec::new()).await.unwrap();
w.write_task_start("tsk_ran", "ctx_ran", "a2a", event_provenance(), 3)
.await
.unwrap();
w.write_task_end("tsk_ran", "ok", 0).await.unwrap();
w.write_task_rejected(
"tsk_1",
"ctx_1",
"detached_shell",
TASK_REJECTED_SESSION_ENDED,
"task rejected: the session ended before this task started",
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let session = events
.iter()
.find(|e| e["event_type"] == "session_start")
.unwrap();
let rejected: Vec<_> = events
.iter()
.filter(|e| e["event_type"] == "task_rejected")
.collect();
assert_eq!(rejected.len(), 1);
let rejected = rejected[0];
assert_eq!(rejected["parent_id"], session["event_id"], "{rejected}");
assert_eq!(rejected["session_id"], session["session_id"]);
assert_eq!(rejected["task_id"], "tsk_1");
assert_eq!(rejected["context_id"], "ctx_1");
assert_eq!(rejected["source"], "detached_shell");
assert_eq!(rejected["cause"], "session_ended");
assert_eq!(
rejected["reason"],
"task rejected: the session ended before this task started"
);
assert!(rejected["event_id"].is_string());
assert!(rejected["timestamp"].is_u64());
assert_eq!(rejected.as_object().unwrap().len(), 10, "{rejected}");
}
#[tokio::test]
async fn task_failed_names_the_task_turn_cause_and_capped_reason() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, Vec::new()).await.unwrap();
assert_eq!(w.last_task_failure(), None);
w.write_task_failed(None, TASK_FAILED_RUNTIME_ERROR, "before any task")
.await
.unwrap();
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
let long = "é".repeat(MAX_TASK_FAILURE_REASON_BYTES);
w.write_task_failed(Some(2), TASK_FAILED_DRIVER_ERROR, &long)
.await
.unwrap();
w.write_task_end("tsk_1", "failed", 0).await.unwrap();
w.flush().await.unwrap();
assert_eq!(w.task_failures_written(), 2);
let events = read_events(dir.path());
let failed: Vec<_> = events
.iter()
.filter(|e| e["event_type"] == "task_failed")
.collect();
assert_eq!(failed.len(), 2);
assert!(failed[0].get("task_id").is_none(), "{}", failed[0]);
assert!(failed[0].get("turn").is_none(), "{}", failed[0]);
assert_eq!(failed[0]["cause"], "runtime_error");
assert_eq!(failed[0]["reason"], "before any task");
let start = events
.iter()
.find(|e| e["event_type"] == "task_start")
.unwrap();
assert_eq!(failed[1]["task_id"], "tsk_1");
assert_eq!(failed[1]["turn"], 2);
assert_eq!(failed[1]["cause"], "driver_error");
assert_eq!(failed[1]["parent_id"], start["event_id"]);
assert_eq!(failed[1]["session_id"], start["session_id"]);
assert!(failed[1]["event_id"].is_string());
assert!(failed[1]["timestamp"].is_u64());
let reason = failed[1]["reason"].as_str().unwrap();
assert!(reason.len() <= MAX_TASK_FAILURE_REASON_BYTES);
assert!(reason.len() > MAX_TASK_FAILURE_REASON_BYTES - 2);
assert_eq!(w.last_task_failure(), Some(reason));
}
#[tokio::test]
async fn turn_level_events_carry_the_active_task_id() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, Vec::new()).await.unwrap();
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
w.write_inference(
0,
Some(10),
Some(5),
"tool_call".to_string(),
Some("tool_call"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_skill_call(
0,
"house-style".to_string(),
&LockOrigin::Operator,
12,
1,
"ok".to_string(),
)
.await
.unwrap();
w.write_compaction(0, 100, 40).await.unwrap();
w.write_task_end("tsk_1", "ok", 0).await.unwrap();
w.write_inference(
1,
Some(1),
Some(1),
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
for event in events
.iter()
.filter(|e| {
matches!(
e["event_type"].as_str(),
Some("inference" | "skill_call" | "compaction")
)
})
.take(3)
{
assert_eq!(event["task_id"], "tsk_1", "{event}");
}
let after = events.last().unwrap();
assert_eq!(after["event_type"], "inference");
assert!(
after["task_id"].is_null(),
"an inference outside any task carries a null task_id, not the last task's"
);
assert_eq!(after["parent_id"], events[0]["event_id"]);
}
#[tokio::test]
async fn tool_call_records_the_provider_id_verbatim_and_empty_as_null() {
for (given, expected) in [
(Some("toolu_1".to_string()), Some("toolu_1")),
(Some(String::new()), None),
(None, None),
] {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_tool_call(
0,
"bash".to_string(),
given.clone(),
serde_json::json!({}),
2,
"",
0,
1,
"ok".to_string(),
None,
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
match expected {
Some(id) => assert_eq!(e["tool_call_id"], id, "given={given:?}"),
None => assert!(e["tool_call_id"].is_null(), "given={given:?}"),
}
}
}
#[tokio::test]
async fn tool_input_refused_records_its_fields_in_every_capture_mode() {
for capture in [
TraceCapture::None,
TraceCapture::Meta,
TraceCapture::Content,
] {
for (given, expected) in [
(Some("toolu_1"), Some("toolu_1")),
(Some(""), None),
(None, None),
] {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), capture).await;
w.write_tool_input_refused(
2,
"murmur-tool-editor",
given,
&["operation".to_string(), "dest_path".to_string()],
"murmur-tool-editor: missing required fields",
)
.await
.unwrap();
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
assert_eq!(e["event_type"], "tool_input_refused", "{capture:?}");
assert!(e["event_id"].is_string());
assert!(e["session_id"].is_string());
assert!(e["timestamp"].is_u64());
assert_eq!(e["turn"], 2);
assert_eq!(e["tool_name"], "murmur-tool-editor");
match expected {
Some(id) => assert_eq!(e["tool_call_id"], id, "given={given:?}"),
None => assert!(e["tool_call_id"].is_null(), "given={given:?}"),
}
assert_eq!(e["missing"], serde_json::json!(["operation", "dest_path"]));
assert_eq!(e["reason"], "murmur-tool-editor: missing required fields");
}
}
}
#[tokio::test]
async fn tools_refreshed_event_records_trigger_names_and_task_parent() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.set_tool_refresh(Some("immediate"));
w.write_session_start(10, Vec::new()).await.unwrap();
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
w.write_inference(
0,
Some(10),
Some(5),
"tool_call".to_string(),
Some("tool_call"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_tools_refreshed(
1,
"immediate",
&["aaa-late-skill".to_string()],
&[],
&[
"aaa-late-skill".to_string(),
"zzz-existing-tool".to_string(),
],
)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events[0]["tool_refresh"], "immediate");
let task = events
.iter()
.find(|e| e["event_type"] == "task_start")
.unwrap();
let refreshed: Vec<&Value> = events
.iter()
.filter(|e| e["event_type"] == "tools_refreshed")
.collect();
assert_eq!(refreshed.len(), 1);
let r = refreshed[0];
assert!(r["event_id"].as_str().unwrap().starts_with("evt_"));
assert_eq!(r["parent_id"], task["event_id"]);
assert_eq!(r["session_id"], "test-session-id");
assert!(r["timestamp"].as_u64().is_some());
assert_eq!(r["turn"], 1);
assert_eq!(r["task_id"], "tsk_1");
assert_eq!(r["trigger"], "immediate");
assert_eq!(r["added"], serde_json::json!(["aaa-late-skill"]));
assert_eq!(r["removed"], serde_json::json!([]));
assert_eq!(
r["tools"],
serde_json::json!(["aaa-late-skill", "zzz-existing-tool"])
);
let process_dir = tempfile::tempdir().unwrap();
let mut process = make_writer(process_dir.path()).await;
process.set_tool_refresh(None);
process.write_session_start(10, Vec::new()).await.unwrap();
process.flush().await.unwrap();
let start = &read_events(process_dir.path())[0];
assert!(start.as_object().unwrap().contains_key("tool_refresh"));
assert!(start["tool_refresh"].is_null());
}
#[tokio::test]
async fn compaction_declined_records_turn_task_tokens_and_reason() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_session_start(10, Vec::new()).await.unwrap();
w.write_task_start("tsk_1", "ctx_1", "a2a", event_provenance(), 3)
.await
.unwrap();
w.write_inference(
2,
Some(10),
Some(5),
"tool_call".to_string(),
Some("tool_call"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.write_compaction_declined(2, 4321, COMPACTION_DECLINED_UNRESOLVED_TOOL_CALL)
.await
.unwrap();
w.flush().await.unwrap();
let events = read_events(dir.path());
let declined: Vec<&Value> = events
.iter()
.filter(|e| e["event_type"] == "compaction_declined")
.collect();
assert_eq!(declined.len(), 1);
let d = declined[0];
assert_eq!(d["reason"], "unresolved_tool_call");
assert_eq!(d["turn"], 2);
assert_eq!(d["task_id"], "tsk_1");
assert_eq!(d["tokens"], 4321);
assert!(d["event_id"].as_str().unwrap().starts_with("evt_"));
assert_eq!(
d["parent_id"],
events
.iter()
.find(|e| e["event_type"] == "inference")
.unwrap()["event_id"]
);
}
#[tokio::test]
async fn event_ids_sort_by_mint_time() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
for turn in 0..5 {
w.write_inference(
turn,
Some(1),
Some(1),
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
}
w.flush().await.unwrap();
let ids: Vec<String> = read_events(dir.path())
.iter()
.map(|e| e["event_id"].as_str().unwrap().to_string())
.collect();
let mut sorted = ids.clone();
sorted.sort();
assert_eq!(ids, sorted, "evt_ ids must sort into mint order");
}
fn wire_payload(system: &str, messages: usize) -> Value {
let messages: Vec<Value> = (0..messages)
.map(|i| serde_json::json!({"role": "user", "content": format!("m{i}")}))
.collect();
serde_json::json!({
"model": "claude-test",
"max_tokens": 1024,
"messages": messages,
"tools": [{"name": "bash"}],
"params": {},
"system": system,
})
}
async fn write_turn(w: &mut TraceWriter, turn: u32, payload: &Value, response: &str) {
let wire = WireCapture::from_driver_payload(payload, response);
w.write_inference(
turn,
Some(10),
Some(5),
"end_turn".to_string(),
Some("end_turn"),
None,
None,
None,
vec!["msg_0000000000000000000000000000000a".to_string()],
Some(&wire),
)
.await
.unwrap();
}
fn blob_path(dir: &std::path::Path, sha: &str) -> std::path::PathBuf {
dir.join(BLOB_DIR_NAME).join(sha)
}
#[tokio::test]
async fn content_capture_names_blobs_that_exist_and_rehash_to_their_names() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let payload = wire_payload("You are a capsule.", 3);
write_turn(&mut w, 0, &payload, r#"{"stop_reason":"end_turn"}"#).await;
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
let mut named: Vec<String> = ["system_sha", "tools_sha", "response_sha"]
.iter()
.map(|key| {
e[*key]
.as_str()
.unwrap_or_else(|| panic!("{key} missing from {e}"))
.to_string()
})
.collect();
let shas = e["message_shas"].as_array().expect("message_shas");
assert_eq!(shas.len(), 3);
named.extend(shas.iter().map(|s| s.as_str().unwrap().to_string()));
for sha in &named {
let path = blob_path(dir.path(), sha);
let bytes =
std::fs::read(&path).unwrap_or_else(|err| panic!("blob {sha} must exist: {err}"));
assert_eq!(murmur_artifact::sha256_hex(&bytes), *sha);
assert_eq!(sha.len(), 64, "{sha}");
assert!(
sha.chars()
.all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()),
"{sha}"
);
}
}
#[tokio::test]
async fn the_system_blob_holds_the_prompt_as_readable_text() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let payload = wire_payload("[Capsule] you are cap.\nBe terse.", 1);
write_turn(&mut w, 0, &payload, "{}").await;
w.flush().await.unwrap();
let sha = read_events(dir.path())[0]["system_sha"]
.as_str()
.unwrap()
.to_string();
assert_eq!(
std::fs::read_to_string(blob_path(dir.path(), &sha)).unwrap(),
"[Capsule] you are cap.\nBe terse."
);
}
#[tokio::test]
async fn meta_writes_the_same_hashes_and_no_bodies() {
let payload = wire_payload("You are a capsule.", 2);
let response = r#"{"stop_reason":"end_turn"}"#;
let meta = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(meta.path(), TraceCapture::Meta).await;
write_turn(&mut w, 0, &payload, response).await;
w.flush().await.unwrap();
let content = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(content.path(), TraceCapture::Content).await;
write_turn(&mut w, 0, &payload, response).await;
w.flush().await.unwrap();
let meta_event = read_events(meta.path()).remove(0);
let content_event = read_events(content.path()).remove(0);
for key in ["system_sha", "tools_sha", "response_sha", "message_shas"] {
assert_eq!(meta_event[key], content_event[key], "{key}");
}
assert!(
!meta.path().join(BLOB_DIR_NAME).exists(),
"meta must not create the blob directory"
);
assert!(content.path().join(BLOB_DIR_NAME).is_dir());
}
#[tokio::test]
async fn the_default_capture_mode_is_meta() {
assert_eq!(TraceCapture::default(), TraceCapture::Meta);
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_prompt(dir.path(), TraceCapture::default(), None, false).await;
write_turn(&mut w, 0, &wire_payload("s", 1), "{}").await;
w.flush().await.unwrap();
let e = &read_events(dir.path())[0];
assert!(e.get("system_sha").is_some());
assert!(!dir.path().join(BLOB_DIR_NAME).exists());
}
#[tokio::test]
async fn none_omits_every_hash_and_leaves_the_rest_of_the_event_intact() {
let payload = wire_payload("You are a capsule.", 2);
let off = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(off.path(), TraceCapture::None).await;
write_turn(&mut w, 3, &payload, "{}").await;
w.flush().await.unwrap();
let e = read_events(off.path()).remove(0);
for key in ["system_sha", "tools_sha", "response_sha", "message_shas"] {
assert!(
e.get(key).is_none(),
"{key} must be absent under none, got {e}"
);
}
assert!(!off.path().join(BLOB_DIR_NAME).exists());
for key in [
"event_type",
"event_id",
"parent_id",
"session_id",
"timestamp",
"turn",
"task_id",
"input_tokens",
"output_tokens",
"decision",
"tool_name",
"message_ids",
] {
assert!(
e.get(key).is_some(),
"{key} must survive under none, got {e}"
);
}
assert_eq!(e["turn"], 3);
assert_eq!(e["message_ids"].as_array().unwrap().len(), 1);
}
#[tokio::test]
async fn an_unchanged_system_prompt_is_stored_once() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let payload = wire_payload("You are a capsule.", 1);
write_turn(&mut w, 0, &payload, "{}").await;
let sha = {
w.flush().await.unwrap();
read_events(dir.path())[0]["system_sha"]
.as_str()
.unwrap()
.to_string()
};
let path = blob_path(dir.path(), &sha);
std::fs::write(&path, b"sentinel").unwrap();
for turn in 1..10 {
write_turn(&mut w, turn, &payload, "{}").await;
}
w.flush().await.unwrap();
let events = read_events(dir.path());
assert_eq!(events.len(), 10);
for e in &events {
assert_eq!(e["system_sha"], sha.as_str());
}
assert_eq!(
std::fs::read(&path).unwrap(),
b"sentinel",
"an existing blob is never rewritten"
);
let matching = std::fs::read_dir(dir.path().join(BLOB_DIR_NAME))
.unwrap()
.filter(|entry| entry.as_ref().unwrap().file_name() == sha.as_str())
.count();
assert_eq!(matching, 1, "exactly one file bears that name");
}
#[tokio::test]
async fn stop_reason_is_absent_when_no_driver_response_was_parsed() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer(dir.path()).await;
w.write_inference(
0,
Some(10),
Some(5),
"end_turn".to_string(),
None,
None,
None,
None,
Vec::new(),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let event = read_events(dir.path()).remove(0);
assert!(
event.get("stop_reason").is_none(),
"stop_reason must be absent, got {event}"
);
}
#[tokio::test]
async fn a_hook_origin_record_carries_no_hashes() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let origin = InferenceOrigin {
source: "hook:gatekeeper".to_string(),
model: "claude-test".to_string(),
};
w.write_inference(
0,
Some(1),
Some(1),
"end_turn".to_string(),
None,
None,
Some(&origin),
None,
Vec::new(),
None,
)
.await
.unwrap();
w.flush().await.unwrap();
let e = read_events(dir.path()).remove(0);
assert_eq!(e["origin"], "hook:gatekeeper");
for key in ["system_sha", "tools_sha", "response_sha", "message_shas"] {
assert!(e.get(key).is_none(), "{key} must be absent, got {e}");
}
assert!(!dir.path().join(BLOB_DIR_NAME).exists());
}
#[tokio::test]
async fn message_shas_locate_the_index_two_runs_diverge_at() {
async fn shas_for(payload: &Value) -> Vec<String> {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Meta).await;
write_turn(&mut w, 0, payload, "{}").await;
w.flush().await.unwrap();
read_events(dir.path())[0]["message_shas"]
.as_array()
.unwrap()
.iter()
.map(|s| s.as_str().unwrap().to_string())
.collect()
}
let first = wire_payload("You are a capsule.", 4);
let mut second = first.clone();
second["messages"][2]["content"] = serde_json::json!("diverged");
let a = shas_for(&first).await;
let b = shas_for(&second).await;
assert_eq!(a.len(), 4);
let divergence = a.iter().zip(&b).position(|(x, y)| x != y);
assert_eq!(divergence, Some(2));
assert_eq!(a[3], b[3], "only the changed message changes its own sha");
}
#[tokio::test]
async fn identical_messages_share_one_sha_and_one_blob() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let payload = serde_json::json!({
"messages": [{"role": "user", "content": "same"}, {"role": "user", "content": "same"}],
"tools": [],
"system": "s",
});
write_turn(&mut w, 0, &payload, "{}").await;
w.flush().await.unwrap();
let shas = read_events(dir.path())[0]["message_shas"]
.as_array()
.unwrap()
.iter()
.map(|s| s.as_str().unwrap().to_string())
.collect::<Vec<_>>();
assert_eq!(shas[0], shas[1]);
assert!(blob_path(dir.path(), &shas[0]).is_file());
}
#[tokio::test]
async fn a_message_blob_is_the_body_verbatim() {
let dir = tempfile::tempdir().unwrap();
let mut w = make_writer_with_opts(dir.path(), TraceCapture::Content).await;
let message = serde_json::json!({"role": "user", "content": "hello"});
let payload = serde_json::json!({
"messages": [message.clone()],
"tools": [],
"system": "s",
});
write_turn(&mut w, 0, &payload, "{}").await;
w.flush().await.unwrap();
let sha = read_events(dir.path())[0]["message_shas"][0]
.as_str()
.unwrap()
.to_string();
assert_eq!(
std::fs::read(blob_path(dir.path(), &sha)).unwrap(),
serde_json::to_vec(&message).unwrap()
);
}
#[tokio::test]
async fn a_plan_written_delegation_record_matches_a_delegate_task_one() {
let plan_dir = tempfile::tempdir().unwrap();
let tool_dir = tempfile::tempdir().unwrap();
let session = "ses_0000000000004000800000000000plan".to_string();
let node = "evt_0000000000004000800000000000node".to_string();
let plan = PlanTraceAppender::open(plan_dir.path(), session.clone(), node.clone()).unwrap();
let tool = ResourceTraceAppender::open(tool_dir.path(), session.clone(), node.clone())
.await
.unwrap();
plan.write_delegation_start("dlg_abc123", "worker", "0.1.0", "ses_child", ".murmur/c");
tool.write_delegation_start("dlg_abc123", "worker", "0.1.0", "ses_child", ".murmur/c")
.await;
plan.write_delegation(
"worker",
"0.1.0",
Some("dlg_abc123".to_string()),
Some("ses_child".to_string()),
42,
"completed",
None,
);
tool.write_delegation(
"worker",
"0.1.0",
Some("dlg_abc123".to_string()),
Some("ses_child".to_string()),
42,
"completed",
None,
)
.await;
let read = |dir: &Path| -> Vec<serde_json::Map<String, Value>> {
std::fs::read_to_string(dir.join("trace.jsonl"))
.unwrap()
.lines()
.map(|line| serde_json::from_str::<serde_json::Map<_, _>>(line).unwrap())
.collect()
};
let from_plan = read(plan_dir.path());
let from_tool = read(tool_dir.path());
assert_eq!(from_plan.len(), 2);
assert_eq!(from_tool.len(), 2);
for (planned, called, event_type) in [
(&from_plan[0], &from_tool[0], "delegation_start"),
(&from_plan[1], &from_tool[1], "delegation"),
] {
assert_eq!(planned["event_type"], event_type);
assert_eq!(called["event_type"], event_type);
assert_eq!(
planned.keys().collect::<Vec<_>>(),
called.keys().collect::<Vec<_>>(),
"{planned:?} vs {called:?}"
);
assert_eq!(planned["parent_id"], node.as_str());
assert_eq!(called["parent_id"], node.as_str());
for (key, value) in planned {
if key == "event_id" || key == "timestamp" {
continue;
}
assert_eq!(value, &called[key], "{key}: {planned:?} vs {called:?}");
}
for key in ["plan_id", "step_id", "surface", "kind"] {
assert!(!planned.contains_key(key), "{key} is on {planned:?}");
assert!(!called.contains_key(key), "{key} is on {called:?}");
}
}
}
#[tokio::test]
async fn spawner_ended_names_the_lineage_or_omits_it() {
let dir = tempfile::tempdir().unwrap();
let session = "ses_0000000000004000800000000child".to_string();
let node = "evt_0000000000004000800000000000node".to_string();
let appender = ResourceTraceAppender::open(dir.path(), session.clone(), node.clone())
.await
.unwrap();
appender
.write_spawner_ended(Some("ses_parent"), Some("dlg_abc123"))
.await;
appender.write_spawner_ended(None, None).await;
let lines: Vec<serde_json::Map<String, Value>> =
std::fs::read_to_string(dir.path().join("trace.jsonl"))
.unwrap()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
assert_eq!(lines.len(), 2);
let named = &lines[0];
assert_eq!(named["event_type"], "spawner_ended");
assert_eq!(named["parent_id"], node.as_str());
assert_eq!(named["session_id"], session.as_str());
assert_eq!(named["spawned_by"], "ses_parent");
assert_eq!(named["delegation_id"], "dlg_abc123");
assert_eq!(
named.keys().collect::<Vec<_>>(),
[
"delegation_id",
"event_id",
"event_type",
"parent_id",
"session_id",
"spawned_by",
"timestamp"
]
);
let bare = &lines[1];
assert_eq!(bare["event_type"], "spawner_ended");
assert!(!bare.contains_key("spawned_by"), "{bare:?}");
assert!(!bare.contains_key("delegation_id"), "{bare:?}");
}
}