use codex_protocol::protocol::AgentStatus;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::SessionSource;
use serde::Serialize;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use tracing::debug;
use tracing::warn;
use uuid::Uuid;
use crate::AgentThreadId;
use crate::CodeCellTraceContext;
use crate::CodexTurnId;
use crate::CompactionId;
use crate::CompactionTraceContext;
use crate::InferenceTraceContext;
use crate::McpCallTraceContext;
use crate::RawPayloadKind;
use crate::RawPayloadRef;
use crate::RawTraceEventContext;
use crate::RawTraceEventPayload;
use crate::RolloutStatus;
use crate::ToolCallId;
use crate::ToolDispatchInvocation;
use crate::ToolDispatchTraceContext;
use crate::TraceWriter;
use crate::protocol_event::codex_turn_trace_event;
use crate::protocol_event::tool_runtime_trace_event;
use crate::protocol_event::wrapped_protocol_event_type;
pub const CODEX_ROLLOUT_TRACE_ROOT_ENV: &str = "CODEX_ROLLOUT_TRACE_ROOT";
#[derive(Serialize)]
pub struct ThreadStartedTraceMetadata {
pub thread_id: String,
pub agent_path: String,
pub task_name: Option<String>,
pub nickname: Option<String>,
pub agent_role: Option<String>,
pub session_source: SessionSource,
pub cwd: std::path::PathBuf,
pub rollout_path: Option<std::path::PathBuf>,
pub model: String,
pub provider_name: String,
pub approval_policy: String,
pub sandbox_policy: String,
}
#[derive(Serialize)]
pub struct AgentResultTracePayload<'a> {
pub child_agent_path: &'a str,
pub message: &'a str,
pub status: &'a AgentStatus,
}
#[derive(Clone, Debug)]
pub struct ThreadTraceContext {
state: ThreadTraceContextState,
}
#[derive(Clone, Debug)]
enum ThreadTraceContextState {
Disabled,
Enabled(EnabledThreadTraceContext),
}
#[derive(Clone, Debug)]
struct EnabledThreadTraceContext {
writer: Arc<TraceWriter>,
root_thread_id: AgentThreadId,
thread_id: AgentThreadId,
}
impl ThreadTraceContext {
pub fn disabled() -> Self {
Self {
state: ThreadTraceContextState::Disabled,
}
}
pub fn start_root_or_disabled(metadata: ThreadStartedTraceMetadata) -> Self {
let Some(root) = std::env::var_os(CODEX_ROLLOUT_TRACE_ROOT_ENV) else {
return Self::disabled();
};
let root = PathBuf::from(root);
match start_root_in_root(root.as_path(), metadata) {
Ok(context) => context,
Err(err) => {
warn!("failed to initialize rollout trace bundle: {err:#}");
Self::disabled()
}
}
}
pub fn start_root_in_root_for_test(
root: &Path,
metadata: ThreadStartedTraceMetadata,
) -> anyhow::Result<Self> {
start_root_in_root(root, metadata)
}
pub(crate) fn start(
writer: Arc<TraceWriter>,
root_thread_id: AgentThreadId,
metadata: ThreadStartedTraceMetadata,
) -> Self {
let context = EnabledThreadTraceContext {
writer,
root_thread_id,
thread_id: metadata.thread_id.clone(),
};
record_thread_started(&context, metadata);
Self {
state: ThreadTraceContextState::Enabled(context),
}
}
pub fn is_enabled(&self) -> bool {
matches!(self.state, ThreadTraceContextState::Enabled(_))
}
pub fn start_child_thread_trace_or_disabled(
&self,
metadata: ThreadStartedTraceMetadata,
) -> Self {
match &self.state {
ThreadTraceContextState::Disabled => Self::disabled(),
ThreadTraceContextState::Enabled(context) => Self::start(
Arc::clone(&context.writer),
context.root_thread_id.clone(),
metadata,
),
}
}
pub fn record_ended(&self, status: RolloutStatus) {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return;
};
context.append_best_effort(RawTraceEventPayload::ThreadEnded {
thread_id: context.thread_id.clone(),
status: status.clone(),
});
if context.thread_id == context.root_thread_id {
context.append_best_effort(RawTraceEventPayload::RolloutEnded { status });
}
}
pub fn record_protocol_event(&self, event: &EventMsg) {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return;
};
let Some(event_type) = wrapped_protocol_event_type(event) else {
return;
};
let Some(event_payload) =
context.write_json_payload_best_effort(RawPayloadKind::ProtocolEvent, event)
else {
return;
};
context.append_best_effort(RawTraceEventPayload::ProtocolEventObserved {
event_type: event_type.to_string(),
event_payload,
});
}
pub fn record_codex_turn_event(&self, default_turn_id: &str, event: &EventMsg) {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return;
};
let Some(trace_event) =
codex_turn_trace_event(context.thread_id.clone(), default_turn_id, event)
else {
return;
};
context.append_with_context_best_effort(
trace_event.context_turn_id.clone(),
trace_event.payload,
);
}
pub fn record_tool_call_event(&self, codex_turn_id: impl Into<CodexTurnId>, event: &EventMsg) {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return;
};
let Some(trace_event) = tool_runtime_trace_event(event) else {
return;
};
let Some(payload) = context.raw_tool_runtime_payload(trace_event) else {
return;
};
context.append_with_context_best_effort(codex_turn_id.into(), payload);
}
pub fn record_agent_result_interaction(
&self,
child_codex_turn_id: impl Into<CodexTurnId>,
parent_thread_id: impl Into<AgentThreadId>,
payload: &AgentResultTracePayload<'_>,
) {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return;
};
let child_codex_turn_id = child_codex_turn_id.into();
let parent_thread_id = parent_thread_id.into();
let carried_payload =
context.write_json_payload_best_effort(RawPayloadKind::AgentResult, payload);
context.append_with_context_best_effort(
child_codex_turn_id.clone(),
RawTraceEventPayload::AgentResultObserved {
edge_id: format!(
"edge:agent_result:{}:{child_codex_turn_id}:{parent_thread_id}",
context.thread_id
),
child_thread_id: context.thread_id.clone(),
child_codex_turn_id,
parent_thread_id,
message: payload.message.to_string(),
carried_payload,
},
);
}
pub fn record_codex_turn_started(&self, codex_turn_id: impl Into<CodexTurnId>) {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return;
};
let codex_turn_id = codex_turn_id.into();
context.append_with_context_best_effort(
codex_turn_id.clone(),
RawTraceEventPayload::CodexTurnStarted {
codex_turn_id,
thread_id: context.thread_id.clone(),
},
);
}
pub fn start_code_cell_trace(
&self,
codex_turn_id: impl Into<CodexTurnId>,
runtime_cell_id: impl Into<String>,
model_visible_call_id: impl Into<String>,
source_js: impl Into<String>,
) -> CodeCellTraceContext {
let context = self.code_cell_trace_context(codex_turn_id, runtime_cell_id);
context.record_started(model_visible_call_id, source_js);
context
}
pub fn code_cell_trace_context(
&self,
codex_turn_id: impl Into<CodexTurnId>,
runtime_cell_id: impl Into<String>,
) -> CodeCellTraceContext {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return CodeCellTraceContext::disabled();
};
CodeCellTraceContext::enabled(
Arc::clone(&context.writer),
context.thread_id.clone(),
codex_turn_id,
runtime_cell_id,
)
}
pub fn start_tool_dispatch_trace(
&self,
invocation: impl FnOnce() -> Option<ToolDispatchInvocation>,
) -> ToolDispatchTraceContext {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return ToolDispatchTraceContext::disabled();
};
let Some(invocation) = invocation() else {
return ToolDispatchTraceContext::disabled();
};
ToolDispatchTraceContext::start(Arc::clone(&context.writer), invocation)
}
pub fn inference_trace_context(
&self,
codex_turn_id: impl Into<CodexTurnId>,
model: impl Into<String>,
provider_name: impl Into<String>,
) -> InferenceTraceContext {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return InferenceTraceContext::disabled();
};
InferenceTraceContext::enabled(
Arc::clone(&context.writer),
context.thread_id.clone(),
codex_turn_id.into(),
model.into(),
provider_name.into(),
)
}
pub fn compaction_trace_context(
&self,
codex_turn_id: impl Into<CodexTurnId>,
compaction_id: impl Into<CompactionId>,
model: impl Into<String>,
provider_name: impl Into<String>,
) -> CompactionTraceContext {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return CompactionTraceContext::disabled();
};
CompactionTraceContext::enabled(
Arc::clone(&context.writer),
context.thread_id.clone(),
codex_turn_id.into(),
compaction_id.into(),
model.into(),
provider_name.into(),
)
}
pub fn start_mcp_call_trace(&self, tool_call_id: impl Into<ToolCallId>) -> McpCallTraceContext {
let ThreadTraceContextState::Enabled(context) = &self.state else {
return McpCallTraceContext::disabled();
};
let mcp_call_id = Uuid::new_v4().to_string();
let trace = McpCallTraceContext::enabled(mcp_call_id.clone());
context.append_best_effort(RawTraceEventPayload::McpToolCallCorrelationAssigned {
tool_call_id: tool_call_id.into(),
mcp_call_id,
});
trace
}
}
fn start_root_in_root(
root: &Path,
metadata: ThreadStartedTraceMetadata,
) -> anyhow::Result<ThreadTraceContext> {
let trace_id = Uuid::new_v4().to_string();
let thread_id = metadata.thread_id.clone();
let bundle_dir = root.join(format!("trace-{trace_id}-{thread_id}"));
let writer = TraceWriter::create(
&bundle_dir,
trace_id.clone(),
thread_id.clone(),
thread_id.clone(),
)?;
let writer = Arc::new(writer);
if let Err(err) = writer.append(RawTraceEventPayload::RolloutStarted {
trace_id,
root_thread_id: thread_id.clone(),
}) {
warn!("failed to append rollout trace event: {err:#}");
}
debug!("recording rollout trace at {}", bundle_dir.display());
Ok(ThreadTraceContext::start(writer, thread_id, metadata))
}
fn record_thread_started(
context: &EnabledThreadTraceContext,
metadata: ThreadStartedTraceMetadata,
) {
let metadata_payload =
context.write_json_payload_best_effort(RawPayloadKind::SessionMetadata, &metadata);
context.append_best_effort(RawTraceEventPayload::ThreadStarted {
thread_id: metadata.thread_id,
agent_path: metadata.agent_path,
metadata_payload,
});
}
impl EnabledThreadTraceContext {
fn write_json_payload_best_effort(
&self,
kind: RawPayloadKind,
payload: &impl Serialize,
) -> Option<RawPayloadRef> {
match self.writer.write_json_payload(kind, payload) {
Ok(payload_ref) => Some(payload_ref),
Err(err) => {
warn!("failed to write rollout trace payload: {err:#}");
None
}
}
}
fn raw_tool_runtime_payload(
&self,
trace_event: crate::protocol_event::ToolRuntimeTraceEvent<'_>,
) -> Option<RawTraceEventPayload> {
match trace_event {
crate::protocol_event::ToolRuntimeTraceEvent::Started {
tool_call_id,
payload,
} => {
let runtime_payload = self
.write_json_payload_best_effort(RawPayloadKind::ToolRuntimeEvent, &payload)?;
Some(RawTraceEventPayload::ToolCallRuntimeStarted {
tool_call_id: tool_call_id.to_string(),
runtime_payload,
})
}
crate::protocol_event::ToolRuntimeTraceEvent::Ended {
tool_call_id,
status,
payload,
} => {
let runtime_payload = self
.write_json_payload_best_effort(RawPayloadKind::ToolRuntimeEvent, &payload)?;
Some(RawTraceEventPayload::ToolCallRuntimeEnded {
tool_call_id: tool_call_id.to_string(),
status,
runtime_payload,
})
}
}
}
fn append_best_effort(&self, payload: RawTraceEventPayload) {
if let Err(err) = self.writer.append(payload) {
warn!("failed to append rollout trace event: {err:#}");
}
}
fn append_with_context_best_effort(
&self,
codex_turn_id: CodexTurnId,
payload: RawTraceEventPayload,
) {
let event_context = RawTraceEventContext {
thread_id: Some(self.thread_id.clone()),
codex_turn_id: Some(codex_turn_id),
};
if let Err(err) = self.writer.append_with_context(event_context, payload) {
warn!("failed to append rollout trace event: {err:#}");
}
}
}
#[cfg(test)]
#[path = "thread_tests.rs"]
mod tests;