use std::collections::BTreeMap;
use std::sync::{Arc, Mutex, MutexGuard};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use crate::checkpoint::{Checkpoint, Checkpointer, InMemoryCheckpointer};
use crate::effects::EffectRequest;
use crate::error::{Result, RustyError};
use crate::executor::{ExecutionOutcome, Executor, RunConfig, DEFAULT_MAX_STEPS};
use crate::graph::Graph;
use crate::journal::{Clock, EventDraft, Journal, JournalSnapshot, RngSource};
use crate::llm::{ChatMessage, ChatModel, ChatResponse, ToolCall, Usage};
use crate::record::{sha256_hex, Effect, EventStatus, PayloadRef, RunEvent, RunEventKind};
use crate::state::{State, StateSpec};
use crate::tool::Tool;
pub const FIXTURE_FORMAT_VERSION: u32 = 1;
pub const SERVABLE_KINDS: [RunEventKind; 4] = [
RunEventKind::ModelCall,
RunEventKind::ToolCall,
RunEventKind::RemoteCall,
RunEventKind::WasmCall,
];
fn is_servable(kind: RunEventKind) -> bool {
SERVABLE_KINDS.contains(&kind)
}
fn replay_error(message: impl Into<String>) -> RustyError {
RustyError::Replay(message.into())
}
pub fn model_call_request(messages: &[ChatMessage], tools: &[Value]) -> Value {
json!({ "messages": messages, "tools": tools })
}
pub fn model_call_response(response: &ChatResponse) -> Value {
json!({
"message": response.message,
"model": response.model,
"usage": response.usage,
})
}
pub fn tool_call_request(name: &str, arguments: &Value) -> Value {
json!({ "tool": name, "arguments": arguments })
}
fn chat_response_from_recorded(seq: u64, value: &Value) -> Result<ChatResponse> {
let decode = |key: &str| -> Result<Value> {
value.get(key).cloned().ok_or_else(|| {
replay_error(format!(
"recorded model call at seq {seq} is malformed: output payload has no `{key}` key"
))
})
};
let message: ChatMessage = serde_json::from_value(decode("message")?)?;
let model: Option<String> = serde_json::from_value(decode("model")?)?;
let usage: Option<Usage> = serde_json::from_value(decode("usage")?)?;
Ok(ChatResponse {
message,
model,
usage,
})
}
fn recorded_error_text(served: &ServedEffect) -> String {
served
.output
.as_ref()
.and_then(|value| value.get("error"))
.and_then(Value::as_str)
.unwrap_or("<no recorded error payload>")
.to_owned()
}
fn verify_snapshot(snapshot: &JournalSnapshot) -> Result<()> {
Journal::from_snapshot(clone_snapshot(snapshot), Clock::System)?;
for (hash, value) in &snapshot.artifacts {
let actual = sha256_hex(&serde_json::to_vec(value)?);
if actual != *hash {
return Err(replay_error(format!(
"journal snapshot artifact integrity failure: artifact stored under {hash} \
hashes to {actual} — the snapshot was tampered with or corrupted"
)));
}
}
for event in &snapshot.events {
for payload in [&event.input, &event.output].into_iter().flatten() {
if let PayloadRef::Artifact(reference) = payload {
if !snapshot.artifacts.contains_key(&reference.sha256) {
return Err(replay_error(format!(
"journal snapshot is truncated: event {} references artifact {} which \
is absent from the artifact map",
event.id, reference.sha256
)));
}
}
}
}
Ok(())
}
fn clone_snapshot(snapshot: &JournalSnapshot) -> JournalSnapshot {
snapshot.clone()
}
fn resolve_in(snapshot: &JournalSnapshot, payload: &PayloadRef) -> Option<Value> {
match payload {
PayloadRef::Inline(value) => Some(value.clone()),
PayloadRef::Artifact(reference) => snapshot.artifacts.get(&reference.sha256).cloned(),
}
}
fn resolve_opt(snapshot: &JournalSnapshot, payload: Option<&PayloadRef>) -> Option<Value> {
payload.and_then(|payload| resolve_in(snapshot, payload))
}
fn request_hash(request: &Value) -> Result<String> {
Ok(sha256_hex(&serde_json::to_vec(request)?))
}
#[derive(Debug, Clone)]
pub struct ServedEffect {
pub event: RunEvent,
pub input: Option<Value>,
pub output: Option<Value>,
}
impl ServedEffect {
pub fn rejournal(&self, journal: &Journal, parent: impl Into<String>) -> String {
let mut draft = EventDraft::new(self.event.kind, self.event.effect)
.status(self.event.status)
.parent(parent);
if let Some(node) = &self.event.node_id {
draft = draft.node(node.clone());
}
if let Some(input) = &self.input {
draft = draft.input(input.clone());
}
if let Some(output) = &self.output {
draft = draft.output(output.clone());
}
if let Some(latency_ms) = self.event.latency_ms {
draft = draft.latency_ms(latency_ms);
}
if let Some(tokens) = self.event.tokens {
draft = draft.tokens(tokens);
}
if let Some(cost_usd) = self.event.cost_usd {
draft = draft.cost_usd(cost_usd);
}
journal.record(draft)
}
}
#[derive(Debug)]
struct ReplaySourceInner {
servable: Vec<RunEvent>,
artifacts: BTreeMap<String, Value>,
cursor: usize,
}
#[derive(Debug, Clone)]
pub struct ReplaySource {
inner: Arc<Mutex<ReplaySourceInner>>,
}
impl ReplaySource {
pub fn new(snapshot: &JournalSnapshot) -> Self {
Self {
inner: Arc::new(Mutex::new(ReplaySourceInner {
servable: snapshot
.events
.iter()
.filter(|event| is_servable(event.kind))
.cloned()
.collect(),
artifacts: snapshot.artifacts.clone(),
cursor: 0,
})),
}
}
fn lock(&self) -> MutexGuard<'_, ReplaySourceInner> {
self.inner.lock().unwrap_or_else(|e| e.into_inner())
}
pub fn serve(&self, kind: RunEventKind, request: &Value) -> Result<ServedEffect> {
if !is_servable(kind) {
return Err(replay_error(format!(
"{kind:?} is not a servable effect kind; exact replay serves only {SERVABLE_KINDS:?} \
— executor-derived events are reproduced by re-execution, not served"
)));
}
let issued_hash = request_hash(request)?;
let mut inner = self.lock();
let Some(event) = inner.servable.get(inner.cursor) else {
return Err(replay_error(format!(
"replay exhaustion: the run issued a {kind:?} call (request hash {issued_hash}), \
but the journal holds no further recorded effects to serve — the replayed run \
is doing more work than the recorded one"
)));
};
if event.kind != kind {
return Err(replay_error(format!(
"replay order violation at recorded seq {}: the run issued a {kind:?} call, but \
the next unserved recorded effect is {:?} — exact replay requires effects in \
journaled sequence order",
event.seq, event.kind
)));
}
let recorded_input = event
.input
.as_ref()
.and_then(|payload| match payload {
PayloadRef::Inline(value) => Some(value.clone()),
PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
})
.ok_or_else(|| {
replay_error(format!(
"recorded {:?} at seq {} has no input payload to match against",
event.kind, event.seq
))
})?;
let recorded_hash = request_hash(&recorded_input)?;
if recorded_hash != issued_hash {
return Err(replay_error(format!(
"replay divergence at recorded seq {} ({:?}): the run issued a request whose \
canonical hash is {issued_hash}, but the journaled request hashes to \
{recorded_hash} — the replayed run has diverged from its evidence",
event.seq, event.kind
)));
}
let served = ServedEffect {
input: Some(recorded_input),
output: event.output.as_ref().and_then(|payload| match payload {
PayloadRef::Inline(value) => Some(value.clone()),
PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
}),
event: event.clone(),
};
inner.cursor += 1;
Ok(served)
}
pub fn is_exhausted(&self) -> bool {
let inner = self.lock();
inner.cursor == inner.servable.len()
}
pub fn remaining(&self) -> Vec<(u64, RunEventKind)> {
let inner = self.lock();
inner.servable[inner.cursor..]
.iter()
.map(|event| (event.seq, event.kind))
.collect()
}
}
pub struct RecordingChatModel {
inner: Arc<dyn ChatModel>,
journal: Journal,
parent: String,
node_id: Option<String>,
}
impl RecordingChatModel {
pub fn new(inner: Arc<dyn ChatModel>, journal: Journal, parent: impl Into<String>) -> Self {
Self {
inner,
journal,
parent: parent.into(),
node_id: None,
}
}
pub fn node(mut self, node_id: impl Into<String>) -> Self {
self.node_id = Some(node_id.into());
self
}
fn draft(&self) -> EventDraft {
let mut draft = EventDraft::new(RunEventKind::ModelCall, self.inner.effect())
.parent(self.parent.clone());
if let Some(node) = &self.node_id {
draft = draft.node(node.clone());
}
draft
}
}
#[async_trait]
impl ChatModel for RecordingChatModel {
async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
let started = self.journal.clock().now();
let result = self.inner.chat(messages, tools).await;
let latency_ms = (self.journal.clock().now() - started)
.num_milliseconds()
.max(0) as u64;
let draft = self
.draft()
.input(model_call_request(messages, tools))
.latency_ms(latency_ms);
match result {
Ok(response) => {
let mut draft = draft.output(model_call_response(&response));
if let Some(usage) = response.usage {
draft = draft.tokens(usage);
}
self.journal.record(draft);
Ok(response)
}
Err(error) => {
self.journal.record(
draft
.status(EventStatus::Error)
.output(json!({ "error": error.to_string() })),
);
Err(error)
}
}
}
fn effect(&self) -> Effect {
self.inner.effect()
}
}
pub struct ReplayingChatModel {
inner: Arc<dyn ChatModel>,
source: ReplaySource,
journal: Journal,
parent: String,
}
impl ReplayingChatModel {
pub fn new(
inner: Arc<dyn ChatModel>,
source: ReplaySource,
journal: Journal,
parent: impl Into<String>,
) -> Self {
Self {
inner,
source,
journal,
parent: parent.into(),
}
}
}
#[async_trait]
impl ChatModel for ReplayingChatModel {
async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
let served = self.source.serve(
RunEventKind::ModelCall,
&model_call_request(messages, tools),
)?;
let _started = self.journal.clock().now();
let _ended = self.journal.clock().now();
served.rejournal(&self.journal, self.parent.clone());
match served.event.status {
EventStatus::Ok => {
let output = served.output.as_ref().ok_or_else(|| {
replay_error(format!(
"recorded model call at seq {} succeeded but carries no output payload",
served.event.seq
))
})?;
chat_response_from_recorded(served.event.seq, output)
}
EventStatus::Error => Err(RustyError::Llm(format!(
"replayed model call at seq {} failed as recorded: {}",
served.event.seq,
recorded_error_text(&served)
))),
EventStatus::Interrupted => Err(replay_error(format!(
"recorded model call at seq {} has status `interrupted`, which model calls \
never produce — the journal is inconsistent",
served.event.seq
))),
}
}
fn effect(&self) -> Effect {
self.inner.effect()
}
}
pub struct RecordingTool {
inner: Arc<dyn Tool>,
journal: Journal,
parent: String,
node_id: Option<String>,
}
impl RecordingTool {
pub fn new(inner: Arc<dyn Tool>, journal: Journal, parent: impl Into<String>) -> Self {
Self {
inner,
journal,
parent: parent.into(),
node_id: None,
}
}
pub fn node(mut self, node_id: impl Into<String>) -> Self {
self.node_id = Some(node_id.into());
self
}
}
#[async_trait]
impl Tool for RecordingTool {
fn name(&self) -> &str {
self.inner.name()
}
fn description(&self) -> &str {
self.inner.description()
}
fn parameters_schema(&self) -> Value {
self.inner.parameters_schema()
}
fn effect(&self) -> Effect {
self.inner.effect()
}
fn effect_kind(&self) -> &str {
self.inner.effect_kind()
}
fn idempotency_key(&self, args: &Value) -> Option<String> {
self.inner.idempotency_key(args)
}
fn effect_request(&self, call: &ToolCall) -> EffectRequest {
self.inner.effect_request(call)
}
async fn call(&self, args: Value) -> Result<Value> {
let started = self.journal.clock().now();
let result = self.inner.call(args.clone()).await;
let latency_ms = (self.journal.clock().now() - started)
.num_milliseconds()
.max(0) as u64;
let mut draft = EventDraft::new(RunEventKind::ToolCall, self.inner.effect())
.input(tool_call_request(self.inner.name(), &args))
.latency_ms(latency_ms)
.parent(self.parent.clone());
if let Some(node) = &self.node_id {
draft = draft.node(node.clone());
}
match result {
Ok(value) => {
self.journal.record(draft.output(value.clone()));
Ok(value)
}
Err(error) => {
self.journal.record(
draft
.status(EventStatus::Error)
.output(json!({ "error": error.to_string() })),
);
Err(error)
}
}
}
}
pub struct ReplayingTool {
inner: Arc<dyn Tool>,
source: ReplaySource,
journal: Journal,
parent: String,
}
impl ReplayingTool {
pub fn new(
inner: Arc<dyn Tool>,
source: ReplaySource,
journal: Journal,
parent: impl Into<String>,
) -> Self {
Self {
inner,
source,
journal,
parent: parent.into(),
}
}
}
#[async_trait]
impl Tool for ReplayingTool {
fn name(&self) -> &str {
self.inner.name()
}
fn description(&self) -> &str {
self.inner.description()
}
fn parameters_schema(&self) -> Value {
self.inner.parameters_schema()
}
fn effect(&self) -> Effect {
self.inner.effect()
}
fn effect_kind(&self) -> &str {
self.inner.effect_kind()
}
fn idempotency_key(&self, args: &Value) -> Option<String> {
self.inner.idempotency_key(args)
}
fn effect_request(&self, call: &ToolCall) -> EffectRequest {
self.inner.effect_request(call)
}
async fn call(&self, args: Value) -> Result<Value> {
let request = tool_call_request(self.inner.name(), &args);
let served = self.source.serve(RunEventKind::ToolCall, &request)?;
let _started = self.journal.clock().now();
let _ended = self.journal.clock().now();
served.rejournal(&self.journal, self.parent.clone());
match served.event.status {
EventStatus::Ok => served.output.clone().ok_or_else(|| {
replay_error(format!(
"recorded tool call at seq {} succeeded but carries no output payload",
served.event.seq
))
}),
EventStatus::Error => Err(RustyError::Tool(format!(
"replayed tool call at seq {} failed as recorded: {}",
served.event.seq,
recorded_error_text(&served)
))),
EventStatus::Interrupted => Err(replay_error(format!(
"recorded tool call at seq {} has status `interrupted`, which tool calls \
never produce — the journal is inconsistent",
served.event.seq
))),
}
}
}
#[derive(Debug)]
pub struct ExactReplay {
snapshot: JournalSnapshot,
source: ReplaySource,
}
impl ExactReplay {
pub fn new(snapshot: JournalSnapshot) -> Result<Self> {
verify_snapshot(&snapshot)?;
if snapshot
.events
.first()
.is_some_and(|event| event.kind == RunEventKind::Resume)
{
return Err(replay_error(
"exact replay of resumed runs is not supported: the journal begins with a \
resume event, whose pre-resume state lives in a checkpoint the journal does \
not carry — replay the original run's journal instead",
));
}
let source = ReplaySource::new(&snapshot);
Ok(Self { snapshot, source })
}
pub fn snapshot(&self) -> &JournalSnapshot {
&self.snapshot
}
pub fn source(&self) -> ReplaySource {
self.source.clone()
}
pub fn fresh_journal(&self, clock: Clock) -> Journal {
Journal::new(
self.snapshot.run_id.clone(),
self.snapshot.thread_id.clone(),
clock,
)
}
pub async fn run(
&self,
graph: &Graph,
spec: &StateSpec,
initial_state: State,
params: ReplayParams,
) -> Result<ReplayOutcome> {
if params.journal.run_id() != self.snapshot.run_id
|| params.journal.thread_id() != self.snapshot.thread_id
{
return Err(replay_error(format!(
"replay journal identity mismatch: the snapshot records run `{}` on thread \
`{}`, but the supplied journal is run `{}` on thread `{}` — build the replay \
journal with ExactReplay::fresh_journal",
self.snapshot.run_id,
self.snapshot.thread_id,
params.journal.run_id(),
params.journal.thread_id()
)));
}
let executor = match ¶ms.checkpointer {
Some(checkpointer) => Executor::with_checkpointer(checkpointer.clone()),
None => Executor::new(),
};
let config = RunConfig::new(self.snapshot.thread_id.clone())
.with_max_steps(params.max_steps)
.with_rng(params.rng.clone())
.with_journal(params.journal.clone());
let outcome = executor.run(graph, spec, initial_state, config).await?;
Ok(ReplayOutcome {
outcome,
journal: params.journal.snapshot(),
})
}
pub fn verify(&self, replayed: &JournalSnapshot) -> Result<()> {
if !self.source.is_exhausted() {
return Err(replay_error(format!(
"exact replay left recorded effects unserved: {:?} — the replayed run stopped \
short of the journaled history",
self.source.remaining()
)));
}
let recorded = &self.snapshot;
if recorded.run_id != replayed.run_id || recorded.thread_id != replayed.thread_id {
return Err(replay_error(format!(
"replay identity mismatch: recorded run `{}`/thread `{}` vs replayed \
`{}`/`{}`",
recorded.run_id, recorded.thread_id, replayed.run_id, replayed.thread_id
)));
}
if recorded.events.len() != replayed.events.len() {
return Err(replay_error(format!(
"replay event-count mismatch: the recorded journal holds {} events but the \
replayed journal holds {}",
recorded.events.len(),
replayed.events.len()
)));
}
for (recorded_event, replayed_event) in recorded.events.iter().zip(&replayed.events) {
if recorded_event != replayed_event {
return Err(replay_error(format!(
"replay event mismatch at seq {}: recorded {} but replayed {}",
recorded_event.seq,
summarize_event(recorded_event),
summarize_event(replayed_event),
)));
}
}
if recorded.artifacts != replayed.artifacts {
return Err(replay_error(
"replay artifact mismatch: the replayed journal's artifact payloads differ \
from the recorded ones",
));
}
if recorded.head_hash != replayed.head_hash {
return Err(replay_error(format!(
"replay head-hash mismatch: recorded {} but replayed {} — event content \
agrees but the evidence chain does not (this should be unreachable; please \
report it)",
recorded.head_hash, replayed.head_hash
)));
}
Ok(())
}
pub async fn run_and_verify(
&self,
graph: &Graph,
spec: &StateSpec,
initial_state: State,
params: ReplayParams,
) -> Result<ReplayOutcome> {
let replayed = self.run(graph, spec, initial_state, params).await?;
self.verify(&replayed.journal)?;
Ok(replayed)
}
}
fn summarize_event(event: &RunEvent) -> String {
format!(
"{:?}(node={:?}, status={:?})",
event.kind, event.node_id, event.status
)
}
pub struct ReplayParams {
pub journal: Journal,
pub rng: RngSource,
pub checkpointer: Option<Arc<dyn Checkpointer>>,
pub max_steps: usize,
}
impl ReplayParams {
pub fn new(journal: Journal, rng: RngSource) -> Self {
Self {
journal,
rng,
checkpointer: None,
max_steps: DEFAULT_MAX_STEPS,
}
}
pub fn with_checkpointer(mut self, checkpointer: Arc<dyn Checkpointer>) -> Self {
self.checkpointer = Some(checkpointer);
self
}
pub fn with_max_steps(mut self, max_steps: usize) -> Self {
self.max_steps = max_steps;
self
}
}
#[derive(Debug)]
pub struct ReplayOutcome {
pub outcome: ExecutionOutcome,
pub journal: JournalSnapshot,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
pub struct BranchTotals {
pub events: usize,
pub tokens: Usage,
pub cost_usd: f64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ChannelDiff {
pub channel: String,
pub base: Option<Value>,
pub branch: Option<Value>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct StepDiff {
pub step: u64,
pub channels: Vec<ChannelDiff>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BranchDiff {
pub first_divergent_seq: Option<u64>,
pub added: Vec<RunEvent>,
pub removed: Vec<RunEvent>,
pub step_diffs: Vec<StepDiff>,
pub base_totals: BranchTotals,
pub branch_totals: BranchTotals,
}
impl BranchDiff {
pub fn between(base: &JournalSnapshot, branch: &JournalSnapshot) -> BranchDiff {
let divergence = (0..base.events.len().max(branch.events.len()))
.find(|&i| {
match (base.events.get(i), branch.events.get(i)) {
(Some(a), Some(b)) => !logically_equal(a, base, b, branch),
_ => true,
}
})
.map(|i| {
base.events
.get(i)
.or_else(|| branch.events.get(i))
.map(|event| event.seq)
.unwrap_or(i as u64)
});
let (removed, added) = match divergence {
Some(seq) => (
base.events
.iter()
.filter(|event| event.seq >= seq)
.cloned()
.collect(),
branch
.events
.iter()
.filter(|event| event.seq >= seq)
.cloned()
.collect(),
),
None => (Vec::new(), Vec::new()),
};
BranchDiff {
first_divergent_seq: divergence,
added,
removed,
step_diffs: diff_steps(base, branch),
base_totals: totals(base),
branch_totals: totals(branch),
}
}
pub fn is_identical(&self) -> bool {
self.first_divergent_seq.is_none()
}
}
fn logically_equal(
a: &RunEvent,
snapshot_a: &JournalSnapshot,
b: &RunEvent,
snapshot_b: &JournalSnapshot,
) -> bool {
a.seq == b.seq
&& a.kind == b.kind
&& a.node_id == b.node_id
&& a.effect == b.effect
&& a.latency_ms == b.latency_ms
&& a.tokens == b.tokens
&& a.cost_usd == b.cost_usd
&& a.status == b.status
&& resolve_opt(snapshot_a, a.input.as_ref()) == resolve_opt(snapshot_b, b.input.as_ref())
&& resolve_opt(snapshot_a, a.output.as_ref()) == resolve_opt(snapshot_b, b.output.as_ref())
}
fn totals(snapshot: &JournalSnapshot) -> BranchTotals {
let mut totals = BranchTotals {
events: snapshot.events.len(),
..BranchTotals::default()
};
for event in &snapshot.events {
if let Some(usage) = event.tokens {
totals.tokens.prompt_tokens = totals
.tokens
.prompt_tokens
.saturating_add(usage.prompt_tokens);
totals.tokens.completion_tokens = totals
.tokens
.completion_tokens
.saturating_add(usage.completion_tokens);
totals.tokens.total_tokens = totals
.tokens
.total_tokens
.saturating_add(usage.total_tokens);
}
if let Some(cost) = event.cost_usd {
totals.cost_usd += cost;
}
}
totals
}
fn step_channel_values(snapshot: &JournalSnapshot) -> BTreeMap<u64, BTreeMap<String, Value>> {
let mut steps = BTreeMap::new();
let mut current_step: Option<u64> = None;
for event in &snapshot.events {
match event.kind {
RunEventKind::SuperStepStart => {
current_step = resolve_opt(snapshot, event.input.as_ref())
.and_then(|input| input.get("step").and_then(Value::as_u64));
}
RunEventKind::SuperStepEnd => {
if let Some(step) = current_step.take() {
let channels = resolve_opt(snapshot, event.output.as_ref())
.and_then(|output| output.as_object().cloned())
.unwrap_or_default()
.into_iter()
.collect();
steps.insert(step, channels);
}
}
_ => {}
}
}
steps
}
fn diff_steps(base: &JournalSnapshot, branch: &JournalSnapshot) -> Vec<StepDiff> {
let base_steps = step_channel_values(base);
let branch_steps = step_channel_values(branch);
let mut diffs = Vec::new();
for step in base_steps
.keys()
.chain(branch_steps.keys())
.copied()
.collect::<std::collections::BTreeSet<u64>>()
{
let empty = BTreeMap::new();
let base_channels = base_steps.get(&step).unwrap_or(&empty);
let branch_channels = branch_steps.get(&step).unwrap_or(&empty);
let mut channels = Vec::new();
for channel in base_channels
.keys()
.chain(branch_channels.keys())
.cloned()
.collect::<std::collections::BTreeSet<String>>()
{
let base_value = base_channels.get(&channel).cloned();
let branch_value = branch_channels.get(&channel).cloned();
if base_value != branch_value {
channels.push(ChannelDiff {
channel,
base: base_value,
branch: branch_value,
});
}
}
if !channels.is_empty() {
diffs.push(StepDiff { step, channels });
}
}
diffs
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct LogicalClockParams {
pub start_ms: u64,
pub tick_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct FixtureMetadata {
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub clock: Option<LogicalClockParams>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rng_seed: Option<u64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReplayFixture {
pub format_version: u32,
pub graph_hash: String,
pub graph_version: String,
pub journal: JournalSnapshot,
pub final_checkpoint: Option<Checkpoint>,
pub metadata: FixtureMetadata,
}
impl ReplayFixture {
pub fn capture(
name: impl Into<String>,
graph: &Graph,
graph_version: impl Into<String>,
journal: JournalSnapshot,
final_checkpoint: Option<Checkpoint>,
clock: Option<LogicalClockParams>,
rng_seed: Option<u64>,
) -> Self {
Self {
format_version: FIXTURE_FORMAT_VERSION,
graph_hash: graph.topology_hash(),
graph_version: graph_version.into(),
journal,
final_checkpoint,
metadata: FixtureMetadata {
name: name.into(),
clock,
rng_seed,
},
}
}
pub fn export(&self) -> Result<String> {
Ok(serde_json::to_string_pretty(self)?)
}
pub fn import(json: &str) -> Result<Self> {
let fixture: Self = serde_json::from_str(json)?;
if fixture.format_version != FIXTURE_FORMAT_VERSION {
return Err(replay_error(format!(
"unsupported fixture format version {} (this build supports \
{FIXTURE_FORMAT_VERSION}) — regenerate the fixture or upgrade the runtime",
fixture.format_version
)));
}
verify_snapshot(&fixture.journal)?;
Ok(fixture)
}
pub fn exact_replay(&self) -> Result<ExactReplay> {
ExactReplay::new(self.journal.clone())
}
pub fn replay_params(&self, replay: &ExactReplay) -> Result<ReplayParams> {
let clock = self.metadata.clock.ok_or_else(|| {
replay_error(format!(
"fixture `{}` carries no logical-clock parameters; exact CI replay requires \
the recorded run's determinism seams",
self.metadata.name
))
})?;
let seed = self.metadata.rng_seed.ok_or_else(|| {
replay_error(format!(
"fixture `{}` carries no RNG seed; exact CI replay requires the recorded \
run's determinism seams",
self.metadata.name
))
})?;
let journal = replay.fresh_journal(Clock::logical(clock.start_ms, clock.tick_ms));
Ok(ReplayParams::new(journal, RngSource::seeded(seed))
.with_checkpointer(Arc::new(InMemoryCheckpointer::new())))
}
pub async fn replay_in_ci(
&self,
replay: ExactReplay,
graph: &Graph,
spec: &StateSpec,
initial_state: State,
params: ReplayParams,
) -> Result<ReplayOutcome> {
let actual_hash = graph.topology_hash();
if actual_hash != self.graph_hash {
return Err(replay_error(format!(
"fixture `{}` was recorded against graph topology {}, but the graph under \
replay hashes to {actual_hash} — rebuild the recorded topology (same node \
names, edges, and entry point)",
self.metadata.name, self.graph_hash
)));
}
let replayed = replay
.run_and_verify(graph, spec, initial_state, params)
.await?;
if let Some(final_checkpoint) = &self.final_checkpoint {
if replayed.outcome.state() != &final_checkpoint.state {
return Err(replay_error(format!(
"fixture `{}`: the replayed final state diverges from the recorded final \
checkpoint (journal evidence agreed; state did not)",
self.metadata.name
)));
}
}
Ok(replayed)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::journal::Journal;
use serde_json::json;
fn snapshot_with(drafts: Vec<EventDraft>) -> JournalSnapshot {
let journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
for draft in drafts {
journal.record(draft);
}
journal.snapshot()
}
fn model_draft(request: Value, response: Value) -> EventDraft {
EventDraft::new(RunEventKind::ModelCall, Effect::NonIdempotent)
.node("agent")
.input(request)
.output(response)
.latency_ms(2)
.tokens(Usage {
prompt_tokens: 10,
completion_tokens: 5,
total_tokens: 15,
})
}
#[test]
fn canonical_shapes_have_stable_hashes() {
let request = model_call_request(&[ChatMessage::user("ping")], &[]);
assert_eq!(
request,
json!({"messages": [{"role": "user", "content": "ping"}], "tools": []})
);
let first = request_hash(&request).unwrap();
let second = request_hash(&model_call_request(&[ChatMessage::user("ping")], &[])).unwrap();
assert_eq!(first, second);
let other = request_hash(&model_call_request(&[ChatMessage::user("pong")], &[])).unwrap();
assert_ne!(first, other);
let tool = tool_call_request("echo", &json!({"text": "hi"}));
assert_eq!(tool, json!({"tool": "echo", "arguments": {"text": "hi"}}));
}
#[test]
fn serve_matches_by_sequence_and_request_hash() {
let snapshot = snapshot_with(vec![
model_draft(
model_call_request(&[ChatMessage::user("a")], &[]),
json!({"r": 1}),
),
model_draft(
model_call_request(&[ChatMessage::user("b")], &[]),
json!({"r": 2}),
),
]);
let source = ReplaySource::new(&snapshot);
assert!(!source.is_exhausted());
let error = source
.serve(
RunEventKind::ModelCall,
&model_call_request(&[ChatMessage::user("zzz")], &[]),
)
.unwrap_err();
assert!(matches!(error, RustyError::Replay(_)));
assert!(error.to_string().contains("divergence"));
assert_eq!(source.remaining().len(), 2);
let error = source
.serve(RunEventKind::ToolCall, &tool_call_request("t", &json!({})))
.unwrap_err();
assert!(error.to_string().contains("order violation"));
let first = source
.serve(
RunEventKind::ModelCall,
&model_call_request(&[ChatMessage::user("a")], &[]),
)
.unwrap();
assert_eq!(first.event.seq, 0);
assert_eq!(first.output, Some(json!({"r": 1})));
let second = source
.serve(
RunEventKind::ModelCall,
&model_call_request(&[ChatMessage::user("b")], &[]),
)
.unwrap();
assert_eq!(second.event.seq, 1);
assert!(source.is_exhausted());
let error = source
.serve(
RunEventKind::ModelCall,
&model_call_request(&[ChatMessage::user("a")], &[]),
)
.unwrap_err();
assert!(error.to_string().contains("exhaustion"));
}
#[test]
fn serve_rejects_non_servable_kinds() {
let snapshot = snapshot_with(vec![EventDraft::new(
RunEventKind::SuperStepStart,
Effect::Pure,
)]);
let error = ReplaySource::new(&snapshot)
.serve(RunEventKind::NodeInput, &json!({}))
.unwrap_err();
assert!(error.to_string().contains("not a servable"));
}
#[test]
fn artifact_backed_requests_match_and_resolve() {
let big_request =
json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
let big_response = json!({"message": {"role": "assistant", "content": "y".repeat(9000)}, "model": null, "usage": null});
let snapshot = snapshot_with(vec![model_draft(big_request.clone(), big_response.clone())]);
assert!(matches!(
snapshot.events[0].input,
Some(PayloadRef::Artifact(_))
));
let served = ReplaySource::new(&snapshot)
.serve(RunEventKind::ModelCall, &big_request)
.unwrap();
assert_eq!(served.output, Some(big_response));
}
#[test]
fn tampered_snapshots_are_rejected_at_the_boundary() {
let snapshot = snapshot_with(vec![model_draft(
model_call_request(&[ChatMessage::user("a")], &[]),
json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
)]);
let mut tampered = snapshot.clone();
tampered.events[0].status = EventStatus::Error;
assert!(ExactReplay::new(tampered).is_err());
let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
let artifact_snapshot = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
let mut tampered = artifact_snapshot;
*tampered.artifacts.values_mut().next().unwrap() = json!({"forged": true});
let error = ExactReplay::new(tampered).unwrap_err();
assert!(error.to_string().contains("integrity"));
let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
let truncated = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
let mut truncated = truncated;
truncated.artifacts.clear();
assert!(ExactReplay::new(truncated).is_err());
}
#[test]
fn resumed_run_journals_are_rejected() {
let snapshot = snapshot_with(vec![EventDraft::new(RunEventKind::Resume, Effect::Pure)
.input(json!({"checkpoint_id": "c"}))]);
let error = ExactReplay::new(snapshot).unwrap_err();
assert!(error.to_string().contains("resumed"));
}
#[test]
fn rejournal_reproduces_the_recorded_event() {
let recorded = snapshot_with(vec![model_draft(
model_call_request(&[ChatMessage::user("a")], &[]),
json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
)]);
let source = ReplaySource::new(&recorded);
let served = source
.serve(
RunEventKind::ModelCall,
&model_call_request(&[ChatMessage::user("a")], &[]),
)
.unwrap();
let replay_journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
served.rejournal(&replay_journal, "run-t:parent");
let replayed = replay_journal.snapshot();
assert_eq!(replayed.events.len(), 1);
let mut expected = recorded.events[0].clone();
expected.parent = Some("run-t:parent".into());
assert_eq!(replayed.events[0], expected);
}
#[test]
fn branch_diff_reports_divergence_steps_and_totals() {
let shared = || {
vec![
EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
.input(json!({"step": 0, "active_nodes": ["a"]})),
EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
.output(json!({"log": ["a"]})),
]
};
let mut base_drafts = shared();
base_drafts.push(
EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
.input(json!({"step": 1, "active_nodes": ["b"]})),
);
base_drafts.push(
EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
.output(json!({"log": ["a", "b"]})),
);
base_drafts.push(
model_draft(
model_call_request(&[ChatMessage::user("q")], &[]),
json!({"r": 1}),
)
.cost_usd(0.001),
);
let base = snapshot_with(base_drafts);
let mut branch_drafts = shared();
branch_drafts.push(
EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
.input(json!({"step": 1, "active_nodes": ["c"]})),
);
branch_drafts.push(
EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
.output(json!({"log": ["a", "c"], "extra": 1})),
);
let branch = snapshot_with(branch_drafts);
let diff = BranchDiff::between(&base, &branch);
assert!(!diff.is_identical());
assert_eq!(diff.first_divergent_seq, Some(2));
assert_eq!(diff.removed.len(), 3);
assert_eq!(diff.added.len(), 2);
assert_eq!(diff.base_totals.events, 5);
assert_eq!(diff.branch_totals.events, 4);
assert_eq!(diff.base_totals.tokens.total_tokens, 15);
assert_eq!(diff.branch_totals.tokens.total_tokens, 0);
assert_eq!(diff.base_totals.cost_usd, 0.001);
assert_eq!(diff.branch_totals.cost_usd, 0.0);
assert_eq!(diff.step_diffs.len(), 1);
let step = &diff.step_diffs[0];
assert_eq!(step.step, 1);
assert_eq!(step.channels.len(), 2);
assert_eq!(step.channels[0].channel, "extra");
assert_eq!(step.channels[0].base, None);
assert_eq!(step.channels[0].branch, Some(json!(1)));
assert_eq!(step.channels[1].channel, "log");
assert_eq!(step.channels[1].base, Some(json!(["a", "b"])));
assert_eq!(step.channels[1].branch, Some(json!(["a", "c"])));
}
#[test]
fn branch_diff_of_identical_snapshots_is_empty() {
let snapshot = snapshot_with(vec![EventDraft::new(
RunEventKind::SuperStepStart,
Effect::Pure,
)
.input(json!({"step": 0, "active_nodes": ["a"]}))]);
let diff = BranchDiff::between(&snapshot, &snapshot);
assert!(diff.is_identical());
assert!(diff.added.is_empty() && diff.removed.is_empty());
assert!(diff.step_diffs.is_empty());
}
#[test]
fn fixture_roundtrip_and_version_guard() {
let journal = snapshot_with(vec![model_draft(
model_call_request(&[ChatMessage::user("a")], &[]),
json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
)]);
let mut builder = crate::graph::GraphBuilder::new();
builder.add_node("agent", |_ctx: crate::node::NodeContext| async {
Ok(crate::node::NodeOutput::empty())
});
builder.set_entry_point("agent");
let graph = builder.compile().unwrap();
let fixture = ReplayFixture::capture(
"roundtrip",
&graph,
"v1",
journal,
None,
Some(LogicalClockParams {
start_ms: 1_000_000,
tick_ms: 5,
}),
Some(7),
);
let wire = fixture.export().unwrap();
let imported = ReplayFixture::import(&wire).unwrap();
assert_eq!(imported.export().unwrap(), wire);
let mut value: Value = serde_json::from_str(&wire).unwrap();
value["format_version"] = json!(99);
let error = ReplayFixture::import(&value.to_string()).unwrap_err();
assert!(error.to_string().contains("format version 99"));
}
}