Skip to main content

rusty_agent_runtime/
replay.rs

1//! The replay engine: exact replay, branch diff, and portable fixtures.
2//!
3//! This module is the second work item of the R0.5 Flight Recorder. Wave 1
4//! landed the evidence contracts ([`crate::record`]), the journal, and the
5//! determinism seams ([`Clock`], [`RngSource`]); this module consumes them to
6//! re-drive a recorded run **exactly**: model, tool, remote, and WASM calls
7//! are served from the journal instead of executed, and the replayed run's
8//! journal and final state must reproduce the recorded ones byte-for-byte.
9//!
10//! # The exact-replay contract
11//!
12//! Exact replay re-runs the *same* graph topology with the *same* determinism
13//! seams (logical clock, seeded RNG) and swaps only the effect
14//! implementations: wherever the recorded run talked to the world, the replay
15//! talks to the journal. Node bodies still execute — `Pure` computation is
16//! freely repeatable — but every outbound effect is matched against the
17//! journaled record **by sequence and request hash** and answered with the
18//! recorded response. Three failures are detected loudly:
19//!
20//! - **Divergence** — the replayed run issues a request whose canonical hash
21//!   differs from the journaled request at the next unserved sequence
22//!   position ([`RustyError::Replay`]).
23//! - **Order violation** — effects arrive in a different order or kind than
24//!   the journal recorded (exact replay serves effects in `seq` order only).
25//! - **Exhaustion / shortfall** — the replayed run issues more effects than
26//!   the journal holds, or finishes with recorded effects unserved
27//!   (verification fails).
28//!
29//! Interrupts need no serving: with every effect answered deterministically
30//! from the journal, node logic re-derives the same interrupt and the
31//! executor journals it identically.
32//!
33//! # Zero outbound calls
34//!
35//! [`ReplayingChatModel`] and [`ReplayingTool`] wrap the real implementations
36//! but **never invoke them** — there is no code path from `chat`/`call` to
37//! the wrapped value. Tests prove it by replaying with panic-on-call
38//! sentinels as the inner implementations. The wrappers exist so the *same
39//! graph code* runs in record and replay mode; only the effect
40//! implementations are swapped.
41//!
42//! # Determinism prerequisites
43//!
44//! Byte-identical replay requires the replay run to observe the same time and
45//! id sequences as the recorded run: attach a fresh journal with the recorded
46//! run's identity (`run_id`, `thread_id`) and a logical clock with the same
47//! parameters ([`ExactReplay::fresh_journal`]), and seed the RNG with the
48//! recorded run's seed ([`ReplayParams`]). [`ReplayFixture`] carries these
49//! parameters in its metadata so CI replay is self-contained. The recording
50//! and replaying wrappers perform the same number of journal-clock reads per
51//! call, keeping the tick sequence aligned; per the journal's determinism
52//! model, super-steps with several parallel nodes interleave clock reads by
53//! schedule, so byte-identical replay is guaranteed for runs whose steps run
54//! one node at a time.
55
56use std::collections::BTreeMap;
57use std::sync::{Arc, Mutex, MutexGuard};
58
59use async_trait::async_trait;
60use serde::{Deserialize, Serialize};
61use serde_json::{json, Value};
62
63use crate::checkpoint::{Checkpoint, Checkpointer, InMemoryCheckpointer};
64use crate::effects::EffectRequest;
65use crate::error::{Result, RustyError};
66use crate::executor::{ExecutionOutcome, Executor, RunConfig, DEFAULT_MAX_STEPS};
67use crate::graph::Graph;
68use crate::journal::{Clock, EventDraft, Journal, JournalSnapshot, RngSource};
69use crate::llm::{ChatMessage, ChatModel, ChatResponse, ToolCall, Usage};
70use crate::record::{sha256_hex, Effect, EventStatus, PayloadRef, RunEvent, RunEventKind};
71use crate::state::{State, StateSpec};
72use crate::tool::Tool;
73
74/// The current on-disk format version of [`ReplayFixture`].
75///
76/// Bump only on a breaking change to the fixture envelope; additive evolution
77/// uses serde defaults so previously written fixtures keep deserializing.
78pub const FIXTURE_FORMAT_VERSION: u32 = 1;
79
80/// The effect kinds exact replay can serve from the journal: outbound calls
81/// whose recorded responses replace live execution. Super-step boundaries,
82/// node inputs/outputs, routing decisions, interrupts, and checkpoint writes
83/// are *re-derived* by the executor, never served.
84const SERVABLE_KINDS: [RunEventKind; 4] = [
85    RunEventKind::ModelCall,
86    RunEventKind::ToolCall,
87    RunEventKind::RemoteCall,
88    RunEventKind::WasmCall,
89];
90
91fn is_servable(kind: RunEventKind) -> bool {
92    SERVABLE_KINDS.contains(&kind)
93}
94
95fn replay_error(message: impl Into<String>) -> RustyError {
96    RustyError::Replay(message.into())
97}
98
99/// The canonical JSON shape a [`RecordingChatModel`] journals (and a
100/// [`ReplayingChatModel`] matches) as a model call's **request**: the
101/// messages and tool schemas exactly as passed to [`ChatModel::chat`].
102///
103/// The hash of this value's canonical serialization is the request identity
104/// exact replay matches on — node code that journals model calls by hand must
105/// use this shape for its journals to be replayable.
106pub fn model_call_request(messages: &[ChatMessage], tools: &[Value]) -> Value {
107    json!({ "messages": messages, "tools": tools })
108}
109
110/// The canonical JSON shape a [`RecordingChatModel`] journals (and a
111/// [`ReplayingChatModel`] reconstructs a [`ChatResponse`] from) as a model
112/// call's **response**: the assistant message, the reported model identity,
113/// and token usage. Keys are always present (`null` when unreported) so the
114/// shape is stable.
115pub fn model_call_response(response: &ChatResponse) -> Value {
116    json!({
117        "message": response.message,
118        "model": response.model,
119        "usage": response.usage,
120    })
121}
122
123/// The canonical JSON shape a [`RecordingTool`] journals (and a
124/// [`ReplayingTool`] matches) as a tool call's **request**: the tool name and
125/// the model-supplied arguments. The recorded **output** is the tool's result
126/// value verbatim.
127pub fn tool_call_request(name: &str, arguments: &Value) -> Value {
128    json!({ "tool": name, "arguments": arguments })
129}
130
131/// Reconstruct a [`ChatResponse`] from a journaled [`model_call_response`]
132/// payload.
133fn chat_response_from_recorded(seq: u64, value: &Value) -> Result<ChatResponse> {
134    let decode = |key: &str| -> Result<Value> {
135        value.get(key).cloned().ok_or_else(|| {
136            replay_error(format!(
137                "recorded model call at seq {seq} is malformed: output payload has no `{key}` key"
138            ))
139        })
140    };
141    let message: ChatMessage = serde_json::from_value(decode("message")?)?;
142    let model: Option<String> = serde_json::from_value(decode("model")?)?;
143    let usage: Option<Usage> = serde_json::from_value(decode("usage")?)?;
144    Ok(ChatResponse {
145        message,
146        model,
147        usage,
148    })
149}
150
151/// The recorded error text of a failed served effect (`{"error": …}` output
152/// payloads, as written by the recording wrappers and the executor's
153/// node-failure path).
154fn recorded_error_text(served: &ServedEffect) -> String {
155    served
156        .output
157        .as_ref()
158        .and_then(|value| value.get("error"))
159        .and_then(Value::as_str)
160        .unwrap_or("<no recorded error payload>")
161        .to_owned()
162}
163
164/// Verify the integrity of a journal snapshot: the chained head hash must
165/// recompute from the events (as [`Journal::from_snapshot`] enforces), every
166/// content-addressed artifact must hash to its own key, and every artifact
167/// reference on an event must resolve. Anything less is not evidence a replay
168/// may trust.
169fn verify_snapshot(snapshot: &JournalSnapshot) -> Result<()> {
170    // Head-hash re-verification (the clock is irrelevant to hashing).
171    Journal::from_snapshot(clone_snapshot(snapshot), Clock::System)?;
172    for (hash, value) in &snapshot.artifacts {
173        let actual = sha256_hex(&serde_json::to_vec(value)?);
174        if actual != *hash {
175            return Err(replay_error(format!(
176                "journal snapshot artifact integrity failure: artifact stored under {hash} \
177                 hashes to {actual} — the snapshot was tampered with or corrupted"
178            )));
179        }
180    }
181    for event in &snapshot.events {
182        for payload in [&event.input, &event.output].into_iter().flatten() {
183            if let PayloadRef::Artifact(reference) = payload {
184                if !snapshot.artifacts.contains_key(&reference.sha256) {
185                    return Err(replay_error(format!(
186                        "journal snapshot is truncated: event {} references artifact {} which \
187                         is absent from the artifact map",
188                        event.id, reference.sha256
189                    )));
190                }
191            }
192        }
193    }
194    Ok(())
195}
196
197// `JournalSnapshot` is `Clone`; the explicit helper documents that the clone
198// exists only because `Journal::from_snapshot` consumes.
199fn clone_snapshot(snapshot: &JournalSnapshot) -> JournalSnapshot {
200    snapshot.clone()
201}
202
203fn resolve_in(snapshot: &JournalSnapshot, payload: &PayloadRef) -> Option<Value> {
204    match payload {
205        PayloadRef::Inline(value) => Some(value.clone()),
206        PayloadRef::Artifact(reference) => snapshot.artifacts.get(&reference.sha256).cloned(),
207    }
208}
209
210fn resolve_opt(snapshot: &JournalSnapshot, payload: Option<&PayloadRef>) -> Option<Value> {
211    payload.and_then(|payload| resolve_in(snapshot, payload))
212}
213
214/// Canonical-content hash of a request value: `sha256` of its `serde_json`
215/// serialization (map keys sort deterministically, so equal values hash
216/// equal). This is the same computation [`PayloadRef::content_hash`] applies
217/// to inline payloads, which is what makes journaled requests matchable.
218fn request_hash(request: &Value) -> Result<String> {
219    Ok(sha256_hex(&serde_json::to_vec(request)?))
220}
221
222/// A recorded effect served during exact replay: the matching journaled event
223/// plus its resolved input/output payloads (artifact references already
224/// looked through).
225#[derive(Debug, Clone)]
226pub struct ServedEffect {
227    /// The journaled event that matched the issued request.
228    pub event: RunEvent,
229
230    /// The event's resolved input payload (the journaled request).
231    pub input: Option<Value>,
232
233    /// The event's resolved output payload (the journaled response).
234    pub output: Option<Value>,
235}
236
237impl ServedEffect {
238    /// Re-journal this served effect into the replay run's journal under
239    /// causal parent `parent`, returning the new event id.
240    ///
241    /// The replayed event reproduces the recorded one exactly — kind, node,
242    /// effect class, payloads, latency, tokens, cost, status — while the
243    /// journal assigns the replay run's own `seq` and `recorded_at` (which
244    /// match the recorded run's when the determinism seams are aligned).
245    pub fn rejournal(&self, journal: &Journal, parent: impl Into<String>) -> String {
246        let mut draft = EventDraft::new(self.event.kind, self.event.effect)
247            .status(self.event.status)
248            .parent(parent);
249        if let Some(node) = &self.event.node_id {
250            draft = draft.node(node.clone());
251        }
252        if let Some(input) = &self.input {
253            draft = draft.input(input.clone());
254        }
255        if let Some(output) = &self.output {
256            draft = draft.output(output.clone());
257        }
258        if let Some(latency_ms) = self.event.latency_ms {
259            draft = draft.latency_ms(latency_ms);
260        }
261        if let Some(tokens) = self.event.tokens {
262            draft = draft.tokens(tokens);
263        }
264        if let Some(cost_usd) = self.event.cost_usd {
265            draft = draft.cost_usd(cost_usd);
266        }
267        journal.record(draft)
268    }
269}
270
271#[derive(Debug)]
272struct ReplaySourceInner {
273    /// The snapshot's servable events, in `seq` order.
274    servable: Vec<RunEvent>,
275    /// The snapshot's artifact map, for payload resolution.
276    artifacts: BTreeMap<String, Value>,
277    /// Position of the next unserved event. Strictly ordered: exact replay
278    /// answers effects in journaled sequence order or not at all.
279    cursor: usize,
280}
281
282/// The serving side of exact replay: an ordered cursor over a journaled run's
283/// recorded effects.
284///
285/// Cheap to clone (shared cursor behind one `Arc`), so every replaying
286/// wrapper in a graph can hold a handle. Matching is by **sequence + request
287/// hash**: [`ReplaySource::serve`] takes the next unserved servable event in
288/// `seq` order, requires its kind to match, and requires the issued request's
289/// canonical hash to equal the journaled request's. Anything else fails
290/// loudly with [`RustyError::Replay`] — a replay that has drifted from its
291/// evidence must stop, not improvise.
292#[derive(Debug, Clone)]
293pub struct ReplaySource {
294    inner: Arc<Mutex<ReplaySourceInner>>,
295}
296
297impl ReplaySource {
298    /// A source over the servable events of `snapshot`. The snapshot is
299    /// assumed already integrity-verified (via [`ExactReplay::new`] or
300    /// [`ReplayFixture::import`]); unresolved artifact references simply fail
301    /// to match.
302    pub fn new(snapshot: &JournalSnapshot) -> Self {
303        Self {
304            inner: Arc::new(Mutex::new(ReplaySourceInner {
305                servable: snapshot
306                    .events
307                    .iter()
308                    .filter(|event| is_servable(event.kind))
309                    .cloned()
310                    .collect(),
311                artifacts: snapshot.artifacts.clone(),
312                cursor: 0,
313            })),
314        }
315    }
316
317    fn lock(&self) -> MutexGuard<'_, ReplaySourceInner> {
318        // Poison means a serving path panicked mid-read; the cursor value is
319        // plain data and stays coherent, so recovering is safe.
320        self.inner.lock().unwrap_or_else(|e| e.into_inner())
321    }
322
323    /// Serve one outbound effect from the journal.
324    ///
325    /// `kind` must be a servable effect kind (model, tool, remote, or WASM
326    /// call); `request` is the canonical request value (see
327    /// [`model_call_request`] / [`tool_call_request`]). On success the cursor
328    /// advances past the matched event and the returned [`ServedEffect`
329    /// carries the recorded response. On any mismatch — wrong kind at the
330    /// cursor, different request hash, no events left — a descriptive
331    /// [`RustyError::Replay`] names the sequence position and both sides of
332    /// the comparison.
333    pub fn serve(&self, kind: RunEventKind, request: &Value) -> Result<ServedEffect> {
334        if !is_servable(kind) {
335            return Err(replay_error(format!(
336                "{kind:?} is not a servable effect kind; exact replay serves only {SERVABLE_KINDS:?} \
337                 — executor-derived events are reproduced by re-execution, not served"
338            )));
339        }
340        let issued_hash = request_hash(request)?;
341        let mut inner = self.lock();
342        let Some(event) = inner.servable.get(inner.cursor) else {
343            return Err(replay_error(format!(
344                "replay exhaustion: the run issued a {kind:?} call (request hash {issued_hash}), \
345                 but the journal holds no further recorded effects to serve — the replayed run \
346                 is doing more work than the recorded one"
347            )));
348        };
349        if event.kind != kind {
350            return Err(replay_error(format!(
351                "replay order violation at recorded seq {}: the run issued a {kind:?} call, but \
352                 the next unserved recorded effect is {:?} — exact replay requires effects in \
353                 journaled sequence order",
354                event.seq, event.kind
355            )));
356        }
357        let recorded_input = event
358            .input
359            .as_ref()
360            .and_then(|payload| match payload {
361                PayloadRef::Inline(value) => Some(value.clone()),
362                PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
363            })
364            .ok_or_else(|| {
365                replay_error(format!(
366                    "recorded {:?} at seq {} has no input payload to match against",
367                    event.kind, event.seq
368                ))
369            })?;
370        let recorded_hash = request_hash(&recorded_input)?;
371        if recorded_hash != issued_hash {
372            return Err(replay_error(format!(
373                "replay divergence at recorded seq {} ({:?}): the run issued a request whose \
374                 canonical hash is {issued_hash}, but the journaled request hashes to \
375                 {recorded_hash} — the replayed run has diverged from its evidence",
376                event.seq, event.kind
377            )));
378        }
379        let served = ServedEffect {
380            input: Some(recorded_input),
381            output: event.output.as_ref().and_then(|payload| match payload {
382                PayloadRef::Inline(value) => Some(value.clone()),
383                PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
384            }),
385            event: event.clone(),
386        };
387        inner.cursor += 1;
388        Ok(served)
389    }
390
391    /// `true` when every recorded effect has been served. Verification treats
392    /// a non-exhausted source as a replay that stopped short of its evidence.
393    pub fn is_exhausted(&self) -> bool {
394        let inner = self.lock();
395        inner.cursor == inner.servable.len()
396    }
397
398    /// The `(seq, kind)` pairs still waiting to be served, in order. Empty
399    /// when [`ReplaySource::is_exhausted`] is `true`.
400    pub fn remaining(&self) -> Vec<(u64, RunEventKind)> {
401        let inner = self.lock();
402        inner.servable[inner.cursor..]
403            .iter()
404            .map(|event| (event.seq, event.kind))
405            .collect()
406    }
407}
408
409/// A [`ChatModel`] that records every call into the run's journal in the
410/// canonical replay-compatible shape, then returns the real response.
411///
412/// Construct per node invocation (it is a cheap handle): the causal parent is
413/// the invocation's node-input event id, delivered to node code via
414/// [`crate::journal::PARENT_EVENT_KEY`]. This is the supported way to produce
415/// journals that [`ReplayingChatModel`] can later replay — hand-rolled
416/// journaling must reproduce the same canonical shapes to be replayable.
417pub struct RecordingChatModel {
418    inner: Arc<dyn ChatModel>,
419    journal: Journal,
420    parent: String,
421    node_id: Option<String>,
422}
423
424impl RecordingChatModel {
425    /// A recording wrapper around `inner`, journaling into `journal` with
426    /// causal parent `parent` (the current invocation's node-input event id).
427    pub fn new(inner: Arc<dyn ChatModel>, journal: Journal, parent: impl Into<String>) -> Self {
428        Self {
429            inner,
430            journal,
431            parent: parent.into(),
432            node_id: None,
433        }
434    }
435
436    /// Builder-style: the node this model call is about (recorded as the
437    /// event's `node_id`).
438    pub fn node(mut self, node_id: impl Into<String>) -> Self {
439        self.node_id = Some(node_id.into());
440        self
441    }
442
443    fn draft(&self) -> EventDraft {
444        let mut draft = EventDraft::new(RunEventKind::ModelCall, self.inner.effect())
445            .parent(self.parent.clone());
446        if let Some(node) = &self.node_id {
447            draft = draft.node(node.clone());
448        }
449        draft
450    }
451}
452
453#[async_trait]
454impl ChatModel for RecordingChatModel {
455    async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
456        let started = self.journal.clock().now();
457        let result = self.inner.chat(messages, tools).await;
458        let latency_ms = (self.journal.clock().now() - started)
459            .num_milliseconds()
460            .max(0) as u64;
461        let draft = self
462            .draft()
463            .input(model_call_request(messages, tools))
464            .latency_ms(latency_ms);
465        match result {
466            Ok(response) => {
467                let mut draft = draft.output(model_call_response(&response));
468                if let Some(usage) = response.usage {
469                    draft = draft.tokens(usage);
470                }
471                self.journal.record(draft);
472                Ok(response)
473            }
474            Err(error) => {
475                self.journal.record(
476                    draft
477                        .status(EventStatus::Error)
478                        .output(json!({ "error": error.to_string() })),
479                );
480                Err(error)
481            }
482        }
483    }
484
485    fn effect(&self) -> Effect {
486        self.inner.effect()
487    }
488}
489
490/// A [`ChatModel`] that answers every call from a recorded journal instead of
491/// executing it.
492///
493/// The wrapped implementation is carried for its identity ([`ChatModel::effect`])
494/// and is **never invoked**: there is no code path from `chat` to `inner`, so
495/// replaying with a panic-on-call sentinel (or a client with no credentials
496/// and no network) is safe. Each call is matched against the journal by
497/// sequence + request hash and answered with the recorded response; the
498/// served event is re-journaled into the replay run's journal so the
499/// replayed evidence reproduces the recorded evidence byte-for-byte.
500pub struct ReplayingChatModel {
501    inner: Arc<dyn ChatModel>,
502    source: ReplaySource,
503    journal: Journal,
504    parent: String,
505}
506
507impl ReplayingChatModel {
508    /// A replaying wrapper around `inner` (never invoked), serving from
509    /// `source` and re-journaling into `journal` with causal parent `parent`
510    /// (the current invocation's node-input event id).
511    pub fn new(
512        inner: Arc<dyn ChatModel>,
513        source: ReplaySource,
514        journal: Journal,
515        parent: impl Into<String>,
516    ) -> Self {
517        Self {
518            inner,
519            source,
520            journal,
521            parent: parent.into(),
522        }
523    }
524}
525
526#[async_trait]
527impl ChatModel for ReplayingChatModel {
528    async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
529        let served = self.source.serve(
530            RunEventKind::ModelCall,
531            &model_call_request(messages, tools),
532        )?;
533        // Clock-read parity with RecordingChatModel (two reads per call)
534        // keeps the logical clock's tick sequence aligned with the recorded
535        // run, so the replayed journal's timestamps reproduce it exactly.
536        let _started = self.journal.clock().now();
537        let _ended = self.journal.clock().now();
538        served.rejournal(&self.journal, self.parent.clone());
539        match served.event.status {
540            EventStatus::Ok => {
541                let output = served.output.as_ref().ok_or_else(|| {
542                    replay_error(format!(
543                        "recorded model call at seq {} succeeded but carries no output payload",
544                        served.event.seq
545                    ))
546                })?;
547                chat_response_from_recorded(served.event.seq, output)
548            }
549            EventStatus::Error => Err(RustyError::Llm(format!(
550                "replayed model call at seq {} failed as recorded: {}",
551                served.event.seq,
552                recorded_error_text(&served)
553            ))),
554            EventStatus::Interrupted => Err(replay_error(format!(
555                "recorded model call at seq {} has status `interrupted`, which model calls \
556                 never produce — the journal is inconsistent",
557                served.event.seq
558            ))),
559        }
560    }
561
562    fn effect(&self) -> Effect {
563        self.inner.effect()
564    }
565}
566
567/// A [`Tool`] that records every call into the run's journal in the canonical
568/// replay-compatible shape, then returns the real result.
569///
570/// Construct per node invocation like [`RecordingChatModel`]; identity
571/// (name, description, schema, effect class) delegates to the wrapped tool.
572pub struct RecordingTool {
573    inner: Arc<dyn Tool>,
574    journal: Journal,
575    parent: String,
576    node_id: Option<String>,
577}
578
579impl RecordingTool {
580    /// A recording wrapper around `inner`, journaling into `journal` with
581    /// causal parent `parent` (the current invocation's node-input event id).
582    pub fn new(inner: Arc<dyn Tool>, journal: Journal, parent: impl Into<String>) -> Self {
583        Self {
584            inner,
585            journal,
586            parent: parent.into(),
587            node_id: None,
588        }
589    }
590
591    /// Builder-style: the node this tool call is about (recorded as the
592    /// event's `node_id`).
593    pub fn node(mut self, node_id: impl Into<String>) -> Self {
594        self.node_id = Some(node_id.into());
595        self
596    }
597}
598
599#[async_trait]
600impl Tool for RecordingTool {
601    fn name(&self) -> &str {
602        self.inner.name()
603    }
604
605    fn description(&self) -> &str {
606        self.inner.description()
607    }
608
609    fn parameters_schema(&self) -> Value {
610        self.inner.parameters_schema()
611    }
612
613    fn effect(&self) -> Effect {
614        self.inner.effect()
615    }
616
617    fn effect_kind(&self) -> &str {
618        self.inner.effect_kind()
619    }
620
621    fn idempotency_key(&self, args: &Value) -> Option<String> {
622        self.inner.idempotency_key(args)
623    }
624
625    fn effect_request(&self, call: &ToolCall) -> EffectRequest {
626        self.inner.effect_request(call)
627    }
628
629    async fn call(&self, args: Value) -> Result<Value> {
630        let started = self.journal.clock().now();
631        let result = self.inner.call(args.clone()).await;
632        let latency_ms = (self.journal.clock().now() - started)
633            .num_milliseconds()
634            .max(0) as u64;
635        let mut draft = EventDraft::new(RunEventKind::ToolCall, self.inner.effect())
636            .input(tool_call_request(self.inner.name(), &args))
637            .latency_ms(latency_ms)
638            .parent(self.parent.clone());
639        if let Some(node) = &self.node_id {
640            draft = draft.node(node.clone());
641        }
642        match result {
643            Ok(value) => {
644                self.journal.record(draft.output(value.clone()));
645                Ok(value)
646            }
647            Err(error) => {
648                self.journal.record(
649                    draft
650                        .status(EventStatus::Error)
651                        .output(json!({ "error": error.to_string() })),
652                );
653                Err(error)
654            }
655        }
656    }
657}
658
659/// A [`Tool`] that answers every call from a recorded journal instead of
660/// executing it.
661///
662/// Identity (name, description, schema, effect class) delegates to the
663/// wrapped tool, which is **never invoked** — the replaying analogue of
664/// [`RecordingTool`]. See [`ReplayingChatModel`] for the matching and
665/// zero-outbound guarantees.
666pub struct ReplayingTool {
667    inner: Arc<dyn Tool>,
668    source: ReplaySource,
669    journal: Journal,
670    parent: String,
671}
672
673impl ReplayingTool {
674    /// A replaying wrapper around `inner` (never invoked), serving from
675    /// `source` and re-journaling into `journal` with causal parent `parent`.
676    pub fn new(
677        inner: Arc<dyn Tool>,
678        source: ReplaySource,
679        journal: Journal,
680        parent: impl Into<String>,
681    ) -> Self {
682        Self {
683            inner,
684            source,
685            journal,
686            parent: parent.into(),
687        }
688    }
689}
690
691#[async_trait]
692impl Tool for ReplayingTool {
693    fn name(&self) -> &str {
694        self.inner.name()
695    }
696
697    fn description(&self) -> &str {
698        self.inner.description()
699    }
700
701    fn parameters_schema(&self) -> Value {
702        self.inner.parameters_schema()
703    }
704
705    fn effect(&self) -> Effect {
706        self.inner.effect()
707    }
708
709    fn effect_kind(&self) -> &str {
710        self.inner.effect_kind()
711    }
712
713    fn idempotency_key(&self, args: &Value) -> Option<String> {
714        self.inner.idempotency_key(args)
715    }
716
717    fn effect_request(&self, call: &ToolCall) -> EffectRequest {
718        self.inner.effect_request(call)
719    }
720
721    async fn call(&self, args: Value) -> Result<Value> {
722        let request = tool_call_request(self.inner.name(), &args);
723        let served = self.source.serve(RunEventKind::ToolCall, &request)?;
724        // Clock-read parity with RecordingTool; see ReplayingChatModel.
725        let _started = self.journal.clock().now();
726        let _ended = self.journal.clock().now();
727        served.rejournal(&self.journal, self.parent.clone());
728        match served.event.status {
729            EventStatus::Ok => served.output.clone().ok_or_else(|| {
730                replay_error(format!(
731                    "recorded tool call at seq {} succeeded but carries no output payload",
732                    served.event.seq
733                ))
734            }),
735            EventStatus::Error => Err(RustyError::Tool(format!(
736                "replayed tool call at seq {} failed as recorded: {}",
737                served.event.seq,
738                recorded_error_text(&served)
739            ))),
740            EventStatus::Interrupted => Err(replay_error(format!(
741                "recorded tool call at seq {} has status `interrupted`, which tool calls \
742                 never produce — the journal is inconsistent",
743                served.event.seq
744            ))),
745        }
746    }
747}
748
749/// An exact-replay session over one integrity-verified journal snapshot.
750///
751/// Construction re-verifies the snapshot (chained head hash, artifact
752/// integrity, reference resolution), so a tampered or truncated journal is
753/// rejected at the boundary. The typical flow:
754///
755/// 1. `let replay = ExactReplay::new(snapshot)?;`
756/// 2. `let journal = replay.fresh_journal(clock)` — a fresh journal with the
757///    recorded run's identity and a matching logical clock;
758/// 3. build the same graph topology with [`ReplayingChatModel`] /
759///    [`ReplayingTool`] wrappers sourcing from `replay.source()` and
760///    re-journaling into that journal;
761/// 4. `replay.run_and_verify(&graph, &spec, initial_state, params).await` —
762///    drives the run and checks the replayed journal reproduces the recorded
763///    one event-for-event.
764#[derive(Debug)]
765pub struct ExactReplay {
766    snapshot: JournalSnapshot,
767    source: ReplaySource,
768}
769
770impl ExactReplay {
771    /// Start an exact-replay session over `snapshot`.
772    ///
773    /// Fails with [`RustyError::Replay`] when the snapshot fails integrity
774    /// verification (tampered head hash, corrupted artifacts, dangling
775    /// references), or with [`RustyError::Serialization`] when an event
776    /// cannot be re-hashed. Exact replay of *resumed* runs (journals whose
777    /// first event is a resume) is deferred: their evidence begins mid-run
778    /// against checkpointed state the journal does not carry.
779    pub fn new(snapshot: JournalSnapshot) -> Result<Self> {
780        verify_snapshot(&snapshot)?;
781        if snapshot
782            .events
783            .first()
784            .is_some_and(|event| event.kind == RunEventKind::Resume)
785        {
786            return Err(replay_error(
787                "exact replay of resumed runs is not supported: the journal begins with a \
788                 resume event, whose pre-resume state lives in a checkpoint the journal does \
789                 not carry — replay the original run's journal instead",
790            ));
791        }
792        let source = ReplaySource::new(&snapshot);
793        Ok(Self { snapshot, source })
794    }
795
796    /// The recorded snapshot this session replays.
797    pub fn snapshot(&self) -> &JournalSnapshot {
798        &self.snapshot
799    }
800
801    /// The serving cursor over the recorded effects, for building replaying
802    /// wrappers. Shared: every wrapper in the replayed graph must serve from
803    /// this one cursor.
804    pub fn source(&self) -> ReplaySource {
805        self.source.clone()
806    }
807
808    /// A fresh journal with the recorded run's identity (`run_id`,
809    /// `thread_id`), timestamping from `clock`. For byte-identical replay,
810    /// `clock` must be a logical clock with the recorded run's parameters
811    /// (carried in [`ReplayFixture`] metadata).
812    pub fn fresh_journal(&self, clock: Clock) -> Journal {
813        Journal::new(
814            self.snapshot.run_id.clone(),
815            self.snapshot.thread_id.clone(),
816            clock,
817        )
818    }
819
820    /// Drive `graph` over `initial_state`, answering effects from the
821    /// recorded journal.
822    ///
823    /// `params.journal` must carry the recorded run's identity (build it with
824    /// [`ExactReplay::fresh_journal`]); a mismatched journal is rejected
825    /// rather than silently producing evidence under the wrong run. The
826    /// replayed run's outcome and its journal snapshot are returned for
827    /// inspection or [`ExactReplay::verify`].
828    pub async fn run(
829        &self,
830        graph: &Graph,
831        spec: &StateSpec,
832        initial_state: State,
833        params: ReplayParams,
834    ) -> Result<ReplayOutcome> {
835        if params.journal.run_id() != self.snapshot.run_id
836            || params.journal.thread_id() != self.snapshot.thread_id
837        {
838            return Err(replay_error(format!(
839                "replay journal identity mismatch: the snapshot records run `{}` on thread \
840                 `{}`, but the supplied journal is run `{}` on thread `{}` — build the replay \
841                 journal with ExactReplay::fresh_journal",
842                self.snapshot.run_id,
843                self.snapshot.thread_id,
844                params.journal.run_id(),
845                params.journal.thread_id()
846            )));
847        }
848        let executor = match &params.checkpointer {
849            Some(checkpointer) => Executor::with_checkpointer(checkpointer.clone()),
850            None => Executor::new(),
851        };
852        let config = RunConfig::new(self.snapshot.thread_id.clone())
853            .with_max_steps(params.max_steps)
854            .with_rng(params.rng.clone())
855            .with_journal(params.journal.clone());
856        let outcome = executor.run(graph, spec, initial_state, config).await?;
857        Ok(ReplayOutcome {
858            outcome,
859            journal: params.journal.snapshot(),
860        })
861    }
862
863    /// Verify a replayed journal against the recorded one: every recorded
864    /// effect must have been served, and the two snapshots must agree on run
865    /// identity, every event, every artifact, and the head hash. The first
866    /// disagreement fails with a descriptive [`RustyError::Replay`].
867    pub fn verify(&self, replayed: &JournalSnapshot) -> Result<()> {
868        if !self.source.is_exhausted() {
869            return Err(replay_error(format!(
870                "exact replay left recorded effects unserved: {:?} — the replayed run stopped \
871                 short of the journaled history",
872                self.source.remaining()
873            )));
874        }
875        let recorded = &self.snapshot;
876        if recorded.run_id != replayed.run_id || recorded.thread_id != replayed.thread_id {
877            return Err(replay_error(format!(
878                "replay identity mismatch: recorded run `{}`/thread `{}` vs replayed \
879                 `{}`/`{}`",
880                recorded.run_id, recorded.thread_id, replayed.run_id, replayed.thread_id
881            )));
882        }
883        if recorded.events.len() != replayed.events.len() {
884            return Err(replay_error(format!(
885                "replay event-count mismatch: the recorded journal holds {} events but the \
886                 replayed journal holds {}",
887                recorded.events.len(),
888                replayed.events.len()
889            )));
890        }
891        for (recorded_event, replayed_event) in recorded.events.iter().zip(&replayed.events) {
892            if recorded_event != replayed_event {
893                return Err(replay_error(format!(
894                    "replay event mismatch at seq {}: recorded {} but replayed {}",
895                    recorded_event.seq,
896                    summarize_event(recorded_event),
897                    summarize_event(replayed_event),
898                )));
899            }
900        }
901        if recorded.artifacts != replayed.artifacts {
902            return Err(replay_error(
903                "replay artifact mismatch: the replayed journal's artifact payloads differ \
904                 from the recorded ones",
905            ));
906        }
907        if recorded.head_hash != replayed.head_hash {
908            return Err(replay_error(format!(
909                "replay head-hash mismatch: recorded {} but replayed {} — event content \
910                 agrees but the evidence chain does not (this should be unreachable; please \
911                 report it)",
912                recorded.head_hash, replayed.head_hash
913            )));
914        }
915        Ok(())
916    }
917
918    /// [`ExactReplay::run`] plus [`ExactReplay::verify`]: the replay either
919    /// reproduces the recorded evidence exactly or fails with the first
920    /// disagreement.
921    pub async fn run_and_verify(
922        &self,
923        graph: &Graph,
924        spec: &StateSpec,
925        initial_state: State,
926        params: ReplayParams,
927    ) -> Result<ReplayOutcome> {
928        let replayed = self.run(graph, spec, initial_state, params).await?;
929        self.verify(&replayed.journal)?;
930        Ok(replayed)
931    }
932}
933
934/// One-line event summary for verification errors (kind, node, status —
935/// enough to locate the divergence without dumping payloads).
936fn summarize_event(event: &RunEvent) -> String {
937    format!(
938        "{:?}(node={:?}, status={:?})",
939        event.kind, event.node_id, event.status
940    )
941}
942
943/// Per-run parameters of an exact replay: the fresh journal (recorded run's
944/// identity plus matching logical clock), the RNG (the recorded run's seed
945/// for byte-identical checkpoint ids), an optional checkpointer (present iff
946/// the recorded run had one — checkpoint-written events are part of the
947/// evidence), and the step limit.
948pub struct ReplayParams {
949    /// The replay run's journal; build with [`ExactReplay::fresh_journal`].
950    pub journal: Journal,
951
952    /// The replay run's randomness source. [`RngSource::seeded`] with the
953    /// recorded run's seed reproduces checkpoint ids; anything else makes the
954    /// replayed journal diverge at the first checkpoint-written event.
955    pub rng: RngSource,
956
957    /// Persistence for the replay run. Must be `Some` when the recorded run
958    /// journaled checkpoint-written events, `None` when it did not.
959    pub checkpointer: Option<Arc<dyn Checkpointer>>,
960
961    /// Super-step limit, as in [`RunConfig::max_steps`].
962    pub max_steps: usize,
963}
964
965impl ReplayParams {
966    /// Parameters over `journal` and `rng` with no checkpointer and the
967    /// default step limit.
968    pub fn new(journal: Journal, rng: RngSource) -> Self {
969        Self {
970            journal,
971            rng,
972            checkpointer: None,
973            max_steps: DEFAULT_MAX_STEPS,
974        }
975    }
976
977    /// Builder-style: persist replay checkpoints through `checkpointer`.
978    pub fn with_checkpointer(mut self, checkpointer: Arc<dyn Checkpointer>) -> Self {
979        self.checkpointer = Some(checkpointer);
980        self
981    }
982
983    /// Builder-style: override the step limit.
984    pub fn with_max_steps(mut self, max_steps: usize) -> Self {
985        self.max_steps = max_steps;
986        self
987    }
988}
989
990/// The result of an exact replay: the run's outcome and the replayed journal
991/// (export form, comparable against the recorded snapshot).
992#[derive(Debug)]
993pub struct ReplayOutcome {
994    /// How the replayed run ended. Exact replay of an interrupted recorded
995    /// run ends in the same `Interrupted` outcome with the same payload.
996    pub outcome: ExecutionOutcome,
997
998    /// The replayed run's journal snapshot. After
999    /// [`ExactReplay::run_and_verify`], this is event-for-event identical to
1000    /// the recorded snapshot.
1001    pub journal: JournalSnapshot,
1002}
1003
1004/// Token and cost totals of one branch of a [`BranchDiff`]: the sum over all
1005/// of the branch's events (the `tokens` field is only ever set on model
1006/// calls, so this is the branch's model usage).
1007#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
1008pub struct BranchTotals {
1009    /// Number of events in the branch.
1010    pub events: usize,
1011
1012    /// Summed token usage across the branch's model calls.
1013    pub tokens: Usage,
1014
1015    /// Summed recorded cost in USD across the branch's events.
1016    pub cost_usd: f64,
1017}
1018
1019/// One channel's differing value at one super-step of a [`BranchDiff`].
1020/// `None` means the channel was absent from that branch's super-step-end
1021/// record at that step.
1022#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1023pub struct ChannelDiff {
1024    /// The state channel that differs.
1025    pub channel: String,
1026
1027    /// The post-reducer value in the base branch, when present.
1028    pub base: Option<Value>,
1029
1030    /// The post-reducer value in the branch, when present.
1031    pub branch: Option<Value>,
1032}
1033
1034/// The state-channel differences of one super-step between two branches.
1035#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1036pub struct StepDiff {
1037    /// The super-step index (from the super-step-start event's input).
1038    pub step: u64,
1039
1040    /// Channels whose post-reducer values differ at this step (presence or
1041    /// value), sorted by channel name.
1042    pub channels: Vec<ChannelDiff>,
1043}
1044
1045/// The structural diff between two journal snapshots — typically two branches
1046/// forked from a common point: one replayed an existing history, the other
1047/// continued differently.
1048///
1049/// Events are compared **logically**: kind, node, sequence, effect class,
1050/// resolved input/output payloads, latency, tokens, cost, and status must
1051/// match, while identity and timing fields (`id`, `run_id`, `thread_id`,
1052/// `parent`, `recorded_at`) are excluded — two branches are separate runs,
1053/// so their identity-bearing fields legitimately differ even when their
1054/// evidence is the same. The first logically-unequal sequence position is the
1055/// divergence point; everything after it is reported as removed (base) or
1056/// added (branch).
1057#[derive(Debug, Clone, Serialize, Deserialize)]
1058pub struct BranchDiff {
1059    /// The first sequence number where the branches' evidence differs, or
1060    /// `None` when they are logically identical.
1061    pub first_divergent_seq: Option<u64>,
1062
1063    /// Events present in the branch at and after the divergence point (the
1064    /// branch's new work relative to the base).
1065    pub added: Vec<RunEvent>,
1066
1067    /// Events present in the base at and after the divergence point (the work
1068    /// the branch replaced).
1069    pub removed: Vec<RunEvent>,
1070
1071    /// State-channel differences at each super-step, in step order. Only
1072    /// steps with at least one differing channel appear.
1073    pub step_diffs: Vec<StepDiff>,
1074
1075    /// Totals over the whole base journal.
1076    pub base_totals: BranchTotals,
1077
1078    /// Totals over the whole branch journal.
1079    pub branch_totals: BranchTotals,
1080}
1081
1082impl BranchDiff {
1083    /// Diff `branch` against `base`. When both snapshots share a common
1084    /// history (a fork), the shared prefix compares equal and the divergence
1085    /// point is where the fork's branches parted; `fork_seq`-style explicit
1086    /// fork points are unnecessary — the evidence carries the cut.
1087    pub fn between(base: &JournalSnapshot, branch: &JournalSnapshot) -> BranchDiff {
1088        let divergence = (0..base.events.len().max(branch.events.len()))
1089            .find(|&i| {
1090                match (base.events.get(i), branch.events.get(i)) {
1091                    (Some(a), Some(b)) => !logically_equal(a, base, b, branch),
1092                    // One side exhausted: divergence at the first missing seq.
1093                    _ => true,
1094                }
1095            })
1096            .map(|i| {
1097                base.events
1098                    .get(i)
1099                    .or_else(|| branch.events.get(i))
1100                    .map(|event| event.seq)
1101                    .unwrap_or(i as u64)
1102            });
1103        let (removed, added) = match divergence {
1104            Some(seq) => (
1105                base.events
1106                    .iter()
1107                    .filter(|event| event.seq >= seq)
1108                    .cloned()
1109                    .collect(),
1110                branch
1111                    .events
1112                    .iter()
1113                    .filter(|event| event.seq >= seq)
1114                    .cloned()
1115                    .collect(),
1116            ),
1117            None => (Vec::new(), Vec::new()),
1118        };
1119        BranchDiff {
1120            first_divergent_seq: divergence,
1121            added,
1122            removed,
1123            step_diffs: diff_steps(base, branch),
1124            base_totals: totals(base),
1125            branch_totals: totals(branch),
1126        }
1127    }
1128
1129    /// `true` when the two branches are logically identical (no divergence
1130    /// point, nothing added or removed).
1131    pub fn is_identical(&self) -> bool {
1132        self.first_divergent_seq.is_none()
1133    }
1134}
1135
1136/// Logical evidence equality, excluding identity and timing fields (see
1137/// [`BranchDiff`]). Payloads compare by resolved value, so an inline payload
1138/// and an artifact reference carrying the same bytes compare equal.
1139fn logically_equal(
1140    a: &RunEvent,
1141    snapshot_a: &JournalSnapshot,
1142    b: &RunEvent,
1143    snapshot_b: &JournalSnapshot,
1144) -> bool {
1145    a.seq == b.seq
1146        && a.kind == b.kind
1147        && a.node_id == b.node_id
1148        && a.effect == b.effect
1149        && a.latency_ms == b.latency_ms
1150        && a.tokens == b.tokens
1151        && a.cost_usd == b.cost_usd
1152        && a.status == b.status
1153        && resolve_opt(snapshot_a, a.input.as_ref()) == resolve_opt(snapshot_b, b.input.as_ref())
1154        && resolve_opt(snapshot_a, a.output.as_ref()) == resolve_opt(snapshot_b, b.output.as_ref())
1155}
1156
1157/// Sum token usage and cost over a snapshot's events.
1158fn totals(snapshot: &JournalSnapshot) -> BranchTotals {
1159    let mut totals = BranchTotals {
1160        events: snapshot.events.len(),
1161        ..BranchTotals::default()
1162    };
1163    for event in &snapshot.events {
1164        if let Some(usage) = event.tokens {
1165            totals.tokens.prompt_tokens = totals
1166                .tokens
1167                .prompt_tokens
1168                .saturating_add(usage.prompt_tokens);
1169            totals.tokens.completion_tokens = totals
1170                .tokens
1171                .completion_tokens
1172                .saturating_add(usage.completion_tokens);
1173            totals.tokens.total_tokens = totals
1174                .tokens
1175                .total_tokens
1176                .saturating_add(usage.total_tokens);
1177        }
1178        if let Some(cost) = event.cost_usd {
1179            totals.cost_usd += cost;
1180        }
1181    }
1182    totals
1183}
1184
1185/// The post-reducer channel values of every super-step in a snapshot, keyed
1186/// by step index: super-step-start events supply the index, the following
1187/// super-step-end event supplies the channel values.
1188fn step_channel_values(snapshot: &JournalSnapshot) -> BTreeMap<u64, BTreeMap<String, Value>> {
1189    let mut steps = BTreeMap::new();
1190    let mut current_step: Option<u64> = None;
1191    for event in &snapshot.events {
1192        match event.kind {
1193            RunEventKind::SuperStepStart => {
1194                current_step = resolve_opt(snapshot, event.input.as_ref())
1195                    .and_then(|input| input.get("step").and_then(Value::as_u64));
1196            }
1197            RunEventKind::SuperStepEnd => {
1198                if let Some(step) = current_step.take() {
1199                    let channels = resolve_opt(snapshot, event.output.as_ref())
1200                        .and_then(|output| output.as_object().cloned())
1201                        .unwrap_or_default()
1202                        .into_iter()
1203                        .collect();
1204                    steps.insert(step, channels);
1205                }
1206            }
1207            _ => {}
1208        }
1209    }
1210    steps
1211}
1212
1213/// Per-step channel diffs between two snapshots (only steps with at least one
1214/// differing channel, channels sorted).
1215fn diff_steps(base: &JournalSnapshot, branch: &JournalSnapshot) -> Vec<StepDiff> {
1216    let base_steps = step_channel_values(base);
1217    let branch_steps = step_channel_values(branch);
1218    let mut diffs = Vec::new();
1219    for step in base_steps
1220        .keys()
1221        .chain(branch_steps.keys())
1222        .copied()
1223        .collect::<std::collections::BTreeSet<u64>>()
1224    {
1225        let empty = BTreeMap::new();
1226        let base_channels = base_steps.get(&step).unwrap_or(&empty);
1227        let branch_channels = branch_steps.get(&step).unwrap_or(&empty);
1228        let mut channels = Vec::new();
1229        for channel in base_channels
1230            .keys()
1231            .chain(branch_channels.keys())
1232            .cloned()
1233            .collect::<std::collections::BTreeSet<String>>()
1234        {
1235            let base_value = base_channels.get(&channel).cloned();
1236            let branch_value = branch_channels.get(&channel).cloned();
1237            if base_value != branch_value {
1238                channels.push(ChannelDiff {
1239                    channel,
1240                    base: base_value,
1241                    branch: branch_value,
1242                });
1243            }
1244        }
1245        if !channels.is_empty() {
1246            diffs.push(StepDiff { step, channels });
1247        }
1248    }
1249    diffs
1250}
1251
1252/// Determinism parameters of a recorded run, carried in [`FixtureMetadata`]
1253/// so [`ReplayFixture::replay_in_ci`] can reproduce the run byte-for-byte.
1254#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1255pub struct LogicalClockParams {
1256    /// The logical clock's start (epoch millis), as in [`Clock::logical`].
1257    pub start_ms: u64,
1258
1259    /// The logical clock's per-read tick (millis), as in [`Clock::logical`].
1260    pub tick_ms: u64,
1261}
1262
1263/// Provenance and determinism metadata of a [`ReplayFixture`].
1264#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1265pub struct FixtureMetadata {
1266    /// Human-readable fixture name (used in error messages).
1267    pub name: String,
1268
1269    /// The recorded run's logical-clock parameters. Required for CI replay.
1270    #[serde(default, skip_serializing_if = "Option::is_none")]
1271    pub clock: Option<LogicalClockParams>,
1272
1273    /// The recorded run's RNG seed. Required for CI replay.
1274    #[serde(default, skip_serializing_if = "Option::is_none")]
1275    pub rng_seed: Option<u64>,
1276}
1277
1278/// A portable, self-contained exact-replay bundle: everything CI (or another
1279/// machine, or a later wave's server endpoint) needs to re-drive a recorded
1280/// run and prove it reproduces.
1281///
1282/// A fixture carries the graph's topology hash (replay refuses a structurally
1283/// different graph), the full journal snapshot (integrity-verified on
1284/// import), the recorded final checkpoint (replay compares the outcome state
1285/// against it), and the determinism parameters the run was recorded with.
1286/// Serialization is plain JSON; [`ReplayFixture::export`] /
1287/// [`ReplayFixture::import`] are the wire boundary.
1288#[derive(Debug, Clone, Serialize, Deserialize)]
1289pub struct ReplayFixture {
1290    /// Fixture envelope version; [`FIXTURE_FORMAT_VERSION`] for anything
1291    /// written now.
1292    pub format_version: u32,
1293
1294    /// SHA-256 of the recorded graph's topology (see
1295    /// [`Graph::topology_hash`]). Checked before replay: a different topology
1296    /// cannot reproduce the journaled evidence.
1297    pub graph_hash: String,
1298
1299    /// The application-declared graph version of the recorded run.
1300    pub graph_version: String,
1301
1302    /// The recorded run's complete journal.
1303    pub journal: JournalSnapshot,
1304
1305    /// The recorded run's final checkpoint, when it ran with a checkpointer.
1306    /// CI replay compares the replayed outcome state against it.
1307    pub final_checkpoint: Option<Checkpoint>,
1308
1309    /// Provenance and determinism parameters.
1310    pub metadata: FixtureMetadata,
1311}
1312
1313impl ReplayFixture {
1314    /// Capture a recorded run as a fixture. `graph` supplies the topology
1315    /// hash; `clock`/`rng_seed` are the determinism parameters the run was
1316    /// recorded with (both required for later CI replay).
1317    pub fn capture(
1318        name: impl Into<String>,
1319        graph: &Graph,
1320        graph_version: impl Into<String>,
1321        journal: JournalSnapshot,
1322        final_checkpoint: Option<Checkpoint>,
1323        clock: Option<LogicalClockParams>,
1324        rng_seed: Option<u64>,
1325    ) -> Self {
1326        Self {
1327            format_version: FIXTURE_FORMAT_VERSION,
1328            graph_hash: graph.topology_hash(),
1329            graph_version: graph_version.into(),
1330            journal,
1331            final_checkpoint,
1332            metadata: FixtureMetadata {
1333                name: name.into(),
1334                clock,
1335                rng_seed,
1336            },
1337        }
1338    }
1339
1340    /// Serialize to pretty-printed JSON (the checked-in / on-disk form).
1341    pub fn export(&self) -> Result<String> {
1342        Ok(serde_json::to_string_pretty(self)?)
1343    }
1344
1345    /// Parse a fixture from JSON, rejecting unsupported format versions and
1346    /// integrity failures (tampered or truncated journals) at the boundary.
1347    pub fn import(json: &str) -> Result<Self> {
1348        let fixture: Self = serde_json::from_str(json)?;
1349        if fixture.format_version != FIXTURE_FORMAT_VERSION {
1350            return Err(replay_error(format!(
1351                "unsupported fixture format version {} (this build supports \
1352                 {FIXTURE_FORMAT_VERSION}) — regenerate the fixture or upgrade the runtime",
1353                fixture.format_version
1354            )));
1355        }
1356        verify_snapshot(&fixture.journal)?;
1357        Ok(fixture)
1358    }
1359
1360    /// An exact-replay session over the fixture's journal (integrity already
1361    /// verified by [`ReplayFixture::import`] or construction; re-verified
1362    /// here so a hand-assembled fixture is checked too).
1363    pub fn exact_replay(&self) -> Result<ExactReplay> {
1364        ExactReplay::new(self.journal.clone())
1365    }
1366
1367    /// The replay parameters matching the fixture's determinism metadata: a
1368    /// fresh journal on the recorded identity and logical clock, the recorded
1369    /// RNG seed, and a fresh in-memory checkpointer (fixtures record runs
1370    /// that used one — the checkpoint-written events are part of the
1371    /// evidence).
1372    ///
1373    /// Fails with [`RustyError::Replay`] when the fixture lacks clock or seed
1374    /// metadata (a run recorded without determinism seams cannot be replayed
1375    /// byte-identically).
1376    pub fn replay_params(&self, replay: &ExactReplay) -> Result<ReplayParams> {
1377        let clock = self.metadata.clock.ok_or_else(|| {
1378            replay_error(format!(
1379                "fixture `{}` carries no logical-clock parameters; exact CI replay requires \
1380                 the recorded run's determinism seams",
1381                self.metadata.name
1382            ))
1383        })?;
1384        let seed = self.metadata.rng_seed.ok_or_else(|| {
1385            replay_error(format!(
1386                "fixture `{}` carries no RNG seed; exact CI replay requires the recorded \
1387                 run's determinism seams",
1388                self.metadata.name
1389            ))
1390        })?;
1391        let journal = replay.fresh_journal(Clock::logical(clock.start_ms, clock.tick_ms));
1392        Ok(ReplayParams::new(journal, RngSource::seeded(seed))
1393            .with_checkpointer(Arc::new(InMemoryCheckpointer::new())))
1394    }
1395
1396    /// Replay the fixture end to end and verify everything: the graph's
1397    /// topology matches the recorded one, the run reproduces the journaled
1398    /// evidence event-for-event, and the outcome state matches the recorded
1399    /// final checkpoint.
1400    ///
1401    /// The caller supplies the replay session (`fixture.exact_replay()?`), a
1402    /// graph built with replaying wrappers over `replay.source()`, and the
1403    /// parameters from `fixture.replay_params(&replay)?` — the same journal
1404    /// handle the wrappers re-journal into. This is the `replay_in_ci` entry
1405    /// point: one call from recorded artifact to verified replay.
1406    pub async fn replay_in_ci(
1407        &self,
1408        replay: ExactReplay,
1409        graph: &Graph,
1410        spec: &StateSpec,
1411        initial_state: State,
1412        params: ReplayParams,
1413    ) -> Result<ReplayOutcome> {
1414        let actual_hash = graph.topology_hash();
1415        if actual_hash != self.graph_hash {
1416            return Err(replay_error(format!(
1417                "fixture `{}` was recorded against graph topology {}, but the graph under \
1418                 replay hashes to {actual_hash} — rebuild the recorded topology (same node \
1419                 names, edges, and entry point)",
1420                self.metadata.name, self.graph_hash
1421            )));
1422        }
1423        let replayed = replay
1424            .run_and_verify(graph, spec, initial_state, params)
1425            .await?;
1426        if let Some(final_checkpoint) = &self.final_checkpoint {
1427            if replayed.outcome.state() != &final_checkpoint.state {
1428                return Err(replay_error(format!(
1429                    "fixture `{}`: the replayed final state diverges from the recorded final \
1430                     checkpoint (journal evidence agreed; state did not)",
1431                    self.metadata.name
1432                )));
1433            }
1434        }
1435        Ok(replayed)
1436    }
1437}
1438
1439#[cfg(test)]
1440mod tests {
1441    use super::*;
1442    use crate::journal::Journal;
1443    use serde_json::json;
1444
1445    fn snapshot_with(drafts: Vec<EventDraft>) -> JournalSnapshot {
1446        let journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
1447        for draft in drafts {
1448            journal.record(draft);
1449        }
1450        journal.snapshot()
1451    }
1452
1453    fn model_draft(request: Value, response: Value) -> EventDraft {
1454        EventDraft::new(RunEventKind::ModelCall, Effect::NonIdempotent)
1455            .node("agent")
1456            .input(request)
1457            .output(response)
1458            .latency_ms(2)
1459            .tokens(Usage {
1460                prompt_tokens: 10,
1461                completion_tokens: 5,
1462                total_tokens: 15,
1463            })
1464    }
1465
1466    #[test]
1467    fn canonical_shapes_have_stable_hashes() {
1468        let request = model_call_request(&[ChatMessage::user("ping")], &[]);
1469        assert_eq!(
1470            request,
1471            json!({"messages": [{"role": "user", "content": "ping"}], "tools": []})
1472        );
1473        let first = request_hash(&request).unwrap();
1474        let second = request_hash(&model_call_request(&[ChatMessage::user("ping")], &[])).unwrap();
1475        assert_eq!(first, second);
1476        // A different request hashes differently.
1477        let other = request_hash(&model_call_request(&[ChatMessage::user("pong")], &[])).unwrap();
1478        assert_ne!(first, other);
1479
1480        let tool = tool_call_request("echo", &json!({"text": "hi"}));
1481        assert_eq!(tool, json!({"tool": "echo", "arguments": {"text": "hi"}}));
1482    }
1483
1484    #[test]
1485    fn serve_matches_by_sequence_and_request_hash() {
1486        let snapshot = snapshot_with(vec![
1487            model_draft(
1488                model_call_request(&[ChatMessage::user("a")], &[]),
1489                json!({"r": 1}),
1490            ),
1491            model_draft(
1492                model_call_request(&[ChatMessage::user("b")], &[]),
1493                json!({"r": 2}),
1494            ),
1495        ]);
1496        let source = ReplaySource::new(&snapshot);
1497        assert!(!source.is_exhausted());
1498
1499        // Wrong request at the cursor: divergence, and the cursor must not advance.
1500        let error = source
1501            .serve(
1502                RunEventKind::ModelCall,
1503                &model_call_request(&[ChatMessage::user("zzz")], &[]),
1504            )
1505            .unwrap_err();
1506        assert!(matches!(error, RustyError::Replay(_)));
1507        assert!(error.to_string().contains("divergence"));
1508        assert_eq!(source.remaining().len(), 2);
1509
1510        // Wrong kind at the cursor: order violation.
1511        let error = source
1512            .serve(RunEventKind::ToolCall, &tool_call_request("t", &json!({})))
1513            .unwrap_err();
1514        assert!(error.to_string().contains("order violation"));
1515
1516        // Correct sequence serves in order and exhausts.
1517        let first = source
1518            .serve(
1519                RunEventKind::ModelCall,
1520                &model_call_request(&[ChatMessage::user("a")], &[]),
1521            )
1522            .unwrap();
1523        assert_eq!(first.event.seq, 0);
1524        assert_eq!(first.output, Some(json!({"r": 1})));
1525        let second = source
1526            .serve(
1527                RunEventKind::ModelCall,
1528                &model_call_request(&[ChatMessage::user("b")], &[]),
1529            )
1530            .unwrap();
1531        assert_eq!(second.event.seq, 1);
1532        assert!(source.is_exhausted());
1533
1534        // Serving past the end is a loud exhaustion error.
1535        let error = source
1536            .serve(
1537                RunEventKind::ModelCall,
1538                &model_call_request(&[ChatMessage::user("a")], &[]),
1539            )
1540            .unwrap_err();
1541        assert!(error.to_string().contains("exhaustion"));
1542    }
1543
1544    #[test]
1545    fn serve_rejects_non_servable_kinds() {
1546        let snapshot = snapshot_with(vec![EventDraft::new(
1547            RunEventKind::SuperStepStart,
1548            Effect::Pure,
1549        )]);
1550        let error = ReplaySource::new(&snapshot)
1551            .serve(RunEventKind::NodeInput, &json!({}))
1552            .unwrap_err();
1553        assert!(error.to_string().contains("not a servable"));
1554    }
1555
1556    #[test]
1557    fn artifact_backed_requests_match_and_resolve() {
1558        let big_request =
1559            json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1560        let big_response = json!({"message": {"role": "assistant", "content": "y".repeat(9000)}, "model": null, "usage": null});
1561        let snapshot = snapshot_with(vec![model_draft(big_request.clone(), big_response.clone())]);
1562        // The payloads were promoted to artifacts at record time.
1563        assert!(matches!(
1564            snapshot.events[0].input,
1565            Some(PayloadRef::Artifact(_))
1566        ));
1567        let served = ReplaySource::new(&snapshot)
1568            .serve(RunEventKind::ModelCall, &big_request)
1569            .unwrap();
1570        assert_eq!(served.output, Some(big_response));
1571    }
1572
1573    #[test]
1574    fn tampered_snapshots_are_rejected_at_the_boundary() {
1575        let snapshot = snapshot_with(vec![model_draft(
1576            model_call_request(&[ChatMessage::user("a")], &[]),
1577            json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1578        )]);
1579
1580        // Event tampering breaks the chained head hash.
1581        let mut tampered = snapshot.clone();
1582        tampered.events[0].status = EventStatus::Error;
1583        assert!(ExactReplay::new(tampered).is_err());
1584
1585        // Artifact tampering breaks content addressing (the head hash chains
1586        // over events, so this needs the dedicated integrity check).
1587        let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1588        let artifact_snapshot = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
1589        let mut tampered = artifact_snapshot;
1590        *tampered.artifacts.values_mut().next().unwrap() = json!({"forged": true});
1591        let error = ExactReplay::new(tampered).unwrap_err();
1592        assert!(error.to_string().contains("integrity"));
1593
1594        // A truncated snapshot (artifact removed) is rejected.
1595        let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1596        let truncated = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
1597        let mut truncated = truncated;
1598        truncated.artifacts.clear();
1599        assert!(ExactReplay::new(truncated).is_err());
1600    }
1601
1602    #[test]
1603    fn resumed_run_journals_are_rejected() {
1604        let snapshot = snapshot_with(vec![EventDraft::new(RunEventKind::Resume, Effect::Pure)
1605            .input(json!({"checkpoint_id": "c"}))]);
1606        let error = ExactReplay::new(snapshot).unwrap_err();
1607        assert!(error.to_string().contains("resumed"));
1608    }
1609
1610    #[test]
1611    fn rejournal_reproduces_the_recorded_event() {
1612        let recorded = snapshot_with(vec![model_draft(
1613            model_call_request(&[ChatMessage::user("a")], &[]),
1614            json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1615        )]);
1616        let source = ReplaySource::new(&recorded);
1617        let served = source
1618            .serve(
1619                RunEventKind::ModelCall,
1620                &model_call_request(&[ChatMessage::user("a")], &[]),
1621            )
1622            .unwrap();
1623        let replay_journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
1624        // Same draft sequence: one record read per event on the same clock.
1625        served.rejournal(&replay_journal, "run-t:parent");
1626        let replayed = replay_journal.snapshot();
1627        assert_eq!(replayed.events.len(), 1);
1628        // Identical except for the causal parent we supplied.
1629        let mut expected = recorded.events[0].clone();
1630        expected.parent = Some("run-t:parent".into());
1631        assert_eq!(replayed.events[0], expected);
1632    }
1633
1634    #[test]
1635    fn branch_diff_reports_divergence_steps_and_totals() {
1636        let shared = || {
1637            vec![
1638                EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1639                    .input(json!({"step": 0, "active_nodes": ["a"]})),
1640                EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1641                    .output(json!({"log": ["a"]})),
1642            ]
1643        };
1644        let mut base_drafts = shared();
1645        base_drafts.push(
1646            EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1647                .input(json!({"step": 1, "active_nodes": ["b"]})),
1648        );
1649        base_drafts.push(
1650            EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1651                .output(json!({"log": ["a", "b"]})),
1652        );
1653        base_drafts.push(
1654            model_draft(
1655                model_call_request(&[ChatMessage::user("q")], &[]),
1656                json!({"r": 1}),
1657            )
1658            .cost_usd(0.001),
1659        );
1660        let base = snapshot_with(base_drafts);
1661
1662        let mut branch_drafts = shared();
1663        branch_drafts.push(
1664            EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1665                .input(json!({"step": 1, "active_nodes": ["c"]})),
1666        );
1667        branch_drafts.push(
1668            EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1669                .output(json!({"log": ["a", "c"], "extra": 1})),
1670        );
1671        let branch = snapshot_with(branch_drafts);
1672
1673        let diff = BranchDiff::between(&base, &branch);
1674        assert!(!diff.is_identical());
1675        assert_eq!(diff.first_divergent_seq, Some(2));
1676        assert_eq!(diff.removed.len(), 3);
1677        assert_eq!(diff.added.len(), 2);
1678        assert_eq!(diff.base_totals.events, 5);
1679        assert_eq!(diff.branch_totals.events, 4);
1680        assert_eq!(diff.base_totals.tokens.total_tokens, 15);
1681        assert_eq!(diff.branch_totals.tokens.total_tokens, 0);
1682        assert_eq!(diff.base_totals.cost_usd, 0.001);
1683        assert_eq!(diff.branch_totals.cost_usd, 0.0);
1684
1685        // Step 0 is shared; step 1 differs on `log` and `extra`.
1686        assert_eq!(diff.step_diffs.len(), 1);
1687        let step = &diff.step_diffs[0];
1688        assert_eq!(step.step, 1);
1689        assert_eq!(step.channels.len(), 2);
1690        assert_eq!(step.channels[0].channel, "extra");
1691        assert_eq!(step.channels[0].base, None);
1692        assert_eq!(step.channels[0].branch, Some(json!(1)));
1693        assert_eq!(step.channels[1].channel, "log");
1694        assert_eq!(step.channels[1].base, Some(json!(["a", "b"])));
1695        assert_eq!(step.channels[1].branch, Some(json!(["a", "c"])));
1696    }
1697
1698    #[test]
1699    fn branch_diff_of_identical_snapshots_is_empty() {
1700        // Two separate runs of the same drafts: identity fields (ids,
1701        // timestamps) match here too, but the point is the logical comparison.
1702        let snapshot = snapshot_with(vec![EventDraft::new(
1703            RunEventKind::SuperStepStart,
1704            Effect::Pure,
1705        )
1706        .input(json!({"step": 0, "active_nodes": ["a"]}))]);
1707        let diff = BranchDiff::between(&snapshot, &snapshot);
1708        assert!(diff.is_identical());
1709        assert!(diff.added.is_empty() && diff.removed.is_empty());
1710        assert!(diff.step_diffs.is_empty());
1711    }
1712
1713    #[test]
1714    fn fixture_roundtrip_and_version_guard() {
1715        let journal = snapshot_with(vec![model_draft(
1716            model_call_request(&[ChatMessage::user("a")], &[]),
1717            json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1718        )]);
1719        let mut builder = crate::graph::GraphBuilder::new();
1720        builder.add_node("agent", |_ctx: crate::node::NodeContext| async {
1721            Ok(crate::node::NodeOutput::empty())
1722        });
1723        builder.set_entry_point("agent");
1724        let graph = builder.compile().unwrap();
1725        let fixture = ReplayFixture::capture(
1726            "roundtrip",
1727            &graph,
1728            "v1",
1729            journal,
1730            None,
1731            Some(LogicalClockParams {
1732                start_ms: 1_000_000,
1733                tick_ms: 5,
1734            }),
1735            Some(7),
1736        );
1737        let wire = fixture.export().unwrap();
1738        let imported = ReplayFixture::import(&wire).unwrap();
1739        assert_eq!(imported.export().unwrap(), wire);
1740
1741        // An unsupported format version is rejected with a descriptive error.
1742        let mut value: Value = serde_json::from_str(&wire).unwrap();
1743        value["format_version"] = json!(99);
1744        let error = ReplayFixture::import(&value.to_string()).unwrap_err();
1745        assert!(error.to_string().contains("format version 99"));
1746    }
1747}