Skip to main content

rusty_agent_runtime/
replay.rs

1//! The replay engine: exact replay, branch diff, and portable fixtures.
2//!
3//! This module is the second work item of the R0.5 Flight Recorder. Wave 1
4//! landed the evidence contracts ([`crate::record`]), the journal, and the
5//! determinism seams ([`Clock`], [`RngSource`]); this module consumes them to
6//! re-drive a recorded run **exactly**: model, tool, remote, and WASM calls
7//! are served from the journal instead of executed, and the replayed run's
8//! journal and final state must reproduce the recorded ones byte-for-byte.
9//!
10//! # The exact-replay contract
11//!
12//! Exact replay re-runs the *same* graph topology with the *same* determinism
13//! seams (logical clock, seeded RNG) and swaps only the effect
14//! implementations: wherever the recorded run talked to the world, the replay
15//! talks to the journal. Node bodies still execute — `Pure` computation is
16//! freely repeatable — but every outbound effect is matched against the
17//! journaled record **by sequence and request hash** and answered with the
18//! recorded response. Three failures are detected loudly:
19//!
20//! - **Divergence** — the replayed run issues a request whose canonical hash
21//!   differs from the journaled request at the next unserved sequence
22//!   position ([`RustyError::Replay`]).
23//! - **Order violation** — effects arrive in a different order or kind than
24//!   the journal recorded (exact replay serves effects in `seq` order only).
25//! - **Exhaustion / shortfall** — the replayed run issues more effects than
26//!   the journal holds, or finishes with recorded effects unserved
27//!   (verification fails).
28//!
29//! Interrupts need no serving: with every effect answered deterministically
30//! from the journal, node logic re-derives the same interrupt and the
31//! executor journals it identically.
32//!
33//! # Zero outbound calls
34//!
35//! [`ReplayingChatModel`] and [`ReplayingTool`] wrap the real implementations
36//! but **never invoke them** — there is no code path from `chat`/`call` to
37//! the wrapped value. Tests prove it by replaying with panic-on-call
38//! sentinels as the inner implementations. The wrappers exist so the *same
39//! graph code* runs in record and replay mode; only the effect
40//! implementations are swapped.
41//!
42//! # Determinism prerequisites
43//!
44//! Byte-identical replay requires the replay run to observe the same time and
45//! id sequences as the recorded run: attach a fresh journal with the recorded
46//! run's identity (`run_id`, `thread_id`) and a logical clock with the same
47//! parameters ([`ExactReplay::fresh_journal`]), and seed the RNG with the
48//! recorded run's seed ([`ReplayParams`]). [`ReplayFixture`] carries these
49//! parameters in its metadata so CI replay is self-contained. The recording
50//! and replaying wrappers perform the same number of journal-clock reads per
51//! call, keeping the tick sequence aligned; per the journal's determinism
52//! model, super-steps with several parallel nodes interleave clock reads by
53//! schedule, so byte-identical replay is guaranteed for runs whose steps run
54//! one node at a time.
55
56use std::collections::BTreeMap;
57use std::sync::{Arc, Mutex, MutexGuard};
58
59use async_trait::async_trait;
60use serde::{Deserialize, Serialize};
61use serde_json::{json, Value};
62
63use crate::checkpoint::{Checkpoint, Checkpointer, InMemoryCheckpointer};
64use crate::effects::EffectRequest;
65use crate::error::{Result, RustyError};
66use crate::executor::{ExecutionOutcome, Executor, RunConfig, DEFAULT_MAX_STEPS};
67use crate::graph::Graph;
68use crate::journal::{Clock, EventDraft, Journal, JournalSnapshot, RngSource};
69use crate::llm::{ChatMessage, ChatModel, ChatResponse, ToolCall, Usage};
70use crate::record::{sha256_hex, Effect, EventStatus, PayloadRef, RunEvent, RunEventKind};
71use crate::state::{State, StateSpec};
72use crate::tool::Tool;
73
74/// The current on-disk format version of [`ReplayFixture`].
75///
76/// Bump only on a breaking change to the fixture envelope; additive evolution
77/// uses serde defaults so previously written fixtures keep deserializing.
78pub const FIXTURE_FORMAT_VERSION: u32 = 1;
79
80/// The effect kinds exact replay can serve from the journal: outbound calls
81/// whose recorded responses replace live execution. Super-step boundaries,
82/// node inputs/outputs, routing decisions, interrupts, and checkpoint writes
83/// are *re-derived* by the executor, never served.
84///
85/// Public so the digital twin (`crate::twin`) decomposes a recorded run into
86/// the same effect set replay serves — one servable-kind vocabulary, shared
87/// by replay and the twin, rather than two lists that can drift.
88pub const SERVABLE_KINDS: [RunEventKind; 4] = [
89    RunEventKind::ModelCall,
90    RunEventKind::ToolCall,
91    RunEventKind::RemoteCall,
92    RunEventKind::WasmCall,
93];
94
95fn is_servable(kind: RunEventKind) -> bool {
96    SERVABLE_KINDS.contains(&kind)
97}
98
99fn replay_error(message: impl Into<String>) -> RustyError {
100    RustyError::Replay(message.into())
101}
102
103/// The canonical JSON shape a [`RecordingChatModel`] journals (and a
104/// [`ReplayingChatModel`] matches) as a model call's **request**: the
105/// messages and tool schemas exactly as passed to [`ChatModel::chat`].
106///
107/// The hash of this value's canonical serialization is the request identity
108/// exact replay matches on — node code that journals model calls by hand must
109/// use this shape for its journals to be replayable.
110pub fn model_call_request(messages: &[ChatMessage], tools: &[Value]) -> Value {
111    json!({ "messages": messages, "tools": tools })
112}
113
114/// The canonical JSON shape a [`RecordingChatModel`] journals (and a
115/// [`ReplayingChatModel`] reconstructs a [`ChatResponse`] from) as a model
116/// call's **response**: the assistant message, the reported model identity,
117/// and token usage. Keys are always present (`null` when unreported) so the
118/// shape is stable.
119pub fn model_call_response(response: &ChatResponse) -> Value {
120    json!({
121        "message": response.message,
122        "model": response.model,
123        "usage": response.usage,
124    })
125}
126
127/// The canonical JSON shape a [`RecordingTool`] journals (and a
128/// [`ReplayingTool`] matches) as a tool call's **request**: the tool name and
129/// the model-supplied arguments. The recorded **output** is the tool's result
130/// value verbatim.
131pub fn tool_call_request(name: &str, arguments: &Value) -> Value {
132    json!({ "tool": name, "arguments": arguments })
133}
134
135/// Reconstruct a [`ChatResponse`] from a journaled [`model_call_response`]
136/// payload.
137fn chat_response_from_recorded(seq: u64, value: &Value) -> Result<ChatResponse> {
138    let decode = |key: &str| -> Result<Value> {
139        value.get(key).cloned().ok_or_else(|| {
140            replay_error(format!(
141                "recorded model call at seq {seq} is malformed: output payload has no `{key}` key"
142            ))
143        })
144    };
145    let message: ChatMessage = serde_json::from_value(decode("message")?)?;
146    let model: Option<String> = serde_json::from_value(decode("model")?)?;
147    let usage: Option<Usage> = serde_json::from_value(decode("usage")?)?;
148    Ok(ChatResponse {
149        message,
150        model,
151        usage,
152    })
153}
154
155/// The recorded error text of a failed served effect (`{"error": …}` output
156/// payloads, as written by the recording wrappers and the executor's
157/// node-failure path).
158fn recorded_error_text(served: &ServedEffect) -> String {
159    served
160        .output
161        .as_ref()
162        .and_then(|value| value.get("error"))
163        .and_then(Value::as_str)
164        .unwrap_or("<no recorded error payload>")
165        .to_owned()
166}
167
168/// Verify the integrity of a journal snapshot: the chained head hash must
169/// recompute from the events (as [`Journal::from_snapshot`] enforces), every
170/// content-addressed artifact must hash to its own key, and every artifact
171/// reference on an event must resolve. Anything less is not evidence a replay
172/// may trust.
173fn verify_snapshot(snapshot: &JournalSnapshot) -> Result<()> {
174    // Head-hash re-verification (the clock is irrelevant to hashing).
175    Journal::from_snapshot(clone_snapshot(snapshot), Clock::System)?;
176    for (hash, value) in &snapshot.artifacts {
177        let actual = sha256_hex(&serde_json::to_vec(value)?);
178        if actual != *hash {
179            return Err(replay_error(format!(
180                "journal snapshot artifact integrity failure: artifact stored under {hash} \
181                 hashes to {actual} — the snapshot was tampered with or corrupted"
182            )));
183        }
184    }
185    for event in &snapshot.events {
186        for payload in [&event.input, &event.output].into_iter().flatten() {
187            if let PayloadRef::Artifact(reference) = payload {
188                if !snapshot.artifacts.contains_key(&reference.sha256) {
189                    return Err(replay_error(format!(
190                        "journal snapshot is truncated: event {} references artifact {} which \
191                         is absent from the artifact map",
192                        event.id, reference.sha256
193                    )));
194                }
195            }
196        }
197    }
198    Ok(())
199}
200
201// `JournalSnapshot` is `Clone`; the explicit helper documents that the clone
202// exists only because `Journal::from_snapshot` consumes.
203fn clone_snapshot(snapshot: &JournalSnapshot) -> JournalSnapshot {
204    snapshot.clone()
205}
206
207fn resolve_in(snapshot: &JournalSnapshot, payload: &PayloadRef) -> Option<Value> {
208    match payload {
209        PayloadRef::Inline(value) => Some(value.clone()),
210        PayloadRef::Artifact(reference) => snapshot.artifacts.get(&reference.sha256).cloned(),
211    }
212}
213
214fn resolve_opt(snapshot: &JournalSnapshot, payload: Option<&PayloadRef>) -> Option<Value> {
215    payload.and_then(|payload| resolve_in(snapshot, payload))
216}
217
218/// Canonical-content hash of a request value: `sha256` of its `serde_json`
219/// serialization (map keys sort deterministically, so equal values hash
220/// equal). This is the same computation [`PayloadRef::content_hash`] applies
221/// to inline payloads, which is what makes journaled requests matchable.
222fn request_hash(request: &Value) -> Result<String> {
223    Ok(sha256_hex(&serde_json::to_vec(request)?))
224}
225
226/// A recorded effect served during exact replay: the matching journaled event
227/// plus its resolved input/output payloads (artifact references already
228/// looked through).
229#[derive(Debug, Clone)]
230pub struct ServedEffect {
231    /// The journaled event that matched the issued request.
232    pub event: RunEvent,
233
234    /// The event's resolved input payload (the journaled request).
235    pub input: Option<Value>,
236
237    /// The event's resolved output payload (the journaled response).
238    pub output: Option<Value>,
239}
240
241impl ServedEffect {
242    /// Re-journal this served effect into the replay run's journal under
243    /// causal parent `parent`, returning the new event id.
244    ///
245    /// The replayed event reproduces the recorded one exactly — kind, node,
246    /// effect class, payloads, latency, tokens, cost, status — while the
247    /// journal assigns the replay run's own `seq` and `recorded_at` (which
248    /// match the recorded run's when the determinism seams are aligned).
249    pub fn rejournal(&self, journal: &Journal, parent: impl Into<String>) -> String {
250        let mut draft = EventDraft::new(self.event.kind, self.event.effect)
251            .status(self.event.status)
252            .parent(parent);
253        if let Some(node) = &self.event.node_id {
254            draft = draft.node(node.clone());
255        }
256        if let Some(input) = &self.input {
257            draft = draft.input(input.clone());
258        }
259        if let Some(output) = &self.output {
260            draft = draft.output(output.clone());
261        }
262        if let Some(latency_ms) = self.event.latency_ms {
263            draft = draft.latency_ms(latency_ms);
264        }
265        if let Some(tokens) = self.event.tokens {
266            draft = draft.tokens(tokens);
267        }
268        if let Some(cost_usd) = self.event.cost_usd {
269            draft = draft.cost_usd(cost_usd);
270        }
271        journal.record(draft)
272    }
273}
274
275#[derive(Debug)]
276struct ReplaySourceInner {
277    /// The snapshot's servable events, in `seq` order.
278    servable: Vec<RunEvent>,
279    /// The snapshot's artifact map, for payload resolution.
280    artifacts: BTreeMap<String, Value>,
281    /// Position of the next unserved event. Strictly ordered: exact replay
282    /// answers effects in journaled sequence order or not at all.
283    cursor: usize,
284}
285
286/// The serving side of exact replay: an ordered cursor over a journaled run's
287/// recorded effects.
288///
289/// Cheap to clone (shared cursor behind one `Arc`), so every replaying
290/// wrapper in a graph can hold a handle. Matching is by **sequence + request
291/// hash**: [`ReplaySource::serve`] takes the next unserved servable event in
292/// `seq` order, requires its kind to match, and requires the issued request's
293/// canonical hash to equal the journaled request's. Anything else fails
294/// loudly with [`RustyError::Replay`] — a replay that has drifted from its
295/// evidence must stop, not improvise.
296#[derive(Debug, Clone)]
297pub struct ReplaySource {
298    inner: Arc<Mutex<ReplaySourceInner>>,
299}
300
301impl ReplaySource {
302    /// A source over the servable events of `snapshot`. The snapshot is
303    /// assumed already integrity-verified (via [`ExactReplay::new`] or
304    /// [`ReplayFixture::import`]); unresolved artifact references simply fail
305    /// to match.
306    pub fn new(snapshot: &JournalSnapshot) -> Self {
307        Self {
308            inner: Arc::new(Mutex::new(ReplaySourceInner {
309                servable: snapshot
310                    .events
311                    .iter()
312                    .filter(|event| is_servable(event.kind))
313                    .cloned()
314                    .collect(),
315                artifacts: snapshot.artifacts.clone(),
316                cursor: 0,
317            })),
318        }
319    }
320
321    fn lock(&self) -> MutexGuard<'_, ReplaySourceInner> {
322        // Poison means a serving path panicked mid-read; the cursor value is
323        // plain data and stays coherent, so recovering is safe.
324        self.inner.lock().unwrap_or_else(|e| e.into_inner())
325    }
326
327    /// Serve one outbound effect from the journal.
328    ///
329    /// `kind` must be a servable effect kind (model, tool, remote, or WASM
330    /// call); `request` is the canonical request value (see
331    /// [`model_call_request`] / [`tool_call_request`]). On success the cursor
332    /// advances past the matched event and the returned [`ServedEffect`
333    /// carries the recorded response. On any mismatch — wrong kind at the
334    /// cursor, different request hash, no events left — a descriptive
335    /// [`RustyError::Replay`] names the sequence position and both sides of
336    /// the comparison.
337    pub fn serve(&self, kind: RunEventKind, request: &Value) -> Result<ServedEffect> {
338        if !is_servable(kind) {
339            return Err(replay_error(format!(
340                "{kind:?} is not a servable effect kind; exact replay serves only {SERVABLE_KINDS:?} \
341                 — executor-derived events are reproduced by re-execution, not served"
342            )));
343        }
344        let issued_hash = request_hash(request)?;
345        let mut inner = self.lock();
346        let Some(event) = inner.servable.get(inner.cursor) else {
347            return Err(replay_error(format!(
348                "replay exhaustion: the run issued a {kind:?} call (request hash {issued_hash}), \
349                 but the journal holds no further recorded effects to serve — the replayed run \
350                 is doing more work than the recorded one"
351            )));
352        };
353        if event.kind != kind {
354            return Err(replay_error(format!(
355                "replay order violation at recorded seq {}: the run issued a {kind:?} call, but \
356                 the next unserved recorded effect is {:?} — exact replay requires effects in \
357                 journaled sequence order",
358                event.seq, event.kind
359            )));
360        }
361        let recorded_input = event
362            .input
363            .as_ref()
364            .and_then(|payload| match payload {
365                PayloadRef::Inline(value) => Some(value.clone()),
366                PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
367            })
368            .ok_or_else(|| {
369                replay_error(format!(
370                    "recorded {:?} at seq {} has no input payload to match against",
371                    event.kind, event.seq
372                ))
373            })?;
374        let recorded_hash = request_hash(&recorded_input)?;
375        if recorded_hash != issued_hash {
376            return Err(replay_error(format!(
377                "replay divergence at recorded seq {} ({:?}): the run issued a request whose \
378                 canonical hash is {issued_hash}, but the journaled request hashes to \
379                 {recorded_hash} — the replayed run has diverged from its evidence",
380                event.seq, event.kind
381            )));
382        }
383        let served = ServedEffect {
384            input: Some(recorded_input),
385            output: event.output.as_ref().and_then(|payload| match payload {
386                PayloadRef::Inline(value) => Some(value.clone()),
387                PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
388            }),
389            event: event.clone(),
390        };
391        inner.cursor += 1;
392        Ok(served)
393    }
394
395    /// `true` when every recorded effect has been served. Verification treats
396    /// a non-exhausted source as a replay that stopped short of its evidence.
397    pub fn is_exhausted(&self) -> bool {
398        let inner = self.lock();
399        inner.cursor == inner.servable.len()
400    }
401
402    /// The `(seq, kind)` pairs still waiting to be served, in order. Empty
403    /// when [`ReplaySource::is_exhausted`] is `true`.
404    pub fn remaining(&self) -> Vec<(u64, RunEventKind)> {
405        let inner = self.lock();
406        inner.servable[inner.cursor..]
407            .iter()
408            .map(|event| (event.seq, event.kind))
409            .collect()
410    }
411}
412
413/// A [`ChatModel`] that records every call into the run's journal in the
414/// canonical replay-compatible shape, then returns the real response.
415///
416/// Construct per node invocation (it is a cheap handle): the causal parent is
417/// the invocation's node-input event id, delivered to node code via
418/// [`crate::journal::PARENT_EVENT_KEY`]. This is the supported way to produce
419/// journals that [`ReplayingChatModel`] can later replay — hand-rolled
420/// journaling must reproduce the same canonical shapes to be replayable.
421pub struct RecordingChatModel {
422    inner: Arc<dyn ChatModel>,
423    journal: Journal,
424    parent: String,
425    node_id: Option<String>,
426}
427
428impl RecordingChatModel {
429    /// A recording wrapper around `inner`, journaling into `journal` with
430    /// causal parent `parent` (the current invocation's node-input event id).
431    pub fn new(inner: Arc<dyn ChatModel>, journal: Journal, parent: impl Into<String>) -> Self {
432        Self {
433            inner,
434            journal,
435            parent: parent.into(),
436            node_id: None,
437        }
438    }
439
440    /// Builder-style: the node this model call is about (recorded as the
441    /// event's `node_id`).
442    pub fn node(mut self, node_id: impl Into<String>) -> Self {
443        self.node_id = Some(node_id.into());
444        self
445    }
446
447    fn draft(&self) -> EventDraft {
448        let mut draft = EventDraft::new(RunEventKind::ModelCall, self.inner.effect())
449            .parent(self.parent.clone());
450        if let Some(node) = &self.node_id {
451            draft = draft.node(node.clone());
452        }
453        draft
454    }
455}
456
457#[async_trait]
458impl ChatModel for RecordingChatModel {
459    async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
460        let started = self.journal.clock().now();
461        let result = self.inner.chat(messages, tools).await;
462        let latency_ms = (self.journal.clock().now() - started)
463            .num_milliseconds()
464            .max(0) as u64;
465        let draft = self
466            .draft()
467            .input(model_call_request(messages, tools))
468            .latency_ms(latency_ms);
469        match result {
470            Ok(response) => {
471                let mut draft = draft.output(model_call_response(&response));
472                if let Some(usage) = response.usage {
473                    draft = draft.tokens(usage);
474                }
475                self.journal.record(draft);
476                Ok(response)
477            }
478            Err(error) => {
479                self.journal.record(
480                    draft
481                        .status(EventStatus::Error)
482                        .output(json!({ "error": error.to_string() })),
483                );
484                Err(error)
485            }
486        }
487    }
488
489    fn effect(&self) -> Effect {
490        self.inner.effect()
491    }
492}
493
494/// A [`ChatModel`] that answers every call from a recorded journal instead of
495/// executing it.
496///
497/// The wrapped implementation is carried for its identity ([`ChatModel::effect`])
498/// and is **never invoked**: there is no code path from `chat` to `inner`, so
499/// replaying with a panic-on-call sentinel (or a client with no credentials
500/// and no network) is safe. Each call is matched against the journal by
501/// sequence + request hash and answered with the recorded response; the
502/// served event is re-journaled into the replay run's journal so the
503/// replayed evidence reproduces the recorded evidence byte-for-byte.
504pub struct ReplayingChatModel {
505    inner: Arc<dyn ChatModel>,
506    source: ReplaySource,
507    journal: Journal,
508    parent: String,
509}
510
511impl ReplayingChatModel {
512    /// A replaying wrapper around `inner` (never invoked), serving from
513    /// `source` and re-journaling into `journal` with causal parent `parent`
514    /// (the current invocation's node-input event id).
515    pub fn new(
516        inner: Arc<dyn ChatModel>,
517        source: ReplaySource,
518        journal: Journal,
519        parent: impl Into<String>,
520    ) -> Self {
521        Self {
522            inner,
523            source,
524            journal,
525            parent: parent.into(),
526        }
527    }
528}
529
530#[async_trait]
531impl ChatModel for ReplayingChatModel {
532    async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
533        let served = self.source.serve(
534            RunEventKind::ModelCall,
535            &model_call_request(messages, tools),
536        )?;
537        // Clock-read parity with RecordingChatModel (two reads per call)
538        // keeps the logical clock's tick sequence aligned with the recorded
539        // run, so the replayed journal's timestamps reproduce it exactly.
540        let _started = self.journal.clock().now();
541        let _ended = self.journal.clock().now();
542        served.rejournal(&self.journal, self.parent.clone());
543        match served.event.status {
544            EventStatus::Ok => {
545                let output = served.output.as_ref().ok_or_else(|| {
546                    replay_error(format!(
547                        "recorded model call at seq {} succeeded but carries no output payload",
548                        served.event.seq
549                    ))
550                })?;
551                chat_response_from_recorded(served.event.seq, output)
552            }
553            EventStatus::Error => Err(RustyError::Llm(format!(
554                "replayed model call at seq {} failed as recorded: {}",
555                served.event.seq,
556                recorded_error_text(&served)
557            ))),
558            EventStatus::Interrupted => Err(replay_error(format!(
559                "recorded model call at seq {} has status `interrupted`, which model calls \
560                 never produce — the journal is inconsistent",
561                served.event.seq
562            ))),
563        }
564    }
565
566    fn effect(&self) -> Effect {
567        self.inner.effect()
568    }
569}
570
571/// A [`Tool`] that records every call into the run's journal in the canonical
572/// replay-compatible shape, then returns the real result.
573///
574/// Construct per node invocation like [`RecordingChatModel`]; identity
575/// (name, description, schema, effect class) delegates to the wrapped tool.
576pub struct RecordingTool {
577    inner: Arc<dyn Tool>,
578    journal: Journal,
579    parent: String,
580    node_id: Option<String>,
581}
582
583impl RecordingTool {
584    /// A recording wrapper around `inner`, journaling into `journal` with
585    /// causal parent `parent` (the current invocation's node-input event id).
586    pub fn new(inner: Arc<dyn Tool>, journal: Journal, parent: impl Into<String>) -> Self {
587        Self {
588            inner,
589            journal,
590            parent: parent.into(),
591            node_id: None,
592        }
593    }
594
595    /// Builder-style: the node this tool call is about (recorded as the
596    /// event's `node_id`).
597    pub fn node(mut self, node_id: impl Into<String>) -> Self {
598        self.node_id = Some(node_id.into());
599        self
600    }
601}
602
603#[async_trait]
604impl Tool for RecordingTool {
605    fn name(&self) -> &str {
606        self.inner.name()
607    }
608
609    fn description(&self) -> &str {
610        self.inner.description()
611    }
612
613    fn parameters_schema(&self) -> Value {
614        self.inner.parameters_schema()
615    }
616
617    fn effect(&self) -> Effect {
618        self.inner.effect()
619    }
620
621    fn effect_kind(&self) -> &str {
622        self.inner.effect_kind()
623    }
624
625    fn idempotency_key(&self, args: &Value) -> Option<String> {
626        self.inner.idempotency_key(args)
627    }
628
629    fn effect_request(&self, call: &ToolCall) -> EffectRequest {
630        self.inner.effect_request(call)
631    }
632
633    async fn call(&self, args: Value) -> Result<Value> {
634        let started = self.journal.clock().now();
635        let result = self.inner.call(args.clone()).await;
636        let latency_ms = (self.journal.clock().now() - started)
637            .num_milliseconds()
638            .max(0) as u64;
639        let mut draft = EventDraft::new(RunEventKind::ToolCall, self.inner.effect())
640            .input(tool_call_request(self.inner.name(), &args))
641            .latency_ms(latency_ms)
642            .parent(self.parent.clone());
643        if let Some(node) = &self.node_id {
644            draft = draft.node(node.clone());
645        }
646        match result {
647            Ok(value) => {
648                self.journal.record(draft.output(value.clone()));
649                Ok(value)
650            }
651            Err(error) => {
652                self.journal.record(
653                    draft
654                        .status(EventStatus::Error)
655                        .output(json!({ "error": error.to_string() })),
656                );
657                Err(error)
658            }
659        }
660    }
661}
662
663/// A [`Tool`] that answers every call from a recorded journal instead of
664/// executing it.
665///
666/// Identity (name, description, schema, effect class) delegates to the
667/// wrapped tool, which is **never invoked** — the replaying analogue of
668/// [`RecordingTool`]. See [`ReplayingChatModel`] for the matching and
669/// zero-outbound guarantees.
670pub struct ReplayingTool {
671    inner: Arc<dyn Tool>,
672    source: ReplaySource,
673    journal: Journal,
674    parent: String,
675}
676
677impl ReplayingTool {
678    /// A replaying wrapper around `inner` (never invoked), serving from
679    /// `source` and re-journaling into `journal` with causal parent `parent`.
680    pub fn new(
681        inner: Arc<dyn Tool>,
682        source: ReplaySource,
683        journal: Journal,
684        parent: impl Into<String>,
685    ) -> Self {
686        Self {
687            inner,
688            source,
689            journal,
690            parent: parent.into(),
691        }
692    }
693}
694
695#[async_trait]
696impl Tool for ReplayingTool {
697    fn name(&self) -> &str {
698        self.inner.name()
699    }
700
701    fn description(&self) -> &str {
702        self.inner.description()
703    }
704
705    fn parameters_schema(&self) -> Value {
706        self.inner.parameters_schema()
707    }
708
709    fn effect(&self) -> Effect {
710        self.inner.effect()
711    }
712
713    fn effect_kind(&self) -> &str {
714        self.inner.effect_kind()
715    }
716
717    fn idempotency_key(&self, args: &Value) -> Option<String> {
718        self.inner.idempotency_key(args)
719    }
720
721    fn effect_request(&self, call: &ToolCall) -> EffectRequest {
722        self.inner.effect_request(call)
723    }
724
725    async fn call(&self, args: Value) -> Result<Value> {
726        let request = tool_call_request(self.inner.name(), &args);
727        let served = self.source.serve(RunEventKind::ToolCall, &request)?;
728        // Clock-read parity with RecordingTool; see ReplayingChatModel.
729        let _started = self.journal.clock().now();
730        let _ended = self.journal.clock().now();
731        served.rejournal(&self.journal, self.parent.clone());
732        match served.event.status {
733            EventStatus::Ok => served.output.clone().ok_or_else(|| {
734                replay_error(format!(
735                    "recorded tool call at seq {} succeeded but carries no output payload",
736                    served.event.seq
737                ))
738            }),
739            EventStatus::Error => Err(RustyError::Tool(format!(
740                "replayed tool call at seq {} failed as recorded: {}",
741                served.event.seq,
742                recorded_error_text(&served)
743            ))),
744            EventStatus::Interrupted => Err(replay_error(format!(
745                "recorded tool call at seq {} has status `interrupted`, which tool calls \
746                 never produce — the journal is inconsistent",
747                served.event.seq
748            ))),
749        }
750    }
751}
752
753/// An exact-replay session over one integrity-verified journal snapshot.
754///
755/// Construction re-verifies the snapshot (chained head hash, artifact
756/// integrity, reference resolution), so a tampered or truncated journal is
757/// rejected at the boundary. The typical flow:
758///
759/// 1. `let replay = ExactReplay::new(snapshot)?;`
760/// 2. `let journal = replay.fresh_journal(clock)` — a fresh journal with the
761///    recorded run's identity and a matching logical clock;
762/// 3. build the same graph topology with [`ReplayingChatModel`] /
763///    [`ReplayingTool`] wrappers sourcing from `replay.source()` and
764///    re-journaling into that journal;
765/// 4. `replay.run_and_verify(&graph, &spec, initial_state, params).await` —
766///    drives the run and checks the replayed journal reproduces the recorded
767///    one event-for-event.
768#[derive(Debug)]
769pub struct ExactReplay {
770    snapshot: JournalSnapshot,
771    source: ReplaySource,
772}
773
774impl ExactReplay {
775    /// Start an exact-replay session over `snapshot`.
776    ///
777    /// Fails with [`RustyError::Replay`] when the snapshot fails integrity
778    /// verification (tampered head hash, corrupted artifacts, dangling
779    /// references), or with [`RustyError::Serialization`] when an event
780    /// cannot be re-hashed. Exact replay of *resumed* runs (journals whose
781    /// first event is a resume) is deferred: their evidence begins mid-run
782    /// against checkpointed state the journal does not carry.
783    pub fn new(snapshot: JournalSnapshot) -> Result<Self> {
784        verify_snapshot(&snapshot)?;
785        if snapshot
786            .events
787            .first()
788            .is_some_and(|event| event.kind == RunEventKind::Resume)
789        {
790            return Err(replay_error(
791                "exact replay of resumed runs is not supported: the journal begins with a \
792                 resume event, whose pre-resume state lives in a checkpoint the journal does \
793                 not carry — replay the original run's journal instead",
794            ));
795        }
796        let source = ReplaySource::new(&snapshot);
797        Ok(Self { snapshot, source })
798    }
799
800    /// The recorded snapshot this session replays.
801    pub fn snapshot(&self) -> &JournalSnapshot {
802        &self.snapshot
803    }
804
805    /// The serving cursor over the recorded effects, for building replaying
806    /// wrappers. Shared: every wrapper in the replayed graph must serve from
807    /// this one cursor.
808    pub fn source(&self) -> ReplaySource {
809        self.source.clone()
810    }
811
812    /// A fresh journal with the recorded run's identity (`run_id`,
813    /// `thread_id`), timestamping from `clock`. For byte-identical replay,
814    /// `clock` must be a logical clock with the recorded run's parameters
815    /// (carried in [`ReplayFixture`] metadata).
816    pub fn fresh_journal(&self, clock: Clock) -> Journal {
817        Journal::new(
818            self.snapshot.run_id.clone(),
819            self.snapshot.thread_id.clone(),
820            clock,
821        )
822    }
823
824    /// Drive `graph` over `initial_state`, answering effects from the
825    /// recorded journal.
826    ///
827    /// `params.journal` must carry the recorded run's identity (build it with
828    /// [`ExactReplay::fresh_journal`]); a mismatched journal is rejected
829    /// rather than silently producing evidence under the wrong run. The
830    /// replayed run's outcome and its journal snapshot are returned for
831    /// inspection or [`ExactReplay::verify`].
832    pub async fn run(
833        &self,
834        graph: &Graph,
835        spec: &StateSpec,
836        initial_state: State,
837        params: ReplayParams,
838    ) -> Result<ReplayOutcome> {
839        if params.journal.run_id() != self.snapshot.run_id
840            || params.journal.thread_id() != self.snapshot.thread_id
841        {
842            return Err(replay_error(format!(
843                "replay journal identity mismatch: the snapshot records run `{}` on thread \
844                 `{}`, but the supplied journal is run `{}` on thread `{}` — build the replay \
845                 journal with ExactReplay::fresh_journal",
846                self.snapshot.run_id,
847                self.snapshot.thread_id,
848                params.journal.run_id(),
849                params.journal.thread_id()
850            )));
851        }
852        let executor = match &params.checkpointer {
853            Some(checkpointer) => Executor::with_checkpointer(checkpointer.clone()),
854            None => Executor::new(),
855        };
856        let config = RunConfig::new(self.snapshot.thread_id.clone())
857            .with_max_steps(params.max_steps)
858            .with_rng(params.rng.clone())
859            .with_journal(params.journal.clone());
860        let outcome = executor.run(graph, spec, initial_state, config).await?;
861        Ok(ReplayOutcome {
862            outcome,
863            journal: params.journal.snapshot(),
864        })
865    }
866
867    /// Verify a replayed journal against the recorded one: every recorded
868    /// effect must have been served, and the two snapshots must agree on run
869    /// identity, every event, every artifact, and the head hash. The first
870    /// disagreement fails with a descriptive [`RustyError::Replay`].
871    pub fn verify(&self, replayed: &JournalSnapshot) -> Result<()> {
872        if !self.source.is_exhausted() {
873            return Err(replay_error(format!(
874                "exact replay left recorded effects unserved: {:?} — the replayed run stopped \
875                 short of the journaled history",
876                self.source.remaining()
877            )));
878        }
879        let recorded = &self.snapshot;
880        if recorded.run_id != replayed.run_id || recorded.thread_id != replayed.thread_id {
881            return Err(replay_error(format!(
882                "replay identity mismatch: recorded run `{}`/thread `{}` vs replayed \
883                 `{}`/`{}`",
884                recorded.run_id, recorded.thread_id, replayed.run_id, replayed.thread_id
885            )));
886        }
887        if recorded.events.len() != replayed.events.len() {
888            return Err(replay_error(format!(
889                "replay event-count mismatch: the recorded journal holds {} events but the \
890                 replayed journal holds {}",
891                recorded.events.len(),
892                replayed.events.len()
893            )));
894        }
895        for (recorded_event, replayed_event) in recorded.events.iter().zip(&replayed.events) {
896            if recorded_event != replayed_event {
897                return Err(replay_error(format!(
898                    "replay event mismatch at seq {}: recorded {} but replayed {}",
899                    recorded_event.seq,
900                    summarize_event(recorded_event),
901                    summarize_event(replayed_event),
902                )));
903            }
904        }
905        if recorded.artifacts != replayed.artifacts {
906            return Err(replay_error(
907                "replay artifact mismatch: the replayed journal's artifact payloads differ \
908                 from the recorded ones",
909            ));
910        }
911        if recorded.head_hash != replayed.head_hash {
912            return Err(replay_error(format!(
913                "replay head-hash mismatch: recorded {} but replayed {} — event content \
914                 agrees but the evidence chain does not (this should be unreachable; please \
915                 report it)",
916                recorded.head_hash, replayed.head_hash
917            )));
918        }
919        Ok(())
920    }
921
922    /// [`ExactReplay::run`] plus [`ExactReplay::verify`]: the replay either
923    /// reproduces the recorded evidence exactly or fails with the first
924    /// disagreement.
925    pub async fn run_and_verify(
926        &self,
927        graph: &Graph,
928        spec: &StateSpec,
929        initial_state: State,
930        params: ReplayParams,
931    ) -> Result<ReplayOutcome> {
932        let replayed = self.run(graph, spec, initial_state, params).await?;
933        self.verify(&replayed.journal)?;
934        Ok(replayed)
935    }
936}
937
938/// One-line event summary for verification errors (kind, node, status —
939/// enough to locate the divergence without dumping payloads).
940fn summarize_event(event: &RunEvent) -> String {
941    format!(
942        "{:?}(node={:?}, status={:?})",
943        event.kind, event.node_id, event.status
944    )
945}
946
947/// Per-run parameters of an exact replay: the fresh journal (recorded run's
948/// identity plus matching logical clock), the RNG (the recorded run's seed
949/// for byte-identical checkpoint ids), an optional checkpointer (present iff
950/// the recorded run had one — checkpoint-written events are part of the
951/// evidence), and the step limit.
952pub struct ReplayParams {
953    /// The replay run's journal; build with [`ExactReplay::fresh_journal`].
954    pub journal: Journal,
955
956    /// The replay run's randomness source. [`RngSource::seeded`] with the
957    /// recorded run's seed reproduces checkpoint ids; anything else makes the
958    /// replayed journal diverge at the first checkpoint-written event.
959    pub rng: RngSource,
960
961    /// Persistence for the replay run. Must be `Some` when the recorded run
962    /// journaled checkpoint-written events, `None` when it did not.
963    pub checkpointer: Option<Arc<dyn Checkpointer>>,
964
965    /// Super-step limit, as in [`RunConfig::max_steps`].
966    pub max_steps: usize,
967}
968
969impl ReplayParams {
970    /// Parameters over `journal` and `rng` with no checkpointer and the
971    /// default step limit.
972    pub fn new(journal: Journal, rng: RngSource) -> Self {
973        Self {
974            journal,
975            rng,
976            checkpointer: None,
977            max_steps: DEFAULT_MAX_STEPS,
978        }
979    }
980
981    /// Builder-style: persist replay checkpoints through `checkpointer`.
982    pub fn with_checkpointer(mut self, checkpointer: Arc<dyn Checkpointer>) -> Self {
983        self.checkpointer = Some(checkpointer);
984        self
985    }
986
987    /// Builder-style: override the step limit.
988    pub fn with_max_steps(mut self, max_steps: usize) -> Self {
989        self.max_steps = max_steps;
990        self
991    }
992}
993
994/// The result of an exact replay: the run's outcome and the replayed journal
995/// (export form, comparable against the recorded snapshot).
996#[derive(Debug)]
997pub struct ReplayOutcome {
998    /// How the replayed run ended. Exact replay of an interrupted recorded
999    /// run ends in the same `Interrupted` outcome with the same payload.
1000    pub outcome: ExecutionOutcome,
1001
1002    /// The replayed run's journal snapshot. After
1003    /// [`ExactReplay::run_and_verify`], this is event-for-event identical to
1004    /// the recorded snapshot.
1005    pub journal: JournalSnapshot,
1006}
1007
1008/// Token and cost totals of one branch of a [`BranchDiff`]: the sum over all
1009/// of the branch's events (the `tokens` field is only ever set on model
1010/// calls, so this is the branch's model usage).
1011#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
1012pub struct BranchTotals {
1013    /// Number of events in the branch.
1014    pub events: usize,
1015
1016    /// Summed token usage across the branch's model calls.
1017    pub tokens: Usage,
1018
1019    /// Summed recorded cost in USD across the branch's events.
1020    pub cost_usd: f64,
1021}
1022
1023/// One channel's differing value at one super-step of a [`BranchDiff`].
1024/// `None` means the channel was absent from that branch's super-step-end
1025/// record at that step.
1026#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1027pub struct ChannelDiff {
1028    /// The state channel that differs.
1029    pub channel: String,
1030
1031    /// The post-reducer value in the base branch, when present.
1032    pub base: Option<Value>,
1033
1034    /// The post-reducer value in the branch, when present.
1035    pub branch: Option<Value>,
1036}
1037
1038/// The state-channel differences of one super-step between two branches.
1039#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1040pub struct StepDiff {
1041    /// The super-step index (from the super-step-start event's input).
1042    pub step: u64,
1043
1044    /// Channels whose post-reducer values differ at this step (presence or
1045    /// value), sorted by channel name.
1046    pub channels: Vec<ChannelDiff>,
1047}
1048
1049/// The structural diff between two journal snapshots — typically two branches
1050/// forked from a common point: one replayed an existing history, the other
1051/// continued differently.
1052///
1053/// Events are compared **logically**: kind, node, sequence, effect class,
1054/// resolved input/output payloads, latency, tokens, cost, and status must
1055/// match, while identity and timing fields (`id`, `run_id`, `thread_id`,
1056/// `parent`, `recorded_at`) are excluded — two branches are separate runs,
1057/// so their identity-bearing fields legitimately differ even when their
1058/// evidence is the same. The first logically-unequal sequence position is the
1059/// divergence point; everything after it is reported as removed (base) or
1060/// added (branch).
1061#[derive(Debug, Clone, Serialize, Deserialize)]
1062pub struct BranchDiff {
1063    /// The first sequence number where the branches' evidence differs, or
1064    /// `None` when they are logically identical.
1065    pub first_divergent_seq: Option<u64>,
1066
1067    /// Events present in the branch at and after the divergence point (the
1068    /// branch's new work relative to the base).
1069    pub added: Vec<RunEvent>,
1070
1071    /// Events present in the base at and after the divergence point (the work
1072    /// the branch replaced).
1073    pub removed: Vec<RunEvent>,
1074
1075    /// State-channel differences at each super-step, in step order. Only
1076    /// steps with at least one differing channel appear.
1077    pub step_diffs: Vec<StepDiff>,
1078
1079    /// Totals over the whole base journal.
1080    pub base_totals: BranchTotals,
1081
1082    /// Totals over the whole branch journal.
1083    pub branch_totals: BranchTotals,
1084}
1085
1086impl BranchDiff {
1087    /// Diff `branch` against `base`. When both snapshots share a common
1088    /// history (a fork), the shared prefix compares equal and the divergence
1089    /// point is where the fork's branches parted; `fork_seq`-style explicit
1090    /// fork points are unnecessary — the evidence carries the cut.
1091    pub fn between(base: &JournalSnapshot, branch: &JournalSnapshot) -> BranchDiff {
1092        let divergence = (0..base.events.len().max(branch.events.len()))
1093            .find(|&i| {
1094                match (base.events.get(i), branch.events.get(i)) {
1095                    (Some(a), Some(b)) => !logically_equal(a, base, b, branch),
1096                    // One side exhausted: divergence at the first missing seq.
1097                    _ => true,
1098                }
1099            })
1100            .map(|i| {
1101                base.events
1102                    .get(i)
1103                    .or_else(|| branch.events.get(i))
1104                    .map(|event| event.seq)
1105                    .unwrap_or(i as u64)
1106            });
1107        let (removed, added) = match divergence {
1108            Some(seq) => (
1109                base.events
1110                    .iter()
1111                    .filter(|event| event.seq >= seq)
1112                    .cloned()
1113                    .collect(),
1114                branch
1115                    .events
1116                    .iter()
1117                    .filter(|event| event.seq >= seq)
1118                    .cloned()
1119                    .collect(),
1120            ),
1121            None => (Vec::new(), Vec::new()),
1122        };
1123        BranchDiff {
1124            first_divergent_seq: divergence,
1125            added,
1126            removed,
1127            step_diffs: diff_steps(base, branch),
1128            base_totals: totals(base),
1129            branch_totals: totals(branch),
1130        }
1131    }
1132
1133    /// `true` when the two branches are logically identical (no divergence
1134    /// point, nothing added or removed).
1135    pub fn is_identical(&self) -> bool {
1136        self.first_divergent_seq.is_none()
1137    }
1138}
1139
1140/// Logical evidence equality, excluding identity and timing fields (see
1141/// [`BranchDiff`]). Payloads compare by resolved value, so an inline payload
1142/// and an artifact reference carrying the same bytes compare equal.
1143fn logically_equal(
1144    a: &RunEvent,
1145    snapshot_a: &JournalSnapshot,
1146    b: &RunEvent,
1147    snapshot_b: &JournalSnapshot,
1148) -> bool {
1149    a.seq == b.seq
1150        && a.kind == b.kind
1151        && a.node_id == b.node_id
1152        && a.effect == b.effect
1153        && a.latency_ms == b.latency_ms
1154        && a.tokens == b.tokens
1155        && a.cost_usd == b.cost_usd
1156        && a.status == b.status
1157        && resolve_opt(snapshot_a, a.input.as_ref()) == resolve_opt(snapshot_b, b.input.as_ref())
1158        && resolve_opt(snapshot_a, a.output.as_ref()) == resolve_opt(snapshot_b, b.output.as_ref())
1159}
1160
1161/// Sum token usage and cost over a snapshot's events.
1162fn totals(snapshot: &JournalSnapshot) -> BranchTotals {
1163    let mut totals = BranchTotals {
1164        events: snapshot.events.len(),
1165        ..BranchTotals::default()
1166    };
1167    for event in &snapshot.events {
1168        if let Some(usage) = event.tokens {
1169            totals.tokens.prompt_tokens = totals
1170                .tokens
1171                .prompt_tokens
1172                .saturating_add(usage.prompt_tokens);
1173            totals.tokens.completion_tokens = totals
1174                .tokens
1175                .completion_tokens
1176                .saturating_add(usage.completion_tokens);
1177            totals.tokens.total_tokens = totals
1178                .tokens
1179                .total_tokens
1180                .saturating_add(usage.total_tokens);
1181        }
1182        if let Some(cost) = event.cost_usd {
1183            totals.cost_usd += cost;
1184        }
1185    }
1186    totals
1187}
1188
1189/// The post-reducer channel values of every super-step in a snapshot, keyed
1190/// by step index: super-step-start events supply the index, the following
1191/// super-step-end event supplies the channel values.
1192fn step_channel_values(snapshot: &JournalSnapshot) -> BTreeMap<u64, BTreeMap<String, Value>> {
1193    let mut steps = BTreeMap::new();
1194    let mut current_step: Option<u64> = None;
1195    for event in &snapshot.events {
1196        match event.kind {
1197            RunEventKind::SuperStepStart => {
1198                current_step = resolve_opt(snapshot, event.input.as_ref())
1199                    .and_then(|input| input.get("step").and_then(Value::as_u64));
1200            }
1201            RunEventKind::SuperStepEnd => {
1202                if let Some(step) = current_step.take() {
1203                    let channels = resolve_opt(snapshot, event.output.as_ref())
1204                        .and_then(|output| output.as_object().cloned())
1205                        .unwrap_or_default()
1206                        .into_iter()
1207                        .collect();
1208                    steps.insert(step, channels);
1209                }
1210            }
1211            _ => {}
1212        }
1213    }
1214    steps
1215}
1216
1217/// Per-step channel diffs between two snapshots (only steps with at least one
1218/// differing channel, channels sorted).
1219fn diff_steps(base: &JournalSnapshot, branch: &JournalSnapshot) -> Vec<StepDiff> {
1220    let base_steps = step_channel_values(base);
1221    let branch_steps = step_channel_values(branch);
1222    let mut diffs = Vec::new();
1223    for step in base_steps
1224        .keys()
1225        .chain(branch_steps.keys())
1226        .copied()
1227        .collect::<std::collections::BTreeSet<u64>>()
1228    {
1229        let empty = BTreeMap::new();
1230        let base_channels = base_steps.get(&step).unwrap_or(&empty);
1231        let branch_channels = branch_steps.get(&step).unwrap_or(&empty);
1232        let mut channels = Vec::new();
1233        for channel in base_channels
1234            .keys()
1235            .chain(branch_channels.keys())
1236            .cloned()
1237            .collect::<std::collections::BTreeSet<String>>()
1238        {
1239            let base_value = base_channels.get(&channel).cloned();
1240            let branch_value = branch_channels.get(&channel).cloned();
1241            if base_value != branch_value {
1242                channels.push(ChannelDiff {
1243                    channel,
1244                    base: base_value,
1245                    branch: branch_value,
1246                });
1247            }
1248        }
1249        if !channels.is_empty() {
1250            diffs.push(StepDiff { step, channels });
1251        }
1252    }
1253    diffs
1254}
1255
1256/// Determinism parameters of a recorded run, carried in [`FixtureMetadata`]
1257/// so [`ReplayFixture::replay_in_ci`] can reproduce the run byte-for-byte.
1258#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1259pub struct LogicalClockParams {
1260    /// The logical clock's start (epoch millis), as in [`Clock::logical`].
1261    pub start_ms: u64,
1262
1263    /// The logical clock's per-read tick (millis), as in [`Clock::logical`].
1264    pub tick_ms: u64,
1265}
1266
1267/// Provenance and determinism metadata of a [`ReplayFixture`].
1268#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1269pub struct FixtureMetadata {
1270    /// Human-readable fixture name (used in error messages).
1271    pub name: String,
1272
1273    /// The recorded run's logical-clock parameters. Required for CI replay.
1274    #[serde(default, skip_serializing_if = "Option::is_none")]
1275    pub clock: Option<LogicalClockParams>,
1276
1277    /// The recorded run's RNG seed. Required for CI replay.
1278    #[serde(default, skip_serializing_if = "Option::is_none")]
1279    pub rng_seed: Option<u64>,
1280}
1281
1282/// A portable, self-contained exact-replay bundle: everything CI (or another
1283/// machine, or a later wave's server endpoint) needs to re-drive a recorded
1284/// run and prove it reproduces.
1285///
1286/// A fixture carries the graph's topology hash (replay refuses a structurally
1287/// different graph), the full journal snapshot (integrity-verified on
1288/// import), the recorded final checkpoint (replay compares the outcome state
1289/// against it), and the determinism parameters the run was recorded with.
1290/// Serialization is plain JSON; [`ReplayFixture::export`] /
1291/// [`ReplayFixture::import`] are the wire boundary.
1292#[derive(Debug, Clone, Serialize, Deserialize)]
1293pub struct ReplayFixture {
1294    /// Fixture envelope version; [`FIXTURE_FORMAT_VERSION`] for anything
1295    /// written now.
1296    pub format_version: u32,
1297
1298    /// SHA-256 of the recorded graph's topology (see
1299    /// [`Graph::topology_hash`]). Checked before replay: a different topology
1300    /// cannot reproduce the journaled evidence.
1301    pub graph_hash: String,
1302
1303    /// The application-declared graph version of the recorded run.
1304    pub graph_version: String,
1305
1306    /// The recorded run's complete journal.
1307    pub journal: JournalSnapshot,
1308
1309    /// The recorded run's final checkpoint, when it ran with a checkpointer.
1310    /// CI replay compares the replayed outcome state against it.
1311    pub final_checkpoint: Option<Checkpoint>,
1312
1313    /// Provenance and determinism parameters.
1314    pub metadata: FixtureMetadata,
1315}
1316
1317impl ReplayFixture {
1318    /// Capture a recorded run as a fixture. `graph` supplies the topology
1319    /// hash; `clock`/`rng_seed` are the determinism parameters the run was
1320    /// recorded with (both required for later CI replay).
1321    pub fn capture(
1322        name: impl Into<String>,
1323        graph: &Graph,
1324        graph_version: impl Into<String>,
1325        journal: JournalSnapshot,
1326        final_checkpoint: Option<Checkpoint>,
1327        clock: Option<LogicalClockParams>,
1328        rng_seed: Option<u64>,
1329    ) -> Self {
1330        Self {
1331            format_version: FIXTURE_FORMAT_VERSION,
1332            graph_hash: graph.topology_hash(),
1333            graph_version: graph_version.into(),
1334            journal,
1335            final_checkpoint,
1336            metadata: FixtureMetadata {
1337                name: name.into(),
1338                clock,
1339                rng_seed,
1340            },
1341        }
1342    }
1343
1344    /// Serialize to pretty-printed JSON (the checked-in / on-disk form).
1345    pub fn export(&self) -> Result<String> {
1346        Ok(serde_json::to_string_pretty(self)?)
1347    }
1348
1349    /// Parse a fixture from JSON, rejecting unsupported format versions and
1350    /// integrity failures (tampered or truncated journals) at the boundary.
1351    pub fn import(json: &str) -> Result<Self> {
1352        let fixture: Self = serde_json::from_str(json)?;
1353        if fixture.format_version != FIXTURE_FORMAT_VERSION {
1354            return Err(replay_error(format!(
1355                "unsupported fixture format version {} (this build supports \
1356                 {FIXTURE_FORMAT_VERSION}) — regenerate the fixture or upgrade the runtime",
1357                fixture.format_version
1358            )));
1359        }
1360        verify_snapshot(&fixture.journal)?;
1361        Ok(fixture)
1362    }
1363
1364    /// An exact-replay session over the fixture's journal (integrity already
1365    /// verified by [`ReplayFixture::import`] or construction; re-verified
1366    /// here so a hand-assembled fixture is checked too).
1367    pub fn exact_replay(&self) -> Result<ExactReplay> {
1368        ExactReplay::new(self.journal.clone())
1369    }
1370
1371    /// The replay parameters matching the fixture's determinism metadata: a
1372    /// fresh journal on the recorded identity and logical clock, the recorded
1373    /// RNG seed, and a fresh in-memory checkpointer (fixtures record runs
1374    /// that used one — the checkpoint-written events are part of the
1375    /// evidence).
1376    ///
1377    /// Fails with [`RustyError::Replay`] when the fixture lacks clock or seed
1378    /// metadata (a run recorded without determinism seams cannot be replayed
1379    /// byte-identically).
1380    pub fn replay_params(&self, replay: &ExactReplay) -> Result<ReplayParams> {
1381        let clock = self.metadata.clock.ok_or_else(|| {
1382            replay_error(format!(
1383                "fixture `{}` carries no logical-clock parameters; exact CI replay requires \
1384                 the recorded run's determinism seams",
1385                self.metadata.name
1386            ))
1387        })?;
1388        let seed = self.metadata.rng_seed.ok_or_else(|| {
1389            replay_error(format!(
1390                "fixture `{}` carries no RNG seed; exact CI replay requires the recorded \
1391                 run's determinism seams",
1392                self.metadata.name
1393            ))
1394        })?;
1395        let journal = replay.fresh_journal(Clock::logical(clock.start_ms, clock.tick_ms));
1396        Ok(ReplayParams::new(journal, RngSource::seeded(seed))
1397            .with_checkpointer(Arc::new(InMemoryCheckpointer::new())))
1398    }
1399
1400    /// Replay the fixture end to end and verify everything: the graph's
1401    /// topology matches the recorded one, the run reproduces the journaled
1402    /// evidence event-for-event, and the outcome state matches the recorded
1403    /// final checkpoint.
1404    ///
1405    /// The caller supplies the replay session (`fixture.exact_replay()?`), a
1406    /// graph built with replaying wrappers over `replay.source()`, and the
1407    /// parameters from `fixture.replay_params(&replay)?` — the same journal
1408    /// handle the wrappers re-journal into. This is the `replay_in_ci` entry
1409    /// point: one call from recorded artifact to verified replay.
1410    pub async fn replay_in_ci(
1411        &self,
1412        replay: ExactReplay,
1413        graph: &Graph,
1414        spec: &StateSpec,
1415        initial_state: State,
1416        params: ReplayParams,
1417    ) -> Result<ReplayOutcome> {
1418        let actual_hash = graph.topology_hash();
1419        if actual_hash != self.graph_hash {
1420            return Err(replay_error(format!(
1421                "fixture `{}` was recorded against graph topology {}, but the graph under \
1422                 replay hashes to {actual_hash} — rebuild the recorded topology (same node \
1423                 names, edges, and entry point)",
1424                self.metadata.name, self.graph_hash
1425            )));
1426        }
1427        let replayed = replay
1428            .run_and_verify(graph, spec, initial_state, params)
1429            .await?;
1430        if let Some(final_checkpoint) = &self.final_checkpoint {
1431            if replayed.outcome.state() != &final_checkpoint.state {
1432                return Err(replay_error(format!(
1433                    "fixture `{}`: the replayed final state diverges from the recorded final \
1434                     checkpoint (journal evidence agreed; state did not)",
1435                    self.metadata.name
1436                )));
1437            }
1438        }
1439        Ok(replayed)
1440    }
1441}
1442
1443#[cfg(test)]
1444mod tests {
1445    use super::*;
1446    use crate::journal::Journal;
1447    use serde_json::json;
1448
1449    fn snapshot_with(drafts: Vec<EventDraft>) -> JournalSnapshot {
1450        let journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
1451        for draft in drafts {
1452            journal.record(draft);
1453        }
1454        journal.snapshot()
1455    }
1456
1457    fn model_draft(request: Value, response: Value) -> EventDraft {
1458        EventDraft::new(RunEventKind::ModelCall, Effect::NonIdempotent)
1459            .node("agent")
1460            .input(request)
1461            .output(response)
1462            .latency_ms(2)
1463            .tokens(Usage {
1464                prompt_tokens: 10,
1465                completion_tokens: 5,
1466                total_tokens: 15,
1467            })
1468    }
1469
1470    #[test]
1471    fn canonical_shapes_have_stable_hashes() {
1472        let request = model_call_request(&[ChatMessage::user("ping")], &[]);
1473        assert_eq!(
1474            request,
1475            json!({"messages": [{"role": "user", "content": "ping"}], "tools": []})
1476        );
1477        let first = request_hash(&request).unwrap();
1478        let second = request_hash(&model_call_request(&[ChatMessage::user("ping")], &[])).unwrap();
1479        assert_eq!(first, second);
1480        // A different request hashes differently.
1481        let other = request_hash(&model_call_request(&[ChatMessage::user("pong")], &[])).unwrap();
1482        assert_ne!(first, other);
1483
1484        let tool = tool_call_request("echo", &json!({"text": "hi"}));
1485        assert_eq!(tool, json!({"tool": "echo", "arguments": {"text": "hi"}}));
1486    }
1487
1488    #[test]
1489    fn serve_matches_by_sequence_and_request_hash() {
1490        let snapshot = snapshot_with(vec![
1491            model_draft(
1492                model_call_request(&[ChatMessage::user("a")], &[]),
1493                json!({"r": 1}),
1494            ),
1495            model_draft(
1496                model_call_request(&[ChatMessage::user("b")], &[]),
1497                json!({"r": 2}),
1498            ),
1499        ]);
1500        let source = ReplaySource::new(&snapshot);
1501        assert!(!source.is_exhausted());
1502
1503        // Wrong request at the cursor: divergence, and the cursor must not advance.
1504        let error = source
1505            .serve(
1506                RunEventKind::ModelCall,
1507                &model_call_request(&[ChatMessage::user("zzz")], &[]),
1508            )
1509            .unwrap_err();
1510        assert!(matches!(error, RustyError::Replay(_)));
1511        assert!(error.to_string().contains("divergence"));
1512        assert_eq!(source.remaining().len(), 2);
1513
1514        // Wrong kind at the cursor: order violation.
1515        let error = source
1516            .serve(RunEventKind::ToolCall, &tool_call_request("t", &json!({})))
1517            .unwrap_err();
1518        assert!(error.to_string().contains("order violation"));
1519
1520        // Correct sequence serves in order and exhausts.
1521        let first = source
1522            .serve(
1523                RunEventKind::ModelCall,
1524                &model_call_request(&[ChatMessage::user("a")], &[]),
1525            )
1526            .unwrap();
1527        assert_eq!(first.event.seq, 0);
1528        assert_eq!(first.output, Some(json!({"r": 1})));
1529        let second = source
1530            .serve(
1531                RunEventKind::ModelCall,
1532                &model_call_request(&[ChatMessage::user("b")], &[]),
1533            )
1534            .unwrap();
1535        assert_eq!(second.event.seq, 1);
1536        assert!(source.is_exhausted());
1537
1538        // Serving past the end is a loud exhaustion error.
1539        let error = source
1540            .serve(
1541                RunEventKind::ModelCall,
1542                &model_call_request(&[ChatMessage::user("a")], &[]),
1543            )
1544            .unwrap_err();
1545        assert!(error.to_string().contains("exhaustion"));
1546    }
1547
1548    #[test]
1549    fn serve_rejects_non_servable_kinds() {
1550        let snapshot = snapshot_with(vec![EventDraft::new(
1551            RunEventKind::SuperStepStart,
1552            Effect::Pure,
1553        )]);
1554        let error = ReplaySource::new(&snapshot)
1555            .serve(RunEventKind::NodeInput, &json!({}))
1556            .unwrap_err();
1557        assert!(error.to_string().contains("not a servable"));
1558    }
1559
1560    #[test]
1561    fn artifact_backed_requests_match_and_resolve() {
1562        let big_request =
1563            json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1564        let big_response = json!({"message": {"role": "assistant", "content": "y".repeat(9000)}, "model": null, "usage": null});
1565        let snapshot = snapshot_with(vec![model_draft(big_request.clone(), big_response.clone())]);
1566        // The payloads were promoted to artifacts at record time.
1567        assert!(matches!(
1568            snapshot.events[0].input,
1569            Some(PayloadRef::Artifact(_))
1570        ));
1571        let served = ReplaySource::new(&snapshot)
1572            .serve(RunEventKind::ModelCall, &big_request)
1573            .unwrap();
1574        assert_eq!(served.output, Some(big_response));
1575    }
1576
1577    #[test]
1578    fn tampered_snapshots_are_rejected_at_the_boundary() {
1579        let snapshot = snapshot_with(vec![model_draft(
1580            model_call_request(&[ChatMessage::user("a")], &[]),
1581            json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1582        )]);
1583
1584        // Event tampering breaks the chained head hash.
1585        let mut tampered = snapshot.clone();
1586        tampered.events[0].status = EventStatus::Error;
1587        assert!(ExactReplay::new(tampered).is_err());
1588
1589        // Artifact tampering breaks content addressing (the head hash chains
1590        // over events, so this needs the dedicated integrity check).
1591        let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1592        let artifact_snapshot = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
1593        let mut tampered = artifact_snapshot;
1594        *tampered.artifacts.values_mut().next().unwrap() = json!({"forged": true});
1595        let error = ExactReplay::new(tampered).unwrap_err();
1596        assert!(error.to_string().contains("integrity"));
1597
1598        // A truncated snapshot (artifact removed) is rejected.
1599        let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1600        let truncated = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
1601        let mut truncated = truncated;
1602        truncated.artifacts.clear();
1603        assert!(ExactReplay::new(truncated).is_err());
1604    }
1605
1606    #[test]
1607    fn resumed_run_journals_are_rejected() {
1608        let snapshot = snapshot_with(vec![EventDraft::new(RunEventKind::Resume, Effect::Pure)
1609            .input(json!({"checkpoint_id": "c"}))]);
1610        let error = ExactReplay::new(snapshot).unwrap_err();
1611        assert!(error.to_string().contains("resumed"));
1612    }
1613
1614    #[test]
1615    fn rejournal_reproduces_the_recorded_event() {
1616        let recorded = snapshot_with(vec![model_draft(
1617            model_call_request(&[ChatMessage::user("a")], &[]),
1618            json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1619        )]);
1620        let source = ReplaySource::new(&recorded);
1621        let served = source
1622            .serve(
1623                RunEventKind::ModelCall,
1624                &model_call_request(&[ChatMessage::user("a")], &[]),
1625            )
1626            .unwrap();
1627        let replay_journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
1628        // Same draft sequence: one record read per event on the same clock.
1629        served.rejournal(&replay_journal, "run-t:parent");
1630        let replayed = replay_journal.snapshot();
1631        assert_eq!(replayed.events.len(), 1);
1632        // Identical except for the causal parent we supplied.
1633        let mut expected = recorded.events[0].clone();
1634        expected.parent = Some("run-t:parent".into());
1635        assert_eq!(replayed.events[0], expected);
1636    }
1637
1638    #[test]
1639    fn branch_diff_reports_divergence_steps_and_totals() {
1640        let shared = || {
1641            vec![
1642                EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1643                    .input(json!({"step": 0, "active_nodes": ["a"]})),
1644                EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1645                    .output(json!({"log": ["a"]})),
1646            ]
1647        };
1648        let mut base_drafts = shared();
1649        base_drafts.push(
1650            EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1651                .input(json!({"step": 1, "active_nodes": ["b"]})),
1652        );
1653        base_drafts.push(
1654            EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1655                .output(json!({"log": ["a", "b"]})),
1656        );
1657        base_drafts.push(
1658            model_draft(
1659                model_call_request(&[ChatMessage::user("q")], &[]),
1660                json!({"r": 1}),
1661            )
1662            .cost_usd(0.001),
1663        );
1664        let base = snapshot_with(base_drafts);
1665
1666        let mut branch_drafts = shared();
1667        branch_drafts.push(
1668            EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1669                .input(json!({"step": 1, "active_nodes": ["c"]})),
1670        );
1671        branch_drafts.push(
1672            EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1673                .output(json!({"log": ["a", "c"], "extra": 1})),
1674        );
1675        let branch = snapshot_with(branch_drafts);
1676
1677        let diff = BranchDiff::between(&base, &branch);
1678        assert!(!diff.is_identical());
1679        assert_eq!(diff.first_divergent_seq, Some(2));
1680        assert_eq!(diff.removed.len(), 3);
1681        assert_eq!(diff.added.len(), 2);
1682        assert_eq!(diff.base_totals.events, 5);
1683        assert_eq!(diff.branch_totals.events, 4);
1684        assert_eq!(diff.base_totals.tokens.total_tokens, 15);
1685        assert_eq!(diff.branch_totals.tokens.total_tokens, 0);
1686        assert_eq!(diff.base_totals.cost_usd, 0.001);
1687        assert_eq!(diff.branch_totals.cost_usd, 0.0);
1688
1689        // Step 0 is shared; step 1 differs on `log` and `extra`.
1690        assert_eq!(diff.step_diffs.len(), 1);
1691        let step = &diff.step_diffs[0];
1692        assert_eq!(step.step, 1);
1693        assert_eq!(step.channels.len(), 2);
1694        assert_eq!(step.channels[0].channel, "extra");
1695        assert_eq!(step.channels[0].base, None);
1696        assert_eq!(step.channels[0].branch, Some(json!(1)));
1697        assert_eq!(step.channels[1].channel, "log");
1698        assert_eq!(step.channels[1].base, Some(json!(["a", "b"])));
1699        assert_eq!(step.channels[1].branch, Some(json!(["a", "c"])));
1700    }
1701
1702    #[test]
1703    fn branch_diff_of_identical_snapshots_is_empty() {
1704        // Two separate runs of the same drafts: identity fields (ids,
1705        // timestamps) match here too, but the point is the logical comparison.
1706        let snapshot = snapshot_with(vec![EventDraft::new(
1707            RunEventKind::SuperStepStart,
1708            Effect::Pure,
1709        )
1710        .input(json!({"step": 0, "active_nodes": ["a"]}))]);
1711        let diff = BranchDiff::between(&snapshot, &snapshot);
1712        assert!(diff.is_identical());
1713        assert!(diff.added.is_empty() && diff.removed.is_empty());
1714        assert!(diff.step_diffs.is_empty());
1715    }
1716
1717    #[test]
1718    fn fixture_roundtrip_and_version_guard() {
1719        let journal = snapshot_with(vec![model_draft(
1720            model_call_request(&[ChatMessage::user("a")], &[]),
1721            json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1722        )]);
1723        let mut builder = crate::graph::GraphBuilder::new();
1724        builder.add_node("agent", |_ctx: crate::node::NodeContext| async {
1725            Ok(crate::node::NodeOutput::empty())
1726        });
1727        builder.set_entry_point("agent");
1728        let graph = builder.compile().unwrap();
1729        let fixture = ReplayFixture::capture(
1730            "roundtrip",
1731            &graph,
1732            "v1",
1733            journal,
1734            None,
1735            Some(LogicalClockParams {
1736                start_ms: 1_000_000,
1737                tick_ms: 5,
1738            }),
1739            Some(7),
1740        );
1741        let wire = fixture.export().unwrap();
1742        let imported = ReplayFixture::import(&wire).unwrap();
1743        assert_eq!(imported.export().unwrap(), wire);
1744
1745        // An unsupported format version is rejected with a descriptive error.
1746        let mut value: Value = serde_json::from_str(&wire).unwrap();
1747        value["format_version"] = json!(99);
1748        let error = ReplayFixture::import(&value.to_string()).unwrap_err();
1749        assert!(error.to_string().contains("format version 99"));
1750    }
1751}