Skip to main content

rusty_agent_runtime/
replay.rs

1//! The replay engine: exact replay, branch diff, and portable fixtures.
2//!
3//! This module is the second work item of the R0.5 Flight Recorder. Wave 1
4//! landed the evidence contracts ([`crate::record`]), the journal, and the
5//! determinism seams ([`Clock`], [`RngSource`]); this module consumes them to
6//! re-drive a recorded run **exactly**: model, tool, remote, and WASM calls
7//! are served from the journal instead of executed, and the replayed run's
8//! journal and final state must reproduce the recorded ones byte-for-byte.
9//!
10//! # The exact-replay contract
11//!
12//! Exact replay re-runs the *same* graph topology with the *same* determinism
13//! seams (logical clock, seeded RNG) and swaps only the effect
14//! implementations: wherever the recorded run talked to the world, the replay
15//! talks to the journal. Node bodies still execute — `Pure` computation is
16//! freely repeatable — but every outbound effect is matched against the
17//! journaled record **by sequence and request hash** and answered with the
18//! recorded response. Three failures are detected loudly:
19//!
20//! - **Divergence** — the replayed run issues a request whose canonical hash
21//!   differs from the journaled request at the next unserved sequence
22//!   position ([`RustyError::Replay`]).
23//! - **Order violation** — effects arrive in a different order or kind than
24//!   the journal recorded (exact replay serves effects in `seq` order only).
25//! - **Exhaustion / shortfall** — the replayed run issues more effects than
26//!   the journal holds, or finishes with recorded effects unserved
27//!   (verification fails).
28//!
29//! Interrupts need no serving: with every effect answered deterministically
30//! from the journal, node logic re-derives the same interrupt and the
31//! executor journals it identically.
32//!
33//! # Zero outbound calls
34//!
35//! [`ReplayingChatModel`] and [`ReplayingTool`] wrap the real implementations
36//! but **never invoke them** — there is no code path from `chat`/`call` to
37//! the wrapped value. Tests prove it by replaying with panic-on-call
38//! sentinels as the inner implementations. The wrappers exist so the *same
39//! graph code* runs in record and replay mode; only the effect
40//! implementations are swapped.
41//!
42//! # Determinism prerequisites
43//!
44//! Byte-identical replay requires the replay run to observe the same time and
45//! id sequences as the recorded run: attach a fresh journal with the recorded
46//! run's identity (`run_id`, `thread_id`) and a logical clock with the same
47//! parameters ([`ExactReplay::fresh_journal`]), and seed the RNG with the
48//! recorded run's seed ([`ReplayParams`]). [`ReplayFixture`] carries these
49//! parameters in its metadata so CI replay is self-contained. The recording
50//! and replaying wrappers perform the same number of journal-clock reads per
51//! call, keeping the tick sequence aligned; per the journal's determinism
52//! model, super-steps with several parallel nodes interleave clock reads by
53//! schedule, so byte-identical replay is guaranteed for runs whose steps run
54//! one node at a time.
55
56use std::collections::{BTreeMap, VecDeque};
57use std::sync::{Arc, Mutex, MutexGuard};
58
59use async_trait::async_trait;
60use serde::{Deserialize, Serialize};
61use serde_json::{json, Value};
62
63use crate::checkpoint::{Checkpoint, Checkpointer, InMemoryCheckpointer};
64use crate::effects::EffectRequest;
65use crate::error::{Result, RustyError};
66use crate::executor::{ExecutionOutcome, Executor, RunConfig, DEFAULT_MAX_STEPS};
67use crate::graph::Graph;
68use crate::journal::{Clock, EventDraft, Journal, JournalSnapshot, RngSource};
69use crate::llm::{ChatMessage, ChatModel, ChatResponse, ToolCall, Usage};
70use crate::record::{sha256_hex, Effect, EventStatus, PayloadRef, RunEvent, RunEventKind};
71use crate::state::{State, StateSpec};
72use crate::tool::Tool;
73
74/// The current on-disk format version of [`ReplayFixture`].
75///
76/// Bump only on a breaking change to the fixture envelope; additive evolution
77/// uses serde defaults so previously written fixtures keep deserializing.
78pub const FIXTURE_FORMAT_VERSION: u32 = 1;
79
80/// The effect kinds exact replay can serve from the journal: outbound calls
81/// whose recorded responses replace live execution. Super-step boundaries,
82/// node inputs/outputs, routing decisions, interrupts, and checkpoint writes
83/// are *re-derived* by the executor, never served.
84///
85/// Public so the digital twin (`crate::twin`) decomposes a recorded run into
86/// the same effect set replay serves — one servable-kind vocabulary, shared
87/// by replay and the twin, rather than two lists that can drift.
88pub const SERVABLE_KINDS: [RunEventKind; 4] = [
89    RunEventKind::ModelCall,
90    RunEventKind::ToolCall,
91    RunEventKind::RemoteCall,
92    RunEventKind::WasmCall,
93];
94
95fn is_servable(kind: RunEventKind) -> bool {
96    SERVABLE_KINDS.contains(&kind)
97}
98
99fn replay_error(message: impl Into<String>) -> RustyError {
100    RustyError::Replay(message.into())
101}
102
103/// The canonical JSON shape a [`RecordingChatModel`] journals (and a
104/// [`ReplayingChatModel`] matches) as a model call's **request**: the
105/// messages and tool schemas exactly as passed to [`ChatModel::chat`].
106///
107/// The hash of this value's canonical serialization is the request identity
108/// exact replay matches on — node code that journals model calls by hand must
109/// use this shape for its journals to be replayable.
110pub fn model_call_request(messages: &[ChatMessage], tools: &[Value]) -> Value {
111    json!({ "messages": messages, "tools": tools })
112}
113
114/// The canonical JSON shape a [`RecordingChatModel`] journals (and a
115/// [`ReplayingChatModel`] reconstructs a [`ChatResponse`] from) as a model
116/// call's **response**: the assistant message, the reported model identity,
117/// and token usage. Keys are always present (`null` when unreported) so the
118/// shape is stable.
119pub fn model_call_response(response: &ChatResponse) -> Value {
120    json!({
121        "message": response.message,
122        "model": response.model,
123        "usage": response.usage,
124    })
125}
126
127/// The canonical JSON shape a [`RecordingTool`] journals (and a
128/// [`ReplayingTool`] matches) as a tool call's **request**: the tool name and
129/// the model-supplied arguments. The recorded **output** is the tool's result
130/// value verbatim.
131pub fn tool_call_request(name: &str, arguments: &Value) -> Value {
132    json!({ "tool": name, "arguments": arguments })
133}
134
135/// Reconstruct a [`ChatResponse`] from a journaled [`model_call_response`]
136/// payload.
137fn chat_response_from_recorded(seq: u64, value: &Value) -> Result<ChatResponse> {
138    let decode = |key: &str| -> Result<Value> {
139        value.get(key).cloned().ok_or_else(|| {
140            replay_error(format!(
141                "recorded model call at seq {seq} is malformed: output payload has no `{key}` key"
142            ))
143        })
144    };
145    let message: ChatMessage = serde_json::from_value(decode("message")?)?;
146    let model: Option<String> = serde_json::from_value(decode("model")?)?;
147    let usage: Option<Usage> = serde_json::from_value(decode("usage")?)?;
148    Ok(ChatResponse {
149        message,
150        model,
151        usage,
152    })
153}
154
155/// The recorded error text of a failed served effect (`{"error": …}` output
156/// payloads, as written by the recording wrappers and the executor's
157/// node-failure path).
158fn recorded_error_text(served: &ServedEffect) -> String {
159    served
160        .output
161        .as_ref()
162        .and_then(|value| value.get("error"))
163        .and_then(Value::as_str)
164        .unwrap_or("<no recorded error payload>")
165        .to_owned()
166}
167
168/// Verify the integrity of a journal snapshot: the chained head hash must
169/// recompute from the events (as [`Journal::from_snapshot`] enforces), every
170/// content-addressed artifact must hash to its own key, and every artifact
171/// reference on an event must resolve. Anything less is not evidence a replay
172/// may trust.
173fn verify_snapshot(snapshot: &JournalSnapshot) -> Result<()> {
174    // Head-hash re-verification (the clock is irrelevant to hashing).
175    Journal::from_snapshot(clone_snapshot(snapshot), Clock::System)?;
176    for (hash, value) in &snapshot.artifacts {
177        let actual = sha256_hex(&serde_json::to_vec(value)?);
178        if actual != *hash {
179            return Err(replay_error(format!(
180                "journal snapshot artifact integrity failure: artifact stored under {hash} \
181                 hashes to {actual} — the snapshot was tampered with or corrupted"
182            )));
183        }
184    }
185    for event in &snapshot.events {
186        for payload in [&event.input, &event.output].into_iter().flatten() {
187            if let PayloadRef::Artifact(reference) = payload {
188                if !snapshot.artifacts.contains_key(&reference.sha256) {
189                    return Err(replay_error(format!(
190                        "journal snapshot is truncated: event {} references artifact {} which \
191                         is absent from the artifact map",
192                        event.id, reference.sha256
193                    )));
194                }
195            }
196        }
197    }
198    Ok(())
199}
200
201// `JournalSnapshot` is `Clone`; the explicit helper documents that the clone
202// exists only because `Journal::from_snapshot` consumes.
203fn clone_snapshot(snapshot: &JournalSnapshot) -> JournalSnapshot {
204    snapshot.clone()
205}
206
207fn resolve_in(snapshot: &JournalSnapshot, payload: &PayloadRef) -> Option<Value> {
208    match payload {
209        PayloadRef::Inline(value) => Some(value.clone()),
210        PayloadRef::Artifact(reference) => snapshot.artifacts.get(&reference.sha256).cloned(),
211    }
212}
213
214fn resolve_opt(snapshot: &JournalSnapshot, payload: Option<&PayloadRef>) -> Option<Value> {
215    payload.and_then(|payload| resolve_in(snapshot, payload))
216}
217
218/// Canonical-content hash of a request value: `sha256` of its `serde_json`
219/// serialization (map keys sort deterministically, so equal values hash
220/// equal). This is the same computation [`PayloadRef::content_hash`] applies
221/// to inline payloads, which is what makes journaled requests matchable.
222fn request_hash(request: &Value) -> Result<String> {
223    Ok(sha256_hex(&serde_json::to_vec(request)?))
224}
225
226/// A recorded effect served during exact replay: the matching journaled event
227/// plus its resolved input/output payloads (artifact references already
228/// looked through).
229#[derive(Debug, Clone)]
230pub struct ServedEffect {
231    /// The journaled event that matched the issued request.
232    pub event: RunEvent,
233
234    /// The event's resolved input payload (the journaled request).
235    pub input: Option<Value>,
236
237    /// The event's resolved output payload (the journaled response).
238    pub output: Option<Value>,
239}
240
241impl ServedEffect {
242    /// Re-journal this served effect into the replay run's journal under
243    /// causal parent `parent`, returning the new event id.
244    ///
245    /// The replayed event reproduces the recorded one exactly — kind, node,
246    /// effect class, payloads, latency, tokens, cost, status — while the
247    /// journal assigns the replay run's own `seq` and `recorded_at` (which
248    /// match the recorded run's when the determinism seams are aligned).
249    pub fn rejournal(&self, journal: &Journal, parent: impl Into<String>) -> String {
250        let mut draft = EventDraft::new(self.event.kind, self.event.effect)
251            .status(self.event.status)
252            .parent(parent);
253        if let Some(node) = &self.event.node_id {
254            draft = draft.node(node.clone());
255        }
256        if let Some(input) = &self.input {
257            draft = draft.input(input.clone());
258        }
259        if let Some(output) = &self.output {
260            draft = draft.output(output.clone());
261        }
262        if let Some(latency_ms) = self.event.latency_ms {
263            draft = draft.latency_ms(latency_ms);
264        }
265        if let Some(tokens) = self.event.tokens {
266            draft = draft.tokens(tokens);
267        }
268        if let Some(cost_usd) = self.event.cost_usd {
269            draft = draft.cost_usd(cost_usd);
270        }
271        journal.record(draft)
272    }
273}
274
275#[derive(Debug)]
276struct ReplaySourceInner {
277    /// The snapshot's servable events, in `seq` order.
278    servable: Vec<RunEvent>,
279    /// The snapshot's artifact map, for payload resolution.
280    artifacts: BTreeMap<String, Value>,
281    /// Position of the next unserved event. Strictly ordered: exact replay
282    /// answers effects in journaled sequence order or not at all.
283    cursor: usize,
284}
285
286/// The serving side of exact replay: an ordered cursor over a journaled run's
287/// recorded effects.
288///
289/// Cheap to clone (shared cursor behind one `Arc`), so every replaying
290/// wrapper in a graph can hold a handle. Matching is by **sequence + request
291/// hash**: [`ReplaySource::serve`] takes the next unserved servable event in
292/// `seq` order, requires its kind to match, and requires the issued request's
293/// canonical hash to equal the journaled request's. Anything else fails
294/// loudly with [`RustyError::Replay`] — a replay that has drifted from its
295/// evidence must stop, not improvise.
296#[derive(Debug, Clone)]
297pub struct ReplaySource {
298    inner: Arc<Mutex<ReplaySourceInner>>,
299}
300
301impl ReplaySource {
302    /// A source over the servable events of `snapshot`. The snapshot is
303    /// assumed already integrity-verified (via [`ExactReplay::new`] or
304    /// [`ReplayFixture::import`]); unresolved artifact references simply fail
305    /// to match.
306    pub fn new(snapshot: &JournalSnapshot) -> Self {
307        Self {
308            inner: Arc::new(Mutex::new(ReplaySourceInner {
309                servable: snapshot
310                    .events
311                    .iter()
312                    .filter(|event| is_servable(event.kind))
313                    .cloned()
314                    .collect(),
315                artifacts: snapshot.artifacts.clone(),
316                cursor: 0,
317            })),
318        }
319    }
320
321    fn lock(&self) -> MutexGuard<'_, ReplaySourceInner> {
322        // Poison means a serving path panicked mid-read; the cursor value is
323        // plain data and stays coherent, so recovering is safe.
324        self.inner.lock().unwrap_or_else(|e| e.into_inner())
325    }
326
327    /// Serve one outbound effect from the journal.
328    ///
329    /// `kind` must be a servable effect kind (model, tool, remote, or WASM
330    /// call); `request` is the canonical request value (see
331    /// [`model_call_request`] / [`tool_call_request`]). On success the cursor
332    /// advances past the matched event and the returned [`ServedEffect`
333    /// carries the recorded response. On any mismatch — wrong kind at the
334    /// cursor, different request hash, no events left — a descriptive
335    /// [`RustyError::Replay`] names the sequence position and both sides of
336    /// the comparison.
337    pub fn serve(&self, kind: RunEventKind, request: &Value) -> Result<ServedEffect> {
338        if !is_servable(kind) {
339            return Err(replay_error(format!(
340                "{kind:?} is not a servable effect kind; exact replay serves only {SERVABLE_KINDS:?} \
341                 — executor-derived events are reproduced by re-execution, not served"
342            )));
343        }
344        let issued_hash = request_hash(request)?;
345        let mut inner = self.lock();
346        let Some(event) = inner.servable.get(inner.cursor) else {
347            return Err(replay_error(format!(
348                "replay exhaustion: the run issued a {kind:?} call (request hash {issued_hash}), \
349                 but the journal holds no further recorded effects to serve — the replayed run \
350                 is doing more work than the recorded one"
351            )));
352        };
353        if event.kind != kind {
354            return Err(replay_error(format!(
355                "replay order violation at recorded seq {}: the run issued a {kind:?} call, but \
356                 the next unserved recorded effect is {:?} — exact replay requires effects in \
357                 journaled sequence order",
358                event.seq, event.kind
359            )));
360        }
361        let recorded_input = event
362            .input
363            .as_ref()
364            .and_then(|payload| match payload {
365                PayloadRef::Inline(value) => Some(value.clone()),
366                PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
367            })
368            .ok_or_else(|| {
369                replay_error(format!(
370                    "recorded {:?} at seq {} has no input payload to match against",
371                    event.kind, event.seq
372                ))
373            })?;
374        let recorded_hash = request_hash(&recorded_input)?;
375        if recorded_hash != issued_hash {
376            return Err(replay_error(format!(
377                "replay divergence at recorded seq {} ({:?}): the run issued a request whose \
378                 canonical hash is {issued_hash}, but the journaled request hashes to \
379                 {recorded_hash} — the replayed run has diverged from its evidence",
380                event.seq, event.kind
381            )));
382        }
383        let served = ServedEffect {
384            input: Some(recorded_input),
385            output: event.output.as_ref().and_then(|payload| match payload {
386                PayloadRef::Inline(value) => Some(value.clone()),
387                PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
388            }),
389            event: event.clone(),
390        };
391        inner.cursor += 1;
392        Ok(served)
393    }
394
395    /// `true` when every recorded effect has been served. Verification treats
396    /// a non-exhausted source as a replay that stopped short of its evidence.
397    pub fn is_exhausted(&self) -> bool {
398        let inner = self.lock();
399        inner.cursor == inner.servable.len()
400    }
401
402    /// The `(seq, kind)` pairs still waiting to be served, in order. Empty
403    /// when [`ReplaySource::is_exhausted`] is `true`.
404    pub fn remaining(&self) -> Vec<(u64, RunEventKind)> {
405        let inner = self.lock();
406        inner.servable[inner.cursor..]
407            .iter()
408            .map(|event| (event.seq, event.kind))
409            .collect()
410    }
411}
412
413/// A [`ChatModel`] that records every call into the run's journal in the
414/// canonical replay-compatible shape, then returns the real response.
415///
416/// Construct per node invocation (it is a cheap handle): the causal parent is
417/// the invocation's node-input event id, delivered to node code via
418/// [`crate::journal::PARENT_EVENT_KEY`]. This is the supported way to produce
419/// journals that [`ReplayingChatModel`] can later replay — hand-rolled
420/// journaling must reproduce the same canonical shapes to be replayable.
421pub struct RecordingChatModel {
422    inner: Arc<dyn ChatModel>,
423    journal: Journal,
424    parent: String,
425    node_id: Option<String>,
426}
427
428impl RecordingChatModel {
429    /// A recording wrapper around `inner`, journaling into `journal` with
430    /// causal parent `parent` (the current invocation's node-input event id).
431    pub fn new(inner: Arc<dyn ChatModel>, journal: Journal, parent: impl Into<String>) -> Self {
432        Self {
433            inner,
434            journal,
435            parent: parent.into(),
436            node_id: None,
437        }
438    }
439
440    /// Builder-style: the node this model call is about (recorded as the
441    /// event's `node_id`).
442    pub fn node(mut self, node_id: impl Into<String>) -> Self {
443        self.node_id = Some(node_id.into());
444        self
445    }
446
447    fn draft(&self) -> EventDraft {
448        let mut draft = EventDraft::new(RunEventKind::ModelCall, self.inner.effect())
449            .parent(self.parent.clone());
450        if let Some(node) = &self.node_id {
451            draft = draft.node(node.clone());
452        }
453        draft
454    }
455}
456
457#[async_trait]
458impl ChatModel for RecordingChatModel {
459    async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
460        let started = self.journal.clock().now();
461        let result = self.inner.chat(messages, tools).await;
462        let latency_ms = (self.journal.clock().now() - started)
463            .num_milliseconds()
464            .max(0) as u64;
465        let draft = self
466            .draft()
467            .input(model_call_request(messages, tools))
468            .latency_ms(latency_ms);
469        match result {
470            Ok(response) => {
471                let mut draft = draft.output(model_call_response(&response));
472                if let Some(usage) = response.usage {
473                    draft = draft.tokens(usage);
474                }
475                self.journal.record(draft);
476                Ok(response)
477            }
478            Err(error) => {
479                self.journal.record(
480                    draft
481                        .status(EventStatus::Error)
482                        .output(json!({ "error": error.to_string() })),
483                );
484                Err(error)
485            }
486        }
487    }
488
489    fn effect(&self) -> Effect {
490        self.inner.effect()
491    }
492}
493
494/// A [`ChatModel`] that answers every call from a recorded journal instead of
495/// executing it.
496///
497/// The wrapped implementation is carried for its identity ([`ChatModel::effect`])
498/// and is **never invoked**: there is no code path from `chat` to `inner`, so
499/// replaying with a panic-on-call sentinel (or a client with no credentials
500/// and no network) is safe. Each call is matched against the journal by
501/// sequence + request hash and answered with the recorded response; the
502/// served event is re-journaled into the replay run's journal so the
503/// replayed evidence reproduces the recorded evidence byte-for-byte.
504pub struct ReplayingChatModel {
505    inner: Arc<dyn ChatModel>,
506    source: ReplaySource,
507    journal: Journal,
508    parent: String,
509}
510
511impl ReplayingChatModel {
512    /// A replaying wrapper around `inner` (never invoked), serving from
513    /// `source` and re-journaling into `journal` with causal parent `parent`
514    /// (the current invocation's node-input event id).
515    pub fn new(
516        inner: Arc<dyn ChatModel>,
517        source: ReplaySource,
518        journal: Journal,
519        parent: impl Into<String>,
520    ) -> Self {
521        Self {
522            inner,
523            source,
524            journal,
525            parent: parent.into(),
526        }
527    }
528}
529
530#[async_trait]
531impl ChatModel for ReplayingChatModel {
532    async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
533        let served = self.source.serve(
534            RunEventKind::ModelCall,
535            &model_call_request(messages, tools),
536        )?;
537        // Clock-read parity with RecordingChatModel (two reads per call)
538        // keeps the logical clock's tick sequence aligned with the recorded
539        // run, so the replayed journal's timestamps reproduce it exactly.
540        let _started = self.journal.clock().now();
541        let _ended = self.journal.clock().now();
542        served.rejournal(&self.journal, self.parent.clone());
543        match served.event.status {
544            EventStatus::Ok => {
545                let output = served.output.as_ref().ok_or_else(|| {
546                    replay_error(format!(
547                        "recorded model call at seq {} succeeded but carries no output payload",
548                        served.event.seq
549                    ))
550                })?;
551                chat_response_from_recorded(served.event.seq, output)
552            }
553            EventStatus::Error => Err(RustyError::Llm(format!(
554                "replayed model call at seq {} failed as recorded: {}",
555                served.event.seq,
556                recorded_error_text(&served)
557            ))),
558            EventStatus::Interrupted => Err(replay_error(format!(
559                "recorded model call at seq {} has status `interrupted`, which model calls \
560                 never produce — the journal is inconsistent",
561                served.event.seq
562            ))),
563        }
564    }
565
566    fn effect(&self) -> Effect {
567        self.inner.effect()
568    }
569}
570
571/// A [`Tool`] that records every call into the run's journal in the canonical
572/// replay-compatible shape, then returns the real result.
573///
574/// Construct per node invocation like [`RecordingChatModel`]; identity
575/// (name, description, schema, effect class) delegates to the wrapped tool.
576pub struct RecordingTool {
577    inner: Arc<dyn Tool>,
578    journal: Journal,
579    parent: String,
580    node_id: Option<String>,
581}
582
583impl RecordingTool {
584    /// A recording wrapper around `inner`, journaling into `journal` with
585    /// causal parent `parent` (the current invocation's node-input event id).
586    pub fn new(inner: Arc<dyn Tool>, journal: Journal, parent: impl Into<String>) -> Self {
587        Self {
588            inner,
589            journal,
590            parent: parent.into(),
591            node_id: None,
592        }
593    }
594
595    /// Builder-style: the node this tool call is about (recorded as the
596    /// event's `node_id`).
597    pub fn node(mut self, node_id: impl Into<String>) -> Self {
598        self.node_id = Some(node_id.into());
599        self
600    }
601}
602
603#[async_trait]
604impl Tool for RecordingTool {
605    fn name(&self) -> &str {
606        self.inner.name()
607    }
608
609    fn description(&self) -> &str {
610        self.inner.description()
611    }
612
613    fn parameters_schema(&self) -> Value {
614        self.inner.parameters_schema()
615    }
616
617    fn effect(&self) -> Effect {
618        self.inner.effect()
619    }
620
621    fn effect_kind(&self) -> &str {
622        self.inner.effect_kind()
623    }
624
625    fn idempotency_key(&self, args: &Value) -> Option<String> {
626        self.inner.idempotency_key(args)
627    }
628
629    fn effect_request(&self, call: &ToolCall) -> EffectRequest {
630        self.inner.effect_request(call)
631    }
632
633    async fn call(&self, args: Value) -> Result<Value> {
634        let started = self.journal.clock().now();
635        let result = self.inner.call(args.clone()).await;
636        let latency_ms = (self.journal.clock().now() - started)
637            .num_milliseconds()
638            .max(0) as u64;
639        let mut draft = EventDraft::new(RunEventKind::ToolCall, self.inner.effect())
640            .input(tool_call_request(self.inner.name(), &args))
641            .latency_ms(latency_ms)
642            .parent(self.parent.clone());
643        if let Some(node) = &self.node_id {
644            draft = draft.node(node.clone());
645        }
646        match result {
647            Ok(value) => {
648                self.journal.record(draft.output(value.clone()));
649                Ok(value)
650            }
651            Err(error) => {
652                self.journal.record(
653                    draft
654                        .status(EventStatus::Error)
655                        .output(json!({ "error": error.to_string() })),
656                );
657                Err(error)
658            }
659        }
660    }
661}
662
663/// A [`Tool`] that answers every call from a recorded journal instead of
664/// executing it.
665///
666/// Identity (name, description, schema, effect class) delegates to the
667/// wrapped tool, which is **never invoked** — the replaying analogue of
668/// [`RecordingTool`]. See [`ReplayingChatModel`] for the matching and
669/// zero-outbound guarantees.
670pub struct ReplayingTool {
671    inner: Arc<dyn Tool>,
672    source: ReplaySource,
673    journal: Journal,
674    parent: String,
675}
676
677impl ReplayingTool {
678    /// A replaying wrapper around `inner` (never invoked), serving from
679    /// `source` and re-journaling into `journal` with causal parent `parent`.
680    pub fn new(
681        inner: Arc<dyn Tool>,
682        source: ReplaySource,
683        journal: Journal,
684        parent: impl Into<String>,
685    ) -> Self {
686        Self {
687            inner,
688            source,
689            journal,
690            parent: parent.into(),
691        }
692    }
693}
694
695#[async_trait]
696impl Tool for ReplayingTool {
697    fn name(&self) -> &str {
698        self.inner.name()
699    }
700
701    fn description(&self) -> &str {
702        self.inner.description()
703    }
704
705    fn parameters_schema(&self) -> Value {
706        self.inner.parameters_schema()
707    }
708
709    fn effect(&self) -> Effect {
710        self.inner.effect()
711    }
712
713    fn effect_kind(&self) -> &str {
714        self.inner.effect_kind()
715    }
716
717    fn idempotency_key(&self, args: &Value) -> Option<String> {
718        self.inner.idempotency_key(args)
719    }
720
721    fn effect_request(&self, call: &ToolCall) -> EffectRequest {
722        self.inner.effect_request(call)
723    }
724
725    async fn call(&self, args: Value) -> Result<Value> {
726        let request = tool_call_request(self.inner.name(), &args);
727        let served = self.source.serve(RunEventKind::ToolCall, &request)?;
728        // Clock-read parity with RecordingTool; see ReplayingChatModel.
729        let _started = self.journal.clock().now();
730        let _ended = self.journal.clock().now();
731        served.rejournal(&self.journal, self.parent.clone());
732        match served.event.status {
733            EventStatus::Ok => served.output.clone().ok_or_else(|| {
734                replay_error(format!(
735                    "recorded tool call at seq {} succeeded but carries no output payload",
736                    served.event.seq
737                ))
738            }),
739            EventStatus::Error => Err(RustyError::Tool(format!(
740                "replayed tool call at seq {} failed as recorded: {}",
741                served.event.seq,
742                recorded_error_text(&served)
743            ))),
744            EventStatus::Interrupted => Err(replay_error(format!(
745                "recorded tool call at seq {} has status `interrupted`, which tool calls \
746                 never produce — the journal is inconsistent",
747                served.event.seq
748            ))),
749        }
750    }
751}
752
753/// An exact-replay session over one integrity-verified journal snapshot.
754///
755/// Construction re-verifies the snapshot (chained head hash, artifact
756/// integrity, reference resolution), so a tampered or truncated journal is
757/// rejected at the boundary. The typical flow:
758///
759/// 1. `let replay = ExactReplay::new(snapshot)?;`
760/// 2. `let journal = replay.fresh_journal(clock)` — a fresh journal with the
761///    recorded run's identity and a matching logical clock;
762/// 3. build the same graph topology with [`ReplayingChatModel`] /
763///    [`ReplayingTool`] wrappers sourcing from `replay.source()` and
764///    re-journaling into that journal;
765/// 4. `replay.run_and_verify(&graph, &spec, initial_state, params).await` —
766///    drives the run and checks the replayed journal reproduces the recorded
767///    one event-for-event.
768#[derive(Debug)]
769pub struct ExactReplay {
770    snapshot: JournalSnapshot,
771    source: ReplaySource,
772}
773
774impl ExactReplay {
775    /// Start an exact-replay session over `snapshot`.
776    ///
777    /// Fails with [`RustyError::Replay`] when the snapshot fails integrity
778    /// verification (tampered head hash, corrupted artifacts, dangling
779    /// references), or with [`RustyError::Serialization`] when an event
780    /// cannot be re-hashed. Exact replay of *resumed* runs (journals whose
781    /// first event is a resume) is deferred: their evidence begins mid-run
782    /// against checkpointed state the journal does not carry.
783    pub fn new(snapshot: JournalSnapshot) -> Result<Self> {
784        verify_snapshot(&snapshot)?;
785        if snapshot
786            .events
787            .first()
788            .is_some_and(|event| event.kind == RunEventKind::Resume)
789        {
790            return Err(replay_error(
791                "exact replay of resumed runs is not supported: the journal begins with a \
792                 resume event, whose pre-resume state lives in a checkpoint the journal does \
793                 not carry — replay the original run's journal instead",
794            ));
795        }
796        let source = ReplaySource::new(&snapshot);
797        Ok(Self { snapshot, source })
798    }
799
800    /// The recorded snapshot this session replays.
801    pub fn snapshot(&self) -> &JournalSnapshot {
802        &self.snapshot
803    }
804
805    /// The serving cursor over the recorded effects, for building replaying
806    /// wrappers. Shared: every wrapper in the replayed graph must serve from
807    /// this one cursor.
808    pub fn source(&self) -> ReplaySource {
809        self.source.clone()
810    }
811
812    /// A fresh journal with the recorded run's identity (`run_id`,
813    /// `thread_id`), timestamping from `clock`. For byte-identical replay,
814    /// `clock` must be a logical clock with the recorded run's parameters
815    /// (carried in [`ReplayFixture`] metadata).
816    pub fn fresh_journal(&self, clock: Clock) -> Journal {
817        Journal::new(
818            self.snapshot.run_id.clone(),
819            self.snapshot.thread_id.clone(),
820            clock,
821        )
822    }
823
824    /// Drive `graph` over `initial_state`, answering effects from the
825    /// recorded journal.
826    ///
827    /// `params.journal` must carry the recorded run's identity (build it with
828    /// [`ExactReplay::fresh_journal`]); a mismatched journal is rejected
829    /// rather than silently producing evidence under the wrong run. The
830    /// replayed run's outcome and its journal snapshot are returned for
831    /// inspection or [`ExactReplay::verify`].
832    pub async fn run(
833        &self,
834        graph: &Graph,
835        spec: &StateSpec,
836        initial_state: State,
837        params: ReplayParams,
838    ) -> Result<ReplayOutcome> {
839        if params.journal.run_id() != self.snapshot.run_id
840            || params.journal.thread_id() != self.snapshot.thread_id
841        {
842            return Err(replay_error(format!(
843                "replay journal identity mismatch: the snapshot records run `{}` on thread \
844                 `{}`, but the supplied journal is run `{}` on thread `{}` — build the replay \
845                 journal with ExactReplay::fresh_journal",
846                self.snapshot.run_id,
847                self.snapshot.thread_id,
848                params.journal.run_id(),
849                params.journal.thread_id()
850            )));
851        }
852        let executor = match &params.checkpointer {
853            Some(checkpointer) => Executor::with_checkpointer(checkpointer.clone()),
854            None => Executor::new(),
855        };
856        let config = RunConfig::new(self.snapshot.thread_id.clone())
857            .with_max_steps(params.max_steps)
858            .with_rng(params.rng.clone())
859            .with_journal(params.journal.clone());
860        let outcome = executor.run(graph, spec, initial_state, config).await?;
861        Ok(ReplayOutcome {
862            outcome,
863            journal: params.journal.snapshot(),
864        })
865    }
866
867    /// Verify a replayed journal against the recorded one: every recorded
868    /// effect must have been served, and the two snapshots must agree on run
869    /// identity, every event, every artifact, and the head hash. The first
870    /// disagreement fails with a descriptive [`RustyError::Replay`].
871    pub fn verify(&self, replayed: &JournalSnapshot) -> Result<()> {
872        if !self.source.is_exhausted() {
873            return Err(replay_error(format!(
874                "exact replay left recorded effects unserved: {:?} — the replayed run stopped \
875                 short of the journaled history",
876                self.source.remaining()
877            )));
878        }
879        let recorded = &self.snapshot;
880        if recorded.run_id != replayed.run_id || recorded.thread_id != replayed.thread_id {
881            return Err(replay_error(format!(
882                "replay identity mismatch: recorded run `{}`/thread `{}` vs replayed \
883                 `{}`/`{}`",
884                recorded.run_id, recorded.thread_id, replayed.run_id, replayed.thread_id
885            )));
886        }
887        if recorded.events.len() != replayed.events.len() {
888            return Err(replay_error(format!(
889                "replay event-count mismatch: the recorded journal holds {} events but the \
890                 replayed journal holds {}",
891                recorded.events.len(),
892                replayed.events.len()
893            )));
894        }
895        for (recorded_event, replayed_event) in recorded.events.iter().zip(&replayed.events) {
896            if recorded_event != replayed_event {
897                return Err(replay_error(format!(
898                    "replay event mismatch at seq {}: recorded {} but replayed {}",
899                    recorded_event.seq,
900                    summarize_event(recorded_event),
901                    summarize_event(replayed_event),
902                )));
903            }
904        }
905        if recorded.artifacts != replayed.artifacts {
906            return Err(replay_error(
907                "replay artifact mismatch: the replayed journal's artifact payloads differ \
908                 from the recorded ones",
909            ));
910        }
911        if recorded.head_hash != replayed.head_hash {
912            return Err(replay_error(format!(
913                "replay head-hash mismatch: recorded {} but replayed {} — event content \
914                 agrees but the evidence chain does not (this should be unreachable; please \
915                 report it)",
916                recorded.head_hash, replayed.head_hash
917            )));
918        }
919        Ok(())
920    }
921
922    /// [`ExactReplay::run`] plus [`ExactReplay::verify`]: the replay either
923    /// reproduces the recorded evidence exactly or fails with the first
924    /// disagreement.
925    pub async fn run_and_verify(
926        &self,
927        graph: &Graph,
928        spec: &StateSpec,
929        initial_state: State,
930        params: ReplayParams,
931    ) -> Result<ReplayOutcome> {
932        let replayed = self.run(graph, spec, initial_state, params).await?;
933        self.verify(&replayed.journal)?;
934        Ok(replayed)
935    }
936}
937
938/// One-line event summary for verification errors (kind, node, status —
939/// enough to locate the divergence without dumping payloads).
940fn summarize_event(event: &RunEvent) -> String {
941    format!(
942        "{:?}(node={:?}, status={:?})",
943        event.kind, event.node_id, event.status
944    )
945}
946
947/// Per-run parameters of an exact replay: the fresh journal (recorded run's
948/// identity plus matching logical clock), the RNG (the recorded run's seed
949/// for byte-identical checkpoint ids), an optional checkpointer (present iff
950/// the recorded run had one — checkpoint-written events are part of the
951/// evidence), and the step limit.
952pub struct ReplayParams {
953    /// The replay run's journal; build with [`ExactReplay::fresh_journal`].
954    pub journal: Journal,
955
956    /// The replay run's randomness source. [`RngSource::seeded`] with the
957    /// recorded run's seed reproduces checkpoint ids; anything else makes the
958    /// replayed journal diverge at the first checkpoint-written event.
959    pub rng: RngSource,
960
961    /// Persistence for the replay run. Must be `Some` when the recorded run
962    /// journaled checkpoint-written events, `None` when it did not.
963    pub checkpointer: Option<Arc<dyn Checkpointer>>,
964
965    /// Super-step limit, as in [`RunConfig::max_steps`].
966    pub max_steps: usize,
967}
968
969impl ReplayParams {
970    /// Parameters over `journal` and `rng` with no checkpointer and the
971    /// default step limit.
972    pub fn new(journal: Journal, rng: RngSource) -> Self {
973        Self {
974            journal,
975            rng,
976            checkpointer: None,
977            max_steps: DEFAULT_MAX_STEPS,
978        }
979    }
980
981    /// Builder-style: persist replay checkpoints through `checkpointer`.
982    pub fn with_checkpointer(mut self, checkpointer: Arc<dyn Checkpointer>) -> Self {
983        self.checkpointer = Some(checkpointer);
984        self
985    }
986
987    /// Builder-style: override the step limit.
988    pub fn with_max_steps(mut self, max_steps: usize) -> Self {
989        self.max_steps = max_steps;
990        self
991    }
992}
993
994/// The result of an exact replay: the run's outcome and the replayed journal
995/// (export form, comparable against the recorded snapshot).
996#[derive(Debug)]
997pub struct ReplayOutcome {
998    /// How the replayed run ended. Exact replay of an interrupted recorded
999    /// run ends in the same `Interrupted` outcome with the same payload.
1000    pub outcome: ExecutionOutcome,
1001
1002    /// The replayed run's journal snapshot. After
1003    /// [`ExactReplay::run_and_verify`], this is event-for-event identical to
1004    /// the recorded snapshot.
1005    pub journal: JournalSnapshot,
1006}
1007
1008/// Token and cost totals of one branch of a [`BranchDiff`]: the sum over all
1009/// of the branch's events (the `tokens` field is only ever set on model
1010/// calls, so this is the branch's model usage).
1011#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
1012pub struct BranchTotals {
1013    /// Number of events in the branch.
1014    pub events: usize,
1015
1016    /// Summed token usage across the branch's model calls.
1017    pub tokens: Usage,
1018
1019    /// Summed recorded cost in USD across the branch's events.
1020    pub cost_usd: f64,
1021}
1022
1023/// One channel's differing value at one super-step of a [`BranchDiff`].
1024/// `None` means the channel was absent from that branch's super-step-end
1025/// record at that step.
1026#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1027pub struct ChannelDiff {
1028    /// The state channel that differs.
1029    pub channel: String,
1030
1031    /// The post-reducer value in the base branch, when present.
1032    pub base: Option<Value>,
1033
1034    /// The post-reducer value in the branch, when present.
1035    pub branch: Option<Value>,
1036}
1037
1038/// The state-channel differences of one super-step between two branches.
1039#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1040pub struct StepDiff {
1041    /// The super-step index (from the super-step-start event's input).
1042    pub step: u64,
1043
1044    /// Channels whose post-reducer values differ at this step (presence or
1045    /// value), sorted by channel name.
1046    pub channels: Vec<ChannelDiff>,
1047}
1048
1049/// The structural diff between two journal snapshots — typically two branches
1050/// forked from a common point: one replayed an existing history, the other
1051/// continued differently.
1052///
1053/// Events are compared **logically**: kind, node, sequence, effect class,
1054/// resolved input/output payloads, latency, tokens, cost, and status must
1055/// match, while identity and timing fields (`id`, `run_id`, `thread_id`,
1056/// `parent`, `recorded_at`) are excluded — two branches are separate runs,
1057/// so their identity-bearing fields legitimately differ even when their
1058/// evidence is the same. The first logically-unequal sequence position is the
1059/// divergence point; everything after it is reported as removed (base) or
1060/// added (branch).
1061#[derive(Debug, Clone, Serialize, Deserialize)]
1062pub struct BranchDiff {
1063    /// The first sequence number where the branches' evidence differs, or
1064    /// `None` when they are logically identical.
1065    pub first_divergent_seq: Option<u64>,
1066
1067    /// Events present in the branch at and after the divergence point (the
1068    /// branch's new work relative to the base).
1069    pub added: Vec<RunEvent>,
1070
1071    /// Events present in the base at and after the divergence point (the work
1072    /// the branch replaced).
1073    pub removed: Vec<RunEvent>,
1074
1075    /// State-channel differences at each super-step, in step order. Only
1076    /// steps with at least one differing channel appear.
1077    pub step_diffs: Vec<StepDiff>,
1078
1079    /// Totals over the whole base journal.
1080    pub base_totals: BranchTotals,
1081
1082    /// Totals over the whole branch journal.
1083    pub branch_totals: BranchTotals,
1084}
1085
1086impl BranchDiff {
1087    /// Diff `branch` against `base`. When both snapshots share a common
1088    /// history (a fork), the shared prefix compares equal and the divergence
1089    /// point is where the fork's branches parted; `fork_seq`-style explicit
1090    /// fork points are unnecessary — the evidence carries the cut.
1091    pub fn between(base: &JournalSnapshot, branch: &JournalSnapshot) -> BranchDiff {
1092        let divergence = (0..base.events.len().max(branch.events.len()))
1093            .find(|&i| {
1094                match (base.events.get(i), branch.events.get(i)) {
1095                    (Some(a), Some(b)) => !logically_equal(a, base, b, branch),
1096                    // One side exhausted: divergence at the first missing seq.
1097                    _ => true,
1098                }
1099            })
1100            .map(|i| {
1101                base.events
1102                    .get(i)
1103                    .or_else(|| branch.events.get(i))
1104                    .map(|event| event.seq)
1105                    .unwrap_or(i as u64)
1106            });
1107        let (removed, added) = match divergence {
1108            Some(seq) => (
1109                base.events
1110                    .iter()
1111                    .filter(|event| event.seq >= seq)
1112                    .cloned()
1113                    .collect(),
1114                branch
1115                    .events
1116                    .iter()
1117                    .filter(|event| event.seq >= seq)
1118                    .cloned()
1119                    .collect(),
1120            ),
1121            None => (Vec::new(), Vec::new()),
1122        };
1123        BranchDiff {
1124            first_divergent_seq: divergence,
1125            added,
1126            removed,
1127            step_diffs: diff_steps(base, branch),
1128            base_totals: totals(base),
1129            branch_totals: totals(branch),
1130        }
1131    }
1132
1133    /// `true` when the two branches are logically identical (no divergence
1134    /// point, nothing added or removed).
1135    pub fn is_identical(&self) -> bool {
1136        self.first_divergent_seq.is_none()
1137    }
1138}
1139
1140/// Logical evidence equality, excluding identity and timing fields (see
1141/// [`BranchDiff`]). Payloads compare by resolved value, so an inline payload
1142/// and an artifact reference carrying the same bytes compare equal.
1143fn logically_equal(
1144    a: &RunEvent,
1145    snapshot_a: &JournalSnapshot,
1146    b: &RunEvent,
1147    snapshot_b: &JournalSnapshot,
1148) -> bool {
1149    a.seq == b.seq
1150        && a.kind == b.kind
1151        && a.node_id == b.node_id
1152        && a.effect == b.effect
1153        && a.latency_ms == b.latency_ms
1154        && a.tokens == b.tokens
1155        && a.cost_usd == b.cost_usd
1156        && a.status == b.status
1157        && resolve_opt(snapshot_a, a.input.as_ref()) == resolve_opt(snapshot_b, b.input.as_ref())
1158        && resolve_opt(snapshot_a, a.output.as_ref()) == resolve_opt(snapshot_b, b.output.as_ref())
1159}
1160
1161/// Sum token usage and cost over a snapshot's events.
1162fn totals(snapshot: &JournalSnapshot) -> BranchTotals {
1163    let mut totals = BranchTotals {
1164        events: snapshot.events.len(),
1165        ..BranchTotals::default()
1166    };
1167    for event in &snapshot.events {
1168        if let Some(usage) = event.tokens {
1169            totals.tokens.prompt_tokens = totals
1170                .tokens
1171                .prompt_tokens
1172                .saturating_add(usage.prompt_tokens);
1173            totals.tokens.completion_tokens = totals
1174                .tokens
1175                .completion_tokens
1176                .saturating_add(usage.completion_tokens);
1177            totals.tokens.total_tokens = totals
1178                .tokens
1179                .total_tokens
1180                .saturating_add(usage.total_tokens);
1181        }
1182        if let Some(cost) = event.cost_usd {
1183            totals.cost_usd += cost;
1184        }
1185    }
1186    totals
1187}
1188
1189/// The post-reducer channel values of every super-step in a snapshot, keyed
1190/// by step index: super-step-start events supply the index, the following
1191/// super-step-end event supplies the channel values.
1192fn step_channel_values(snapshot: &JournalSnapshot) -> BTreeMap<u64, BTreeMap<String, Value>> {
1193    let mut steps = BTreeMap::new();
1194    let mut current_step: Option<u64> = None;
1195    for event in &snapshot.events {
1196        match event.kind {
1197            RunEventKind::SuperStepStart => {
1198                current_step = resolve_opt(snapshot, event.input.as_ref())
1199                    .and_then(|input| input.get("step").and_then(Value::as_u64));
1200            }
1201            RunEventKind::SuperStepEnd => {
1202                if let Some(step) = current_step.take() {
1203                    let channels = resolve_opt(snapshot, event.output.as_ref())
1204                        .and_then(|output| output.as_object().cloned())
1205                        .unwrap_or_default()
1206                        .into_iter()
1207                        .collect();
1208                    steps.insert(step, channels);
1209                }
1210            }
1211            _ => {}
1212        }
1213    }
1214    steps
1215}
1216
1217/// Per-step channel diffs between two snapshots (only steps with at least one
1218/// differing channel, channels sorted).
1219fn diff_steps(base: &JournalSnapshot, branch: &JournalSnapshot) -> Vec<StepDiff> {
1220    let base_steps = step_channel_values(base);
1221    let branch_steps = step_channel_values(branch);
1222    let mut diffs = Vec::new();
1223    for step in base_steps
1224        .keys()
1225        .chain(branch_steps.keys())
1226        .copied()
1227        .collect::<std::collections::BTreeSet<u64>>()
1228    {
1229        let empty = BTreeMap::new();
1230        let base_channels = base_steps.get(&step).unwrap_or(&empty);
1231        let branch_channels = branch_steps.get(&step).unwrap_or(&empty);
1232        let mut channels = Vec::new();
1233        for channel in base_channels
1234            .keys()
1235            .chain(branch_channels.keys())
1236            .cloned()
1237            .collect::<std::collections::BTreeSet<String>>()
1238        {
1239            let base_value = base_channels.get(&channel).cloned();
1240            let branch_value = branch_channels.get(&channel).cloned();
1241            if base_value != branch_value {
1242                channels.push(ChannelDiff {
1243                    channel,
1244                    base: base_value,
1245                    branch: branch_value,
1246                });
1247            }
1248        }
1249        if !channels.is_empty() {
1250            diffs.push(StepDiff { step, channels });
1251        }
1252    }
1253    diffs
1254}
1255
1256/// Determinism parameters of a recorded run, carried in [`FixtureMetadata`]
1257/// so [`ReplayFixture::replay_in_ci`] can reproduce the run byte-for-byte.
1258#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1259pub struct LogicalClockParams {
1260    /// The logical clock's start (epoch millis), as in [`Clock::logical`].
1261    pub start_ms: u64,
1262
1263    /// The logical clock's per-read tick (millis), as in [`Clock::logical`].
1264    pub tick_ms: u64,
1265}
1266
1267/// Provenance and determinism metadata of a [`ReplayFixture`].
1268#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1269pub struct FixtureMetadata {
1270    /// Human-readable fixture name (used in error messages).
1271    pub name: String,
1272
1273    /// The recorded run's logical-clock parameters. Required for CI replay.
1274    #[serde(default, skip_serializing_if = "Option::is_none")]
1275    pub clock: Option<LogicalClockParams>,
1276
1277    /// The recorded run's RNG seed. Required for CI replay.
1278    #[serde(default, skip_serializing_if = "Option::is_none")]
1279    pub rng_seed: Option<u64>,
1280}
1281
1282/// A portable, self-contained exact-replay bundle: everything CI (or another
1283/// machine, or a later wave's server endpoint) needs to re-drive a recorded
1284/// run and prove it reproduces.
1285///
1286/// A fixture carries the graph's topology hash (replay refuses a structurally
1287/// different graph), the full journal snapshot (integrity-verified on
1288/// import), the recorded final checkpoint (replay compares the outcome state
1289/// against it), and the determinism parameters the run was recorded with.
1290/// Serialization is plain JSON; [`ReplayFixture::export`] /
1291/// [`ReplayFixture::import`] are the wire boundary.
1292#[derive(Debug, Clone, Serialize, Deserialize)]
1293pub struct ReplayFixture {
1294    /// Fixture envelope version; [`FIXTURE_FORMAT_VERSION`] for anything
1295    /// written now.
1296    pub format_version: u32,
1297
1298    /// SHA-256 of the recorded graph's topology (see
1299    /// [`Graph::topology_hash`]). Checked before replay: a different topology
1300    /// cannot reproduce the journaled evidence.
1301    pub graph_hash: String,
1302
1303    /// The application-declared graph version of the recorded run.
1304    pub graph_version: String,
1305
1306    /// The recorded run's complete journal.
1307    pub journal: JournalSnapshot,
1308
1309    /// The recorded run's final checkpoint, when it ran with a checkpointer.
1310    /// CI replay compares the replayed outcome state against it.
1311    pub final_checkpoint: Option<Checkpoint>,
1312
1313    /// Provenance and determinism parameters.
1314    pub metadata: FixtureMetadata,
1315}
1316
1317impl ReplayFixture {
1318    /// Capture a recorded run as a fixture. `graph` supplies the topology
1319    /// hash; `clock`/`rng_seed` are the determinism parameters the run was
1320    /// recorded with (both required for later CI replay).
1321    pub fn capture(
1322        name: impl Into<String>,
1323        graph: &Graph,
1324        graph_version: impl Into<String>,
1325        journal: JournalSnapshot,
1326        final_checkpoint: Option<Checkpoint>,
1327        clock: Option<LogicalClockParams>,
1328        rng_seed: Option<u64>,
1329    ) -> Self {
1330        Self {
1331            format_version: FIXTURE_FORMAT_VERSION,
1332            graph_hash: graph.topology_hash(),
1333            graph_version: graph_version.into(),
1334            journal,
1335            final_checkpoint,
1336            metadata: FixtureMetadata {
1337                name: name.into(),
1338                clock,
1339                rng_seed,
1340            },
1341        }
1342    }
1343
1344    /// Serialize to pretty-printed JSON (the checked-in / on-disk form).
1345    pub fn export(&self) -> Result<String> {
1346        Ok(serde_json::to_string_pretty(self)?)
1347    }
1348
1349    /// Parse a fixture from JSON, rejecting unsupported format versions and
1350    /// integrity failures (tampered or truncated journals) at the boundary.
1351    pub fn import(json: &str) -> Result<Self> {
1352        let fixture: Self = serde_json::from_str(json)?;
1353        if fixture.format_version != FIXTURE_FORMAT_VERSION {
1354            return Err(replay_error(format!(
1355                "unsupported fixture format version {} (this build supports \
1356                 {FIXTURE_FORMAT_VERSION}) — regenerate the fixture or upgrade the runtime",
1357                fixture.format_version
1358            )));
1359        }
1360        verify_snapshot(&fixture.journal)?;
1361        Ok(fixture)
1362    }
1363
1364    /// An exact-replay session over the fixture's journal (integrity already
1365    /// verified by [`ReplayFixture::import`] or construction; re-verified
1366    /// here so a hand-assembled fixture is checked too).
1367    pub fn exact_replay(&self) -> Result<ExactReplay> {
1368        ExactReplay::new(self.journal.clone())
1369    }
1370
1371    /// The replay parameters matching the fixture's determinism metadata: a
1372    /// fresh journal on the recorded identity and logical clock, the recorded
1373    /// RNG seed, and a fresh in-memory checkpointer (fixtures record runs
1374    /// that used one — the checkpoint-written events are part of the
1375    /// evidence).
1376    ///
1377    /// Fails with [`RustyError::Replay`] when the fixture lacks clock or seed
1378    /// metadata (a run recorded without determinism seams cannot be replayed
1379    /// byte-identically).
1380    pub fn replay_params(&self, replay: &ExactReplay) -> Result<ReplayParams> {
1381        let clock = self.metadata.clock.ok_or_else(|| {
1382            replay_error(format!(
1383                "fixture `{}` carries no logical-clock parameters; exact CI replay requires \
1384                 the recorded run's determinism seams",
1385                self.metadata.name
1386            ))
1387        })?;
1388        let seed = self.metadata.rng_seed.ok_or_else(|| {
1389            replay_error(format!(
1390                "fixture `{}` carries no RNG seed; exact CI replay requires the recorded \
1391                 run's determinism seams",
1392                self.metadata.name
1393            ))
1394        })?;
1395        let journal = replay.fresh_journal(Clock::logical(clock.start_ms, clock.tick_ms));
1396        Ok(ReplayParams::new(journal, RngSource::seeded(seed))
1397            .with_checkpointer(Arc::new(InMemoryCheckpointer::new())))
1398    }
1399
1400    /// Replay the fixture end to end and verify everything: the graph's
1401    /// topology matches the recorded one, the run reproduces the journaled
1402    /// evidence event-for-event, and the outcome state matches the recorded
1403    /// final checkpoint.
1404    ///
1405    /// The caller supplies the replay session (`fixture.exact_replay()?`), a
1406    /// graph built with replaying wrappers over `replay.source()`, and the
1407    /// parameters from `fixture.replay_params(&replay)?` — the same journal
1408    /// handle the wrappers re-journal into. This is the `replay_in_ci` entry
1409    /// point: one call from recorded artifact to verified replay.
1410    pub async fn replay_in_ci(
1411        &self,
1412        replay: ExactReplay,
1413        graph: &Graph,
1414        spec: &StateSpec,
1415        initial_state: State,
1416        params: ReplayParams,
1417    ) -> Result<ReplayOutcome> {
1418        let actual_hash = graph.topology_hash();
1419        if actual_hash != self.graph_hash {
1420            return Err(replay_error(format!(
1421                "fixture `{}` was recorded against graph topology {}, but the graph under \
1422                 replay hashes to {actual_hash} — rebuild the recorded topology (same node \
1423                 names, edges, and entry point)",
1424                self.metadata.name, self.graph_hash
1425            )));
1426        }
1427        let replayed = replay
1428            .run_and_verify(graph, spec, initial_state, params)
1429            .await?;
1430        if let Some(final_checkpoint) = &self.final_checkpoint {
1431            if replayed.outcome.state() != &final_checkpoint.state {
1432                return Err(replay_error(format!(
1433                    "fixture `{}`: the replayed final state diverges from the recorded final \
1434                     checkpoint (journal evidence agreed; state did not)",
1435                    self.metadata.name
1436                )));
1437            }
1438        }
1439        Ok(replayed)
1440    }
1441}
1442
1443// --------------------------------------------------------------------- //
1444// The shadow's recorded world (R0.12 wave 4)
1445// --------------------------------------------------------------------- //
1446
1447/// One source run's journal, indexed as the recorded world a shadow run
1448/// serves refused effects from — the [`crate::effects::ShadowOutcomeSource`]
1449/// over the Flight Recorder's own evidence.
1450///
1451/// The index is per-tool, in sequence order: every recorded
1452/// [`RunEventKind::ToolCall`] queues its request hash and recorded output
1453/// under the tool's name. Serving pops the **front** of the tool's queue
1454/// and only when the live request hashes to the same value — the hybrid
1455/// rule applied to ordering: the shadow serves the recorded outcome of
1456/// exactly the call the source made next under that tool, never a later
1457/// one and never a different argument set. A call the recorded world
1458/// never saw, or saw in a different order, is a divergence and answers
1459/// `None`.
1460///
1461/// After the run, [`JournalShadowSource::unrequested`] names the tools
1462/// whose recorded calls the shadow never made — divergence in the other
1463/// direction, journaled into the shadow's verdict.
1464#[derive(Debug)]
1465pub struct JournalShadowSource {
1466    queues: Mutex<BTreeMap<String, VecDeque<(String, Value)>>>,
1467}
1468
1469impl JournalShadowSource {
1470    /// Index a recorded run's journal for shadow serving. Events whose
1471    /// payloads the snapshot cannot resolve (externalized artifact
1472    /// references) are skipped rather than guessed at: an unresolvable
1473    /// recorded call is one the shadow cannot serve, and serving it
1474    /// anyway would fabricate evidence.
1475    pub fn new(snapshot: &JournalSnapshot) -> Self {
1476        let mut queues: BTreeMap<String, VecDeque<(String, Value)>> = BTreeMap::new();
1477        for event in &snapshot.events {
1478            if event.kind != RunEventKind::ToolCall {
1479                continue;
1480            }
1481            let Some(input) = event.input.as_ref().and_then(|p| resolve_in(snapshot, p)) else {
1482                continue;
1483            };
1484            let Some(tool) = input.get("tool").and_then(Value::as_str).map(str::to_owned) else {
1485                continue;
1486            };
1487            let Ok(hash) = PayloadRef::Inline(input).content_hash() else {
1488                continue;
1489            };
1490            let output = event
1491                .output
1492                .as_ref()
1493                .and_then(|p| resolve_in(snapshot, p))
1494                .unwrap_or(Value::Null);
1495            queues.entry(tool).or_default().push_back((hash, output));
1496        }
1497        Self {
1498            queues: Mutex::new(queues),
1499        }
1500    }
1501
1502    /// The tools holding recorded calls no shadow requested, sorted.
1503    /// Called once the shadow's run has ended — while it runs, a
1504    /// non-empty queue is simply the recorded future.
1505    pub fn unrequested(&self) -> Vec<String> {
1506        let queues = self.queues.lock().unwrap_or_else(|p| p.into_inner());
1507        queues
1508            .iter()
1509            .filter(|(_, queue)| !queue.is_empty())
1510            .map(|(tool, _)| tool.clone())
1511            .collect()
1512    }
1513}
1514
1515impl crate::effects::ShadowOutcomeSource for JournalShadowSource {
1516    fn serve(&self, kind: &str, recorded_request: &Value) -> Option<Value> {
1517        let hash = PayloadRef::Inline(recorded_request.clone())
1518            .content_hash()
1519            .ok()?;
1520        let mut queues = self.queues.lock().unwrap_or_else(|p| p.into_inner());
1521        let queue = queues.get_mut(kind)?;
1522        if queue.front().is_some_and(|(front, _)| *front == hash) {
1523            queue.pop_front().map(|(_, output)| output)
1524        } else {
1525            None
1526        }
1527    }
1528}
1529
1530#[cfg(test)]
1531mod tests {
1532    use super::*;
1533    use crate::journal::Journal;
1534    use serde_json::json;
1535
1536    fn snapshot_with(drafts: Vec<EventDraft>) -> JournalSnapshot {
1537        let journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
1538        for draft in drafts {
1539            journal.record(draft);
1540        }
1541        journal.snapshot()
1542    }
1543
1544    fn model_draft(request: Value, response: Value) -> EventDraft {
1545        EventDraft::new(RunEventKind::ModelCall, Effect::NonIdempotent)
1546            .node("agent")
1547            .input(request)
1548            .output(response)
1549            .latency_ms(2)
1550            .tokens(Usage {
1551                prompt_tokens: 10,
1552                completion_tokens: 5,
1553                total_tokens: 15,
1554            })
1555    }
1556
1557    #[test]
1558    fn canonical_shapes_have_stable_hashes() {
1559        let request = model_call_request(&[ChatMessage::user("ping")], &[]);
1560        assert_eq!(
1561            request,
1562            json!({"messages": [{"role": "user", "content": "ping"}], "tools": []})
1563        );
1564        let first = request_hash(&request).unwrap();
1565        let second = request_hash(&model_call_request(&[ChatMessage::user("ping")], &[])).unwrap();
1566        assert_eq!(first, second);
1567        // A different request hashes differently.
1568        let other = request_hash(&model_call_request(&[ChatMessage::user("pong")], &[])).unwrap();
1569        assert_ne!(first, other);
1570
1571        let tool = tool_call_request("echo", &json!({"text": "hi"}));
1572        assert_eq!(tool, json!({"tool": "echo", "arguments": {"text": "hi"}}));
1573    }
1574
1575    #[test]
1576    fn serve_matches_by_sequence_and_request_hash() {
1577        let snapshot = snapshot_with(vec![
1578            model_draft(
1579                model_call_request(&[ChatMessage::user("a")], &[]),
1580                json!({"r": 1}),
1581            ),
1582            model_draft(
1583                model_call_request(&[ChatMessage::user("b")], &[]),
1584                json!({"r": 2}),
1585            ),
1586        ]);
1587        let source = ReplaySource::new(&snapshot);
1588        assert!(!source.is_exhausted());
1589
1590        // Wrong request at the cursor: divergence, and the cursor must not advance.
1591        let error = source
1592            .serve(
1593                RunEventKind::ModelCall,
1594                &model_call_request(&[ChatMessage::user("zzz")], &[]),
1595            )
1596            .unwrap_err();
1597        assert!(matches!(error, RustyError::Replay(_)));
1598        assert!(error.to_string().contains("divergence"));
1599        assert_eq!(source.remaining().len(), 2);
1600
1601        // Wrong kind at the cursor: order violation.
1602        let error = source
1603            .serve(RunEventKind::ToolCall, &tool_call_request("t", &json!({})))
1604            .unwrap_err();
1605        assert!(error.to_string().contains("order violation"));
1606
1607        // Correct sequence serves in order and exhausts.
1608        let first = source
1609            .serve(
1610                RunEventKind::ModelCall,
1611                &model_call_request(&[ChatMessage::user("a")], &[]),
1612            )
1613            .unwrap();
1614        assert_eq!(first.event.seq, 0);
1615        assert_eq!(first.output, Some(json!({"r": 1})));
1616        let second = source
1617            .serve(
1618                RunEventKind::ModelCall,
1619                &model_call_request(&[ChatMessage::user("b")], &[]),
1620            )
1621            .unwrap();
1622        assert_eq!(second.event.seq, 1);
1623        assert!(source.is_exhausted());
1624
1625        // Serving past the end is a loud exhaustion error.
1626        let error = source
1627            .serve(
1628                RunEventKind::ModelCall,
1629                &model_call_request(&[ChatMessage::user("a")], &[]),
1630            )
1631            .unwrap_err();
1632        assert!(error.to_string().contains("exhaustion"));
1633    }
1634
1635    #[test]
1636    fn serve_rejects_non_servable_kinds() {
1637        let snapshot = snapshot_with(vec![EventDraft::new(
1638            RunEventKind::SuperStepStart,
1639            Effect::Pure,
1640        )]);
1641        let error = ReplaySource::new(&snapshot)
1642            .serve(RunEventKind::NodeInput, &json!({}))
1643            .unwrap_err();
1644        assert!(error.to_string().contains("not a servable"));
1645    }
1646
1647    #[test]
1648    fn artifact_backed_requests_match_and_resolve() {
1649        let big_request =
1650            json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1651        let big_response = json!({"message": {"role": "assistant", "content": "y".repeat(9000)}, "model": null, "usage": null});
1652        let snapshot = snapshot_with(vec![model_draft(big_request.clone(), big_response.clone())]);
1653        // The payloads were promoted to artifacts at record time.
1654        assert!(matches!(
1655            snapshot.events[0].input,
1656            Some(PayloadRef::Artifact(_))
1657        ));
1658        let served = ReplaySource::new(&snapshot)
1659            .serve(RunEventKind::ModelCall, &big_request)
1660            .unwrap();
1661        assert_eq!(served.output, Some(big_response));
1662    }
1663
1664    #[test]
1665    fn tampered_snapshots_are_rejected_at_the_boundary() {
1666        let snapshot = snapshot_with(vec![model_draft(
1667            model_call_request(&[ChatMessage::user("a")], &[]),
1668            json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1669        )]);
1670
1671        // Event tampering breaks the chained head hash.
1672        let mut tampered = snapshot.clone();
1673        tampered.events[0].status = EventStatus::Error;
1674        assert!(ExactReplay::new(tampered).is_err());
1675
1676        // Artifact tampering breaks content addressing (the head hash chains
1677        // over events, so this needs the dedicated integrity check).
1678        let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1679        let artifact_snapshot = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
1680        let mut tampered = artifact_snapshot;
1681        *tampered.artifacts.values_mut().next().unwrap() = json!({"forged": true});
1682        let error = ExactReplay::new(tampered).unwrap_err();
1683        assert!(error.to_string().contains("integrity"));
1684
1685        // A truncated snapshot (artifact removed) is rejected.
1686        let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1687        let truncated = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
1688        let mut truncated = truncated;
1689        truncated.artifacts.clear();
1690        assert!(ExactReplay::new(truncated).is_err());
1691    }
1692
1693    #[test]
1694    fn resumed_run_journals_are_rejected() {
1695        let snapshot = snapshot_with(vec![EventDraft::new(RunEventKind::Resume, Effect::Pure)
1696            .input(json!({"checkpoint_id": "c"}))]);
1697        let error = ExactReplay::new(snapshot).unwrap_err();
1698        assert!(error.to_string().contains("resumed"));
1699    }
1700
1701    #[test]
1702    fn rejournal_reproduces_the_recorded_event() {
1703        let recorded = snapshot_with(vec![model_draft(
1704            model_call_request(&[ChatMessage::user("a")], &[]),
1705            json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1706        )]);
1707        let source = ReplaySource::new(&recorded);
1708        let served = source
1709            .serve(
1710                RunEventKind::ModelCall,
1711                &model_call_request(&[ChatMessage::user("a")], &[]),
1712            )
1713            .unwrap();
1714        let replay_journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
1715        // Same draft sequence: one record read per event on the same clock.
1716        served.rejournal(&replay_journal, "run-t:parent");
1717        let replayed = replay_journal.snapshot();
1718        assert_eq!(replayed.events.len(), 1);
1719        // Identical except for the causal parent we supplied.
1720        let mut expected = recorded.events[0].clone();
1721        expected.parent = Some("run-t:parent".into());
1722        assert_eq!(replayed.events[0], expected);
1723    }
1724
1725    #[test]
1726    fn branch_diff_reports_divergence_steps_and_totals() {
1727        let shared = || {
1728            vec![
1729                EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1730                    .input(json!({"step": 0, "active_nodes": ["a"]})),
1731                EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1732                    .output(json!({"log": ["a"]})),
1733            ]
1734        };
1735        let mut base_drafts = shared();
1736        base_drafts.push(
1737            EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1738                .input(json!({"step": 1, "active_nodes": ["b"]})),
1739        );
1740        base_drafts.push(
1741            EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1742                .output(json!({"log": ["a", "b"]})),
1743        );
1744        base_drafts.push(
1745            model_draft(
1746                model_call_request(&[ChatMessage::user("q")], &[]),
1747                json!({"r": 1}),
1748            )
1749            .cost_usd(0.001),
1750        );
1751        let base = snapshot_with(base_drafts);
1752
1753        let mut branch_drafts = shared();
1754        branch_drafts.push(
1755            EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1756                .input(json!({"step": 1, "active_nodes": ["c"]})),
1757        );
1758        branch_drafts.push(
1759            EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1760                .output(json!({"log": ["a", "c"], "extra": 1})),
1761        );
1762        let branch = snapshot_with(branch_drafts);
1763
1764        let diff = BranchDiff::between(&base, &branch);
1765        assert!(!diff.is_identical());
1766        assert_eq!(diff.first_divergent_seq, Some(2));
1767        assert_eq!(diff.removed.len(), 3);
1768        assert_eq!(diff.added.len(), 2);
1769        assert_eq!(diff.base_totals.events, 5);
1770        assert_eq!(diff.branch_totals.events, 4);
1771        assert_eq!(diff.base_totals.tokens.total_tokens, 15);
1772        assert_eq!(diff.branch_totals.tokens.total_tokens, 0);
1773        assert_eq!(diff.base_totals.cost_usd, 0.001);
1774        assert_eq!(diff.branch_totals.cost_usd, 0.0);
1775
1776        // Step 0 is shared; step 1 differs on `log` and `extra`.
1777        assert_eq!(diff.step_diffs.len(), 1);
1778        let step = &diff.step_diffs[0];
1779        assert_eq!(step.step, 1);
1780        assert_eq!(step.channels.len(), 2);
1781        assert_eq!(step.channels[0].channel, "extra");
1782        assert_eq!(step.channels[0].base, None);
1783        assert_eq!(step.channels[0].branch, Some(json!(1)));
1784        assert_eq!(step.channels[1].channel, "log");
1785        assert_eq!(step.channels[1].base, Some(json!(["a", "b"])));
1786        assert_eq!(step.channels[1].branch, Some(json!(["a", "c"])));
1787    }
1788
1789    #[test]
1790    fn branch_diff_of_identical_snapshots_is_empty() {
1791        // Two separate runs of the same drafts: identity fields (ids,
1792        // timestamps) match here too, but the point is the logical comparison.
1793        let snapshot = snapshot_with(vec![EventDraft::new(
1794            RunEventKind::SuperStepStart,
1795            Effect::Pure,
1796        )
1797        .input(json!({"step": 0, "active_nodes": ["a"]}))]);
1798        let diff = BranchDiff::between(&snapshot, &snapshot);
1799        assert!(diff.is_identical());
1800        assert!(diff.added.is_empty() && diff.removed.is_empty());
1801        assert!(diff.step_diffs.is_empty());
1802    }
1803
1804    #[test]
1805    fn fixture_roundtrip_and_version_guard() {
1806        let journal = snapshot_with(vec![model_draft(
1807            model_call_request(&[ChatMessage::user("a")], &[]),
1808            json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1809        )]);
1810        let mut builder = crate::graph::GraphBuilder::new();
1811        builder.add_node("agent", |_ctx: crate::node::NodeContext| async {
1812            Ok(crate::node::NodeOutput::empty())
1813        });
1814        builder.set_entry_point("agent");
1815        let graph = builder.compile().unwrap();
1816        let fixture = ReplayFixture::capture(
1817            "roundtrip",
1818            &graph,
1819            "v1",
1820            journal,
1821            None,
1822            Some(LogicalClockParams {
1823                start_ms: 1_000_000,
1824                tick_ms: 5,
1825            }),
1826            Some(7),
1827        );
1828        let wire = fixture.export().unwrap();
1829        let imported = ReplayFixture::import(&wire).unwrap();
1830        assert_eq!(imported.export().unwrap(), wire);
1831
1832        // An unsupported format version is rejected with a descriptive error.
1833        let mut value: Value = serde_json::from_str(&wire).unwrap();
1834        value["format_version"] = json!(99);
1835        let error = ReplayFixture::import(&value.to_string()).unwrap_err();
1836        assert!(error.to_string().contains("format version 99"));
1837    }
1838
1839    // ---------- the shadow's recorded world (R0.12 wave 4) ----------
1840
1841    fn tool_draft(name: &str, arguments: Value, result: Value) -> EventDraft {
1842        EventDraft::new(RunEventKind::ToolCall, Effect::NonIdempotent)
1843            .node("tools")
1844            .input(tool_call_request(name, &arguments))
1845            .output(result)
1846    }
1847
1848    fn shadow_snapshot() -> JournalSnapshot {
1849        snapshot_with(vec![
1850            tool_draft("charge", json!({"amount": 10}), json!({"receipt": "r-1"})),
1851            tool_draft("charge", json!({"amount": 20}), json!({"receipt": "r-2"})),
1852            tool_draft("lookup", json!({"id": 7}), json!({"row": "x"})),
1853        ])
1854    }
1855
1856    #[test]
1857    fn the_shadow_source_serves_in_sequence_and_only_the_matching_call() {
1858        use crate::effects::ShadowOutcomeSource;
1859        let source = JournalShadowSource::new(&shadow_snapshot());
1860
1861        // The recorded next call under the tool serves; canonicalization
1862        // makes argument key order irrelevant.
1863        assert_eq!(
1864            source.serve(
1865                "charge",
1866                &tool_call_request("charge", &json!({"amount": 10}))
1867            ),
1868            Some(json!({"receipt": "r-1"}))
1869        );
1870        // A replay of the same call now diverges — the front of the queue
1871        // is the second charge, and a mismatch never pops it.
1872        assert_eq!(
1873            source.serve(
1874                "charge",
1875                &tool_call_request("charge", &json!({"amount": 10}))
1876            ),
1877            None
1878        );
1879        assert_eq!(
1880            source.serve(
1881                "charge",
1882                &tool_call_request("charge", &json!({"amount": 20}))
1883            ),
1884            Some(json!({"receipt": "r-2"}))
1885        );
1886        // A tool the recorded world never called answers None.
1887        assert_eq!(
1888            source.serve("refund", &tool_call_request("refund", &json!({}))),
1889            None
1890        );
1891    }
1892
1893    #[test]
1894    fn the_shadow_source_reports_what_the_candidate_never_requested() {
1895        use crate::effects::ShadowOutcomeSource;
1896        let source = JournalShadowSource::new(&shadow_snapshot());
1897        assert_eq!(source.unrequested(), vec!["charge", "lookup"]);
1898        source.serve(
1899            "charge",
1900            &tool_call_request("charge", &json!({"amount": 10})),
1901        );
1902        source.serve(
1903            "charge",
1904            &tool_call_request("charge", &json!({"amount": 20})),
1905        );
1906        assert_eq!(
1907            source.unrequested(),
1908            vec!["lookup"],
1909            "served calls leave the queue; the rest is divergence evidence"
1910        );
1911        source.serve("lookup", &tool_call_request("lookup", &json!({"id": 7})));
1912        assert!(source.unrequested().is_empty());
1913    }
1914
1915    #[test]
1916    fn the_shadow_source_ignores_non_tool_events() {
1917        use crate::effects::ShadowOutcomeSource;
1918        let snapshot = snapshot_with(vec![model_draft(
1919            model_call_request(&[ChatMessage::user("a")], &[]),
1920            json!({"r": 1}),
1921        )]);
1922        let source = JournalShadowSource::new(&snapshot);
1923        assert_eq!(
1924            source.serve("agent", &json!({"tool": "agent", "arguments": {}})),
1925            None,
1926            "only recorded tool calls are servable"
1927        );
1928        assert!(source.unrequested().is_empty());
1929    }
1930}