1use std::collections::BTreeMap;
57use std::sync::{Arc, Mutex, MutexGuard};
58
59use async_trait::async_trait;
60use serde::{Deserialize, Serialize};
61use serde_json::{json, Value};
62
63use crate::checkpoint::{Checkpoint, Checkpointer, InMemoryCheckpointer};
64use crate::effects::EffectRequest;
65use crate::error::{Result, RustyError};
66use crate::executor::{ExecutionOutcome, Executor, RunConfig, DEFAULT_MAX_STEPS};
67use crate::graph::Graph;
68use crate::journal::{Clock, EventDraft, Journal, JournalSnapshot, RngSource};
69use crate::llm::{ChatMessage, ChatModel, ChatResponse, ToolCall, Usage};
70use crate::record::{sha256_hex, Effect, EventStatus, PayloadRef, RunEvent, RunEventKind};
71use crate::state::{State, StateSpec};
72use crate::tool::Tool;
73
74pub const FIXTURE_FORMAT_VERSION: u32 = 1;
79
80pub const SERVABLE_KINDS: [RunEventKind; 4] = [
89 RunEventKind::ModelCall,
90 RunEventKind::ToolCall,
91 RunEventKind::RemoteCall,
92 RunEventKind::WasmCall,
93];
94
95fn is_servable(kind: RunEventKind) -> bool {
96 SERVABLE_KINDS.contains(&kind)
97}
98
99fn replay_error(message: impl Into<String>) -> RustyError {
100 RustyError::Replay(message.into())
101}
102
103pub fn model_call_request(messages: &[ChatMessage], tools: &[Value]) -> Value {
111 json!({ "messages": messages, "tools": tools })
112}
113
114pub fn model_call_response(response: &ChatResponse) -> Value {
120 json!({
121 "message": response.message,
122 "model": response.model,
123 "usage": response.usage,
124 })
125}
126
127pub fn tool_call_request(name: &str, arguments: &Value) -> Value {
132 json!({ "tool": name, "arguments": arguments })
133}
134
135fn chat_response_from_recorded(seq: u64, value: &Value) -> Result<ChatResponse> {
138 let decode = |key: &str| -> Result<Value> {
139 value.get(key).cloned().ok_or_else(|| {
140 replay_error(format!(
141 "recorded model call at seq {seq} is malformed: output payload has no `{key}` key"
142 ))
143 })
144 };
145 let message: ChatMessage = serde_json::from_value(decode("message")?)?;
146 let model: Option<String> = serde_json::from_value(decode("model")?)?;
147 let usage: Option<Usage> = serde_json::from_value(decode("usage")?)?;
148 Ok(ChatResponse {
149 message,
150 model,
151 usage,
152 })
153}
154
155fn recorded_error_text(served: &ServedEffect) -> String {
159 served
160 .output
161 .as_ref()
162 .and_then(|value| value.get("error"))
163 .and_then(Value::as_str)
164 .unwrap_or("<no recorded error payload>")
165 .to_owned()
166}
167
168fn verify_snapshot(snapshot: &JournalSnapshot) -> Result<()> {
174 Journal::from_snapshot(clone_snapshot(snapshot), Clock::System)?;
176 for (hash, value) in &snapshot.artifacts {
177 let actual = sha256_hex(&serde_json::to_vec(value)?);
178 if actual != *hash {
179 return Err(replay_error(format!(
180 "journal snapshot artifact integrity failure: artifact stored under {hash} \
181 hashes to {actual} — the snapshot was tampered with or corrupted"
182 )));
183 }
184 }
185 for event in &snapshot.events {
186 for payload in [&event.input, &event.output].into_iter().flatten() {
187 if let PayloadRef::Artifact(reference) = payload {
188 if !snapshot.artifacts.contains_key(&reference.sha256) {
189 return Err(replay_error(format!(
190 "journal snapshot is truncated: event {} references artifact {} which \
191 is absent from the artifact map",
192 event.id, reference.sha256
193 )));
194 }
195 }
196 }
197 }
198 Ok(())
199}
200
201fn clone_snapshot(snapshot: &JournalSnapshot) -> JournalSnapshot {
204 snapshot.clone()
205}
206
207fn resolve_in(snapshot: &JournalSnapshot, payload: &PayloadRef) -> Option<Value> {
208 match payload {
209 PayloadRef::Inline(value) => Some(value.clone()),
210 PayloadRef::Artifact(reference) => snapshot.artifacts.get(&reference.sha256).cloned(),
211 }
212}
213
214fn resolve_opt(snapshot: &JournalSnapshot, payload: Option<&PayloadRef>) -> Option<Value> {
215 payload.and_then(|payload| resolve_in(snapshot, payload))
216}
217
218fn request_hash(request: &Value) -> Result<String> {
223 Ok(sha256_hex(&serde_json::to_vec(request)?))
224}
225
226#[derive(Debug, Clone)]
230pub struct ServedEffect {
231 pub event: RunEvent,
233
234 pub input: Option<Value>,
236
237 pub output: Option<Value>,
239}
240
241impl ServedEffect {
242 pub fn rejournal(&self, journal: &Journal, parent: impl Into<String>) -> String {
250 let mut draft = EventDraft::new(self.event.kind, self.event.effect)
251 .status(self.event.status)
252 .parent(parent);
253 if let Some(node) = &self.event.node_id {
254 draft = draft.node(node.clone());
255 }
256 if let Some(input) = &self.input {
257 draft = draft.input(input.clone());
258 }
259 if let Some(output) = &self.output {
260 draft = draft.output(output.clone());
261 }
262 if let Some(latency_ms) = self.event.latency_ms {
263 draft = draft.latency_ms(latency_ms);
264 }
265 if let Some(tokens) = self.event.tokens {
266 draft = draft.tokens(tokens);
267 }
268 if let Some(cost_usd) = self.event.cost_usd {
269 draft = draft.cost_usd(cost_usd);
270 }
271 journal.record(draft)
272 }
273}
274
275#[derive(Debug)]
276struct ReplaySourceInner {
277 servable: Vec<RunEvent>,
279 artifacts: BTreeMap<String, Value>,
281 cursor: usize,
284}
285
286#[derive(Debug, Clone)]
297pub struct ReplaySource {
298 inner: Arc<Mutex<ReplaySourceInner>>,
299}
300
301impl ReplaySource {
302 pub fn new(snapshot: &JournalSnapshot) -> Self {
307 Self {
308 inner: Arc::new(Mutex::new(ReplaySourceInner {
309 servable: snapshot
310 .events
311 .iter()
312 .filter(|event| is_servable(event.kind))
313 .cloned()
314 .collect(),
315 artifacts: snapshot.artifacts.clone(),
316 cursor: 0,
317 })),
318 }
319 }
320
321 fn lock(&self) -> MutexGuard<'_, ReplaySourceInner> {
322 self.inner.lock().unwrap_or_else(|e| e.into_inner())
325 }
326
327 pub fn serve(&self, kind: RunEventKind, request: &Value) -> Result<ServedEffect> {
338 if !is_servable(kind) {
339 return Err(replay_error(format!(
340 "{kind:?} is not a servable effect kind; exact replay serves only {SERVABLE_KINDS:?} \
341 — executor-derived events are reproduced by re-execution, not served"
342 )));
343 }
344 let issued_hash = request_hash(request)?;
345 let mut inner = self.lock();
346 let Some(event) = inner.servable.get(inner.cursor) else {
347 return Err(replay_error(format!(
348 "replay exhaustion: the run issued a {kind:?} call (request hash {issued_hash}), \
349 but the journal holds no further recorded effects to serve — the replayed run \
350 is doing more work than the recorded one"
351 )));
352 };
353 if event.kind != kind {
354 return Err(replay_error(format!(
355 "replay order violation at recorded seq {}: the run issued a {kind:?} call, but \
356 the next unserved recorded effect is {:?} — exact replay requires effects in \
357 journaled sequence order",
358 event.seq, event.kind
359 )));
360 }
361 let recorded_input = event
362 .input
363 .as_ref()
364 .and_then(|payload| match payload {
365 PayloadRef::Inline(value) => Some(value.clone()),
366 PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
367 })
368 .ok_or_else(|| {
369 replay_error(format!(
370 "recorded {:?} at seq {} has no input payload to match against",
371 event.kind, event.seq
372 ))
373 })?;
374 let recorded_hash = request_hash(&recorded_input)?;
375 if recorded_hash != issued_hash {
376 return Err(replay_error(format!(
377 "replay divergence at recorded seq {} ({:?}): the run issued a request whose \
378 canonical hash is {issued_hash}, but the journaled request hashes to \
379 {recorded_hash} — the replayed run has diverged from its evidence",
380 event.seq, event.kind
381 )));
382 }
383 let served = ServedEffect {
384 input: Some(recorded_input),
385 output: event.output.as_ref().and_then(|payload| match payload {
386 PayloadRef::Inline(value) => Some(value.clone()),
387 PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
388 }),
389 event: event.clone(),
390 };
391 inner.cursor += 1;
392 Ok(served)
393 }
394
395 pub fn is_exhausted(&self) -> bool {
398 let inner = self.lock();
399 inner.cursor == inner.servable.len()
400 }
401
402 pub fn remaining(&self) -> Vec<(u64, RunEventKind)> {
405 let inner = self.lock();
406 inner.servable[inner.cursor..]
407 .iter()
408 .map(|event| (event.seq, event.kind))
409 .collect()
410 }
411}
412
413pub struct RecordingChatModel {
422 inner: Arc<dyn ChatModel>,
423 journal: Journal,
424 parent: String,
425 node_id: Option<String>,
426}
427
428impl RecordingChatModel {
429 pub fn new(inner: Arc<dyn ChatModel>, journal: Journal, parent: impl Into<String>) -> Self {
432 Self {
433 inner,
434 journal,
435 parent: parent.into(),
436 node_id: None,
437 }
438 }
439
440 pub fn node(mut self, node_id: impl Into<String>) -> Self {
443 self.node_id = Some(node_id.into());
444 self
445 }
446
447 fn draft(&self) -> EventDraft {
448 let mut draft = EventDraft::new(RunEventKind::ModelCall, self.inner.effect())
449 .parent(self.parent.clone());
450 if let Some(node) = &self.node_id {
451 draft = draft.node(node.clone());
452 }
453 draft
454 }
455}
456
457#[async_trait]
458impl ChatModel for RecordingChatModel {
459 async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
460 let started = self.journal.clock().now();
461 let result = self.inner.chat(messages, tools).await;
462 let latency_ms = (self.journal.clock().now() - started)
463 .num_milliseconds()
464 .max(0) as u64;
465 let draft = self
466 .draft()
467 .input(model_call_request(messages, tools))
468 .latency_ms(latency_ms);
469 match result {
470 Ok(response) => {
471 let mut draft = draft.output(model_call_response(&response));
472 if let Some(usage) = response.usage {
473 draft = draft.tokens(usage);
474 }
475 self.journal.record(draft);
476 Ok(response)
477 }
478 Err(error) => {
479 self.journal.record(
480 draft
481 .status(EventStatus::Error)
482 .output(json!({ "error": error.to_string() })),
483 );
484 Err(error)
485 }
486 }
487 }
488
489 fn effect(&self) -> Effect {
490 self.inner.effect()
491 }
492}
493
494pub struct ReplayingChatModel {
505 inner: Arc<dyn ChatModel>,
506 source: ReplaySource,
507 journal: Journal,
508 parent: String,
509}
510
511impl ReplayingChatModel {
512 pub fn new(
516 inner: Arc<dyn ChatModel>,
517 source: ReplaySource,
518 journal: Journal,
519 parent: impl Into<String>,
520 ) -> Self {
521 Self {
522 inner,
523 source,
524 journal,
525 parent: parent.into(),
526 }
527 }
528}
529
530#[async_trait]
531impl ChatModel for ReplayingChatModel {
532 async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
533 let served = self.source.serve(
534 RunEventKind::ModelCall,
535 &model_call_request(messages, tools),
536 )?;
537 let _started = self.journal.clock().now();
541 let _ended = self.journal.clock().now();
542 served.rejournal(&self.journal, self.parent.clone());
543 match served.event.status {
544 EventStatus::Ok => {
545 let output = served.output.as_ref().ok_or_else(|| {
546 replay_error(format!(
547 "recorded model call at seq {} succeeded but carries no output payload",
548 served.event.seq
549 ))
550 })?;
551 chat_response_from_recorded(served.event.seq, output)
552 }
553 EventStatus::Error => Err(RustyError::Llm(format!(
554 "replayed model call at seq {} failed as recorded: {}",
555 served.event.seq,
556 recorded_error_text(&served)
557 ))),
558 EventStatus::Interrupted => Err(replay_error(format!(
559 "recorded model call at seq {} has status `interrupted`, which model calls \
560 never produce — the journal is inconsistent",
561 served.event.seq
562 ))),
563 }
564 }
565
566 fn effect(&self) -> Effect {
567 self.inner.effect()
568 }
569}
570
571pub struct RecordingTool {
577 inner: Arc<dyn Tool>,
578 journal: Journal,
579 parent: String,
580 node_id: Option<String>,
581}
582
583impl RecordingTool {
584 pub fn new(inner: Arc<dyn Tool>, journal: Journal, parent: impl Into<String>) -> Self {
587 Self {
588 inner,
589 journal,
590 parent: parent.into(),
591 node_id: None,
592 }
593 }
594
595 pub fn node(mut self, node_id: impl Into<String>) -> Self {
598 self.node_id = Some(node_id.into());
599 self
600 }
601}
602
603#[async_trait]
604impl Tool for RecordingTool {
605 fn name(&self) -> &str {
606 self.inner.name()
607 }
608
609 fn description(&self) -> &str {
610 self.inner.description()
611 }
612
613 fn parameters_schema(&self) -> Value {
614 self.inner.parameters_schema()
615 }
616
617 fn effect(&self) -> Effect {
618 self.inner.effect()
619 }
620
621 fn effect_kind(&self) -> &str {
622 self.inner.effect_kind()
623 }
624
625 fn idempotency_key(&self, args: &Value) -> Option<String> {
626 self.inner.idempotency_key(args)
627 }
628
629 fn effect_request(&self, call: &ToolCall) -> EffectRequest {
630 self.inner.effect_request(call)
631 }
632
633 async fn call(&self, args: Value) -> Result<Value> {
634 let started = self.journal.clock().now();
635 let result = self.inner.call(args.clone()).await;
636 let latency_ms = (self.journal.clock().now() - started)
637 .num_milliseconds()
638 .max(0) as u64;
639 let mut draft = EventDraft::new(RunEventKind::ToolCall, self.inner.effect())
640 .input(tool_call_request(self.inner.name(), &args))
641 .latency_ms(latency_ms)
642 .parent(self.parent.clone());
643 if let Some(node) = &self.node_id {
644 draft = draft.node(node.clone());
645 }
646 match result {
647 Ok(value) => {
648 self.journal.record(draft.output(value.clone()));
649 Ok(value)
650 }
651 Err(error) => {
652 self.journal.record(
653 draft
654 .status(EventStatus::Error)
655 .output(json!({ "error": error.to_string() })),
656 );
657 Err(error)
658 }
659 }
660 }
661}
662
663pub struct ReplayingTool {
671 inner: Arc<dyn Tool>,
672 source: ReplaySource,
673 journal: Journal,
674 parent: String,
675}
676
677impl ReplayingTool {
678 pub fn new(
681 inner: Arc<dyn Tool>,
682 source: ReplaySource,
683 journal: Journal,
684 parent: impl Into<String>,
685 ) -> Self {
686 Self {
687 inner,
688 source,
689 journal,
690 parent: parent.into(),
691 }
692 }
693}
694
695#[async_trait]
696impl Tool for ReplayingTool {
697 fn name(&self) -> &str {
698 self.inner.name()
699 }
700
701 fn description(&self) -> &str {
702 self.inner.description()
703 }
704
705 fn parameters_schema(&self) -> Value {
706 self.inner.parameters_schema()
707 }
708
709 fn effect(&self) -> Effect {
710 self.inner.effect()
711 }
712
713 fn effect_kind(&self) -> &str {
714 self.inner.effect_kind()
715 }
716
717 fn idempotency_key(&self, args: &Value) -> Option<String> {
718 self.inner.idempotency_key(args)
719 }
720
721 fn effect_request(&self, call: &ToolCall) -> EffectRequest {
722 self.inner.effect_request(call)
723 }
724
725 async fn call(&self, args: Value) -> Result<Value> {
726 let request = tool_call_request(self.inner.name(), &args);
727 let served = self.source.serve(RunEventKind::ToolCall, &request)?;
728 let _started = self.journal.clock().now();
730 let _ended = self.journal.clock().now();
731 served.rejournal(&self.journal, self.parent.clone());
732 match served.event.status {
733 EventStatus::Ok => served.output.clone().ok_or_else(|| {
734 replay_error(format!(
735 "recorded tool call at seq {} succeeded but carries no output payload",
736 served.event.seq
737 ))
738 }),
739 EventStatus::Error => Err(RustyError::Tool(format!(
740 "replayed tool call at seq {} failed as recorded: {}",
741 served.event.seq,
742 recorded_error_text(&served)
743 ))),
744 EventStatus::Interrupted => Err(replay_error(format!(
745 "recorded tool call at seq {} has status `interrupted`, which tool calls \
746 never produce — the journal is inconsistent",
747 served.event.seq
748 ))),
749 }
750 }
751}
752
753#[derive(Debug)]
769pub struct ExactReplay {
770 snapshot: JournalSnapshot,
771 source: ReplaySource,
772}
773
774impl ExactReplay {
775 pub fn new(snapshot: JournalSnapshot) -> Result<Self> {
784 verify_snapshot(&snapshot)?;
785 if snapshot
786 .events
787 .first()
788 .is_some_and(|event| event.kind == RunEventKind::Resume)
789 {
790 return Err(replay_error(
791 "exact replay of resumed runs is not supported: the journal begins with a \
792 resume event, whose pre-resume state lives in a checkpoint the journal does \
793 not carry — replay the original run's journal instead",
794 ));
795 }
796 let source = ReplaySource::new(&snapshot);
797 Ok(Self { snapshot, source })
798 }
799
800 pub fn snapshot(&self) -> &JournalSnapshot {
802 &self.snapshot
803 }
804
805 pub fn source(&self) -> ReplaySource {
809 self.source.clone()
810 }
811
812 pub fn fresh_journal(&self, clock: Clock) -> Journal {
817 Journal::new(
818 self.snapshot.run_id.clone(),
819 self.snapshot.thread_id.clone(),
820 clock,
821 )
822 }
823
824 pub async fn run(
833 &self,
834 graph: &Graph,
835 spec: &StateSpec,
836 initial_state: State,
837 params: ReplayParams,
838 ) -> Result<ReplayOutcome> {
839 if params.journal.run_id() != self.snapshot.run_id
840 || params.journal.thread_id() != self.snapshot.thread_id
841 {
842 return Err(replay_error(format!(
843 "replay journal identity mismatch: the snapshot records run `{}` on thread \
844 `{}`, but the supplied journal is run `{}` on thread `{}` — build the replay \
845 journal with ExactReplay::fresh_journal",
846 self.snapshot.run_id,
847 self.snapshot.thread_id,
848 params.journal.run_id(),
849 params.journal.thread_id()
850 )));
851 }
852 let executor = match ¶ms.checkpointer {
853 Some(checkpointer) => Executor::with_checkpointer(checkpointer.clone()),
854 None => Executor::new(),
855 };
856 let config = RunConfig::new(self.snapshot.thread_id.clone())
857 .with_max_steps(params.max_steps)
858 .with_rng(params.rng.clone())
859 .with_journal(params.journal.clone());
860 let outcome = executor.run(graph, spec, initial_state, config).await?;
861 Ok(ReplayOutcome {
862 outcome,
863 journal: params.journal.snapshot(),
864 })
865 }
866
867 pub fn verify(&self, replayed: &JournalSnapshot) -> Result<()> {
872 if !self.source.is_exhausted() {
873 return Err(replay_error(format!(
874 "exact replay left recorded effects unserved: {:?} — the replayed run stopped \
875 short of the journaled history",
876 self.source.remaining()
877 )));
878 }
879 let recorded = &self.snapshot;
880 if recorded.run_id != replayed.run_id || recorded.thread_id != replayed.thread_id {
881 return Err(replay_error(format!(
882 "replay identity mismatch: recorded run `{}`/thread `{}` vs replayed \
883 `{}`/`{}`",
884 recorded.run_id, recorded.thread_id, replayed.run_id, replayed.thread_id
885 )));
886 }
887 if recorded.events.len() != replayed.events.len() {
888 return Err(replay_error(format!(
889 "replay event-count mismatch: the recorded journal holds {} events but the \
890 replayed journal holds {}",
891 recorded.events.len(),
892 replayed.events.len()
893 )));
894 }
895 for (recorded_event, replayed_event) in recorded.events.iter().zip(&replayed.events) {
896 if recorded_event != replayed_event {
897 return Err(replay_error(format!(
898 "replay event mismatch at seq {}: recorded {} but replayed {}",
899 recorded_event.seq,
900 summarize_event(recorded_event),
901 summarize_event(replayed_event),
902 )));
903 }
904 }
905 if recorded.artifacts != replayed.artifacts {
906 return Err(replay_error(
907 "replay artifact mismatch: the replayed journal's artifact payloads differ \
908 from the recorded ones",
909 ));
910 }
911 if recorded.head_hash != replayed.head_hash {
912 return Err(replay_error(format!(
913 "replay head-hash mismatch: recorded {} but replayed {} — event content \
914 agrees but the evidence chain does not (this should be unreachable; please \
915 report it)",
916 recorded.head_hash, replayed.head_hash
917 )));
918 }
919 Ok(())
920 }
921
922 pub async fn run_and_verify(
926 &self,
927 graph: &Graph,
928 spec: &StateSpec,
929 initial_state: State,
930 params: ReplayParams,
931 ) -> Result<ReplayOutcome> {
932 let replayed = self.run(graph, spec, initial_state, params).await?;
933 self.verify(&replayed.journal)?;
934 Ok(replayed)
935 }
936}
937
938fn summarize_event(event: &RunEvent) -> String {
941 format!(
942 "{:?}(node={:?}, status={:?})",
943 event.kind, event.node_id, event.status
944 )
945}
946
947pub struct ReplayParams {
953 pub journal: Journal,
955
956 pub rng: RngSource,
960
961 pub checkpointer: Option<Arc<dyn Checkpointer>>,
964
965 pub max_steps: usize,
967}
968
969impl ReplayParams {
970 pub fn new(journal: Journal, rng: RngSource) -> Self {
973 Self {
974 journal,
975 rng,
976 checkpointer: None,
977 max_steps: DEFAULT_MAX_STEPS,
978 }
979 }
980
981 pub fn with_checkpointer(mut self, checkpointer: Arc<dyn Checkpointer>) -> Self {
983 self.checkpointer = Some(checkpointer);
984 self
985 }
986
987 pub fn with_max_steps(mut self, max_steps: usize) -> Self {
989 self.max_steps = max_steps;
990 self
991 }
992}
993
994#[derive(Debug)]
997pub struct ReplayOutcome {
998 pub outcome: ExecutionOutcome,
1001
1002 pub journal: JournalSnapshot,
1006}
1007
1008#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
1012pub struct BranchTotals {
1013 pub events: usize,
1015
1016 pub tokens: Usage,
1018
1019 pub cost_usd: f64,
1021}
1022
1023#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1027pub struct ChannelDiff {
1028 pub channel: String,
1030
1031 pub base: Option<Value>,
1033
1034 pub branch: Option<Value>,
1036}
1037
1038#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1040pub struct StepDiff {
1041 pub step: u64,
1043
1044 pub channels: Vec<ChannelDiff>,
1047}
1048
1049#[derive(Debug, Clone, Serialize, Deserialize)]
1062pub struct BranchDiff {
1063 pub first_divergent_seq: Option<u64>,
1066
1067 pub added: Vec<RunEvent>,
1070
1071 pub removed: Vec<RunEvent>,
1074
1075 pub step_diffs: Vec<StepDiff>,
1078
1079 pub base_totals: BranchTotals,
1081
1082 pub branch_totals: BranchTotals,
1084}
1085
1086impl BranchDiff {
1087 pub fn between(base: &JournalSnapshot, branch: &JournalSnapshot) -> BranchDiff {
1092 let divergence = (0..base.events.len().max(branch.events.len()))
1093 .find(|&i| {
1094 match (base.events.get(i), branch.events.get(i)) {
1095 (Some(a), Some(b)) => !logically_equal(a, base, b, branch),
1096 _ => true,
1098 }
1099 })
1100 .map(|i| {
1101 base.events
1102 .get(i)
1103 .or_else(|| branch.events.get(i))
1104 .map(|event| event.seq)
1105 .unwrap_or(i as u64)
1106 });
1107 let (removed, added) = match divergence {
1108 Some(seq) => (
1109 base.events
1110 .iter()
1111 .filter(|event| event.seq >= seq)
1112 .cloned()
1113 .collect(),
1114 branch
1115 .events
1116 .iter()
1117 .filter(|event| event.seq >= seq)
1118 .cloned()
1119 .collect(),
1120 ),
1121 None => (Vec::new(), Vec::new()),
1122 };
1123 BranchDiff {
1124 first_divergent_seq: divergence,
1125 added,
1126 removed,
1127 step_diffs: diff_steps(base, branch),
1128 base_totals: totals(base),
1129 branch_totals: totals(branch),
1130 }
1131 }
1132
1133 pub fn is_identical(&self) -> bool {
1136 self.first_divergent_seq.is_none()
1137 }
1138}
1139
1140fn logically_equal(
1144 a: &RunEvent,
1145 snapshot_a: &JournalSnapshot,
1146 b: &RunEvent,
1147 snapshot_b: &JournalSnapshot,
1148) -> bool {
1149 a.seq == b.seq
1150 && a.kind == b.kind
1151 && a.node_id == b.node_id
1152 && a.effect == b.effect
1153 && a.latency_ms == b.latency_ms
1154 && a.tokens == b.tokens
1155 && a.cost_usd == b.cost_usd
1156 && a.status == b.status
1157 && resolve_opt(snapshot_a, a.input.as_ref()) == resolve_opt(snapshot_b, b.input.as_ref())
1158 && resolve_opt(snapshot_a, a.output.as_ref()) == resolve_opt(snapshot_b, b.output.as_ref())
1159}
1160
1161fn totals(snapshot: &JournalSnapshot) -> BranchTotals {
1163 let mut totals = BranchTotals {
1164 events: snapshot.events.len(),
1165 ..BranchTotals::default()
1166 };
1167 for event in &snapshot.events {
1168 if let Some(usage) = event.tokens {
1169 totals.tokens.prompt_tokens = totals
1170 .tokens
1171 .prompt_tokens
1172 .saturating_add(usage.prompt_tokens);
1173 totals.tokens.completion_tokens = totals
1174 .tokens
1175 .completion_tokens
1176 .saturating_add(usage.completion_tokens);
1177 totals.tokens.total_tokens = totals
1178 .tokens
1179 .total_tokens
1180 .saturating_add(usage.total_tokens);
1181 }
1182 if let Some(cost) = event.cost_usd {
1183 totals.cost_usd += cost;
1184 }
1185 }
1186 totals
1187}
1188
1189fn step_channel_values(snapshot: &JournalSnapshot) -> BTreeMap<u64, BTreeMap<String, Value>> {
1193 let mut steps = BTreeMap::new();
1194 let mut current_step: Option<u64> = None;
1195 for event in &snapshot.events {
1196 match event.kind {
1197 RunEventKind::SuperStepStart => {
1198 current_step = resolve_opt(snapshot, event.input.as_ref())
1199 .and_then(|input| input.get("step").and_then(Value::as_u64));
1200 }
1201 RunEventKind::SuperStepEnd => {
1202 if let Some(step) = current_step.take() {
1203 let channels = resolve_opt(snapshot, event.output.as_ref())
1204 .and_then(|output| output.as_object().cloned())
1205 .unwrap_or_default()
1206 .into_iter()
1207 .collect();
1208 steps.insert(step, channels);
1209 }
1210 }
1211 _ => {}
1212 }
1213 }
1214 steps
1215}
1216
1217fn diff_steps(base: &JournalSnapshot, branch: &JournalSnapshot) -> Vec<StepDiff> {
1220 let base_steps = step_channel_values(base);
1221 let branch_steps = step_channel_values(branch);
1222 let mut diffs = Vec::new();
1223 for step in base_steps
1224 .keys()
1225 .chain(branch_steps.keys())
1226 .copied()
1227 .collect::<std::collections::BTreeSet<u64>>()
1228 {
1229 let empty = BTreeMap::new();
1230 let base_channels = base_steps.get(&step).unwrap_or(&empty);
1231 let branch_channels = branch_steps.get(&step).unwrap_or(&empty);
1232 let mut channels = Vec::new();
1233 for channel in base_channels
1234 .keys()
1235 .chain(branch_channels.keys())
1236 .cloned()
1237 .collect::<std::collections::BTreeSet<String>>()
1238 {
1239 let base_value = base_channels.get(&channel).cloned();
1240 let branch_value = branch_channels.get(&channel).cloned();
1241 if base_value != branch_value {
1242 channels.push(ChannelDiff {
1243 channel,
1244 base: base_value,
1245 branch: branch_value,
1246 });
1247 }
1248 }
1249 if !channels.is_empty() {
1250 diffs.push(StepDiff { step, channels });
1251 }
1252 }
1253 diffs
1254}
1255
1256#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1259pub struct LogicalClockParams {
1260 pub start_ms: u64,
1262
1263 pub tick_ms: u64,
1265}
1266
1267#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1269pub struct FixtureMetadata {
1270 pub name: String,
1272
1273 #[serde(default, skip_serializing_if = "Option::is_none")]
1275 pub clock: Option<LogicalClockParams>,
1276
1277 #[serde(default, skip_serializing_if = "Option::is_none")]
1279 pub rng_seed: Option<u64>,
1280}
1281
1282#[derive(Debug, Clone, Serialize, Deserialize)]
1293pub struct ReplayFixture {
1294 pub format_version: u32,
1297
1298 pub graph_hash: String,
1302
1303 pub graph_version: String,
1305
1306 pub journal: JournalSnapshot,
1308
1309 pub final_checkpoint: Option<Checkpoint>,
1312
1313 pub metadata: FixtureMetadata,
1315}
1316
1317impl ReplayFixture {
1318 pub fn capture(
1322 name: impl Into<String>,
1323 graph: &Graph,
1324 graph_version: impl Into<String>,
1325 journal: JournalSnapshot,
1326 final_checkpoint: Option<Checkpoint>,
1327 clock: Option<LogicalClockParams>,
1328 rng_seed: Option<u64>,
1329 ) -> Self {
1330 Self {
1331 format_version: FIXTURE_FORMAT_VERSION,
1332 graph_hash: graph.topology_hash(),
1333 graph_version: graph_version.into(),
1334 journal,
1335 final_checkpoint,
1336 metadata: FixtureMetadata {
1337 name: name.into(),
1338 clock,
1339 rng_seed,
1340 },
1341 }
1342 }
1343
1344 pub fn export(&self) -> Result<String> {
1346 Ok(serde_json::to_string_pretty(self)?)
1347 }
1348
1349 pub fn import(json: &str) -> Result<Self> {
1352 let fixture: Self = serde_json::from_str(json)?;
1353 if fixture.format_version != FIXTURE_FORMAT_VERSION {
1354 return Err(replay_error(format!(
1355 "unsupported fixture format version {} (this build supports \
1356 {FIXTURE_FORMAT_VERSION}) — regenerate the fixture or upgrade the runtime",
1357 fixture.format_version
1358 )));
1359 }
1360 verify_snapshot(&fixture.journal)?;
1361 Ok(fixture)
1362 }
1363
1364 pub fn exact_replay(&self) -> Result<ExactReplay> {
1368 ExactReplay::new(self.journal.clone())
1369 }
1370
1371 pub fn replay_params(&self, replay: &ExactReplay) -> Result<ReplayParams> {
1381 let clock = self.metadata.clock.ok_or_else(|| {
1382 replay_error(format!(
1383 "fixture `{}` carries no logical-clock parameters; exact CI replay requires \
1384 the recorded run's determinism seams",
1385 self.metadata.name
1386 ))
1387 })?;
1388 let seed = self.metadata.rng_seed.ok_or_else(|| {
1389 replay_error(format!(
1390 "fixture `{}` carries no RNG seed; exact CI replay requires the recorded \
1391 run's determinism seams",
1392 self.metadata.name
1393 ))
1394 })?;
1395 let journal = replay.fresh_journal(Clock::logical(clock.start_ms, clock.tick_ms));
1396 Ok(ReplayParams::new(journal, RngSource::seeded(seed))
1397 .with_checkpointer(Arc::new(InMemoryCheckpointer::new())))
1398 }
1399
1400 pub async fn replay_in_ci(
1411 &self,
1412 replay: ExactReplay,
1413 graph: &Graph,
1414 spec: &StateSpec,
1415 initial_state: State,
1416 params: ReplayParams,
1417 ) -> Result<ReplayOutcome> {
1418 let actual_hash = graph.topology_hash();
1419 if actual_hash != self.graph_hash {
1420 return Err(replay_error(format!(
1421 "fixture `{}` was recorded against graph topology {}, but the graph under \
1422 replay hashes to {actual_hash} — rebuild the recorded topology (same node \
1423 names, edges, and entry point)",
1424 self.metadata.name, self.graph_hash
1425 )));
1426 }
1427 let replayed = replay
1428 .run_and_verify(graph, spec, initial_state, params)
1429 .await?;
1430 if let Some(final_checkpoint) = &self.final_checkpoint {
1431 if replayed.outcome.state() != &final_checkpoint.state {
1432 return Err(replay_error(format!(
1433 "fixture `{}`: the replayed final state diverges from the recorded final \
1434 checkpoint (journal evidence agreed; state did not)",
1435 self.metadata.name
1436 )));
1437 }
1438 }
1439 Ok(replayed)
1440 }
1441}
1442
1443#[cfg(test)]
1444mod tests {
1445 use super::*;
1446 use crate::journal::Journal;
1447 use serde_json::json;
1448
1449 fn snapshot_with(drafts: Vec<EventDraft>) -> JournalSnapshot {
1450 let journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
1451 for draft in drafts {
1452 journal.record(draft);
1453 }
1454 journal.snapshot()
1455 }
1456
1457 fn model_draft(request: Value, response: Value) -> EventDraft {
1458 EventDraft::new(RunEventKind::ModelCall, Effect::NonIdempotent)
1459 .node("agent")
1460 .input(request)
1461 .output(response)
1462 .latency_ms(2)
1463 .tokens(Usage {
1464 prompt_tokens: 10,
1465 completion_tokens: 5,
1466 total_tokens: 15,
1467 })
1468 }
1469
1470 #[test]
1471 fn canonical_shapes_have_stable_hashes() {
1472 let request = model_call_request(&[ChatMessage::user("ping")], &[]);
1473 assert_eq!(
1474 request,
1475 json!({"messages": [{"role": "user", "content": "ping"}], "tools": []})
1476 );
1477 let first = request_hash(&request).unwrap();
1478 let second = request_hash(&model_call_request(&[ChatMessage::user("ping")], &[])).unwrap();
1479 assert_eq!(first, second);
1480 let other = request_hash(&model_call_request(&[ChatMessage::user("pong")], &[])).unwrap();
1482 assert_ne!(first, other);
1483
1484 let tool = tool_call_request("echo", &json!({"text": "hi"}));
1485 assert_eq!(tool, json!({"tool": "echo", "arguments": {"text": "hi"}}));
1486 }
1487
1488 #[test]
1489 fn serve_matches_by_sequence_and_request_hash() {
1490 let snapshot = snapshot_with(vec![
1491 model_draft(
1492 model_call_request(&[ChatMessage::user("a")], &[]),
1493 json!({"r": 1}),
1494 ),
1495 model_draft(
1496 model_call_request(&[ChatMessage::user("b")], &[]),
1497 json!({"r": 2}),
1498 ),
1499 ]);
1500 let source = ReplaySource::new(&snapshot);
1501 assert!(!source.is_exhausted());
1502
1503 let error = source
1505 .serve(
1506 RunEventKind::ModelCall,
1507 &model_call_request(&[ChatMessage::user("zzz")], &[]),
1508 )
1509 .unwrap_err();
1510 assert!(matches!(error, RustyError::Replay(_)));
1511 assert!(error.to_string().contains("divergence"));
1512 assert_eq!(source.remaining().len(), 2);
1513
1514 let error = source
1516 .serve(RunEventKind::ToolCall, &tool_call_request("t", &json!({})))
1517 .unwrap_err();
1518 assert!(error.to_string().contains("order violation"));
1519
1520 let first = source
1522 .serve(
1523 RunEventKind::ModelCall,
1524 &model_call_request(&[ChatMessage::user("a")], &[]),
1525 )
1526 .unwrap();
1527 assert_eq!(first.event.seq, 0);
1528 assert_eq!(first.output, Some(json!({"r": 1})));
1529 let second = source
1530 .serve(
1531 RunEventKind::ModelCall,
1532 &model_call_request(&[ChatMessage::user("b")], &[]),
1533 )
1534 .unwrap();
1535 assert_eq!(second.event.seq, 1);
1536 assert!(source.is_exhausted());
1537
1538 let error = source
1540 .serve(
1541 RunEventKind::ModelCall,
1542 &model_call_request(&[ChatMessage::user("a")], &[]),
1543 )
1544 .unwrap_err();
1545 assert!(error.to_string().contains("exhaustion"));
1546 }
1547
1548 #[test]
1549 fn serve_rejects_non_servable_kinds() {
1550 let snapshot = snapshot_with(vec![EventDraft::new(
1551 RunEventKind::SuperStepStart,
1552 Effect::Pure,
1553 )]);
1554 let error = ReplaySource::new(&snapshot)
1555 .serve(RunEventKind::NodeInput, &json!({}))
1556 .unwrap_err();
1557 assert!(error.to_string().contains("not a servable"));
1558 }
1559
1560 #[test]
1561 fn artifact_backed_requests_match_and_resolve() {
1562 let big_request =
1563 json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1564 let big_response = json!({"message": {"role": "assistant", "content": "y".repeat(9000)}, "model": null, "usage": null});
1565 let snapshot = snapshot_with(vec![model_draft(big_request.clone(), big_response.clone())]);
1566 assert!(matches!(
1568 snapshot.events[0].input,
1569 Some(PayloadRef::Artifact(_))
1570 ));
1571 let served = ReplaySource::new(&snapshot)
1572 .serve(RunEventKind::ModelCall, &big_request)
1573 .unwrap();
1574 assert_eq!(served.output, Some(big_response));
1575 }
1576
1577 #[test]
1578 fn tampered_snapshots_are_rejected_at_the_boundary() {
1579 let snapshot = snapshot_with(vec![model_draft(
1580 model_call_request(&[ChatMessage::user("a")], &[]),
1581 json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1582 )]);
1583
1584 let mut tampered = snapshot.clone();
1586 tampered.events[0].status = EventStatus::Error;
1587 assert!(ExactReplay::new(tampered).is_err());
1588
1589 let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1592 let artifact_snapshot = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
1593 let mut tampered = artifact_snapshot;
1594 *tampered.artifacts.values_mut().next().unwrap() = json!({"forged": true});
1595 let error = ExactReplay::new(tampered).unwrap_err();
1596 assert!(error.to_string().contains("integrity"));
1597
1598 let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1600 let truncated = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
1601 let mut truncated = truncated;
1602 truncated.artifacts.clear();
1603 assert!(ExactReplay::new(truncated).is_err());
1604 }
1605
1606 #[test]
1607 fn resumed_run_journals_are_rejected() {
1608 let snapshot = snapshot_with(vec![EventDraft::new(RunEventKind::Resume, Effect::Pure)
1609 .input(json!({"checkpoint_id": "c"}))]);
1610 let error = ExactReplay::new(snapshot).unwrap_err();
1611 assert!(error.to_string().contains("resumed"));
1612 }
1613
1614 #[test]
1615 fn rejournal_reproduces_the_recorded_event() {
1616 let recorded = snapshot_with(vec![model_draft(
1617 model_call_request(&[ChatMessage::user("a")], &[]),
1618 json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1619 )]);
1620 let source = ReplaySource::new(&recorded);
1621 let served = source
1622 .serve(
1623 RunEventKind::ModelCall,
1624 &model_call_request(&[ChatMessage::user("a")], &[]),
1625 )
1626 .unwrap();
1627 let replay_journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
1628 served.rejournal(&replay_journal, "run-t:parent");
1630 let replayed = replay_journal.snapshot();
1631 assert_eq!(replayed.events.len(), 1);
1632 let mut expected = recorded.events[0].clone();
1634 expected.parent = Some("run-t:parent".into());
1635 assert_eq!(replayed.events[0], expected);
1636 }
1637
1638 #[test]
1639 fn branch_diff_reports_divergence_steps_and_totals() {
1640 let shared = || {
1641 vec![
1642 EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1643 .input(json!({"step": 0, "active_nodes": ["a"]})),
1644 EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1645 .output(json!({"log": ["a"]})),
1646 ]
1647 };
1648 let mut base_drafts = shared();
1649 base_drafts.push(
1650 EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1651 .input(json!({"step": 1, "active_nodes": ["b"]})),
1652 );
1653 base_drafts.push(
1654 EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1655 .output(json!({"log": ["a", "b"]})),
1656 );
1657 base_drafts.push(
1658 model_draft(
1659 model_call_request(&[ChatMessage::user("q")], &[]),
1660 json!({"r": 1}),
1661 )
1662 .cost_usd(0.001),
1663 );
1664 let base = snapshot_with(base_drafts);
1665
1666 let mut branch_drafts = shared();
1667 branch_drafts.push(
1668 EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1669 .input(json!({"step": 1, "active_nodes": ["c"]})),
1670 );
1671 branch_drafts.push(
1672 EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1673 .output(json!({"log": ["a", "c"], "extra": 1})),
1674 );
1675 let branch = snapshot_with(branch_drafts);
1676
1677 let diff = BranchDiff::between(&base, &branch);
1678 assert!(!diff.is_identical());
1679 assert_eq!(diff.first_divergent_seq, Some(2));
1680 assert_eq!(diff.removed.len(), 3);
1681 assert_eq!(diff.added.len(), 2);
1682 assert_eq!(diff.base_totals.events, 5);
1683 assert_eq!(diff.branch_totals.events, 4);
1684 assert_eq!(diff.base_totals.tokens.total_tokens, 15);
1685 assert_eq!(diff.branch_totals.tokens.total_tokens, 0);
1686 assert_eq!(diff.base_totals.cost_usd, 0.001);
1687 assert_eq!(diff.branch_totals.cost_usd, 0.0);
1688
1689 assert_eq!(diff.step_diffs.len(), 1);
1691 let step = &diff.step_diffs[0];
1692 assert_eq!(step.step, 1);
1693 assert_eq!(step.channels.len(), 2);
1694 assert_eq!(step.channels[0].channel, "extra");
1695 assert_eq!(step.channels[0].base, None);
1696 assert_eq!(step.channels[0].branch, Some(json!(1)));
1697 assert_eq!(step.channels[1].channel, "log");
1698 assert_eq!(step.channels[1].base, Some(json!(["a", "b"])));
1699 assert_eq!(step.channels[1].branch, Some(json!(["a", "c"])));
1700 }
1701
1702 #[test]
1703 fn branch_diff_of_identical_snapshots_is_empty() {
1704 let snapshot = snapshot_with(vec![EventDraft::new(
1707 RunEventKind::SuperStepStart,
1708 Effect::Pure,
1709 )
1710 .input(json!({"step": 0, "active_nodes": ["a"]}))]);
1711 let diff = BranchDiff::between(&snapshot, &snapshot);
1712 assert!(diff.is_identical());
1713 assert!(diff.added.is_empty() && diff.removed.is_empty());
1714 assert!(diff.step_diffs.is_empty());
1715 }
1716
1717 #[test]
1718 fn fixture_roundtrip_and_version_guard() {
1719 let journal = snapshot_with(vec![model_draft(
1720 model_call_request(&[ChatMessage::user("a")], &[]),
1721 json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1722 )]);
1723 let mut builder = crate::graph::GraphBuilder::new();
1724 builder.add_node("agent", |_ctx: crate::node::NodeContext| async {
1725 Ok(crate::node::NodeOutput::empty())
1726 });
1727 builder.set_entry_point("agent");
1728 let graph = builder.compile().unwrap();
1729 let fixture = ReplayFixture::capture(
1730 "roundtrip",
1731 &graph,
1732 "v1",
1733 journal,
1734 None,
1735 Some(LogicalClockParams {
1736 start_ms: 1_000_000,
1737 tick_ms: 5,
1738 }),
1739 Some(7),
1740 );
1741 let wire = fixture.export().unwrap();
1742 let imported = ReplayFixture::import(&wire).unwrap();
1743 assert_eq!(imported.export().unwrap(), wire);
1744
1745 let mut value: Value = serde_json::from_str(&wire).unwrap();
1747 value["format_version"] = json!(99);
1748 let error = ReplayFixture::import(&value.to_string()).unwrap_err();
1749 assert!(error.to_string().contains("format version 99"));
1750 }
1751}