1use std::collections::BTreeMap;
57use std::sync::{Arc, Mutex, MutexGuard};
58
59use async_trait::async_trait;
60use serde::{Deserialize, Serialize};
61use serde_json::{json, Value};
62
63use crate::checkpoint::{Checkpoint, Checkpointer, InMemoryCheckpointer};
64use crate::effects::EffectRequest;
65use crate::error::{Result, RustyError};
66use crate::executor::{ExecutionOutcome, Executor, RunConfig, DEFAULT_MAX_STEPS};
67use crate::graph::Graph;
68use crate::journal::{Clock, EventDraft, Journal, JournalSnapshot, RngSource};
69use crate::llm::{ChatMessage, ChatModel, ChatResponse, ToolCall, Usage};
70use crate::record::{sha256_hex, Effect, EventStatus, PayloadRef, RunEvent, RunEventKind};
71use crate::state::{State, StateSpec};
72use crate::tool::Tool;
73
74pub const FIXTURE_FORMAT_VERSION: u32 = 1;
79
80const SERVABLE_KINDS: [RunEventKind; 4] = [
85 RunEventKind::ModelCall,
86 RunEventKind::ToolCall,
87 RunEventKind::RemoteCall,
88 RunEventKind::WasmCall,
89];
90
91fn is_servable(kind: RunEventKind) -> bool {
92 SERVABLE_KINDS.contains(&kind)
93}
94
95fn replay_error(message: impl Into<String>) -> RustyError {
96 RustyError::Replay(message.into())
97}
98
99pub fn model_call_request(messages: &[ChatMessage], tools: &[Value]) -> Value {
107 json!({ "messages": messages, "tools": tools })
108}
109
110pub fn model_call_response(response: &ChatResponse) -> Value {
116 json!({
117 "message": response.message,
118 "model": response.model,
119 "usage": response.usage,
120 })
121}
122
123pub fn tool_call_request(name: &str, arguments: &Value) -> Value {
128 json!({ "tool": name, "arguments": arguments })
129}
130
131fn chat_response_from_recorded(seq: u64, value: &Value) -> Result<ChatResponse> {
134 let decode = |key: &str| -> Result<Value> {
135 value.get(key).cloned().ok_or_else(|| {
136 replay_error(format!(
137 "recorded model call at seq {seq} is malformed: output payload has no `{key}` key"
138 ))
139 })
140 };
141 let message: ChatMessage = serde_json::from_value(decode("message")?)?;
142 let model: Option<String> = serde_json::from_value(decode("model")?)?;
143 let usage: Option<Usage> = serde_json::from_value(decode("usage")?)?;
144 Ok(ChatResponse {
145 message,
146 model,
147 usage,
148 })
149}
150
151fn recorded_error_text(served: &ServedEffect) -> String {
155 served
156 .output
157 .as_ref()
158 .and_then(|value| value.get("error"))
159 .and_then(Value::as_str)
160 .unwrap_or("<no recorded error payload>")
161 .to_owned()
162}
163
164fn verify_snapshot(snapshot: &JournalSnapshot) -> Result<()> {
170 Journal::from_snapshot(clone_snapshot(snapshot), Clock::System)?;
172 for (hash, value) in &snapshot.artifacts {
173 let actual = sha256_hex(&serde_json::to_vec(value)?);
174 if actual != *hash {
175 return Err(replay_error(format!(
176 "journal snapshot artifact integrity failure: artifact stored under {hash} \
177 hashes to {actual} — the snapshot was tampered with or corrupted"
178 )));
179 }
180 }
181 for event in &snapshot.events {
182 for payload in [&event.input, &event.output].into_iter().flatten() {
183 if let PayloadRef::Artifact(reference) = payload {
184 if !snapshot.artifacts.contains_key(&reference.sha256) {
185 return Err(replay_error(format!(
186 "journal snapshot is truncated: event {} references artifact {} which \
187 is absent from the artifact map",
188 event.id, reference.sha256
189 )));
190 }
191 }
192 }
193 }
194 Ok(())
195}
196
197fn clone_snapshot(snapshot: &JournalSnapshot) -> JournalSnapshot {
200 snapshot.clone()
201}
202
203fn resolve_in(snapshot: &JournalSnapshot, payload: &PayloadRef) -> Option<Value> {
204 match payload {
205 PayloadRef::Inline(value) => Some(value.clone()),
206 PayloadRef::Artifact(reference) => snapshot.artifacts.get(&reference.sha256).cloned(),
207 }
208}
209
210fn resolve_opt(snapshot: &JournalSnapshot, payload: Option<&PayloadRef>) -> Option<Value> {
211 payload.and_then(|payload| resolve_in(snapshot, payload))
212}
213
214fn request_hash(request: &Value) -> Result<String> {
219 Ok(sha256_hex(&serde_json::to_vec(request)?))
220}
221
222#[derive(Debug, Clone)]
226pub struct ServedEffect {
227 pub event: RunEvent,
229
230 pub input: Option<Value>,
232
233 pub output: Option<Value>,
235}
236
237impl ServedEffect {
238 pub fn rejournal(&self, journal: &Journal, parent: impl Into<String>) -> String {
246 let mut draft = EventDraft::new(self.event.kind, self.event.effect)
247 .status(self.event.status)
248 .parent(parent);
249 if let Some(node) = &self.event.node_id {
250 draft = draft.node(node.clone());
251 }
252 if let Some(input) = &self.input {
253 draft = draft.input(input.clone());
254 }
255 if let Some(output) = &self.output {
256 draft = draft.output(output.clone());
257 }
258 if let Some(latency_ms) = self.event.latency_ms {
259 draft = draft.latency_ms(latency_ms);
260 }
261 if let Some(tokens) = self.event.tokens {
262 draft = draft.tokens(tokens);
263 }
264 if let Some(cost_usd) = self.event.cost_usd {
265 draft = draft.cost_usd(cost_usd);
266 }
267 journal.record(draft)
268 }
269}
270
271#[derive(Debug)]
272struct ReplaySourceInner {
273 servable: Vec<RunEvent>,
275 artifacts: BTreeMap<String, Value>,
277 cursor: usize,
280}
281
282#[derive(Debug, Clone)]
293pub struct ReplaySource {
294 inner: Arc<Mutex<ReplaySourceInner>>,
295}
296
297impl ReplaySource {
298 pub fn new(snapshot: &JournalSnapshot) -> Self {
303 Self {
304 inner: Arc::new(Mutex::new(ReplaySourceInner {
305 servable: snapshot
306 .events
307 .iter()
308 .filter(|event| is_servable(event.kind))
309 .cloned()
310 .collect(),
311 artifacts: snapshot.artifacts.clone(),
312 cursor: 0,
313 })),
314 }
315 }
316
317 fn lock(&self) -> MutexGuard<'_, ReplaySourceInner> {
318 self.inner.lock().unwrap_or_else(|e| e.into_inner())
321 }
322
323 pub fn serve(&self, kind: RunEventKind, request: &Value) -> Result<ServedEffect> {
334 if !is_servable(kind) {
335 return Err(replay_error(format!(
336 "{kind:?} is not a servable effect kind; exact replay serves only {SERVABLE_KINDS:?} \
337 — executor-derived events are reproduced by re-execution, not served"
338 )));
339 }
340 let issued_hash = request_hash(request)?;
341 let mut inner = self.lock();
342 let Some(event) = inner.servable.get(inner.cursor) else {
343 return Err(replay_error(format!(
344 "replay exhaustion: the run issued a {kind:?} call (request hash {issued_hash}), \
345 but the journal holds no further recorded effects to serve — the replayed run \
346 is doing more work than the recorded one"
347 )));
348 };
349 if event.kind != kind {
350 return Err(replay_error(format!(
351 "replay order violation at recorded seq {}: the run issued a {kind:?} call, but \
352 the next unserved recorded effect is {:?} — exact replay requires effects in \
353 journaled sequence order",
354 event.seq, event.kind
355 )));
356 }
357 let recorded_input = event
358 .input
359 .as_ref()
360 .and_then(|payload| match payload {
361 PayloadRef::Inline(value) => Some(value.clone()),
362 PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
363 })
364 .ok_or_else(|| {
365 replay_error(format!(
366 "recorded {:?} at seq {} has no input payload to match against",
367 event.kind, event.seq
368 ))
369 })?;
370 let recorded_hash = request_hash(&recorded_input)?;
371 if recorded_hash != issued_hash {
372 return Err(replay_error(format!(
373 "replay divergence at recorded seq {} ({:?}): the run issued a request whose \
374 canonical hash is {issued_hash}, but the journaled request hashes to \
375 {recorded_hash} — the replayed run has diverged from its evidence",
376 event.seq, event.kind
377 )));
378 }
379 let served = ServedEffect {
380 input: Some(recorded_input),
381 output: event.output.as_ref().and_then(|payload| match payload {
382 PayloadRef::Inline(value) => Some(value.clone()),
383 PayloadRef::Artifact(reference) => inner.artifacts.get(&reference.sha256).cloned(),
384 }),
385 event: event.clone(),
386 };
387 inner.cursor += 1;
388 Ok(served)
389 }
390
391 pub fn is_exhausted(&self) -> bool {
394 let inner = self.lock();
395 inner.cursor == inner.servable.len()
396 }
397
398 pub fn remaining(&self) -> Vec<(u64, RunEventKind)> {
401 let inner = self.lock();
402 inner.servable[inner.cursor..]
403 .iter()
404 .map(|event| (event.seq, event.kind))
405 .collect()
406 }
407}
408
409pub struct RecordingChatModel {
418 inner: Arc<dyn ChatModel>,
419 journal: Journal,
420 parent: String,
421 node_id: Option<String>,
422}
423
424impl RecordingChatModel {
425 pub fn new(inner: Arc<dyn ChatModel>, journal: Journal, parent: impl Into<String>) -> Self {
428 Self {
429 inner,
430 journal,
431 parent: parent.into(),
432 node_id: None,
433 }
434 }
435
436 pub fn node(mut self, node_id: impl Into<String>) -> Self {
439 self.node_id = Some(node_id.into());
440 self
441 }
442
443 fn draft(&self) -> EventDraft {
444 let mut draft = EventDraft::new(RunEventKind::ModelCall, self.inner.effect())
445 .parent(self.parent.clone());
446 if let Some(node) = &self.node_id {
447 draft = draft.node(node.clone());
448 }
449 draft
450 }
451}
452
453#[async_trait]
454impl ChatModel for RecordingChatModel {
455 async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
456 let started = self.journal.clock().now();
457 let result = self.inner.chat(messages, tools).await;
458 let latency_ms = (self.journal.clock().now() - started)
459 .num_milliseconds()
460 .max(0) as u64;
461 let draft = self
462 .draft()
463 .input(model_call_request(messages, tools))
464 .latency_ms(latency_ms);
465 match result {
466 Ok(response) => {
467 let mut draft = draft.output(model_call_response(&response));
468 if let Some(usage) = response.usage {
469 draft = draft.tokens(usage);
470 }
471 self.journal.record(draft);
472 Ok(response)
473 }
474 Err(error) => {
475 self.journal.record(
476 draft
477 .status(EventStatus::Error)
478 .output(json!({ "error": error.to_string() })),
479 );
480 Err(error)
481 }
482 }
483 }
484
485 fn effect(&self) -> Effect {
486 self.inner.effect()
487 }
488}
489
490pub struct ReplayingChatModel {
501 inner: Arc<dyn ChatModel>,
502 source: ReplaySource,
503 journal: Journal,
504 parent: String,
505}
506
507impl ReplayingChatModel {
508 pub fn new(
512 inner: Arc<dyn ChatModel>,
513 source: ReplaySource,
514 journal: Journal,
515 parent: impl Into<String>,
516 ) -> Self {
517 Self {
518 inner,
519 source,
520 journal,
521 parent: parent.into(),
522 }
523 }
524}
525
526#[async_trait]
527impl ChatModel for ReplayingChatModel {
528 async fn chat(&self, messages: &[ChatMessage], tools: &[Value]) -> Result<ChatResponse> {
529 let served = self.source.serve(
530 RunEventKind::ModelCall,
531 &model_call_request(messages, tools),
532 )?;
533 let _started = self.journal.clock().now();
537 let _ended = self.journal.clock().now();
538 served.rejournal(&self.journal, self.parent.clone());
539 match served.event.status {
540 EventStatus::Ok => {
541 let output = served.output.as_ref().ok_or_else(|| {
542 replay_error(format!(
543 "recorded model call at seq {} succeeded but carries no output payload",
544 served.event.seq
545 ))
546 })?;
547 chat_response_from_recorded(served.event.seq, output)
548 }
549 EventStatus::Error => Err(RustyError::Llm(format!(
550 "replayed model call at seq {} failed as recorded: {}",
551 served.event.seq,
552 recorded_error_text(&served)
553 ))),
554 EventStatus::Interrupted => Err(replay_error(format!(
555 "recorded model call at seq {} has status `interrupted`, which model calls \
556 never produce — the journal is inconsistent",
557 served.event.seq
558 ))),
559 }
560 }
561
562 fn effect(&self) -> Effect {
563 self.inner.effect()
564 }
565}
566
567pub struct RecordingTool {
573 inner: Arc<dyn Tool>,
574 journal: Journal,
575 parent: String,
576 node_id: Option<String>,
577}
578
579impl RecordingTool {
580 pub fn new(inner: Arc<dyn Tool>, journal: Journal, parent: impl Into<String>) -> Self {
583 Self {
584 inner,
585 journal,
586 parent: parent.into(),
587 node_id: None,
588 }
589 }
590
591 pub fn node(mut self, node_id: impl Into<String>) -> Self {
594 self.node_id = Some(node_id.into());
595 self
596 }
597}
598
599#[async_trait]
600impl Tool for RecordingTool {
601 fn name(&self) -> &str {
602 self.inner.name()
603 }
604
605 fn description(&self) -> &str {
606 self.inner.description()
607 }
608
609 fn parameters_schema(&self) -> Value {
610 self.inner.parameters_schema()
611 }
612
613 fn effect(&self) -> Effect {
614 self.inner.effect()
615 }
616
617 fn effect_kind(&self) -> &str {
618 self.inner.effect_kind()
619 }
620
621 fn idempotency_key(&self, args: &Value) -> Option<String> {
622 self.inner.idempotency_key(args)
623 }
624
625 fn effect_request(&self, call: &ToolCall) -> EffectRequest {
626 self.inner.effect_request(call)
627 }
628
629 async fn call(&self, args: Value) -> Result<Value> {
630 let started = self.journal.clock().now();
631 let result = self.inner.call(args.clone()).await;
632 let latency_ms = (self.journal.clock().now() - started)
633 .num_milliseconds()
634 .max(0) as u64;
635 let mut draft = EventDraft::new(RunEventKind::ToolCall, self.inner.effect())
636 .input(tool_call_request(self.inner.name(), &args))
637 .latency_ms(latency_ms)
638 .parent(self.parent.clone());
639 if let Some(node) = &self.node_id {
640 draft = draft.node(node.clone());
641 }
642 match result {
643 Ok(value) => {
644 self.journal.record(draft.output(value.clone()));
645 Ok(value)
646 }
647 Err(error) => {
648 self.journal.record(
649 draft
650 .status(EventStatus::Error)
651 .output(json!({ "error": error.to_string() })),
652 );
653 Err(error)
654 }
655 }
656 }
657}
658
659pub struct ReplayingTool {
667 inner: Arc<dyn Tool>,
668 source: ReplaySource,
669 journal: Journal,
670 parent: String,
671}
672
673impl ReplayingTool {
674 pub fn new(
677 inner: Arc<dyn Tool>,
678 source: ReplaySource,
679 journal: Journal,
680 parent: impl Into<String>,
681 ) -> Self {
682 Self {
683 inner,
684 source,
685 journal,
686 parent: parent.into(),
687 }
688 }
689}
690
691#[async_trait]
692impl Tool for ReplayingTool {
693 fn name(&self) -> &str {
694 self.inner.name()
695 }
696
697 fn description(&self) -> &str {
698 self.inner.description()
699 }
700
701 fn parameters_schema(&self) -> Value {
702 self.inner.parameters_schema()
703 }
704
705 fn effect(&self) -> Effect {
706 self.inner.effect()
707 }
708
709 fn effect_kind(&self) -> &str {
710 self.inner.effect_kind()
711 }
712
713 fn idempotency_key(&self, args: &Value) -> Option<String> {
714 self.inner.idempotency_key(args)
715 }
716
717 fn effect_request(&self, call: &ToolCall) -> EffectRequest {
718 self.inner.effect_request(call)
719 }
720
721 async fn call(&self, args: Value) -> Result<Value> {
722 let request = tool_call_request(self.inner.name(), &args);
723 let served = self.source.serve(RunEventKind::ToolCall, &request)?;
724 let _started = self.journal.clock().now();
726 let _ended = self.journal.clock().now();
727 served.rejournal(&self.journal, self.parent.clone());
728 match served.event.status {
729 EventStatus::Ok => served.output.clone().ok_or_else(|| {
730 replay_error(format!(
731 "recorded tool call at seq {} succeeded but carries no output payload",
732 served.event.seq
733 ))
734 }),
735 EventStatus::Error => Err(RustyError::Tool(format!(
736 "replayed tool call at seq {} failed as recorded: {}",
737 served.event.seq,
738 recorded_error_text(&served)
739 ))),
740 EventStatus::Interrupted => Err(replay_error(format!(
741 "recorded tool call at seq {} has status `interrupted`, which tool calls \
742 never produce — the journal is inconsistent",
743 served.event.seq
744 ))),
745 }
746 }
747}
748
749#[derive(Debug)]
765pub struct ExactReplay {
766 snapshot: JournalSnapshot,
767 source: ReplaySource,
768}
769
770impl ExactReplay {
771 pub fn new(snapshot: JournalSnapshot) -> Result<Self> {
780 verify_snapshot(&snapshot)?;
781 if snapshot
782 .events
783 .first()
784 .is_some_and(|event| event.kind == RunEventKind::Resume)
785 {
786 return Err(replay_error(
787 "exact replay of resumed runs is not supported: the journal begins with a \
788 resume event, whose pre-resume state lives in a checkpoint the journal does \
789 not carry — replay the original run's journal instead",
790 ));
791 }
792 let source = ReplaySource::new(&snapshot);
793 Ok(Self { snapshot, source })
794 }
795
796 pub fn snapshot(&self) -> &JournalSnapshot {
798 &self.snapshot
799 }
800
801 pub fn source(&self) -> ReplaySource {
805 self.source.clone()
806 }
807
808 pub fn fresh_journal(&self, clock: Clock) -> Journal {
813 Journal::new(
814 self.snapshot.run_id.clone(),
815 self.snapshot.thread_id.clone(),
816 clock,
817 )
818 }
819
820 pub async fn run(
829 &self,
830 graph: &Graph,
831 spec: &StateSpec,
832 initial_state: State,
833 params: ReplayParams,
834 ) -> Result<ReplayOutcome> {
835 if params.journal.run_id() != self.snapshot.run_id
836 || params.journal.thread_id() != self.snapshot.thread_id
837 {
838 return Err(replay_error(format!(
839 "replay journal identity mismatch: the snapshot records run `{}` on thread \
840 `{}`, but the supplied journal is run `{}` on thread `{}` — build the replay \
841 journal with ExactReplay::fresh_journal",
842 self.snapshot.run_id,
843 self.snapshot.thread_id,
844 params.journal.run_id(),
845 params.journal.thread_id()
846 )));
847 }
848 let executor = match ¶ms.checkpointer {
849 Some(checkpointer) => Executor::with_checkpointer(checkpointer.clone()),
850 None => Executor::new(),
851 };
852 let config = RunConfig::new(self.snapshot.thread_id.clone())
853 .with_max_steps(params.max_steps)
854 .with_rng(params.rng.clone())
855 .with_journal(params.journal.clone());
856 let outcome = executor.run(graph, spec, initial_state, config).await?;
857 Ok(ReplayOutcome {
858 outcome,
859 journal: params.journal.snapshot(),
860 })
861 }
862
863 pub fn verify(&self, replayed: &JournalSnapshot) -> Result<()> {
868 if !self.source.is_exhausted() {
869 return Err(replay_error(format!(
870 "exact replay left recorded effects unserved: {:?} — the replayed run stopped \
871 short of the journaled history",
872 self.source.remaining()
873 )));
874 }
875 let recorded = &self.snapshot;
876 if recorded.run_id != replayed.run_id || recorded.thread_id != replayed.thread_id {
877 return Err(replay_error(format!(
878 "replay identity mismatch: recorded run `{}`/thread `{}` vs replayed \
879 `{}`/`{}`",
880 recorded.run_id, recorded.thread_id, replayed.run_id, replayed.thread_id
881 )));
882 }
883 if recorded.events.len() != replayed.events.len() {
884 return Err(replay_error(format!(
885 "replay event-count mismatch: the recorded journal holds {} events but the \
886 replayed journal holds {}",
887 recorded.events.len(),
888 replayed.events.len()
889 )));
890 }
891 for (recorded_event, replayed_event) in recorded.events.iter().zip(&replayed.events) {
892 if recorded_event != replayed_event {
893 return Err(replay_error(format!(
894 "replay event mismatch at seq {}: recorded {} but replayed {}",
895 recorded_event.seq,
896 summarize_event(recorded_event),
897 summarize_event(replayed_event),
898 )));
899 }
900 }
901 if recorded.artifacts != replayed.artifacts {
902 return Err(replay_error(
903 "replay artifact mismatch: the replayed journal's artifact payloads differ \
904 from the recorded ones",
905 ));
906 }
907 if recorded.head_hash != replayed.head_hash {
908 return Err(replay_error(format!(
909 "replay head-hash mismatch: recorded {} but replayed {} — event content \
910 agrees but the evidence chain does not (this should be unreachable; please \
911 report it)",
912 recorded.head_hash, replayed.head_hash
913 )));
914 }
915 Ok(())
916 }
917
918 pub async fn run_and_verify(
922 &self,
923 graph: &Graph,
924 spec: &StateSpec,
925 initial_state: State,
926 params: ReplayParams,
927 ) -> Result<ReplayOutcome> {
928 let replayed = self.run(graph, spec, initial_state, params).await?;
929 self.verify(&replayed.journal)?;
930 Ok(replayed)
931 }
932}
933
934fn summarize_event(event: &RunEvent) -> String {
937 format!(
938 "{:?}(node={:?}, status={:?})",
939 event.kind, event.node_id, event.status
940 )
941}
942
943pub struct ReplayParams {
949 pub journal: Journal,
951
952 pub rng: RngSource,
956
957 pub checkpointer: Option<Arc<dyn Checkpointer>>,
960
961 pub max_steps: usize,
963}
964
965impl ReplayParams {
966 pub fn new(journal: Journal, rng: RngSource) -> Self {
969 Self {
970 journal,
971 rng,
972 checkpointer: None,
973 max_steps: DEFAULT_MAX_STEPS,
974 }
975 }
976
977 pub fn with_checkpointer(mut self, checkpointer: Arc<dyn Checkpointer>) -> Self {
979 self.checkpointer = Some(checkpointer);
980 self
981 }
982
983 pub fn with_max_steps(mut self, max_steps: usize) -> Self {
985 self.max_steps = max_steps;
986 self
987 }
988}
989
990#[derive(Debug)]
993pub struct ReplayOutcome {
994 pub outcome: ExecutionOutcome,
997
998 pub journal: JournalSnapshot,
1002}
1003
1004#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
1008pub struct BranchTotals {
1009 pub events: usize,
1011
1012 pub tokens: Usage,
1014
1015 pub cost_usd: f64,
1017}
1018
1019#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1023pub struct ChannelDiff {
1024 pub channel: String,
1026
1027 pub base: Option<Value>,
1029
1030 pub branch: Option<Value>,
1032}
1033
1034#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1036pub struct StepDiff {
1037 pub step: u64,
1039
1040 pub channels: Vec<ChannelDiff>,
1043}
1044
1045#[derive(Debug, Clone, Serialize, Deserialize)]
1058pub struct BranchDiff {
1059 pub first_divergent_seq: Option<u64>,
1062
1063 pub added: Vec<RunEvent>,
1066
1067 pub removed: Vec<RunEvent>,
1070
1071 pub step_diffs: Vec<StepDiff>,
1074
1075 pub base_totals: BranchTotals,
1077
1078 pub branch_totals: BranchTotals,
1080}
1081
1082impl BranchDiff {
1083 pub fn between(base: &JournalSnapshot, branch: &JournalSnapshot) -> BranchDiff {
1088 let divergence = (0..base.events.len().max(branch.events.len()))
1089 .find(|&i| {
1090 match (base.events.get(i), branch.events.get(i)) {
1091 (Some(a), Some(b)) => !logically_equal(a, base, b, branch),
1092 _ => true,
1094 }
1095 })
1096 .map(|i| {
1097 base.events
1098 .get(i)
1099 .or_else(|| branch.events.get(i))
1100 .map(|event| event.seq)
1101 .unwrap_or(i as u64)
1102 });
1103 let (removed, added) = match divergence {
1104 Some(seq) => (
1105 base.events
1106 .iter()
1107 .filter(|event| event.seq >= seq)
1108 .cloned()
1109 .collect(),
1110 branch
1111 .events
1112 .iter()
1113 .filter(|event| event.seq >= seq)
1114 .cloned()
1115 .collect(),
1116 ),
1117 None => (Vec::new(), Vec::new()),
1118 };
1119 BranchDiff {
1120 first_divergent_seq: divergence,
1121 added,
1122 removed,
1123 step_diffs: diff_steps(base, branch),
1124 base_totals: totals(base),
1125 branch_totals: totals(branch),
1126 }
1127 }
1128
1129 pub fn is_identical(&self) -> bool {
1132 self.first_divergent_seq.is_none()
1133 }
1134}
1135
1136fn logically_equal(
1140 a: &RunEvent,
1141 snapshot_a: &JournalSnapshot,
1142 b: &RunEvent,
1143 snapshot_b: &JournalSnapshot,
1144) -> bool {
1145 a.seq == b.seq
1146 && a.kind == b.kind
1147 && a.node_id == b.node_id
1148 && a.effect == b.effect
1149 && a.latency_ms == b.latency_ms
1150 && a.tokens == b.tokens
1151 && a.cost_usd == b.cost_usd
1152 && a.status == b.status
1153 && resolve_opt(snapshot_a, a.input.as_ref()) == resolve_opt(snapshot_b, b.input.as_ref())
1154 && resolve_opt(snapshot_a, a.output.as_ref()) == resolve_opt(snapshot_b, b.output.as_ref())
1155}
1156
1157fn totals(snapshot: &JournalSnapshot) -> BranchTotals {
1159 let mut totals = BranchTotals {
1160 events: snapshot.events.len(),
1161 ..BranchTotals::default()
1162 };
1163 for event in &snapshot.events {
1164 if let Some(usage) = event.tokens {
1165 totals.tokens.prompt_tokens = totals
1166 .tokens
1167 .prompt_tokens
1168 .saturating_add(usage.prompt_tokens);
1169 totals.tokens.completion_tokens = totals
1170 .tokens
1171 .completion_tokens
1172 .saturating_add(usage.completion_tokens);
1173 totals.tokens.total_tokens = totals
1174 .tokens
1175 .total_tokens
1176 .saturating_add(usage.total_tokens);
1177 }
1178 if let Some(cost) = event.cost_usd {
1179 totals.cost_usd += cost;
1180 }
1181 }
1182 totals
1183}
1184
1185fn step_channel_values(snapshot: &JournalSnapshot) -> BTreeMap<u64, BTreeMap<String, Value>> {
1189 let mut steps = BTreeMap::new();
1190 let mut current_step: Option<u64> = None;
1191 for event in &snapshot.events {
1192 match event.kind {
1193 RunEventKind::SuperStepStart => {
1194 current_step = resolve_opt(snapshot, event.input.as_ref())
1195 .and_then(|input| input.get("step").and_then(Value::as_u64));
1196 }
1197 RunEventKind::SuperStepEnd => {
1198 if let Some(step) = current_step.take() {
1199 let channels = resolve_opt(snapshot, event.output.as_ref())
1200 .and_then(|output| output.as_object().cloned())
1201 .unwrap_or_default()
1202 .into_iter()
1203 .collect();
1204 steps.insert(step, channels);
1205 }
1206 }
1207 _ => {}
1208 }
1209 }
1210 steps
1211}
1212
1213fn diff_steps(base: &JournalSnapshot, branch: &JournalSnapshot) -> Vec<StepDiff> {
1216 let base_steps = step_channel_values(base);
1217 let branch_steps = step_channel_values(branch);
1218 let mut diffs = Vec::new();
1219 for step in base_steps
1220 .keys()
1221 .chain(branch_steps.keys())
1222 .copied()
1223 .collect::<std::collections::BTreeSet<u64>>()
1224 {
1225 let empty = BTreeMap::new();
1226 let base_channels = base_steps.get(&step).unwrap_or(&empty);
1227 let branch_channels = branch_steps.get(&step).unwrap_or(&empty);
1228 let mut channels = Vec::new();
1229 for channel in base_channels
1230 .keys()
1231 .chain(branch_channels.keys())
1232 .cloned()
1233 .collect::<std::collections::BTreeSet<String>>()
1234 {
1235 let base_value = base_channels.get(&channel).cloned();
1236 let branch_value = branch_channels.get(&channel).cloned();
1237 if base_value != branch_value {
1238 channels.push(ChannelDiff {
1239 channel,
1240 base: base_value,
1241 branch: branch_value,
1242 });
1243 }
1244 }
1245 if !channels.is_empty() {
1246 diffs.push(StepDiff { step, channels });
1247 }
1248 }
1249 diffs
1250}
1251
1252#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1255pub struct LogicalClockParams {
1256 pub start_ms: u64,
1258
1259 pub tick_ms: u64,
1261}
1262
1263#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1265pub struct FixtureMetadata {
1266 pub name: String,
1268
1269 #[serde(default, skip_serializing_if = "Option::is_none")]
1271 pub clock: Option<LogicalClockParams>,
1272
1273 #[serde(default, skip_serializing_if = "Option::is_none")]
1275 pub rng_seed: Option<u64>,
1276}
1277
1278#[derive(Debug, Clone, Serialize, Deserialize)]
1289pub struct ReplayFixture {
1290 pub format_version: u32,
1293
1294 pub graph_hash: String,
1298
1299 pub graph_version: String,
1301
1302 pub journal: JournalSnapshot,
1304
1305 pub final_checkpoint: Option<Checkpoint>,
1308
1309 pub metadata: FixtureMetadata,
1311}
1312
1313impl ReplayFixture {
1314 pub fn capture(
1318 name: impl Into<String>,
1319 graph: &Graph,
1320 graph_version: impl Into<String>,
1321 journal: JournalSnapshot,
1322 final_checkpoint: Option<Checkpoint>,
1323 clock: Option<LogicalClockParams>,
1324 rng_seed: Option<u64>,
1325 ) -> Self {
1326 Self {
1327 format_version: FIXTURE_FORMAT_VERSION,
1328 graph_hash: graph.topology_hash(),
1329 graph_version: graph_version.into(),
1330 journal,
1331 final_checkpoint,
1332 metadata: FixtureMetadata {
1333 name: name.into(),
1334 clock,
1335 rng_seed,
1336 },
1337 }
1338 }
1339
1340 pub fn export(&self) -> Result<String> {
1342 Ok(serde_json::to_string_pretty(self)?)
1343 }
1344
1345 pub fn import(json: &str) -> Result<Self> {
1348 let fixture: Self = serde_json::from_str(json)?;
1349 if fixture.format_version != FIXTURE_FORMAT_VERSION {
1350 return Err(replay_error(format!(
1351 "unsupported fixture format version {} (this build supports \
1352 {FIXTURE_FORMAT_VERSION}) — regenerate the fixture or upgrade the runtime",
1353 fixture.format_version
1354 )));
1355 }
1356 verify_snapshot(&fixture.journal)?;
1357 Ok(fixture)
1358 }
1359
1360 pub fn exact_replay(&self) -> Result<ExactReplay> {
1364 ExactReplay::new(self.journal.clone())
1365 }
1366
1367 pub fn replay_params(&self, replay: &ExactReplay) -> Result<ReplayParams> {
1377 let clock = self.metadata.clock.ok_or_else(|| {
1378 replay_error(format!(
1379 "fixture `{}` carries no logical-clock parameters; exact CI replay requires \
1380 the recorded run's determinism seams",
1381 self.metadata.name
1382 ))
1383 })?;
1384 let seed = self.metadata.rng_seed.ok_or_else(|| {
1385 replay_error(format!(
1386 "fixture `{}` carries no RNG seed; exact CI replay requires the recorded \
1387 run's determinism seams",
1388 self.metadata.name
1389 ))
1390 })?;
1391 let journal = replay.fresh_journal(Clock::logical(clock.start_ms, clock.tick_ms));
1392 Ok(ReplayParams::new(journal, RngSource::seeded(seed))
1393 .with_checkpointer(Arc::new(InMemoryCheckpointer::new())))
1394 }
1395
1396 pub async fn replay_in_ci(
1407 &self,
1408 replay: ExactReplay,
1409 graph: &Graph,
1410 spec: &StateSpec,
1411 initial_state: State,
1412 params: ReplayParams,
1413 ) -> Result<ReplayOutcome> {
1414 let actual_hash = graph.topology_hash();
1415 if actual_hash != self.graph_hash {
1416 return Err(replay_error(format!(
1417 "fixture `{}` was recorded against graph topology {}, but the graph under \
1418 replay hashes to {actual_hash} — rebuild the recorded topology (same node \
1419 names, edges, and entry point)",
1420 self.metadata.name, self.graph_hash
1421 )));
1422 }
1423 let replayed = replay
1424 .run_and_verify(graph, spec, initial_state, params)
1425 .await?;
1426 if let Some(final_checkpoint) = &self.final_checkpoint {
1427 if replayed.outcome.state() != &final_checkpoint.state {
1428 return Err(replay_error(format!(
1429 "fixture `{}`: the replayed final state diverges from the recorded final \
1430 checkpoint (journal evidence agreed; state did not)",
1431 self.metadata.name
1432 )));
1433 }
1434 }
1435 Ok(replayed)
1436 }
1437}
1438
1439#[cfg(test)]
1440mod tests {
1441 use super::*;
1442 use crate::journal::Journal;
1443 use serde_json::json;
1444
1445 fn snapshot_with(drafts: Vec<EventDraft>) -> JournalSnapshot {
1446 let journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
1447 for draft in drafts {
1448 journal.record(draft);
1449 }
1450 journal.snapshot()
1451 }
1452
1453 fn model_draft(request: Value, response: Value) -> EventDraft {
1454 EventDraft::new(RunEventKind::ModelCall, Effect::NonIdempotent)
1455 .node("agent")
1456 .input(request)
1457 .output(response)
1458 .latency_ms(2)
1459 .tokens(Usage {
1460 prompt_tokens: 10,
1461 completion_tokens: 5,
1462 total_tokens: 15,
1463 })
1464 }
1465
1466 #[test]
1467 fn canonical_shapes_have_stable_hashes() {
1468 let request = model_call_request(&[ChatMessage::user("ping")], &[]);
1469 assert_eq!(
1470 request,
1471 json!({"messages": [{"role": "user", "content": "ping"}], "tools": []})
1472 );
1473 let first = request_hash(&request).unwrap();
1474 let second = request_hash(&model_call_request(&[ChatMessage::user("ping")], &[])).unwrap();
1475 assert_eq!(first, second);
1476 let other = request_hash(&model_call_request(&[ChatMessage::user("pong")], &[])).unwrap();
1478 assert_ne!(first, other);
1479
1480 let tool = tool_call_request("echo", &json!({"text": "hi"}));
1481 assert_eq!(tool, json!({"tool": "echo", "arguments": {"text": "hi"}}));
1482 }
1483
1484 #[test]
1485 fn serve_matches_by_sequence_and_request_hash() {
1486 let snapshot = snapshot_with(vec![
1487 model_draft(
1488 model_call_request(&[ChatMessage::user("a")], &[]),
1489 json!({"r": 1}),
1490 ),
1491 model_draft(
1492 model_call_request(&[ChatMessage::user("b")], &[]),
1493 json!({"r": 2}),
1494 ),
1495 ]);
1496 let source = ReplaySource::new(&snapshot);
1497 assert!(!source.is_exhausted());
1498
1499 let error = source
1501 .serve(
1502 RunEventKind::ModelCall,
1503 &model_call_request(&[ChatMessage::user("zzz")], &[]),
1504 )
1505 .unwrap_err();
1506 assert!(matches!(error, RustyError::Replay(_)));
1507 assert!(error.to_string().contains("divergence"));
1508 assert_eq!(source.remaining().len(), 2);
1509
1510 let error = source
1512 .serve(RunEventKind::ToolCall, &tool_call_request("t", &json!({})))
1513 .unwrap_err();
1514 assert!(error.to_string().contains("order violation"));
1515
1516 let first = source
1518 .serve(
1519 RunEventKind::ModelCall,
1520 &model_call_request(&[ChatMessage::user("a")], &[]),
1521 )
1522 .unwrap();
1523 assert_eq!(first.event.seq, 0);
1524 assert_eq!(first.output, Some(json!({"r": 1})));
1525 let second = source
1526 .serve(
1527 RunEventKind::ModelCall,
1528 &model_call_request(&[ChatMessage::user("b")], &[]),
1529 )
1530 .unwrap();
1531 assert_eq!(second.event.seq, 1);
1532 assert!(source.is_exhausted());
1533
1534 let error = source
1536 .serve(
1537 RunEventKind::ModelCall,
1538 &model_call_request(&[ChatMessage::user("a")], &[]),
1539 )
1540 .unwrap_err();
1541 assert!(error.to_string().contains("exhaustion"));
1542 }
1543
1544 #[test]
1545 fn serve_rejects_non_servable_kinds() {
1546 let snapshot = snapshot_with(vec![EventDraft::new(
1547 RunEventKind::SuperStepStart,
1548 Effect::Pure,
1549 )]);
1550 let error = ReplaySource::new(&snapshot)
1551 .serve(RunEventKind::NodeInput, &json!({}))
1552 .unwrap_err();
1553 assert!(error.to_string().contains("not a servable"));
1554 }
1555
1556 #[test]
1557 fn artifact_backed_requests_match_and_resolve() {
1558 let big_request =
1559 json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1560 let big_response = json!({"message": {"role": "assistant", "content": "y".repeat(9000)}, "model": null, "usage": null});
1561 let snapshot = snapshot_with(vec![model_draft(big_request.clone(), big_response.clone())]);
1562 assert!(matches!(
1564 snapshot.events[0].input,
1565 Some(PayloadRef::Artifact(_))
1566 ));
1567 let served = ReplaySource::new(&snapshot)
1568 .serve(RunEventKind::ModelCall, &big_request)
1569 .unwrap();
1570 assert_eq!(served.output, Some(big_response));
1571 }
1572
1573 #[test]
1574 fn tampered_snapshots_are_rejected_at_the_boundary() {
1575 let snapshot = snapshot_with(vec![model_draft(
1576 model_call_request(&[ChatMessage::user("a")], &[]),
1577 json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1578 )]);
1579
1580 let mut tampered = snapshot.clone();
1582 tampered.events[0].status = EventStatus::Error;
1583 assert!(ExactReplay::new(tampered).is_err());
1584
1585 let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1588 let artifact_snapshot = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
1589 let mut tampered = artifact_snapshot;
1590 *tampered.artifacts.values_mut().next().unwrap() = json!({"forged": true});
1591 let error = ExactReplay::new(tampered).unwrap_err();
1592 assert!(error.to_string().contains("integrity"));
1593
1594 let big = json!({"messages": [{"role": "user", "content": "x".repeat(9000)}], "tools": []});
1596 let truncated = snapshot_with(vec![model_draft(big, json!({"ok": true}))]);
1597 let mut truncated = truncated;
1598 truncated.artifacts.clear();
1599 assert!(ExactReplay::new(truncated).is_err());
1600 }
1601
1602 #[test]
1603 fn resumed_run_journals_are_rejected() {
1604 let snapshot = snapshot_with(vec![EventDraft::new(RunEventKind::Resume, Effect::Pure)
1605 .input(json!({"checkpoint_id": "c"}))]);
1606 let error = ExactReplay::new(snapshot).unwrap_err();
1607 assert!(error.to_string().contains("resumed"));
1608 }
1609
1610 #[test]
1611 fn rejournal_reproduces_the_recorded_event() {
1612 let recorded = snapshot_with(vec![model_draft(
1613 model_call_request(&[ChatMessage::user("a")], &[]),
1614 json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1615 )]);
1616 let source = ReplaySource::new(&recorded);
1617 let served = source
1618 .serve(
1619 RunEventKind::ModelCall,
1620 &model_call_request(&[ChatMessage::user("a")], &[]),
1621 )
1622 .unwrap();
1623 let replay_journal = Journal::new("run-t", "thread-t", Clock::logical(1_000_000, 5));
1624 served.rejournal(&replay_journal, "run-t:parent");
1626 let replayed = replay_journal.snapshot();
1627 assert_eq!(replayed.events.len(), 1);
1628 let mut expected = recorded.events[0].clone();
1630 expected.parent = Some("run-t:parent".into());
1631 assert_eq!(replayed.events[0], expected);
1632 }
1633
1634 #[test]
1635 fn branch_diff_reports_divergence_steps_and_totals() {
1636 let shared = || {
1637 vec![
1638 EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1639 .input(json!({"step": 0, "active_nodes": ["a"]})),
1640 EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1641 .output(json!({"log": ["a"]})),
1642 ]
1643 };
1644 let mut base_drafts = shared();
1645 base_drafts.push(
1646 EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1647 .input(json!({"step": 1, "active_nodes": ["b"]})),
1648 );
1649 base_drafts.push(
1650 EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1651 .output(json!({"log": ["a", "b"]})),
1652 );
1653 base_drafts.push(
1654 model_draft(
1655 model_call_request(&[ChatMessage::user("q")], &[]),
1656 json!({"r": 1}),
1657 )
1658 .cost_usd(0.001),
1659 );
1660 let base = snapshot_with(base_drafts);
1661
1662 let mut branch_drafts = shared();
1663 branch_drafts.push(
1664 EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure)
1665 .input(json!({"step": 1, "active_nodes": ["c"]})),
1666 );
1667 branch_drafts.push(
1668 EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1669 .output(json!({"log": ["a", "c"], "extra": 1})),
1670 );
1671 let branch = snapshot_with(branch_drafts);
1672
1673 let diff = BranchDiff::between(&base, &branch);
1674 assert!(!diff.is_identical());
1675 assert_eq!(diff.first_divergent_seq, Some(2));
1676 assert_eq!(diff.removed.len(), 3);
1677 assert_eq!(diff.added.len(), 2);
1678 assert_eq!(diff.base_totals.events, 5);
1679 assert_eq!(diff.branch_totals.events, 4);
1680 assert_eq!(diff.base_totals.tokens.total_tokens, 15);
1681 assert_eq!(diff.branch_totals.tokens.total_tokens, 0);
1682 assert_eq!(diff.base_totals.cost_usd, 0.001);
1683 assert_eq!(diff.branch_totals.cost_usd, 0.0);
1684
1685 assert_eq!(diff.step_diffs.len(), 1);
1687 let step = &diff.step_diffs[0];
1688 assert_eq!(step.step, 1);
1689 assert_eq!(step.channels.len(), 2);
1690 assert_eq!(step.channels[0].channel, "extra");
1691 assert_eq!(step.channels[0].base, None);
1692 assert_eq!(step.channels[0].branch, Some(json!(1)));
1693 assert_eq!(step.channels[1].channel, "log");
1694 assert_eq!(step.channels[1].base, Some(json!(["a", "b"])));
1695 assert_eq!(step.channels[1].branch, Some(json!(["a", "c"])));
1696 }
1697
1698 #[test]
1699 fn branch_diff_of_identical_snapshots_is_empty() {
1700 let snapshot = snapshot_with(vec![EventDraft::new(
1703 RunEventKind::SuperStepStart,
1704 Effect::Pure,
1705 )
1706 .input(json!({"step": 0, "active_nodes": ["a"]}))]);
1707 let diff = BranchDiff::between(&snapshot, &snapshot);
1708 assert!(diff.is_identical());
1709 assert!(diff.added.is_empty() && diff.removed.is_empty());
1710 assert!(diff.step_diffs.is_empty());
1711 }
1712
1713 #[test]
1714 fn fixture_roundtrip_and_version_guard() {
1715 let journal = snapshot_with(vec![model_draft(
1716 model_call_request(&[ChatMessage::user("a")], &[]),
1717 json!({"message": {"role": "assistant", "content": "r"}, "model": null, "usage": null}),
1718 )]);
1719 let mut builder = crate::graph::GraphBuilder::new();
1720 builder.add_node("agent", |_ctx: crate::node::NodeContext| async {
1721 Ok(crate::node::NodeOutput::empty())
1722 });
1723 builder.set_entry_point("agent");
1724 let graph = builder.compile().unwrap();
1725 let fixture = ReplayFixture::capture(
1726 "roundtrip",
1727 &graph,
1728 "v1",
1729 journal,
1730 None,
1731 Some(LogicalClockParams {
1732 start_ms: 1_000_000,
1733 tick_ms: 5,
1734 }),
1735 Some(7),
1736 );
1737 let wire = fixture.export().unwrap();
1738 let imported = ReplayFixture::import(&wire).unwrap();
1739 assert_eq!(imported.export().unwrap(), wire);
1740
1741 let mut value: Value = serde_json::from_str(&wire).unwrap();
1743 value["format_version"] = json!(99);
1744 let error = ReplayFixture::import(&value.to_string()).unwrap_err();
1745 assert!(error.to_string().contains("format version 99"));
1746 }
1747}