Skip to main content

rusty_agent_runtime/
executor.rs

1//! The executor: a Pregel/BSP-inspired super-step run loop.
2//!
3//! Execution proceeds in discrete **super-steps** (Google Pregel /
4//! Bulk-Synchronous-Parallel), each super-step being:
5//!
6//! 1. **Plan** — determine the active node set (entry point on step 0;
7//!    afterwards the routing result of the previous step, including
8//!    [`crate::node::Command::goto`] overrides and [`crate::graph::Send`]
9//!    fan-outs).
10//! 2. **Compute** — run all active nodes concurrently in a
11//!    `tokio::task::JoinSet`, each receiving an **immutable snapshot** of
12//!    the state as of the start of the step. No node can observe another's
13//!    in-progress writes.
14//! 3. **Barrier** — wait for all active nodes. The step is *transactional*:
15//!    if any node fails, the step's writes are discarded. An
16//!    [`RustyError::Interrupt`] suspends the whole run instead.
17//! 4. **Merge** — apply all node updates to the state via
18//!    [`crate::state::StateSpec::apply_super_step`] (per-channel reducers +
19//!    `LastValue` single-write validation).
20//! 5. **Route** — evaluate outgoing edges / commands against the
21//!    post-barrier state to determine the next active set; `Route::End` (or
22//!    an empty next set) terminates the run.
23//! 6. **Checkpoint** — persist a [`crate::checkpoint::Checkpoint`] recording
24//!    step, state, and next nodes, and emit [`GraphEvent`]s for streaming.
25//!
26//! A graph *cycle* (e.g. the ReAct loop `agent → tools → agent`) is not
27//! call-stack recursion — it is nodes being re-scheduled across super-steps,
28//! which is why the guard is `max_steps`, not a stack limit.
29//!
30//! # Observability
31//!
32//! The executor emits `tracing` telemetry throughout a run (no subscriber is
33//! installed by the library — the application chooses one):
34//!
35//! - `rusty.run` (INFO span) — one per [`Executor::run`] call, carrying
36//!   `thread_id` and `max_steps`; parent of everything below.
37//! - `rusty.super_step` (DEBUG span) — one per super-step, carrying
38//!   `step` and `active_nodes`; covers plan → barrier → merge → route.
39//! - `rusty.node` (INFO span) — one per spawned node task, carrying
40//!   `node` and `step`; attached to the `JoinSet` task via `.instrument()`.
41//! - DEBUG event on each barrier merge (channels written), INFO events on
42//!   interrupt and run completion (`steps`, `duration_ms`), WARN events on
43//!   node failure (with a `retryable` classification).
44
45use std::collections::{HashMap, HashSet};
46use std::sync::{Arc, Mutex};
47
48use serde::{Deserialize, Serialize};
49use serde_json::Value;
50use tokio::sync::mpsc;
51use tokio::task::JoinSet;
52use tokio_util::sync::CancellationToken;
53use tracing::Instrument;
54
55use crate::checkpoint::{Checkpoint, Checkpointer};
56use crate::effects::{CompensationRegistry, EffectAdmissionContext};
57use crate::error::{Result, RustyError};
58use crate::graph::{Edge, Graph, Route};
59use crate::journal::{Clock, EventDraft, Journal, RngSource};
60use crate::middleware::{Middleware, MiddlewareChain, NodeCall};
61use crate::node::{Command, NodeConfig, NodeContext, NodeOutput};
62use crate::record::{
63    CheckpointHeader, Effect, EventStatus, PolicyVersion, RunEventKind, RunManifest,
64    CURRENT_FORMAT_VERSION,
65};
66use crate::state::{State, StateSpec};
67
68/// How a run ended.
69#[derive(Debug)]
70pub enum ExecutionOutcome {
71    /// The run terminated normally (routing reached `Route::End` or no
72    /// nodes remained active). Carries the final state.
73    Done(State),
74
75    /// A node called `interrupt(payload)`: the run is suspended and
76    /// resumable. Carry on by calling [`Executor::run`] again with the same
77    /// `thread_id` and `RunConfig::resume` set.
78    ///
79    /// Suspension is run-wide, not node-local. The in-flight super-step is
80    /// transactional: every write of the step is discarded (including writes
81    /// from sibling nodes that completed before the interrupt was observed
82    /// at the barrier), still-running siblings are aborted, and the
83    /// suspension checkpoint re-schedules **every** node of the step. On
84    /// resume all of them re-execute from their start, so node logic must be
85    /// idempotent.
86    Interrupted {
87        /// The payload passed to `interrupt()` (surfaced to the caller,
88        /// e.g. a human-approval request).
89        value: Value,
90        /// The state as of the suspension point.
91        state: State,
92        /// The checkpoint persisted at the suspension point, for resuming
93        /// or time travel. When the executor has no checkpointer attached,
94        /// the run still suspends but nothing is persisted: the id is then
95        /// only an opaque handle and can never be replayed.
96        checkpoint_id: String,
97    },
98}
99
100impl ExecutionOutcome {
101    /// The final (or suspension-point) state, regardless of variant.
102    pub fn state(&self) -> &State {
103        match self {
104            ExecutionOutcome::Done(s) => s,
105            ExecutionOutcome::Interrupted { state, .. } => state,
106        }
107    }
108
109    /// Whether the run ended in [`ExecutionOutcome::Interrupted`] (suspended,
110    /// resumable) rather than [`ExecutionOutcome::Done`].
111    pub fn is_interrupted(&self) -> bool {
112        matches!(self, ExecutionOutcome::Interrupted { .. })
113    }
114}
115
116/// Per-run configuration (the LangGraph `RunnableConfig` analog).
117#[derive(Debug, Clone)]
118pub struct RunConfig {
119    /// Thread (session) id. Stable across interrupt/resume; namespaces all
120    /// checkpoints for this run. Required for persistence and resume.
121    pub thread_id: String,
122
123    /// Maximum number of super-steps before the run aborts with
124    /// [`crate::error::RustyError::Graph`] (the LangGraph
125    /// `recursion_limit` / `GraphRecursionError` guard). Default: 1000.
126    pub max_steps: usize,
127
128    /// Resume value for continuing an interrupted run. When set, the
129    /// executor restores the latest checkpoint for `thread_id` and
130    /// re-executes the checkpointed next-node set with
131    /// [`crate::node::NodeContext::resume_value`] returning this value.
132    ///
133    /// The value is **broadcast**: every node scheduled in the first
134    /// super-step after the resume observes it, not only the node that
135    /// originally interrupted (a suspension checkpoint re-schedules the
136    /// whole active set — see [`ExecutionOutcome::Interrupted`]). Nodes that
137    /// should react only when they themselves were resumed must key off
138    /// their own state, not the presence of a resume value.
139    pub resume: Option<Value>,
140
141    /// Replay/time-travel handle: the id of a specific checkpoint of
142    /// `thread_id` to resume from. When set, the executor loads **that**
143    /// checkpoint (not the latest) and continues the run from its state and
144    /// next-node set. Requires a checkpointer on the executor.
145    ///
146    /// Combines with `resume`: `checkpoint_id` selects **where** the run
147    /// restarts, `resume` (when also set) is delivered as the resume value to
148    /// the first super-step, exactly as in interrupt/resume.
149    ///
150    /// Safe pattern: replaying on the *same* thread appends new history on
151    /// top of the old timeline, so prefer forking first —
152    /// [`crate::checkpoint::Checkpointer::fork_thread`] the thread into a new
153    /// thread id, then run the fork with `checkpoint_id` set. Direct replay
154    /// on the original thread is supported for cases where appended history
155    /// is acceptable.
156    pub checkpoint_id: Option<String>,
157
158    /// Optional event sink for streaming: the executor emits [`GraphEvent`]s
159    /// as the run progresses (node start/end, state updates, checkpoints,
160    /// super-step boundaries). Consumers implement LangGraph's stream modes
161    /// (`values` / `updates` / `tasks` / ...) as filters over this stream.
162    pub event_tx: Option<mpsc::Sender<GraphEvent>>,
163
164    /// Flight Recorder determinism seam (R0.5): the run's time source. Every
165    /// executor timestamp — event `recorded_at`, node latencies, checkpoint
166    /// `created_at` — is read through it. `None` (the default) is the system
167    /// wall clock, byte-identical to pre-R0.5 behavior. Attach
168    /// [`Clock::Logical`] to make a recorded run re-drivable.
169    pub clock: Option<Clock>,
170
171    /// Flight Recorder determinism seam (R0.5): the run's randomness source.
172    /// Checkpoint ids (and the run id, when the executor creates the
173    /// journal) are minted through it. `None` (the default) is OS entropy,
174    /// byte-identical to pre-R0.5 behavior; [`RngSource::Seeded`] makes the
175    /// id stream reproducible.
176    pub rng: Option<RngSource>,
177
178    /// Flight Recorder (R0.5): attach a pre-built journal for this run. Node
179    /// closures capture a clone of the same [`Journal`] to record their own
180    /// model/tool/remote/WASM calls into the run's evidence (the journal
181    /// stamps sequence numbers and timestamps itself). When `None`, the
182    /// executor creates a fresh journal per run; either way the run's
183    /// journal is retrievable afterwards via [`Executor::journal`].
184    pub journal: Option<Journal>,
185
186    /// Flight Recorder (R0.5): the executor policy active for this run,
187    /// stamped into every checkpoint header. `None` records the static
188    /// default ([`PolicyVersion::STATIC_V0`]) — except on resume (R0.8 wave
189    /// 4), where the run inherits the version its checkpoint header pins so
190    /// an in-flight run is immune to mid-run promotions. An explicit pin
191    /// always wins over inheritance.
192    pub policy_version: Option<PolicyVersion>,
193
194    /// Flight Recorder (R0.5): the application's own graph version string,
195    /// stamped into every checkpoint header next to the topology hash.
196    /// `None` records `"unversioned"`. Bump it when node bodies change in
197    /// ways the topology hash cannot see.
198    pub graph_version: Option<String>,
199
200    /// Cooperative cancellation (R0.6 wave 2c, drain). When set, the
201    /// executor checks the token **at every super-step boundary** and, once
202    /// it is cancelled, stops the run before starting the next super-step,
203    /// returning [`RustyError::Cancelled`].
204    ///
205    /// The boundary is the only cancellation point, deliberately: a
206    /// super-step is transactional (its nodes run off an immutable snapshot
207    /// and its writes merge atomically), and its boundary checkpoint is
208    /// already persisted by the time the token is observed — so a cancelled
209    /// run resumes from exactly where it stopped, with no torn step and no
210    /// lost writes. Cancellation granularity is therefore one super-step;
211    /// work inside the in-flight step runs to its barrier. This is the hook
212    /// graceful shutdown uses: a draining server cancels the token and every
213    /// in-flight run parks itself at a resumable checkpoint instead of being
214    /// torn down mid-step.
215    pub cancellation: Option<CancellationToken>,
216
217    /// Effect kernel v2 (R0.7): the versioned run manifest — prompts, tool
218    /// schemas, model and parameters, memory schema, capsule versions the run
219    /// pins. Stamped into every checkpoint header so a resumed run keeps
220    /// executing against its pinned versions (see
221    /// [`crate::record::RunManifest`] for the upgrade-safety contract).
222    /// `None` (the default) pins nothing and leaves checkpoint bytes
223    /// byte-identical to R0.5/R0.6.
224    pub manifest: Option<RunManifest>,
225
226    /// Effect kernel v2 (R0.7): the approval tokens this run carries for its
227    /// irreversible effects (see [`crate::effects::ApprovalToken`]). The
228    /// executor consults them when [`Executor::with_effect_admission`] has
229    /// enabled the guarded tool path. Empty by default; an irreversible call
230    /// dispatched through that path then fails admission before its body
231    /// runs.
232    pub effect_approvals: Vec<crate::effects::ApprovalToken>,
233}
234
235impl Default for RunConfig {
236    /// A config with an empty `thread_id` and the default step limit —
237    /// identical to `RunConfig::new("")`. Derived `Default` would zero
238    /// `max_steps`, so any `default()`-built run would instantly trip the
239    /// step guard; keep this impl in sync with [`RunConfig::new`].
240    fn default() -> Self {
241        Self::new(String::new())
242    }
243}
244
245impl RunConfig {
246    /// A config for `thread_id` with the default step limit.
247    pub fn new(thread_id: impl Into<String>) -> Self {
248        Self {
249            thread_id: thread_id.into(),
250            max_steps: DEFAULT_MAX_STEPS,
251            resume: None,
252            checkpoint_id: None,
253            event_tx: None,
254            clock: None,
255            rng: None,
256            journal: None,
257            policy_version: None,
258            graph_version: None,
259            cancellation: None,
260            manifest: None,
261            effect_approvals: Vec::new(),
262        }
263    }
264
265    /// Builder-style: override the step limit.
266    pub fn with_max_steps(mut self, max_steps: usize) -> Self {
267        self.max_steps = max_steps;
268        self
269    }
270
271    /// Builder-style: set the resume value.
272    pub fn with_resume(mut self, value: Value) -> Self {
273        self.resume = Some(value);
274        self
275    }
276
277    /// Builder-style: replay from a specific checkpoint of `thread_id`
278    /// (time travel). See the [`RunConfig::checkpoint_id`] field docs for
279    /// semantics and the fork-first safe pattern.
280    pub fn with_checkpoint_id(mut self, checkpoint_id: impl Into<String>) -> Self {
281        self.checkpoint_id = Some(checkpoint_id.into());
282        self
283    }
284
285    /// Builder-style: attach a streaming event sink.
286    pub fn with_event_tx(mut self, tx: mpsc::Sender<GraphEvent>) -> Self {
287        self.event_tx = Some(tx);
288        self
289    }
290
291    /// Builder-style: source the run's time from `clock` (Flight Recorder
292    /// determinism seam; see the [`RunConfig::clock`] field docs).
293    pub fn with_clock(mut self, clock: Clock) -> Self {
294        self.clock = Some(clock);
295        self
296    }
297
298    /// Builder-style: source the run's randomness from `rng` (Flight
299    /// Recorder determinism seam; see the [`RunConfig::rng`] field docs).
300    pub fn with_rng(mut self, rng: RngSource) -> Self {
301        self.rng = Some(rng);
302        self
303    }
304
305    /// Builder-style: record the run into `journal` (Flight Recorder; see
306    /// the [`RunConfig::journal`] field docs). When the config carries no
307    /// explicit [`RunConfig::clock`], the executor also reads time from the
308    /// attached journal's clock, keeping one time source per run.
309    pub fn with_journal(mut self, journal: Journal) -> Self {
310        self.journal = Some(journal);
311        self
312    }
313
314    /// Builder-style: stamp the active executor policy version into every
315    /// checkpoint header (see the [`RunConfig::policy_version`] field docs).
316    pub fn with_policy_version(mut self, version: PolicyVersion) -> Self {
317        self.policy_version = Some(version);
318        self
319    }
320
321    /// Builder-style: stamp the application's graph version into every
322    /// checkpoint header (see the [`RunConfig::graph_version`] field docs).
323    pub fn with_graph_version(mut self, version: impl Into<String>) -> Self {
324        self.graph_version = Some(version.into());
325        self
326    }
327
328    /// Builder-style: make the run cooperatively cancellable (see the
329    /// [`RunConfig::cancellation`] field docs). The token is observed only at
330    /// super-step boundaries: cancelling it stops the run *after* the
331    /// in-flight step's barrier and boundary checkpoint, so the run is left
332    /// resumable from exactly that checkpoint with [`RustyError::Cancelled`].
333    pub fn with_cancellation(mut self, token: CancellationToken) -> Self {
334        self.cancellation = Some(token);
335        self
336    }
337
338    /// Builder-style: pin the run's versioned manifest into every checkpoint
339    /// header (see the [`RunConfig::manifest`] field docs).
340    pub fn with_manifest(mut self, manifest: RunManifest) -> Self {
341        self.manifest = Some(manifest);
342        self
343    }
344
345    /// Builder-style: carry approval tokens for the run's irreversible
346    /// effects (see the [`RunConfig::effect_approvals`] field docs).
347    pub fn with_effect_approvals(
348        mut self,
349        approvals: impl IntoIterator<Item = crate::effects::ApprovalToken>,
350    ) -> Self {
351        self.effect_approvals = approvals.into_iter().collect();
352        self
353    }
354
355    /// A clone of the event sink sender, for wiring into nodes that stream
356    /// [`GraphEvent::Token`] deltas (LangGraph's `messages` stream mode).
357    ///
358    /// Typical flow: create the channel, call `config.token_tx()` to obtain
359    /// the clone a node closure captures, then hand `config` to
360    /// [`Executor::run`]. `None` when no sink is attached. See the
361    /// [`crate::llm::ChatModel`] rustdoc for the full pattern.
362    pub fn token_tx(&self) -> Option<mpsc::Sender<GraphEvent>> {
363        self.event_tx.clone()
364    }
365}
366
367/// Default super-step limit. Deliberately far above LangGraph's default
368/// `recursion_limit` of 25: ReAct-style loops burn one super-step per
369/// agent/tool hop, so long tool chains legitimately exceed 25.
370pub const DEFAULT_MAX_STEPS: usize = 1000;
371
372/// Streaming events emitted by the executor during a run. All of LangGraph's
373/// stream modes are views over this single typed event stream.
374///
375/// The enum is serializable so event streams can cross process / FFI
376/// boundaries (e.g. a WebSocket bridge or a persisted event log).
377#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
378#[serde(tag = "type", rename_all = "snake_case")]
379pub enum GraphEvent {
380    /// A node began executing.
381    NodeStart {
382        /// Node name.
383        node: String,
384        /// Super-step index.
385        step: usize,
386    },
387    /// A node finished executing (successfully).
388    NodeEnd {
389        /// Node name.
390        node: String,
391        /// Super-step index.
392        step: usize,
393    },
394    /// A single LLM token delta (the LangGraph `messages` stream mode).
395    ///
396    /// The executor itself never emits this variant: tokens originate inside
397    /// nodes that call [`crate::llm::ChatModel::chat_stream`] and forward
398    /// each [`crate::llm::TokenChunk`] into the run's event channel (see the
399    /// `ChatModel` rustdoc for the wiring pattern and
400    /// [`RunConfig::token_tx`] / [`Executor::token_tx`] for sender handles).
401    Token {
402        /// Node that produced the token.
403        node: String,
404        /// Incremental text produced since the previous token.
405        delta: String,
406    },
407    /// State was updated at a super-step barrier (`updates` stream mode).
408    StateUpdate {
409        /// Super-step index at which the update was applied.
410        step: usize,
411        /// Every channel written in this step mapped to its **post-reducer**
412        /// value (e.g. the full appended list for an `Append` channel), read
413        /// back from the merged state — not the raw per-node partials.
414        updates: serde_json::Map<String, Value>,
415    },
416    /// A checkpoint was persisted at a super-step boundary.
417    CheckpointSaved {
418        /// The checkpoint id.
419        checkpoint_id: String,
420        /// Super-step index at the boundary.
421        step: usize,
422    },
423    /// A super-step began; lists the nodes activated in it.
424    SuperStep {
425        /// Super-step index.
426        step: usize,
427        /// Nodes active in this step.
428        active_nodes: Vec<String>,
429    },
430}
431
432/// The graph executor. Holds an optional checkpointer and an optional
433/// middleware chain; stateless with respect to individual runs, so one
434/// `Executor` can drive many concurrent runs (each with its own
435/// `thread_id`).
436#[derive(Default)]
437pub struct Executor {
438    checkpointer: Option<Arc<dyn Checkpointer>>,
439    token_tx: Option<mpsc::Sender<GraphEvent>>,
440    // The most recent run's Flight Recorder journal. Interior mutability
441    // keeps `run` taking `&self`; overwritten at the start of every run.
442    journal: Mutex<Option<Journal>>,
443    // Middleware/Interceptor SDK: ordered layers wrapping every node
444    // invocation (and, via `NodeContext::middleware`, the tool/model calls
445    // node code makes). An empty chain takes the original dispatch path.
446    middleware: MiddlewareChain,
447    // R0.8 wave 1: the governed-memory source this executor's runs answer
448    // memory reads from (live store or replay cursor). The executor never
449    // touches memory itself this wave — the full run-loop integration is a
450    // later wave; this is the installation point node factories use (see
451    // `Executor::memory`), mirroring `with_token_tx`.
452    memory_source: Option<crate::memory::MemorySource>,
453    // Present only when guarded tool effect admission is enabled. The run's
454    // thread id and approval tokens are combined with these rollback handlers
455    // into an EffectAdmissionContext for each node invocation.
456    effect_compensations: Option<CompensationRegistry>,
457}
458
459impl Executor {
460    /// An executor without persistence (runs cannot be interrupted/resumed
461    /// durably; interrupts will still surface but resume requires a
462    /// checkpointer).
463    pub fn new() -> Self {
464        Self::default()
465    }
466
467    /// An executor persisting checkpoints through `checkpointer`.
468    pub fn with_checkpointer(checkpointer: Arc<dyn Checkpointer>) -> Self {
469        Self {
470            checkpointer: Some(checkpointer),
471            ..Self::default()
472        }
473    }
474
475    /// Builder-style: attach a middleware layer (Middleware/Interceptor
476    /// SDK). Layers compose in registration order — before-hooks run in
477    /// `.layer()` order on the way into a node/model/tool call, after-hooks
478    /// in reverse order on the way out. See [`crate::middleware`].
479    pub fn layer<M: Middleware + 'static>(mut self, middleware: M) -> Self {
480        self.middleware.push(Arc::new(middleware));
481        self
482    }
483
484    /// Builder-style: attach a pre-shared middleware layer.
485    pub fn layer_shared(mut self, middleware: Arc<dyn Middleware>) -> Self {
486        self.middleware.push(middleware);
487        self
488    }
489
490    /// The attached middleware chain (empty when no layers were added).
491    pub fn middleware(&self) -> &MiddlewareChain {
492        &self.middleware
493    }
494
495    /// Builder-style: make guarded tool effect admission available to nodes.
496    ///
497    /// The supplied registry may be empty. Pure and read-only calls remain
498    /// automatic; idempotent calls require a key, compensatable calls require
499    /// a handler in this registry, and non-idempotent calls require a matching
500    /// token in [`RunConfig::effect_approvals`]. The prebuilt ReAct tools node
501    /// enforces this automatically. Custom nodes must pass
502    /// [`NodeContext::effect_admission`] to the [`crate::tool::ToolExecutor`]
503    /// they construct; direct [`crate::tool::Tool::call`] invocations are not
504    /// intercepted by the executor.
505    pub fn with_effect_admission(mut self, compensations: CompensationRegistry) -> Self {
506        self.effect_compensations = Some(compensations);
507        self
508    }
509
510    /// Whether this executor enforces the effect admission boundary.
511    pub fn effect_admission_enabled(&self) -> bool {
512        self.effect_compensations.is_some()
513    }
514
515    /// Builder-style: hold a token broadcast sender that nodes can clone to
516    /// publish [`GraphEvent::Token`] deltas (LangGraph's `messages` stream
517    /// mode).
518    ///
519    /// The executor never emits `Token` events itself — tokens originate in
520    /// nodes calling [`crate::llm::ChatModel::chat_stream`]. This is a
521    /// convenience handle so node factories built around an `Executor` can
522    /// fetch the sink via [`Executor::token_tx`] and capture a clone in each
523    /// node closure. When the same channel should also receive the
524    /// executor's own events, attach it to the run via
525    /// [`RunConfig::with_event_tx`] instead (or as well).
526    pub fn with_token_tx(mut self, token_tx: mpsc::Sender<GraphEvent>) -> Self {
527        self.token_tx = Some(token_tx);
528        self
529    }
530
531    /// The token broadcast sender, if one was attached via
532    /// [`Executor::with_token_tx`]. Clone it into node closures to stream
533    /// [`GraphEvent::Token`]s from within nodes.
534    pub fn token_tx(&self) -> Option<&mpsc::Sender<GraphEvent>> {
535        self.token_tx.as_ref()
536    }
537
538    /// Builder-style: install the governed-memory source this executor's
539    /// runs answer memory reads from (R0.8 wave 1; see
540    /// [`crate::memory::MemorySource`]).
541    ///
542    /// The executor never reads or writes memory itself this wave — the
543    /// full run-loop integration is a later wave. This is the seam's
544    /// installation point, mirroring [`Executor::with_token_tx`]: node
545    /// factories built around an `Executor` fetch the journaled handle via
546    /// [`Executor::memory`] once the run has started and capture a clone in
547    /// each node closure. Nodes may equally bind a source to an attached
548    /// journal directly ([`crate::journal::Journal::memory`]); both paths
549    /// journal into the same run evidence.
550    pub fn with_memory_source(mut self, source: crate::memory::MemorySource) -> Self {
551        self.memory_source = Some(source);
552        self
553    }
554
555    /// The configured memory source, if one was installed via
556    /// [`Executor::with_memory_source`].
557    pub fn memory_source(&self) -> Option<&crate::memory::MemorySource> {
558        self.memory_source.as_ref()
559    }
560
561    /// The journaled memory handle for the most recent run: the run's
562    /// journal (set at run start) bound to the configured source. `None`
563    /// before the first run or when no source is installed. The same
564    /// most-recent-run caveat as [`Executor::journal`] applies: driving
565    /// several runs concurrently through one `Executor` leaves this handle
566    /// pointing at whichever run started last — bind per-run handles via
567    /// [`crate::journal::Journal::memory`] when runs overlap.
568    pub fn memory(&self) -> Option<crate::memory::JournaledMemory> {
569        let journal = self.journal()?;
570        let source = self.memory_source.clone()?;
571        Some(crate::memory::JournaledMemory::new(&journal, source))
572    }
573
574    /// The configured checkpointer, if any. Shared (not consumed) so one
575    /// `Executor` can drive many concurrent runs over the same store.
576    pub fn checkpointer(&self) -> Option<&Arc<dyn Checkpointer>> {
577        self.checkpointer.as_ref()
578    }
579
580    /// The Flight Recorder journal of the most recent run started through
581    /// this executor — the one attached via [`RunConfig::with_journal`], or
582    /// the fresh journal the executor created for the run. `None` before the
583    /// first run.
584    ///
585    /// The journal is set at run start, so the evidence of a failed or
586    /// suspended run is retrievable exactly like a completed run's. Note the
587    /// executor is deliberately stateless across runs: driving several runs
588    /// concurrently through one `Executor` leaves this handle pointing at
589    /// whichever run started last — attach per-run journals via the config
590    /// when runs overlap.
591    pub fn journal(&self) -> Option<Journal> {
592        self.journal
593            .lock()
594            .unwrap_or_else(|e| e.into_inner())
595            .clone()
596    }
597
598    /// Run a compiled graph to completion (or interruption).
599    ///
600    /// - `graph`: the compiled, frozen graph topology.
601    /// - `spec`: the state schema (channels + reducers) used to merge node
602    ///   updates at each barrier.
603    /// - `initial_state`: the starting state. When `config.resume` is set and
604    ///   a checkpoint exists for `config.thread_id`, the checkpointed state
605    ///   and next-node set take precedence over this argument. When
606    ///   `config.checkpoint_id` is set, that specific checkpoint (rather than
607    ///   the latest) is restored — replay/time travel; forking into a fresh
608    ///   thread first via
609    ///   [`crate::checkpoint::Checkpointer::fork_thread`] is the safe pattern,
610    ///   since replaying on the same thread appends new history.
611    /// - `config`: run configuration (thread id, step limit, resume value,
612    ///   streaming sink).
613    ///
614    /// # Super-step semantics
615    ///
616    /// Each loop iteration runs one super-step as a transaction: the active
617    /// nodes execute in parallel over an immutable start-of-step snapshot;
618    /// the barrier discards the whole step's writes on any node failure and
619    /// suspends the run on an interrupt; only then are writes merged via the
620    /// channel reducers, routing computed against the post-barrier state,
621    /// and a boundary checkpoint persisted. The module-level docs enumerate
622    /// the six phases; `execute_super_step` is the implementation.
623    ///
624    /// The loop returns [`ExecutionOutcome::Done`] when routing yields an
625    /// empty next set, [`ExecutionOutcome::Interrupted`] when a node
626    /// interrupts, an [`RustyError::Graph`] error once `config.max_steps`
627    /// super-steps have run without termination, and
628    /// [`RustyError::Cancelled`] when [`RunConfig::cancellation`] is
629    /// observed cancelled at a super-step boundary (the boundary checkpoint
630    /// is intact; the run resumes from it).
631    pub async fn run(
632        &self,
633        graph: &Graph,
634        spec: &StateSpec,
635        initial_state: State,
636        config: RunConfig,
637    ) -> Result<ExecutionOutcome> {
638        // Run-level span: every super-step, node, and checkpoint trace in the
639        // run attaches to it. Attached via `.instrument()` (never `.enter()`)
640        // so no span guard is held across `.await` points and the returned
641        // future stays `Send`.
642        let run_span = tracing::info_span!(
643            "rusty.run",
644            thread_id = %config.thread_id,
645            max_steps = config.max_steps,
646            resume = config.resume.is_some(),
647            replay = config.checkpoint_id.is_some(),
648        );
649        self.run_inner(graph, spec, initial_state, config)
650            .instrument(run_span)
651            .await
652    }
653
654    /// The instrumented body of [`Executor::run`]; see that method's docs for
655    /// the super-step algorithm.
656    async fn run_inner(
657        &self,
658        graph: &Graph,
659        spec: &StateSpec,
660        initial_state: State,
661        config: RunConfig,
662    ) -> Result<ExecutionOutcome> {
663        let started = std::time::Instant::now();
664
665        // ---- flight recorder setup ----
666        //
667        // Resolve the determinism seams and the run's journal before any
668        // checkpoint work: every timestamp and id below flows through them.
669        // An attached journal carries its own identity and clock; otherwise
670        // the executor mints a run id from the (possibly seeded) RNG and a
671        // journal over the configured clock. The run reads time from the
672        // journal's clock when no explicit clock is configured, keeping one
673        // time source per run.
674        let rng = config.rng.clone().unwrap_or_default();
675        let journal = match &config.journal {
676            Some(attached) => attached.clone(),
677            None => Journal::new(
678                rng.uuid_string(),
679                config.thread_id.clone(),
680                config.clock.clone().unwrap_or_default(),
681            ),
682        };
683        let mut recorder = Recorder {
684            clock: config
685                .clock
686                .clone()
687                .unwrap_or_else(|| journal.clock().clone()),
688            rng,
689            graph_version: config
690                .graph_version
691                .clone()
692                .unwrap_or_else(|| "unversioned".to_owned()),
693            graph_hash: graph.topology_hash(),
694            policy_version: config.policy_version.clone().unwrap_or_default(),
695            manifest: config.manifest.clone(),
696            journal: journal.clone(),
697        };
698        // Publish before the loop: evidence of a failed or suspended run is
699        // retrievable exactly like a completed run's.
700        *self.journal.lock().unwrap_or_else(|e| e.into_inner()) = Some(journal);
701
702        // ---- initialization / resume ----
703        //
704        // On resume the checkpointed state and next-node set take precedence
705        // over `initial_state`; the resume value is delivered to the first
706        // super-step (whose active set is the checkpointed next-node set —
707        // after an interrupt, every node of the suspended step) via
708        // `NodeContext::resume_value()`.
709        //
710        // Time travel: when `config.checkpoint_id` is set, THAT checkpoint is
711        // restored instead of the latest — this is replay from an arbitrary
712        // history point. The two knobs compose: `checkpoint_id` selects WHERE
713        // the run restarts, `resume` (when also set) supplies the resume value
714        // for the first super-step.
715        let mut state = initial_state;
716        let mut active: Vec<ActiveTask>;
717        let mut step: usize = 0;
718        let mut pending_resume: Option<Value> = None;
719        // Causal parent for the first super-step: the resume event on a
720        // resumed run, nothing on a fresh one.
721        let mut step_parent: Option<String> = None;
722
723        if config.checkpoint_id.is_some() || config.resume.is_some() {
724            let checkpointer = self.checkpointer.as_ref().ok_or_else(|| {
725                RustyError::Checkpoint(
726                    "RunConfig.checkpoint_id/resume is set but no checkpointer is configured \
727                     on the executor"
728                        .into(),
729                )
730            })?;
731            let checkpoint = match &config.checkpoint_id {
732                Some(id) => checkpointer
733                    .get_by_id(&config.thread_id, id)
734                    .await?
735                    .ok_or_else(|| {
736                        RustyError::Checkpoint(format!(
737                            "cannot replay thread `{}`: checkpoint `{id}` not found",
738                            config.thread_id
739                        ))
740                    })?,
741                None => checkpointer
742                    .get_latest(&config.thread_id)
743                    .await?
744                    .ok_or_else(|| {
745                        RustyError::Checkpoint(format!(
746                            "cannot resume thread `{}`: no checkpoint found",
747                            config.thread_id
748                        ))
749                    })?,
750            };
751            // Policy-binding continuity (R0.8 wave 4): a resumed run keeps
752            // the policy version its checkpoint header pins, so a mid-run
753            // promotion never changes behavior under an in-flight execution.
754            // An explicit `RunConfig::with_policy_version` pin wins over
755            // inheritance — the caller asked for that version deliberately.
756            if config.policy_version.is_none() {
757                recorder.policy_version = checkpoint.header.policy_version.clone();
758            }
759            step_parent = Some(recorder.record(
760                EventDraft::new(RunEventKind::Resume, Effect::Pure).input(serde_json::json!({
761                    "checkpoint_id": checkpoint.id,
762                    "step": checkpoint.step,
763                    "resume": config.resume.clone().unwrap_or(Value::Null),
764                })),
765            ));
766            state = checkpoint.state;
767            step = checkpoint.step;
768            active = checkpoint
769                .next_nodes
770                .into_iter()
771                .map(|name| ActiveTask { name, scoped: None })
772                .collect();
773            pending_resume = config.resume.clone();
774            if active.is_empty() {
775                tracing::info!(
776                    steps = 0,
777                    duration_ms = started.elapsed().as_millis() as u64,
778                    "run complete"
779                );
780                return Ok(ExecutionOutcome::Done(state));
781            }
782        } else {
783            active = vec![ActiveTask {
784                name: graph.entry_point().to_owned(),
785                scoped: None,
786            }];
787        }
788
789        // One shared context for the whole run: clones handed to parallel
790        // nodes share its approval ledger, and later super-steps observe
791        // tokens already consumed by earlier calls.
792        let effect_admission = self.effect_compensations.as_ref().map(|compensations| {
793            EffectAdmissionContext::new(config.thread_id.clone())
794                .with_approvals(config.effect_approvals.clone())
795                .with_compensations(compensations.clone())
796        });
797
798        // ---- super-step loop ----
799        let mut steps_run: usize = 0;
800        loop {
801            if steps_run >= config.max_steps {
802                return Err(RustyError::Graph(format!(
803                    "max_steps ({}) exceeded: the graph did not terminate within the step \
804                     budget (possible infinite cycle; raise RunConfig::max_steps or add a \
805                     terminating route)",
806                    config.max_steps
807                )));
808            }
809            // Cooperative cancellation (drain): observed only here, at a
810            // super-step boundary. The last step's barrier merged cleanly
811            // and its boundary checkpoint is already persisted, so
812            // cancelling never tears a step — the run resumes from exactly
813            // this point. Nodes of the in-flight step always run to their
814            // barrier; cancellation granularity is one super-step.
815            if config
816                .cancellation
817                .as_ref()
818                .is_some_and(CancellationToken::is_cancelled)
819            {
820                tracing::warn!(
821                    steps_run,
822                    "run cancelled at a super-step boundary; resumable from the last checkpoint"
823                );
824                return Err(RustyError::Cancelled(format!(
825                    "cancelled after {steps_run} super-step(s); the boundary checkpoint of \
826                     thread `{}` is intact, so re-running the thread resumes from there",
827                    config.thread_id
828                )));
829            }
830
831            // The step body runs in a dedicated method so the whole body is
832            // one instrumented future under the per-step span.
833            let step_span =
834                tracing::debug_span!("rusty.super_step", step = step, active_nodes = active.len(),);
835
836            let transition = self
837                .execute_super_step(
838                    graph,
839                    spec,
840                    &config,
841                    &recorder,
842                    effect_admission.as_ref(),
843                    &mut state,
844                    &active,
845                    step,
846                    &mut pending_resume,
847                    step_parent.clone(),
848                )
849                .instrument(step_span)
850                .await?;
851
852            match transition {
853                StepTransition::Next(next, route_event) => {
854                    active = next;
855                    step += 1;
856                    steps_run += 1;
857                    step_parent = Some(route_event);
858                }
859                StepTransition::Finish(outcome) => {
860                    if !outcome.is_interrupted() {
861                        tracing::info!(
862                            steps = steps_run + 1,
863                            duration_ms = started.elapsed().as_millis() as u64,
864                            "run complete"
865                        );
866                    }
867                    return Ok(outcome);
868                }
869            }
870        }
871    }
872
873    /// Executes one super-step: plan -> compute -> barrier -> merge -> route
874    /// -> boundary checkpoint. Returns the next active set, or
875    /// [`StepTransition::Finish`] with the terminal outcome when the run ends
876    /// (`Done`) or suspends (`Interrupted`).
877    ///
878    /// Flight Recorder: every phase transition is journaled through
879    /// `recorder` — super-step start/end, one node input/output pair per
880    /// invocation (in deterministic active-set order, not finish order), the
881    /// routing decision, and the checkpoint write. Node events carry the
882    /// node's declared [`Effect`] classification.
883    #[allow(clippy::too_many_arguments)]
884    async fn execute_super_step(
885        &self,
886        graph: &Graph,
887        spec: &StateSpec,
888        config: &RunConfig,
889        recorder: &Recorder,
890        effect_admission: Option<&EffectAdmissionContext>,
891        state: &mut State,
892        active: &[ActiveTask],
893        step: usize,
894        pending_resume: &mut Option<Value>,
895        step_parent: Option<String>,
896    ) -> Result<StepTransition> {
897        // -- plan.
898        let active_names: Vec<String> = active.iter().map(|t| t.name.clone()).collect();
899        Self::emit(
900            config,
901            GraphEvent::SuperStep {
902                step,
903                active_nodes: active_names.clone(),
904            },
905        );
906        let mut plan_draft =
907            EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure).input(serde_json::json!({
908                "step": step,
909                "active_nodes": active_names,
910            }));
911        if let Some(parent) = step_parent {
912            plan_draft = plan_draft.parent(parent);
913        }
914        let step_start_event = recorder.record(plan_draft);
915
916        // -- compute. Scoped (Send) state is overlaid onto each invocation's
917        //    private copy of the start-of-step snapshot, so fan-out items
918        //    never collide in the shared state.
919        let snapshot = state.clone();
920        let effect_admission = effect_admission.cloned();
921        let mut join_set: JoinSet<(usize, String, Result<NodeOutput>, u64)> = JoinSet::new();
922        // Per-invocation journal metadata, aligned with `active`: the input
923        // event id (causal parent of the matching output) and the node's
924        // declared effect class. Fan-out invocations of the same node each
925        // get their own entry — index, not name, is the identity.
926        let mut input_events: Vec<String> = Vec::with_capacity(active.len());
927        let mut invocation_effects: Vec<Effect> = Vec::with_capacity(active.len());
928
929        for (index, task) in active.iter().enumerate() {
930            let node = graph.node(&task.name).ok_or_else(|| {
931                RustyError::Graph(format!("routing activated unknown node `{}`", task.name))
932            })?;
933
934            let mut node_state = snapshot.clone();
935            if let Some(scoped) = &task.scoped {
936                match scoped {
937                    Value::Object(map) => {
938                        for (channel, value) in map {
939                            node_state.insert(channel.clone(), value.clone());
940                        }
941                    }
942                    other => {
943                        return Err(RustyError::InvalidUpdate(format!(
944                            "Send scoped state for node `{}` must be a JSON object, \
945                             got {other}",
946                            task.name
947                        )));
948                    }
949                }
950            }
951
952            let input_event = recorder.record(
953                EventDraft::new(RunEventKind::NodeInput, node.effect())
954                    .node(task.name.clone())
955                    .input(node_state.to_value())
956                    .parent(step_start_event.clone()),
957            );
958            invocation_effects.push(node.effect());
959
960            let node_config = NodeConfig {
961                thread_id: config.thread_id.clone(),
962                step,
963                resume: pending_resume.clone(),
964                // Hand the invocation its own journal event id so node
965                // code can parent the effects it records (model/tool
966                // calls) to this invocation.
967                extra: HashMap::from([(
968                    crate::journal::PARENT_EVENT_KEY.to_owned(),
969                    Value::String(input_event.clone()),
970                )]),
971            };
972            input_events.push(input_event);
973            let name = task.name.clone();
974            Self::emit(
975                config,
976                GraphEvent::NodeStart {
977                    node: name.clone(),
978                    step,
979                },
980            );
981            // A JoinSet polls tasks independently of the spawning task's
982            // context, so the per-node span is attached to each spawned
983            // future explicitly via `.instrument()`.
984            let node_span = tracing::info_span!("rusty.node", node = %name, step = step);
985            // Latency is read through the run's clock seam — under a logical
986            // clock the value is reproducible; under the system clock it is
987            // the real elapsed time.
988            let clock = recorder.clock.clone();
989            let chain = self.middleware.clone();
990            let effect_admission = effect_admission.clone();
991            join_set.spawn(
992                async move {
993                    let node_started = clock.now();
994                    // Middleware chain: with layers attached, the invocation
995                    // runs inside the onion — before-hooks may mutate the
996                    // snapshot, reject the run, or short-circuit with a
997                    // substitute output; after-hooks unwind over the result.
998                    // No layers: the original dispatch, byte-identical.
999                    let result = if chain.is_empty() {
1000                        node.run(
1001                            NodeContext::new(node_state, node_config)
1002                                .with_optional_effect_admission(effect_admission),
1003                        )
1004                        .await
1005                    } else {
1006                        let mut call = NodeCall::new(
1007                            node_config.thread_id.clone(),
1008                            name.clone(),
1009                            step,
1010                            node_state,
1011                        );
1012                        chain
1013                            .run_node(&mut call, |call| {
1014                                let ctx = NodeContext::new(call.state().clone(), node_config)
1015                                    .with_middleware(chain.clone())
1016                                    .with_optional_effect_admission(effect_admission);
1017                                let node = Arc::clone(&node);
1018                                async move { node.run(ctx).await }
1019                            })
1020                            .await
1021                    };
1022                    let latency_ms = (clock.now() - node_started).num_milliseconds().max(0) as u64;
1023                    (index, name, result, latency_ms)
1024                }
1025                .instrument(node_span),
1026            );
1027        }
1028        // The resume value is consumed by the first super-step after a resume.
1029        *pending_resume = None;
1030
1031        // -- barrier: collect every node result. The step is
1032        //    transactional: on any failure the JoinSet is dropped
1033        //    (aborting stragglers) and the step's writes are discarded.
1034        let mut writes: Vec<(String, HashMap<String, Value>)> = Vec::new();
1035        let mut commands: Vec<Command> = Vec::new();
1036        let mut ran_nodes: Vec<String> = Vec::new();
1037        let mut interrupted: Option<(usize, String, Value)> = None;
1038        // Journal payloads of finished invocations, in finish order here;
1039        // journaled in active-set order after the barrier.
1040        let mut completed: Vec<(usize, String, u64, Value)> = Vec::new();
1041
1042        while let Some(joined) = join_set.join_next().await {
1043            let (index, name, result, latency_ms) = joined.map_err(|e| {
1044                RustyError::Node(format!(
1045                    "node task failed to join (panic or cancellation): {e}"
1046                ))
1047            })?;
1048            match result {
1049                Ok(output) => {
1050                    Self::emit(
1051                        config,
1052                        GraphEvent::NodeEnd {
1053                            node: name.clone(),
1054                            step,
1055                        },
1056                    );
1057                    completed.push((
1058                        index,
1059                        name.clone(),
1060                        latency_ms,
1061                        serde_json::json!({
1062                            "updates": &output.updates,
1063                            "command": &output.command,
1064                        }),
1065                    ));
1066                    if let Some(command) = output.command {
1067                        if !command.goto.is_empty() {
1068                            commands.push(command);
1069                        }
1070                    }
1071                    ran_nodes.push(name.clone());
1072                    writes.push((name, output.updates));
1073                }
1074                Err(RustyError::Interrupt { value }) => {
1075                    // Record the suspension and stop the barrier loop; the
1076                    // JoinSet is dropped below to abort stragglers.
1077                    interrupted = Some((index, name, value));
1078                    break;
1079                }
1080                Err(e) => {
1081                    // A failed node is still evidence: journal the failure
1082                    // before the error unwinds the run.
1083                    recorder.record(
1084                        EventDraft::new(RunEventKind::NodeOutput, invocation_effects[index])
1085                            .node(name.clone())
1086                            .status(EventStatus::Error)
1087                            .output(serde_json::json!({ "error": e.to_string() }))
1088                            .latency_ms(latency_ms)
1089                            .parent(input_events[index].clone()),
1090                    );
1091                    // LLM and tool failures are the transient, retryable
1092                    // error classes; everything else is a hard failure.
1093                    let retryable = matches!(e, RustyError::Llm(_) | RustyError::Tool(_));
1094                    tracing::warn!(
1095                        node = %name,
1096                        step = step,
1097                        error = %e,
1098                        retryable = retryable,
1099                        "node failed; super-step aborted and its writes discarded"
1100                    );
1101                    return Err(RustyError::Node(format!(
1102                        "node `{name}` failed at super-step {step}: {e}"
1103                    )));
1104                }
1105            }
1106        }
1107
1108        if let Some((index, name, value)) = interrupted {
1109            // Suspend the run. The step is transactional, so no write of
1110            // this step survived — not even from siblings that completed
1111            // before the interrupt reached the barrier. The suspension
1112            // checkpoint therefore re-schedules the ENTIRE active set (the
1113            // interrupting node plus all siblings), otherwise completed
1114            // siblings' discarded writes would be silently lost and aborted
1115            // siblings would never re-run. Dropping the JoinSet first aborts
1116            // stragglers, preserving the transactional suspension point.
1117            drop(join_set);
1118            let interrupt_event = recorder.record(
1119                EventDraft::new(RunEventKind::Interrupt, invocation_effects[index])
1120                    .node(name.clone())
1121                    .input(value.clone())
1122                    .status(EventStatus::Interrupted)
1123                    .parent(input_events[index].clone()),
1124            );
1125            tracing::info!(
1126                node = %name,
1127                step = step,
1128                "node interrupted; run suspended (resumable via RunConfig::resume)"
1129            );
1130            let pending: Vec<String> = active.iter().map(|t| t.name.clone()).collect();
1131            let checkpoint =
1132                recorder.mint_checkpoint(config.thread_id.clone(), step, state.clone(), pending);
1133            let checkpoint_id = checkpoint.id.clone();
1134            if let Some(checkpointer) = &self.checkpointer {
1135                checkpointer.put(checkpoint).await?;
1136                recorder.record(
1137                    EventDraft::new(RunEventKind::CheckpointWritten, Effect::Idempotent)
1138                        .output(serde_json::json!({
1139                            "checkpoint_id": checkpoint_id,
1140                            "step": step,
1141                            "suspension": true,
1142                        }))
1143                        .parent(interrupt_event),
1144                );
1145                Self::emit(
1146                    config,
1147                    GraphEvent::CheckpointSaved {
1148                        checkpoint_id: checkpoint_id.clone(),
1149                        step,
1150                    },
1151                );
1152            }
1153            return Ok(StepTransition::Finish(ExecutionOutcome::Interrupted {
1154                value,
1155                state: state.clone(),
1156                checkpoint_id,
1157            }));
1158        }
1159
1160        // Journal node outputs in active-set order, not JoinSet finish
1161        // order: the finish order is scheduling-dependent, and the journal's
1162        // sequence is evidence — replay compares it.
1163        completed.sort_by_key(|(index, ..)| *index);
1164        let mut last_output_event: Option<String> = None;
1165        for (index, name, latency_ms, output_json) in completed {
1166            last_output_event = Some(
1167                recorder.record(
1168                    EventDraft::new(RunEventKind::NodeOutput, invocation_effects[index])
1169                        .node(name)
1170                        .output(output_json)
1171                        .latency_ms(latency_ms)
1172                        .parent(input_events[index].clone()),
1173                ),
1174            );
1175        }
1176
1177        // -- merge: reducers + LastValue single-write validation. On
1178        //    error the mutated state is dropped with the run
1179        //    (transactional super-step).
1180        //
1181        // The start-of-step snapshot's job ends at the barrier — every node
1182        // has reported (or the step has failed). Dropping it before the
1183        // merge releases its per-channel shared references, so channels no
1184        // checkpoint still shares reach the reducer with refcount 1 and
1185        // merge in place (W4 copy-on-write) instead of cloning.
1186        drop(snapshot);
1187        let written_channels: HashSet<String> = writes
1188            .iter()
1189            .flat_map(|(_, updates)| updates.keys().cloned())
1190            .collect();
1191        spec.apply_super_step(state, writes)?;
1192        // The event carries the post-reducer values read back out of the
1193        // merged state: when several nodes write the same channel in one
1194        // step (the normal Append fan-in case), reporting the raw partials
1195        // would keep only the last write and hide the rest.
1196        let mut merged_updates = serde_json::Map::new();
1197        for channel in &written_channels {
1198            if let Some(value) = state.get(channel) {
1199                merged_updates.insert(channel.clone(), value.clone());
1200            }
1201        }
1202        let channels_written: Vec<&str> = merged_updates.keys().map(String::as_str).collect();
1203        tracing::debug!(
1204            step = step,
1205            channels = ?channels_written,
1206            "merged node updates at super-step barrier"
1207        );
1208        // The super-step's reducer result is journaled even when empty —
1209        // "no channel changed" is evidence too.
1210        let step_end_event = recorder.record(
1211            EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1212                .output(Value::Object(merged_updates.clone()))
1213                .parent(last_output_event.unwrap_or_else(|| step_start_event.clone())),
1214        );
1215        if !merged_updates.is_empty() {
1216            Self::emit(
1217                config,
1218                GraphEvent::StateUpdate {
1219                    step,
1220                    updates: merged_updates,
1221                },
1222            );
1223        }
1224
1225        // -- route: Command::goto overrides the static edge set;
1226        //    otherwise evaluate outgoing edges of every node that ran
1227        //    against the post-barrier state.
1228        let mut next: Vec<ActiveTask> = Vec::new();
1229        let mut planned: HashSet<String> = HashSet::new();
1230
1231        if !commands.is_empty() {
1232            for command in &commands {
1233                for target in &command.goto {
1234                    if !graph.has_node(target) {
1235                        return Err(RustyError::Graph(format!(
1236                            "Command::goto references unknown node `{target}`"
1237                        )));
1238                    }
1239                    if planned.insert(target.clone()) {
1240                        next.push(ActiveTask {
1241                            name: target.clone(),
1242                            scoped: None,
1243                        });
1244                    }
1245                }
1246            }
1247        } else {
1248            let mut evaluated: HashSet<String> = HashSet::new();
1249            for name in &ran_nodes {
1250                // Fan-out invocations of the same node share one edge set;
1251                // evaluate it once.
1252                if !evaluated.insert(name.clone()) {
1253                    continue;
1254                }
1255                for edge in graph.outgoing_edges(name) {
1256                    match edge {
1257                        Edge::Direct { to, .. } => {
1258                            if planned.insert(to.clone()) {
1259                                next.push(ActiveTask {
1260                                    name: to.clone(),
1261                                    scoped: None,
1262                                });
1263                            }
1264                        }
1265                        Edge::Conditional { router, .. } => {
1266                            match router(state.clone()).await? {
1267                                Route::Node(target) => {
1268                                    if !graph.has_node(&target) {
1269                                        return Err(RustyError::Graph(format!(
1270                                            "conditional router from `{name}` returned \
1271                                             unknown node `{target}`"
1272                                        )));
1273                                    }
1274                                    if planned.insert(target.clone()) {
1275                                        next.push(ActiveTask {
1276                                            name: target,
1277                                            scoped: None,
1278                                        });
1279                                    }
1280                                }
1281                                Route::Send(sends) => {
1282                                    for send in sends {
1283                                        if !graph.has_node(&send.node) {
1284                                            return Err(RustyError::Graph(format!(
1285                                                "Route::Send from `{name}` targets unknown \
1286                                                 node `{}`",
1287                                                send.node
1288                                            )));
1289                                        }
1290                                        // Each Send is its own invocation with its own
1291                                        // scoped state, even when several target the
1292                                        // same node.
1293                                        next.push(ActiveTask {
1294                                            name: send.node,
1295                                            scoped: Some(send.state),
1296                                        });
1297                                    }
1298                                }
1299                                Route::End => {}
1300                            }
1301                        }
1302                    }
1303                }
1304            }
1305        }
1306
1307        let route_event = recorder.record(
1308            EventDraft::new(RunEventKind::RoutingDecision, Effect::Pure)
1309                .output(serde_json::json!({
1310                    "next": next
1311                        .iter()
1312                        .map(|t| serde_json::json!({
1313                            "node": &t.name,
1314                            "scoped": &t.scoped,
1315                        }))
1316                        .collect::<Vec<_>>(),
1317                    "goto": commands
1318                        .iter()
1319                        .flat_map(|command| command.goto.iter().cloned())
1320                        .collect::<Vec<String>>(),
1321                }))
1322                .parent(step_end_event.clone()),
1323        );
1324
1325        // -- checkpoint at the super-step boundary.
1326        if let Some(checkpointer) = &self.checkpointer {
1327            let next_names: Vec<String> = next.iter().map(|t| t.name.clone()).collect();
1328            let checkpoint =
1329                recorder.mint_checkpoint(config.thread_id.clone(), step, state.clone(), next_names);
1330            let checkpoint_id = checkpoint.id.clone();
1331            checkpointer.put(checkpoint).await?;
1332            recorder.record(
1333                EventDraft::new(RunEventKind::CheckpointWritten, Effect::Idempotent)
1334                    .output(serde_json::json!({
1335                        "checkpoint_id": checkpoint_id,
1336                        "step": step,
1337                        "suspension": false,
1338                    }))
1339                    .parent(route_event.clone()),
1340            );
1341            Self::emit(
1342                config,
1343                GraphEvent::CheckpointSaved {
1344                    checkpoint_id,
1345                    step,
1346                },
1347            );
1348        }
1349
1350        // -- terminate or schedule the next super-step.
1351        if next.is_empty() {
1352            return Ok(StepTransition::Finish(ExecutionOutcome::Done(
1353                state.clone(),
1354            )));
1355        }
1356        Ok(StepTransition::Next(next, route_event))
1357    }
1358
1359    /// Best-effort event emission: a full or closed channel never aborts a run.
1360    fn emit(config: &RunConfig, event: GraphEvent) {
1361        if let Some(tx) = &config.event_tx {
1362            let _ = tx.try_send(event);
1363        }
1364    }
1365}
1366
1367/// Per-run Flight Recorder state handed to every super-step: the journal,
1368/// the determinism seams, and the frozen provenance stamped into every
1369/// checkpoint of the run.
1370struct Recorder {
1371    journal: Journal,
1372    clock: Clock,
1373    rng: RngSource,
1374    graph_version: String,
1375    graph_hash: String,
1376    policy_version: PolicyVersion,
1377    manifest: Option<RunManifest>,
1378}
1379
1380impl Recorder {
1381    /// Append one event to the run's journal; returns the event id (the
1382    /// causal-parent handle for whatever the event causes).
1383    fn record(&self, draft: EventDraft) -> String {
1384        self.journal.record(draft)
1385    }
1386
1387    /// Mint a boundary checkpoint through the determinism seams: id from the
1388    /// run's RNG, timestamp from the run's clock, the frozen provenance
1389    /// header, and a journal reference pinning the evidence head as it stood
1390    /// *before* this checkpoint's own `checkpoint_written` event is
1391    /// recorded.
1392    fn mint_checkpoint(
1393        &self,
1394        thread_id: String,
1395        step: usize,
1396        state: State,
1397        next_nodes: Vec<String>,
1398    ) -> Checkpoint {
1399        Checkpoint {
1400            id: self.rng.uuid_string(),
1401            thread_id,
1402            step,
1403            state,
1404            next_nodes,
1405            created_at: self.clock.now(),
1406            header: CheckpointHeader {
1407                format_version: CURRENT_FORMAT_VERSION,
1408                graph_version: self.graph_version.clone(),
1409                graph_hash: self.graph_hash.clone(),
1410                policy_version: self.policy_version.clone(),
1411                logical_clock: self.clock.now_ms(),
1412                manifest: self.manifest.clone(),
1413            },
1414            journal_ref: Some(self.journal.head_ref()),
1415            // The executor always mints full snapshots; delta encoding is
1416            // decided inside the checkpointer (`checkpoint::encode_delta`).
1417            base: None,
1418        }
1419    }
1420}
1421
1422/// One scheduled node invocation within a super-step. `scoped` carries the
1423/// per-invocation input of a [`crate::graph::Send`] fan-out, overlaid onto
1424/// that invocation's private state snapshot before the node runs.
1425struct ActiveTask {
1426    name: String,
1427    scoped: Option<Value>,
1428}
1429
1430/// The control-flow result of a single super-step: either the next active
1431/// set (the loop continues) or the terminal run outcome (the loop breaks).
1432enum StepTransition {
1433    /// The next active set plus the routing-decision journal event that
1434    /// produced it (the causal parent of the next super-step's start event).
1435    Next(Vec<ActiveTask>, String),
1436    Finish(ExecutionOutcome),
1437}
1438
1439#[cfg(test)]
1440mod tests {
1441    use super::*;
1442    use crate::checkpoint::InMemoryCheckpointer;
1443    use crate::graph::GraphBuilder;
1444    use crate::llm::ChatModel;
1445    use crate::state::Reducer;
1446    use serde_json::json;
1447    use std::sync::Mutex;
1448
1449    #[tokio::test]
1450    async fn linear_two_node_graph_executes_in_order() {
1451        let spec = StateSpec::new().channel("log", Reducer::Append);
1452
1453        let mut builder = GraphBuilder::new();
1454        builder.add_node("first", |_ctx: NodeContext| async {
1455            Ok(NodeOutput::update("log", json!("first")))
1456        });
1457        builder.add_node("second", |ctx: NodeContext| async move {
1458            // The next super-step observes the previous step's merged writes.
1459            assert_eq!(ctx.state().get("log"), Some(&json!(["first"])));
1460            assert_eq!(ctx.step(), 1);
1461            Ok(NodeOutput::update("log", json!("second")))
1462        });
1463        builder.set_entry_point("first");
1464        builder.add_edge("first", "second");
1465        let graph = builder.compile().unwrap();
1466
1467        let outcome = Executor::new()
1468            .run(&graph, &spec, State::new(), RunConfig::new("t-linear"))
1469            .await
1470            .unwrap();
1471
1472        match outcome {
1473            ExecutionOutcome::Done(state) => {
1474                assert_eq!(state.get("log"), Some(&json!(["first", "second"])));
1475            }
1476            other => panic!("expected Done, got {other:?}"),
1477        }
1478    }
1479
1480    #[tokio::test]
1481    async fn parallel_fan_in_merges_via_reducer() {
1482        let spec = StateSpec::new().channel("results", Reducer::Append);
1483
1484        let mut builder = GraphBuilder::new();
1485        builder.add_node("start", |_ctx: NodeContext| async {
1486            Ok(NodeOutput::empty())
1487        });
1488        builder.add_node("worker_a", |ctx: NodeContext| async move {
1489            // Snapshot isolation: parallel workers cannot see each other.
1490            assert!(!ctx.state().contains("results"));
1491            Ok(NodeOutput::update("results", json!("a")))
1492        });
1493        builder.add_node("worker_b", |ctx: NodeContext| async move {
1494            assert!(!ctx.state().contains("results"));
1495            Ok(NodeOutput::update("results", json!("b")))
1496        });
1497        builder.set_entry_point("start");
1498        builder.add_edge("start", "worker_a");
1499        builder.add_edge("start", "worker_b");
1500        let graph = builder.compile().unwrap();
1501
1502        let outcome = Executor::new()
1503            .run(&graph, &spec, State::new(), RunConfig::new("t-fan-in"))
1504            .await
1505            .unwrap();
1506
1507        match outcome {
1508            ExecutionOutcome::Done(state) => {
1509                let results = state
1510                    .get("results")
1511                    .and_then(Value::as_array)
1512                    .expect("results channel must exist")
1513                    .clone();
1514                // Completion order across the JoinSet is nondeterministic.
1515                let mut items: Vec<&str> = results.iter().map(|v| v.as_str().unwrap()).collect();
1516                items.sort_unstable();
1517                assert_eq!(items, ["a", "b"]);
1518            }
1519            other => panic!("expected Done, got {other:?}"),
1520        }
1521    }
1522
1523    #[tokio::test]
1524    async fn interrupt_returns_interrupted_outcome_and_resume_completes() {
1525        let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
1526
1527        let mut builder = GraphBuilder::new();
1528        builder.add_node("gate", |ctx: NodeContext| async move {
1529            match ctx.resume_value() {
1530                Some(v) => Ok(NodeOutput::update("answer", v.clone())),
1531                None => Err(ctx.interrupt(json!({"question": "approve?"}))),
1532            }
1533        });
1534        builder.set_entry_point("gate");
1535        let graph = builder.compile().unwrap();
1536
1537        let checkpointer = Arc::new(InMemoryCheckpointer::new());
1538        let executor = Executor::with_checkpointer(checkpointer.clone());
1539
1540        // First run: the gate node interrupts and the run suspends.
1541        let outcome = executor
1542            .run(&graph, &spec, State::new(), RunConfig::new("t-hitl"))
1543            .await
1544            .unwrap();
1545
1546        let checkpoint_id = match outcome {
1547            ExecutionOutcome::Interrupted {
1548                value,
1549                checkpoint_id,
1550                ..
1551            } => {
1552                assert_eq!(value, json!({"question": "approve?"}));
1553                assert!(!checkpoint_id.is_empty());
1554                checkpoint_id
1555            }
1556            other => panic!("expected Interrupted, got {other:?}"),
1557        };
1558
1559        // The suspension point was persisted and schedules the gate node.
1560        let stored = checkpointer.get_latest("t-hitl").await.unwrap().unwrap();
1561        assert_eq!(stored.id, checkpoint_id);
1562        assert_eq!(stored.next_nodes, vec!["gate".to_string()]);
1563
1564        // Resume: the gate node re-runs with the resume value, writes its
1565        // answer, and the run terminates.
1566        let outcome = executor
1567            .run(
1568                &graph,
1569                &spec,
1570                State::new(),
1571                RunConfig::new("t-hitl").with_resume(json!(true)),
1572            )
1573            .await
1574            .unwrap();
1575
1576        match outcome {
1577            ExecutionOutcome::Done(state) => {
1578                assert_eq!(state.get("answer"), Some(&json!(true)));
1579            }
1580            other => panic!("expected Done after resume, got {other:?}"),
1581        }
1582    }
1583
1584    #[test]
1585    fn run_config_default_uses_default_step_limit() {
1586        // Regression: a derived `Default` would zero `max_steps`, making
1587        // every `RunConfig::default()` run fail immediately.
1588        let config = RunConfig::default();
1589        assert_eq!(config.max_steps, DEFAULT_MAX_STEPS);
1590        assert!(config.thread_id.is_empty());
1591        assert!(config.resume.is_none() && config.checkpoint_id.is_none());
1592    }
1593
1594    #[tokio::test]
1595    async fn interrupt_reschedules_entire_active_set() {
1596        // Regression: the suspension checkpoint used to schedule only the
1597        // interrupting node, silently dropping parallel siblings — including
1598        // ones that had already completed, whose writes are discarded with
1599        // the aborted step.
1600        let spec = StateSpec::new()
1601            .channel("log", Reducer::Append)
1602            .channel("answer", Reducer::Overwrite);
1603
1604        let mut builder = GraphBuilder::new();
1605        builder.add_node("start", |_ctx: NodeContext| async {
1606            Ok(NodeOutput::empty())
1607        });
1608        builder.add_node("gate", |ctx: NodeContext| async move {
1609            match ctx.resume_value() {
1610                Some(v) => Ok(NodeOutput::update("answer", v.clone())),
1611                None => Err(ctx.interrupt(json!({"question": "approve?"}))),
1612            }
1613        });
1614        // Completes immediately in the interrupted step; its write is
1615        // discarded with the step and must be recomputed on resume.
1616        builder.add_node("fast", |_ctx: NodeContext| async {
1617            Ok(NodeOutput::update("log", json!("fast")))
1618        });
1619        // Still in flight when the interrupt hits; aborted, re-run on resume.
1620        builder.add_node("slow", |ctx: NodeContext| async move {
1621            if ctx.resume_value().is_none() {
1622                tokio::time::sleep(std::time::Duration::from_secs(3600)).await;
1623            }
1624            Ok(NodeOutput::update("log", json!("slow")))
1625        });
1626        builder.set_entry_point("start");
1627        builder.add_edge("start", "gate");
1628        builder.add_edge("start", "slow");
1629        builder.add_edge("start", "fast");
1630        let graph = builder.compile().unwrap();
1631
1632        let checkpointer = Arc::new(InMemoryCheckpointer::new());
1633        let executor = Executor::with_checkpointer(checkpointer.clone());
1634
1635        let outcome = executor
1636            .run(&graph, &spec, State::new(), RunConfig::new("t-par-hitl"))
1637            .await
1638            .unwrap();
1639        match &outcome {
1640            ExecutionOutcome::Interrupted { state, .. } => {
1641                // Transactional suspension: fast's completed write was
1642                // discarded with the rest of the step.
1643                assert_eq!(state.get("log"), None);
1644            }
1645            other => panic!("expected Interrupted, got {other:?}"),
1646        }
1647
1648        // The suspension checkpoint re-schedules every node of the step.
1649        let stored = checkpointer
1650            .get_latest("t-par-hitl")
1651            .await
1652            .unwrap()
1653            .unwrap();
1654        let mut scheduled = stored.next_nodes.clone();
1655        scheduled.sort_unstable();
1656        assert_eq!(scheduled, ["fast", "gate", "slow"]);
1657
1658        // Resume: all three re-run (the resume value is broadcast to the
1659        // whole step); fast's write lands exactly once.
1660        let outcome = executor
1661            .run(
1662                &graph,
1663                &spec,
1664                State::new(),
1665                RunConfig::new("t-par-hitl").with_resume(json!(true)),
1666            )
1667            .await
1668            .unwrap();
1669        match outcome {
1670            ExecutionOutcome::Done(state) => {
1671                assert_eq!(state.get("answer"), Some(&json!(true)));
1672                let mut log: Vec<String> = state
1673                    .get("log")
1674                    .and_then(Value::as_array)
1675                    .expect("log channel must exist")
1676                    .iter()
1677                    .map(|v| v.as_str().unwrap().to_owned())
1678                    .collect();
1679                log.sort_unstable();
1680                assert_eq!(log, ["fast", "slow"]);
1681            }
1682            other => panic!("expected Done after resume, got {other:?}"),
1683        }
1684    }
1685
1686    #[tokio::test]
1687    async fn state_update_event_reports_post_reducer_values() {
1688        // Regression: with several writers on one channel, the event used to
1689        // carry raw per-node partials collapsed by last-write-wins, hiding
1690        // all but one write behind its documented "merged" contract.
1691        let spec = StateSpec::new().channel("results", Reducer::Append);
1692
1693        let mut builder = GraphBuilder::new();
1694        builder.add_node("start", |_ctx: NodeContext| async {
1695            Ok(NodeOutput::empty())
1696        });
1697        builder.add_node("worker_a", |_ctx: NodeContext| async {
1698            Ok(NodeOutput::update("results", json!("a")))
1699        });
1700        builder.add_node("worker_b", |_ctx: NodeContext| async {
1701            Ok(NodeOutput::update("results", json!("b")))
1702        });
1703        builder.set_entry_point("start");
1704        builder.add_edge("start", "worker_a");
1705        builder.add_edge("start", "worker_b");
1706        let graph = builder.compile().unwrap();
1707
1708        let (tx, mut rx) = mpsc::channel::<GraphEvent>(64);
1709        let outcome = Executor::new()
1710            .run(
1711                &graph,
1712                &spec,
1713                State::new(),
1714                RunConfig::new("t-event").with_event_tx(tx),
1715            )
1716            .await
1717            .unwrap();
1718        assert!(matches!(outcome, ExecutionOutcome::Done(_)));
1719
1720        let mut merged: Option<Vec<String>> = None;
1721        while let Ok(event) = rx.try_recv() {
1722            if let GraphEvent::StateUpdate { step: 1, updates } = event {
1723                let values = updates
1724                    .get("results")
1725                    .and_then(Value::as_array)
1726                    .expect("StateUpdate must carry the results channel")
1727                    .iter()
1728                    .map(|v| v.as_str().unwrap().to_owned())
1729                    .collect();
1730                merged = Some(values);
1731            }
1732        }
1733        let mut merged = merged.expect("expected a StateUpdate event for step 1");
1734        merged.sort_unstable();
1735        // Both partial writes are visible in the single post-reducer value.
1736        assert_eq!(merged, ["a", "b"]);
1737    }
1738
1739    #[tokio::test]
1740    async fn max_steps_guard_aborts_infinite_cycles() {
1741        let spec = StateSpec::new().channel("x", Reducer::Overwrite);
1742        let mut builder = GraphBuilder::new();
1743        builder.add_node("loop_node", |_ctx: NodeContext| async {
1744            Ok(NodeOutput::empty())
1745        });
1746        builder.set_entry_point("loop_node");
1747        builder.add_edge("loop_node", "loop_node");
1748        let graph = builder.compile().unwrap();
1749
1750        let err = Executor::new()
1751            .run(
1752                &graph,
1753                &spec,
1754                State::new(),
1755                RunConfig::new("t-loop").with_max_steps(5),
1756            )
1757            .await
1758            .unwrap_err();
1759        assert!(matches!(err, RustyError::Graph(_)));
1760    }
1761
1762    #[test]
1763    fn token_event_serde_roundtrip() {
1764        let event = GraphEvent::Token {
1765            node: "agent".into(),
1766            delta: "Hello".into(),
1767        };
1768        let wire = serde_json::to_string(&event).unwrap();
1769        // Internally tagged: the variant name travels on the wire.
1770        assert!(
1771            wire.contains("\"type\":\"token\""),
1772            "unexpected wire: {wire}"
1773        );
1774        let back: GraphEvent = serde_json::from_str(&wire).unwrap();
1775        assert_eq!(event, back);
1776
1777        // The other variants still roundtrip under the new serde derives.
1778        let step_event = GraphEvent::SuperStep {
1779            step: 3,
1780            active_nodes: vec!["a".into()],
1781        };
1782        let back: GraphEvent =
1783            serde_json::from_str(&serde_json::to_string(&step_event).unwrap()).unwrap();
1784        assert_eq!(step_event, back);
1785    }
1786
1787    /// A mock model whose `chat_stream` override emits real multi-chunk
1788    /// deltas, proving the accumulation contract outside any HTTP client.
1789    struct StreamingMock;
1790
1791    #[async_trait::async_trait]
1792    impl crate::llm::ChatModel for StreamingMock {
1793        async fn chat(
1794            &self,
1795            _messages: &[crate::llm::ChatMessage],
1796            _tools: &[Value],
1797        ) -> Result<crate::llm::ChatResponse> {
1798            Ok(crate::llm::ChatResponse {
1799                message: crate::llm::ChatMessage::assistant("Hello"),
1800                model: None,
1801                usage: None,
1802            })
1803        }
1804
1805        async fn chat_stream(
1806            &self,
1807            messages: &[crate::llm::ChatMessage],
1808            tools: &[Value],
1809            on_token: &mut (dyn FnMut(crate::llm::TokenChunk) + Send),
1810        ) -> Result<crate::llm::ChatResponse> {
1811            for piece in ["Hel", "lo"] {
1812                on_token(crate::llm::TokenChunk {
1813                    delta: piece.into(),
1814                    finish: false,
1815                    raw: None,
1816                });
1817            }
1818            on_token(crate::llm::TokenChunk {
1819                delta: String::new(),
1820                finish: true,
1821                raw: None,
1822            });
1823            self.chat(messages, tools).await
1824        }
1825    }
1826
1827    #[tokio::test]
1828    async fn node_streams_token_events_through_event_sink() {
1829        let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
1830
1831        let (tx, mut rx) = mpsc::channel::<GraphEvent>(64);
1832        // The wiring pattern from the ChatModel rustdoc: the node closure
1833        // captures a clone of the run's event sender and forwards chunks.
1834        let node_tx = tx.clone();
1835
1836        let mut builder = GraphBuilder::new();
1837        builder.add_node("agent", move |_ctx: NodeContext| {
1838            let node_tx = node_tx.clone();
1839            async move {
1840                let model = StreamingMock;
1841                let mut full = String::new();
1842                model
1843                    .chat_stream(&[], &[], &mut |chunk| {
1844                        if !chunk.delta.is_empty() {
1845                            full.push_str(&chunk.delta);
1846                            let _ = node_tx.try_send(GraphEvent::Token {
1847                                node: "agent".into(),
1848                                delta: chunk.delta,
1849                            });
1850                        }
1851                    })
1852                    .await
1853                    .unwrap();
1854                Ok(NodeOutput::update("answer", json!(full)))
1855            }
1856        });
1857        builder.set_entry_point("agent");
1858        let graph = builder.compile().unwrap();
1859
1860        let config = RunConfig::new("t-tokens").with_event_tx(tx);
1861        // The RunConfig helper hands out the same sender for node wiring.
1862        assert!(config.token_tx().is_some());
1863        // The Executor builder/accessor pair stores a broadcast handle.
1864        let executor = Executor::new().with_token_tx(config.token_tx().unwrap());
1865        assert!(executor.token_tx().is_some());
1866
1867        let outcome = executor
1868            .run(&graph, &spec, State::new(), config)
1869            .await
1870            .unwrap();
1871        match outcome {
1872            ExecutionOutcome::Done(state) => {
1873                assert_eq!(state.get("answer"), Some(&json!("Hello")))
1874            }
1875            other => panic!("expected Done, got {other:?}"),
1876        }
1877
1878        // Drain the event stream: token deltas interleave with executor events.
1879        let mut deltas = Vec::new();
1880        while let Ok(event) = rx.try_recv() {
1881            if let GraphEvent::Token { node, delta } = event {
1882                assert_eq!(node, "agent");
1883                deltas.push(delta);
1884            }
1885        }
1886        assert_eq!(deltas, ["Hel", "lo"]);
1887    }
1888
1889    /// A minimal `tracing::Subscriber` that records formatted event fields
1890    /// (`name=value` pairs) into a shared buffer, so tests can assert on the
1891    /// executor's instrumentation. Implemented directly against the
1892    /// `tracing` crate's own `Subscriber` trait (re-exported from
1893    /// `tracing-core`) — no `tracing-subscriber` dependency required.
1894    #[derive(Clone, Default)]
1895    struct EventCapture {
1896        events: Arc<Mutex<Vec<String>>>,
1897    }
1898
1899    /// Formats an event's fields as `"name=value "` pairs.
1900    struct FieldVisitor(String);
1901
1902    impl tracing::field::Visit for FieldVisitor {
1903        fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
1904            use std::fmt::Write as _;
1905            let _ = write!(self.0, "{}={:?} ", field.name(), value);
1906        }
1907    }
1908
1909    impl tracing::Subscriber for EventCapture {
1910        fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
1911            true
1912        }
1913
1914        fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
1915            // Spans are irrelevant to these assertions; one id serves all.
1916            tracing::span::Id::from_u64(1)
1917        }
1918
1919        fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
1920
1921        fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
1922
1923        fn event(&self, event: &tracing::Event<'_>) {
1924            let mut visitor = FieldVisitor(String::new());
1925            event.record(&mut visitor);
1926            self.events.lock().unwrap().push(visitor.0);
1927        }
1928
1929        fn enter(&self, _span: &tracing::span::Id) {}
1930
1931        fn exit(&self, _span: &tracing::span::Id) {}
1932    }
1933
1934    /// The instrumentation must be observability-only: installing a
1935    /// subscriber changes nothing about the run's outcome, and the expected
1936    /// telemetry (merge debug event, run-completion info event) is emitted.
1937    #[tokio::test]
1938    async fn instrumentation_emits_events_without_changing_outcome() {
1939        let capture = EventCapture::default();
1940        let events = capture.events.clone();
1941        // Global default subscriber, deliberately NOT a thread-local
1942        // `set_default`: callsite interest is cached process-wide and lazily
1943        // (re)registered from whichever thread first hits a callsite, so a
1944        // thread-local subscriber races with the other tests in this binary
1945        // that run graphs concurrently (they rebuild the cache against the
1946        // no-subscriber global default and our events get silently dropped).
1947        // A global default makes every thread's rebuild see this subscriber.
1948        // Setting it is additive — other tests neither set nor assert on
1949        // subscribers, and captured events from concurrent runs only help the
1950        // `any()` assertions below. `set_global_default` may only be called
1951        // once per process; this is the only test that installs a subscriber.
1952        tracing::subscriber::set_global_default(capture)
1953            .expect("no other test may install a global tracing subscriber");
1954
1955        let spec = StateSpec::new().channel("log", Reducer::Append);
1956        let mut builder = GraphBuilder::new();
1957        builder.add_node("only", |_ctx: NodeContext| async {
1958            Ok(NodeOutput::update("log", json!("x")))
1959        });
1960        builder.set_entry_point("only");
1961        let graph = builder.compile().unwrap();
1962
1963        let outcome = Executor::new()
1964            .run(&graph, &spec, State::new(), RunConfig::new("t-tracing"))
1965            .await
1966            .unwrap();
1967
1968        // Identical semantics: the run completes with the expected state.
1969        match outcome {
1970            ExecutionOutcome::Done(state) => {
1971                assert_eq!(state.get("log"), Some(&json!(["x"])));
1972            }
1973            other => panic!("expected Done, got {other:?}"),
1974        }
1975
1976        let captured = events.lock().unwrap();
1977        assert!(
1978            captured.iter().any(|e| e.contains("channels")),
1979            "expected a merge debug event listing written channels, got: {captured:?}"
1980        );
1981        assert!(
1982            captured
1983                .iter()
1984                .any(|e| e.contains("steps") && e.contains("duration_ms")),
1985            "expected a run-completion info event with steps and duration_ms, got: {captured:?}"
1986        );
1987    }
1988
1989    /// A 3-node linear graph (`a -> b -> c`) appending each node name to the
1990    /// `log` channel.
1991    fn linear_three_node_graph() -> (Graph, StateSpec) {
1992        let spec = StateSpec::new().channel("log", Reducer::Append);
1993        let mut builder = GraphBuilder::new();
1994        for name in ["a", "b", "c"] {
1995            builder.add_node(name, move |_ctx: NodeContext| async move {
1996                Ok(NodeOutput::update("log", json!(name)))
1997            });
1998        }
1999        builder.set_entry_point("a");
2000        builder.add_edge("a", "b");
2001        builder.add_edge("b", "c");
2002        (builder.compile().unwrap(), spec)
2003    }
2004
2005    #[tokio::test]
2006    async fn run_with_checkpoint_id_replays_from_earlier_state() {
2007        let (graph, spec) = linear_three_node_graph();
2008        let checkpointer = Arc::new(InMemoryCheckpointer::new());
2009        let executor = Executor::with_checkpointer(checkpointer.clone());
2010
2011        // Full run on the source thread: checkpoints at steps 0, 1, 2.
2012        let outcome = executor
2013            .run(&graph, &spec, State::new(), RunConfig::new("t-src"))
2014            .await
2015            .unwrap();
2016        match outcome {
2017            ExecutionOutcome::Done(state) => {
2018                assert_eq!(state.get("log"), Some(&json!(["a", "b", "c"])));
2019            }
2020            other => panic!("expected Done, got {other:?}"),
2021        }
2022
2023        let history = checkpointer.list("t-src").await.unwrap();
2024        assert_eq!(history.len(), 3);
2025        // The step-1 checkpoint: `a` and `b` have run, `c` is scheduled next.
2026        let step1 = history[1].clone();
2027        assert_eq!(step1.step, 1);
2028        assert_eq!(step1.state.get("log"), Some(&json!(["a", "b"])));
2029        assert_eq!(step1.next_nodes, vec!["c".to_string()]);
2030
2031        // Safe pattern: fork the thread at the step-1 checkpoint, then replay
2032        // the fork from that checkpoint (not the fork's latest — here the cut
2033        // point IS the latest, but `checkpoint_id` is what selects it).
2034        let copied = checkpointer
2035            .fork_thread("t-src", "t-fork", Some(&step1.id))
2036            .await
2037            .unwrap();
2038        assert_eq!(copied, 2);
2039
2040        let outcome = executor
2041            .run(
2042                &graph,
2043                &spec,
2044                State::new(),
2045                RunConfig::new("t-fork").with_checkpoint_id(step1.id.clone()),
2046            )
2047            .await
2048            .unwrap();
2049
2050        match outcome {
2051            ExecutionOutcome::Done(state) => {
2052                // Execution continued from the step-1 state: only `c` ran,
2053                // `b` was not re-executed.
2054                assert_eq!(state.get("log"), Some(&json!(["a", "b", "c"])));
2055            }
2056            other => panic!("expected Done, got {other:?}"),
2057        }
2058
2059        // The replay appended its own boundary checkpoint to the fork only.
2060        assert_eq!(checkpointer.list("t-src").await.unwrap().len(), 3);
2061        assert_eq!(checkpointer.list("t-fork").await.unwrap().len(), 3);
2062    }
2063
2064    #[tokio::test]
2065    async fn run_with_checkpoint_id_plus_resume_combined() {
2066        let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
2067
2068        let mut builder = GraphBuilder::new();
2069        builder.add_node("gate", |ctx: NodeContext| async move {
2070            match ctx.resume_value() {
2071                Some(v) => Ok(NodeOutput::update("answer", v.clone())),
2072                None => Err(ctx.interrupt(json!({"question": "approve?"}))),
2073            }
2074        });
2075        builder.set_entry_point("gate");
2076        let graph = builder.compile().unwrap();
2077
2078        let checkpointer = Arc::new(InMemoryCheckpointer::new());
2079        let executor = Executor::with_checkpointer(checkpointer);
2080
2081        // Suspend at the gate and capture the suspension checkpoint id.
2082        let outcome = executor
2083            .run(&graph, &spec, State::new(), RunConfig::new("t-hitl"))
2084            .await
2085            .unwrap();
2086        let checkpoint_id = match outcome {
2087            ExecutionOutcome::Interrupted { checkpoint_id, .. } => checkpoint_id,
2088            other => panic!("expected Interrupted, got {other:?}"),
2089        };
2090
2091        // checkpoint_id selects WHERE (the suspension checkpoint), resume
2092        // supplies the value delivered to the re-running gate node.
2093        let outcome = executor
2094            .run(
2095                &graph,
2096                &spec,
2097                State::new(),
2098                RunConfig::new("t-hitl")
2099                    .with_checkpoint_id(checkpoint_id)
2100                    .with_resume(json!(true)),
2101            )
2102            .await
2103            .unwrap();
2104
2105        match outcome {
2106            ExecutionOutcome::Done(state) => {
2107                assert_eq!(state.get("answer"), Some(&json!(true)));
2108            }
2109            other => panic!("expected Done after replay+resume, got {other:?}"),
2110        }
2111    }
2112
2113    #[tokio::test]
2114    async fn run_with_checkpoint_id_errors_without_checkpointer_or_unknown_id() {
2115        let (graph, spec) = linear_three_node_graph();
2116
2117        // No checkpointer configured: replay is impossible.
2118        let err = Executor::new()
2119            .run(
2120                &graph,
2121                &spec,
2122                State::new(),
2123                RunConfig::new("t-x").with_checkpoint_id("some-id"),
2124            )
2125            .await
2126            .unwrap_err();
2127        assert!(matches!(err, RustyError::Checkpoint(_)));
2128
2129        // Checkpointer present but the id does not exist on the thread.
2130        let checkpointer = Arc::new(InMemoryCheckpointer::new());
2131        let executor = Executor::with_checkpointer(checkpointer.clone());
2132        executor
2133            .run(&graph, &spec, State::new(), RunConfig::new("t-x"))
2134            .await
2135            .unwrap();
2136        let err = executor
2137            .run(
2138                &graph,
2139                &spec,
2140                State::new(),
2141                RunConfig::new("t-x").with_checkpoint_id("no-such-checkpoint"),
2142            )
2143            .await
2144            .unwrap_err();
2145        assert!(matches!(err, RustyError::Checkpoint(_)));
2146    }
2147
2148    /// A self-looping spinner: each super-step increments `n`; the router
2149    /// terminates the run once `n` reaches 5.
2150    fn spinner_graph() -> (Graph, StateSpec) {
2151        let spec = StateSpec::new().channel("n", Reducer::Overwrite);
2152        let mut builder = GraphBuilder::new();
2153        builder.add_node("spin", |ctx: NodeContext| async move {
2154            // A paced step: without it the loop would burn through the
2155            // default max_steps budget faster than a test can cancel it.
2156            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2157            let n = ctx.state().get("n").and_then(Value::as_i64).unwrap_or(0);
2158            Ok(NodeOutput::update("n", json!(n + 1)))
2159        });
2160        builder.set_entry_point("spin");
2161        builder.add_conditional_edges("spin", |state: State| async move {
2162            if state.get("n").and_then(Value::as_i64).unwrap_or(0) >= 5 {
2163                Ok(Route::End)
2164            } else {
2165                Ok(Route::Node("spin".into()))
2166            }
2167        });
2168        (builder.compile().unwrap(), spec)
2169    }
2170
2171    #[tokio::test]
2172    async fn pre_cancelled_token_stops_the_run_before_the_first_step() {
2173        let (graph, spec) = spinner_graph();
2174        let token = CancellationToken::new();
2175        token.cancel();
2176
2177        let err = Executor::new()
2178            .run(
2179                &graph,
2180                &spec,
2181                State::new(),
2182                RunConfig::new("t-cancel").with_cancellation(token),
2183            )
2184            .await
2185            .unwrap_err();
2186        assert!(matches!(err, RustyError::Cancelled(_)));
2187        assert!(
2188            err.to_string().contains("after 0 super-step(s)"),
2189            "a pre-cancelled run must not execute a single step: {err}"
2190        );
2191    }
2192
2193    #[tokio::test]
2194    async fn cancellation_stops_at_a_boundary_and_the_run_resumes_from_it() {
2195        let (graph, spec) = spinner_graph();
2196        let checkpointer = Arc::new(InMemoryCheckpointer::new());
2197        let executor = Executor::with_checkpointer(checkpointer.clone());
2198        let token = CancellationToken::new();
2199
2200        // Cancel as soon as the first boundary checkpoint lands: the stop
2201        // is then provably the token's doing, not the terminating route.
2202        let canceller = token.clone();
2203        let watcher = checkpointer.clone();
2204        tokio::spawn(async move {
2205            loop {
2206                if watcher.get_latest("t-cancel").await.unwrap().is_some() {
2207                    break;
2208                }
2209                tokio::time::sleep(std::time::Duration::from_millis(1)).await;
2210            }
2211            canceller.cancel();
2212        });
2213        let err = executor
2214            .run(
2215                &graph,
2216                &spec,
2217                State::new(),
2218                RunConfig::new("t-cancel").with_cancellation(token.clone()),
2219            )
2220            .await
2221            .unwrap_err();
2222        assert!(matches!(err, RustyError::Cancelled(_)));
2223
2224        // The boundary checkpoint is intact: n counts the completed steps —
2225        // strictly fewer than the terminating route's 5, proving the token
2226        // (not the route) ended the run.
2227        let checkpoint = checkpointer
2228            .get_latest("t-cancel")
2229            .await
2230            .unwrap()
2231            .expect("a boundary checkpoint must survive cancellation");
2232        let steps = checkpoint.state.get("n").and_then(Value::as_i64).unwrap();
2233        assert!(
2234            (1..5).contains(&steps),
2235            "unexpected stop point: n = {steps}"
2236        );
2237
2238        // Re-running the thread resumes the spinner from the checkpoint.
2239        // Drain tokens are one-shot (a cancelled token stays cancelled), so
2240        // the post-deploy process runs under a fresh one.
2241        let outcome = executor
2242            .run(
2243                &graph,
2244                &spec,
2245                State::new(),
2246                RunConfig::new("t-cancel")
2247                    .with_resume(json!(null))
2248                    .with_cancellation(CancellationToken::new()),
2249            )
2250            .await
2251            .unwrap();
2252        match outcome {
2253            ExecutionOutcome::Done(state) => {
2254                assert_eq!(state.get("n"), Some(&json!(5)));
2255            }
2256            other => panic!("expected Done after resume, got {other:?}"),
2257        }
2258    }
2259}