Skip to main content

rusty_agent_runtime/
executor.rs

1//! The executor: a Pregel/BSP-inspired super-step run loop.
2//!
3//! Execution proceeds in discrete **super-steps** (Google Pregel /
4//! Bulk-Synchronous-Parallel), each super-step being:
5//!
6//! 1. **Plan** — determine the active node set (entry point on step 0;
7//!    afterwards the routing result of the previous step, including
8//!    [`crate::node::Command::goto`] overrides and [`crate::graph::Send`]
9//!    fan-outs).
10//! 2. **Compute** — run all active nodes concurrently in a
11//!    `tokio::task::JoinSet`, each receiving an **immutable snapshot** of
12//!    the state as of the start of the step. No node can observe another's
13//!    in-progress writes.
14//! 3. **Barrier** — wait for all active nodes. The step is *transactional*:
15//!    if any node fails, the step's writes are discarded. An
16//!    [`RustyError::Interrupt`] suspends the whole run instead.
17//! 4. **Merge** — apply all node updates to the state via
18//!    [`crate::state::StateSpec::apply_super_step`] (per-channel reducers +
19//!    `LastValue` single-write validation).
20//! 5. **Route** — evaluate outgoing edges / commands against the
21//!    post-barrier state to determine the next active set; `Route::End` (or
22//!    an empty next set) terminates the run.
23//! 6. **Checkpoint** — persist a [`crate::checkpoint::Checkpoint`] recording
24//!    step, state, and next nodes, and emit [`GraphEvent`]s for streaming.
25//!
26//! A graph *cycle* (e.g. the ReAct loop `agent → tools → agent`) is not
27//! call-stack recursion — it is nodes being re-scheduled across super-steps,
28//! which is why the guard is `max_steps`, not a stack limit.
29//!
30//! # Observability
31//!
32//! The executor emits `tracing` telemetry throughout a run (no subscriber is
33//! installed by the library — the application chooses one):
34//!
35//! - `rusty.run` (INFO span) — one per [`Executor::run`] call, carrying
36//!   `thread_id` and `max_steps`; parent of everything below.
37//! - `rusty.super_step` (DEBUG span) — one per super-step, carrying
38//!   `step` and `active_nodes`; covers plan → barrier → merge → route.
39//! - `rusty.node` (INFO span) — one per spawned node task, carrying
40//!   `node` and `step`; attached to the `JoinSet` task via `.instrument()`.
41//! - DEBUG event on each barrier merge (channels written), INFO events on
42//!   interrupt and run completion (`steps`, `duration_ms`), WARN events on
43//!   node failure (with a `retryable` classification).
44
45use std::collections::{HashMap, HashSet};
46use std::sync::{Arc, Mutex};
47
48use serde::{Deserialize, Serialize};
49use serde_json::Value;
50use tokio::sync::mpsc;
51use tokio::task::JoinSet;
52use tokio_util::sync::CancellationToken;
53use tracing::Instrument;
54
55use crate::checkpoint::{Checkpoint, Checkpointer};
56use crate::effects::{CompensationRegistry, EffectAdmissionContext};
57use crate::error::{Result, RustyError};
58use crate::graph::{Edge, Graph, Route};
59use crate::journal::{Clock, EventDraft, Journal, RngSource};
60use crate::middleware::{Middleware, MiddlewareChain, NodeCall};
61use crate::node::{Command, NodeConfig, NodeContext, NodeOutput};
62use crate::record::{
63    CheckpointHeader, Effect, EventStatus, PolicyVersion, RunEventKind, RunManifest,
64    CURRENT_FORMAT_VERSION,
65};
66use crate::state::{State, StateSpec};
67
68/// How a run ended.
69#[derive(Debug)]
70pub enum ExecutionOutcome {
71    /// The run terminated normally (routing reached `Route::End` or no
72    /// nodes remained active). Carries the final state.
73    Done(State),
74
75    /// A node called `interrupt(payload)`: the run is suspended and
76    /// resumable. Carry on by calling [`Executor::run`] again with the same
77    /// `thread_id` and `RunConfig::resume` set.
78    ///
79    /// Suspension is run-wide, not node-local. The in-flight super-step is
80    /// transactional: every write of the step is discarded (including writes
81    /// from sibling nodes that completed before the interrupt was observed
82    /// at the barrier), still-running siblings are aborted, and the
83    /// suspension checkpoint re-schedules **every** node of the step. On
84    /// resume all of them re-execute from their start, so node logic must be
85    /// idempotent.
86    Interrupted {
87        /// The payload passed to `interrupt()` (surfaced to the caller,
88        /// e.g. a human-approval request).
89        value: Value,
90        /// The state as of the suspension point.
91        state: State,
92        /// The checkpoint persisted at the suspension point, for resuming
93        /// or time travel. When the executor has no checkpointer attached,
94        /// the run still suspends but nothing is persisted: the id is then
95        /// only an opaque handle and can never be replayed.
96        checkpoint_id: String,
97    },
98}
99
100impl ExecutionOutcome {
101    /// The final (or suspension-point) state, regardless of variant.
102    pub fn state(&self) -> &State {
103        match self {
104            ExecutionOutcome::Done(s) => s,
105            ExecutionOutcome::Interrupted { state, .. } => state,
106        }
107    }
108
109    /// Whether the run ended in [`ExecutionOutcome::Interrupted`] (suspended,
110    /// resumable) rather than [`ExecutionOutcome::Done`].
111    pub fn is_interrupted(&self) -> bool {
112        matches!(self, ExecutionOutcome::Interrupted { .. })
113    }
114}
115
116/// Per-run configuration (the LangGraph `RunnableConfig` analog).
117#[derive(Debug, Clone)]
118pub struct RunConfig {
119    /// Thread (session) id. Stable across interrupt/resume; namespaces all
120    /// checkpoints for this run. Required for persistence and resume.
121    pub thread_id: String,
122
123    /// Maximum number of super-steps before the run aborts with
124    /// [`crate::error::RustyError::Graph`] (the LangGraph
125    /// `recursion_limit` / `GraphRecursionError` guard). Default: 1000.
126    pub max_steps: usize,
127
128    /// Resume value for continuing an interrupted run. When set, the
129    /// executor restores the latest checkpoint for `thread_id` and
130    /// re-executes the checkpointed next-node set with
131    /// [`crate::node::NodeContext::resume_value`] returning this value.
132    ///
133    /// The value is **broadcast**: every node scheduled in the first
134    /// super-step after the resume observes it, not only the node that
135    /// originally interrupted (a suspension checkpoint re-schedules the
136    /// whole active set — see [`ExecutionOutcome::Interrupted`]). Nodes that
137    /// should react only when they themselves were resumed must key off
138    /// their own state, not the presence of a resume value.
139    pub resume: Option<Value>,
140
141    /// Replay/time-travel handle: the id of a specific checkpoint of
142    /// `thread_id` to resume from. When set, the executor loads **that**
143    /// checkpoint (not the latest) and continues the run from its state and
144    /// next-node set. Requires a checkpointer on the executor.
145    ///
146    /// Combines with `resume`: `checkpoint_id` selects **where** the run
147    /// restarts, `resume` (when also set) is delivered as the resume value to
148    /// the first super-step, exactly as in interrupt/resume.
149    ///
150    /// Safe pattern: replaying on the *same* thread appends new history on
151    /// top of the old timeline, so prefer forking first —
152    /// [`crate::checkpoint::Checkpointer::fork_thread`] the thread into a new
153    /// thread id, then run the fork with `checkpoint_id` set. Direct replay
154    /// on the original thread is supported for cases where appended history
155    /// is acceptable.
156    pub checkpoint_id: Option<String>,
157
158    /// Optional event sink for streaming: the executor emits [`GraphEvent`]s
159    /// as the run progresses (node start/end, state updates, checkpoints,
160    /// super-step boundaries). Consumers implement LangGraph's stream modes
161    /// (`values` / `updates` / `tasks` / ...) as filters over this stream.
162    pub event_tx: Option<mpsc::Sender<GraphEvent>>,
163
164    /// Flight Recorder determinism seam (R0.5): the run's time source. Every
165    /// executor timestamp — event `recorded_at`, node latencies, checkpoint
166    /// `created_at` — is read through it. `None` (the default) is the system
167    /// wall clock, byte-identical to pre-R0.5 behavior. Attach
168    /// [`Clock::Logical`] to make a recorded run re-drivable.
169    pub clock: Option<Clock>,
170
171    /// Flight Recorder determinism seam (R0.5): the run's randomness source.
172    /// Checkpoint ids (and the run id, when the executor creates the
173    /// journal) are minted through it. `None` (the default) is OS entropy,
174    /// byte-identical to pre-R0.5 behavior; [`RngSource::Seeded`] makes the
175    /// id stream reproducible.
176    pub rng: Option<RngSource>,
177
178    /// Flight Recorder (R0.5): attach a pre-built journal for this run. Node
179    /// closures capture a clone of the same [`Journal`] to record their own
180    /// model/tool/remote/WASM calls into the run's evidence (the journal
181    /// stamps sequence numbers and timestamps itself). When `None`, the
182    /// executor creates a fresh journal per run; either way the run's
183    /// journal is retrievable afterwards via [`Executor::journal`].
184    pub journal: Option<Journal>,
185
186    /// Flight Recorder (R0.5): the executor policy active for this run,
187    /// stamped into every checkpoint header. `None` records the static
188    /// default ([`PolicyVersion::STATIC_V0`]) — except on resume (R0.8 wave
189    /// 4), where the run inherits the version its checkpoint header pins so
190    /// an in-flight run is immune to mid-run promotions. An explicit pin
191    /// always wins over inheritance.
192    pub policy_version: Option<PolicyVersion>,
193
194    /// Flight Recorder (R0.5): the application's own graph version string,
195    /// stamped into every checkpoint header next to the topology hash.
196    /// `None` records `"unversioned"`. Bump it when node bodies change in
197    /// ways the topology hash cannot see.
198    pub graph_version: Option<String>,
199
200    /// Cooperative cancellation (R0.6 wave 2c, drain). When set, the
201    /// executor checks the token **at every super-step boundary** and, once
202    /// it is cancelled, stops the run before starting the next super-step,
203    /// returning [`RustyError::Cancelled`].
204    ///
205    /// The boundary is the only cancellation point, deliberately: a
206    /// super-step is transactional (its nodes run off an immutable snapshot
207    /// and its writes merge atomically), and its boundary checkpoint is
208    /// already persisted by the time the token is observed — so a cancelled
209    /// run resumes from exactly where it stopped, with no torn step and no
210    /// lost writes. Cancellation granularity is therefore one super-step;
211    /// work inside the in-flight step runs to its barrier. This is the hook
212    /// graceful shutdown uses: a draining server cancels the token and every
213    /// in-flight run parks itself at a resumable checkpoint instead of being
214    /// torn down mid-step.
215    pub cancellation: Option<CancellationToken>,
216
217    /// Effect kernel v2 (R0.7): the versioned run manifest — prompts, tool
218    /// schemas, model and parameters, memory schema, capsule versions the run
219    /// pins. Stamped into every checkpoint header so a resumed run keeps
220    /// executing against its pinned versions (see
221    /// [`crate::record::RunManifest`] for the upgrade-safety contract).
222    /// `None` (the default) pins nothing and leaves checkpoint bytes
223    /// byte-identical to R0.5/R0.6.
224    pub manifest: Option<RunManifest>,
225
226    /// Effect kernel v2 (R0.7): the approval tokens this run carries for its
227    /// irreversible effects (see [`crate::effects::ApprovalToken`]). The
228    /// executor consults them when [`Executor::with_effect_admission`] has
229    /// enabled the guarded tool path. Empty by default; an irreversible call
230    /// dispatched through that path then fails admission before its body
231    /// runs.
232    pub effect_approvals: Vec<crate::effects::ApprovalToken>,
233
234    /// Shadow runs (R0.12 wave 4): an explicit admission context for this
235    /// run, overriding the one the executor would otherwise build from
236    /// [`Executor::with_effect_admission`]. A shadow deployment builds a
237    /// context whose boundary refuses every effect above read-only and serves
238    /// refused calls from the recorded world (see
239    /// [`crate::effects::EffectAdmissionContext::shadow`]); injecting it here
240    /// is what makes the run execute against that boundary instead of the
241    /// production one. `None` (the default) is byte-identical to prior
242    /// behavior.
243    pub effect_admission: Option<EffectAdmissionContext>,
244}
245
246impl Default for RunConfig {
247    /// A config with an empty `thread_id` and the default step limit —
248    /// identical to `RunConfig::new("")`. Derived `Default` would zero
249    /// `max_steps`, so any `default()`-built run would instantly trip the
250    /// step guard; keep this impl in sync with [`RunConfig::new`].
251    fn default() -> Self {
252        Self::new(String::new())
253    }
254}
255
256impl RunConfig {
257    /// A config for `thread_id` with the default step limit.
258    pub fn new(thread_id: impl Into<String>) -> Self {
259        Self {
260            thread_id: thread_id.into(),
261            max_steps: DEFAULT_MAX_STEPS,
262            resume: None,
263            checkpoint_id: None,
264            event_tx: None,
265            clock: None,
266            rng: None,
267            journal: None,
268            policy_version: None,
269            graph_version: None,
270            cancellation: None,
271            manifest: None,
272            effect_approvals: Vec::new(),
273            effect_admission: None,
274        }
275    }
276
277    /// Builder-style: override the step limit.
278    pub fn with_max_steps(mut self, max_steps: usize) -> Self {
279        self.max_steps = max_steps;
280        self
281    }
282
283    /// Builder-style: set the resume value.
284    pub fn with_resume(mut self, value: Value) -> Self {
285        self.resume = Some(value);
286        self
287    }
288
289    /// Builder-style: replay from a specific checkpoint of `thread_id`
290    /// (time travel). See the [`RunConfig::checkpoint_id`] field docs for
291    /// semantics and the fork-first safe pattern.
292    pub fn with_checkpoint_id(mut self, checkpoint_id: impl Into<String>) -> Self {
293        self.checkpoint_id = Some(checkpoint_id.into());
294        self
295    }
296
297    /// Builder-style: attach a streaming event sink.
298    pub fn with_event_tx(mut self, tx: mpsc::Sender<GraphEvent>) -> Self {
299        self.event_tx = Some(tx);
300        self
301    }
302
303    /// Builder-style: source the run's time from `clock` (Flight Recorder
304    /// determinism seam; see the [`RunConfig::clock`] field docs).
305    pub fn with_clock(mut self, clock: Clock) -> Self {
306        self.clock = Some(clock);
307        self
308    }
309
310    /// Builder-style: source the run's randomness from `rng` (Flight
311    /// Recorder determinism seam; see the [`RunConfig::rng`] field docs).
312    pub fn with_rng(mut self, rng: RngSource) -> Self {
313        self.rng = Some(rng);
314        self
315    }
316
317    /// Builder-style: record the run into `journal` (Flight Recorder; see
318    /// the [`RunConfig::journal`] field docs). When the config carries no
319    /// explicit [`RunConfig::clock`], the executor also reads time from the
320    /// attached journal's clock, keeping one time source per run.
321    pub fn with_journal(mut self, journal: Journal) -> Self {
322        self.journal = Some(journal);
323        self
324    }
325
326    /// Builder-style: stamp the active executor policy version into every
327    /// checkpoint header (see the [`RunConfig::policy_version`] field docs).
328    pub fn with_policy_version(mut self, version: PolicyVersion) -> Self {
329        self.policy_version = Some(version);
330        self
331    }
332
333    /// Builder-style: stamp the application's graph version into every
334    /// checkpoint header (see the [`RunConfig::graph_version`] field docs).
335    pub fn with_graph_version(mut self, version: impl Into<String>) -> Self {
336        self.graph_version = Some(version.into());
337        self
338    }
339
340    /// Builder-style: make the run cooperatively cancellable (see the
341    /// [`RunConfig::cancellation`] field docs). The token is observed only at
342    /// super-step boundaries: cancelling it stops the run *after* the
343    /// in-flight step's barrier and boundary checkpoint, so the run is left
344    /// resumable from exactly that checkpoint with [`RustyError::Cancelled`].
345    pub fn with_cancellation(mut self, token: CancellationToken) -> Self {
346        self.cancellation = Some(token);
347        self
348    }
349
350    /// Builder-style: pin the run's versioned manifest into every checkpoint
351    /// header (see the [`RunConfig::manifest`] field docs).
352    pub fn with_manifest(mut self, manifest: RunManifest) -> Self {
353        self.manifest = Some(manifest);
354        self
355    }
356
357    /// Builder-style: carry approval tokens for the run's irreversible
358    /// effects (see the [`RunConfig::effect_approvals`] field docs).
359    pub fn with_effect_approvals(
360        mut self,
361        approvals: impl IntoIterator<Item = crate::effects::ApprovalToken>,
362    ) -> Self {
363        self.effect_approvals = approvals.into_iter().collect();
364        self
365    }
366
367    /// Builder-style: inject an explicit admission context for this run (see
368    /// the [`RunConfig::effect_admission`] field docs). Shadow deployments
369    /// use it to run the graph against the shadow boundary instead of the
370    /// production one.
371    pub fn with_effect_admission_context(mut self, context: EffectAdmissionContext) -> Self {
372        self.effect_admission = Some(context);
373        self
374    }
375
376    /// A clone of the event sink sender, for wiring into nodes that stream
377    /// [`GraphEvent::Token`] deltas (LangGraph's `messages` stream mode).
378    ///
379    /// Typical flow: create the channel, call `config.token_tx()` to obtain
380    /// the clone a node closure captures, then hand `config` to
381    /// [`Executor::run`]. `None` when no sink is attached. See the
382    /// [`crate::llm::ChatModel`] rustdoc for the full pattern.
383    pub fn token_tx(&self) -> Option<mpsc::Sender<GraphEvent>> {
384        self.event_tx.clone()
385    }
386}
387
388/// Default super-step limit. Deliberately far above LangGraph's default
389/// `recursion_limit` of 25: ReAct-style loops burn one super-step per
390/// agent/tool hop, so long tool chains legitimately exceed 25.
391pub const DEFAULT_MAX_STEPS: usize = 1000;
392
393/// Streaming events emitted by the executor during a run. All of LangGraph's
394/// stream modes are views over this single typed event stream.
395///
396/// The enum is serializable so event streams can cross process / FFI
397/// boundaries (e.g. a WebSocket bridge or a persisted event log).
398#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
399#[serde(tag = "type", rename_all = "snake_case")]
400pub enum GraphEvent {
401    /// A node began executing.
402    NodeStart {
403        /// Node name.
404        node: String,
405        /// Super-step index.
406        step: usize,
407    },
408    /// A node finished executing (successfully).
409    NodeEnd {
410        /// Node name.
411        node: String,
412        /// Super-step index.
413        step: usize,
414    },
415    /// A single LLM token delta (the LangGraph `messages` stream mode).
416    ///
417    /// The executor itself never emits this variant: tokens originate inside
418    /// nodes that call [`crate::llm::ChatModel::chat_stream`] and forward
419    /// each [`crate::llm::TokenChunk`] into the run's event channel (see the
420    /// `ChatModel` rustdoc for the wiring pattern and
421    /// [`RunConfig::token_tx`] / [`Executor::token_tx`] for sender handles).
422    Token {
423        /// Node that produced the token.
424        node: String,
425        /// Incremental text produced since the previous token.
426        delta: String,
427    },
428    /// State was updated at a super-step barrier (`updates` stream mode).
429    StateUpdate {
430        /// Super-step index at which the update was applied.
431        step: usize,
432        /// Every channel written in this step mapped to its **post-reducer**
433        /// value (e.g. the full appended list for an `Append` channel), read
434        /// back from the merged state — not the raw per-node partials.
435        updates: serde_json::Map<String, Value>,
436    },
437    /// A checkpoint was persisted at a super-step boundary.
438    CheckpointSaved {
439        /// The checkpoint id.
440        checkpoint_id: String,
441        /// Super-step index at the boundary.
442        step: usize,
443    },
444    /// A super-step began; lists the nodes activated in it.
445    SuperStep {
446        /// Super-step index.
447        step: usize,
448        /// Nodes active in this step.
449        active_nodes: Vec<String>,
450    },
451}
452
453/// The graph executor. Holds an optional checkpointer and an optional
454/// middleware chain; stateless with respect to individual runs, so one
455/// `Executor` can drive many concurrent runs (each with its own
456/// `thread_id`).
457#[derive(Default)]
458pub struct Executor {
459    checkpointer: Option<Arc<dyn Checkpointer>>,
460    token_tx: Option<mpsc::Sender<GraphEvent>>,
461    // The most recent run's Flight Recorder journal. Interior mutability
462    // keeps `run` taking `&self`; overwritten at the start of every run.
463    journal: Mutex<Option<Journal>>,
464    // Middleware/Interceptor SDK: ordered layers wrapping every node
465    // invocation (and, via `NodeContext::middleware`, the tool/model calls
466    // node code makes). An empty chain takes the original dispatch path.
467    middleware: MiddlewareChain,
468    // R0.8 wave 1: the governed-memory source this executor's runs answer
469    // memory reads from (live store or replay cursor). The executor never
470    // touches memory itself this wave — the full run-loop integration is a
471    // later wave; this is the installation point node factories use (see
472    // `Executor::memory`), mirroring `with_token_tx`.
473    memory_source: Option<crate::memory::MemorySource>,
474    // Present only when guarded tool effect admission is enabled. The run's
475    // thread id and approval tokens are combined with these rollback handlers
476    // into an EffectAdmissionContext for each node invocation.
477    effect_compensations: Option<CompensationRegistry>,
478}
479
480impl Executor {
481    /// An executor without persistence (runs cannot be interrupted/resumed
482    /// durably; interrupts will still surface but resume requires a
483    /// checkpointer).
484    pub fn new() -> Self {
485        Self::default()
486    }
487
488    /// An executor persisting checkpoints through `checkpointer`.
489    pub fn with_checkpointer(checkpointer: Arc<dyn Checkpointer>) -> Self {
490        Self {
491            checkpointer: Some(checkpointer),
492            ..Self::default()
493        }
494    }
495
496    /// Builder-style: attach a middleware layer (Middleware/Interceptor
497    /// SDK). Layers compose in registration order — before-hooks run in
498    /// `.layer()` order on the way into a node/model/tool call, after-hooks
499    /// in reverse order on the way out. See [`crate::middleware`].
500    pub fn layer<M: Middleware + 'static>(mut self, middleware: M) -> Self {
501        self.middleware.push(Arc::new(middleware));
502        self
503    }
504
505    /// Builder-style: attach a pre-shared middleware layer.
506    pub fn layer_shared(mut self, middleware: Arc<dyn Middleware>) -> Self {
507        self.middleware.push(middleware);
508        self
509    }
510
511    /// The attached middleware chain (empty when no layers were added).
512    pub fn middleware(&self) -> &MiddlewareChain {
513        &self.middleware
514    }
515
516    /// Builder-style: make guarded tool effect admission available to nodes.
517    ///
518    /// The supplied registry may be empty. Pure and read-only calls remain
519    /// automatic; idempotent calls require a key, compensatable calls require
520    /// a handler in this registry, and non-idempotent calls require a matching
521    /// token in [`RunConfig::effect_approvals`]. The prebuilt ReAct tools node
522    /// enforces this automatically. Custom nodes must pass
523    /// [`NodeContext::effect_admission`] to the [`crate::tool::ToolExecutor`]
524    /// they construct; direct [`crate::tool::Tool::call`] invocations are not
525    /// intercepted by the executor.
526    pub fn with_effect_admission(mut self, compensations: CompensationRegistry) -> Self {
527        self.effect_compensations = Some(compensations);
528        self
529    }
530
531    /// Whether this executor enforces the effect admission boundary.
532    pub fn effect_admission_enabled(&self) -> bool {
533        self.effect_compensations.is_some()
534    }
535
536    /// Builder-style: hold a token broadcast sender that nodes can clone to
537    /// publish [`GraphEvent::Token`] deltas (LangGraph's `messages` stream
538    /// mode).
539    ///
540    /// The executor never emits `Token` events itself — tokens originate in
541    /// nodes calling [`crate::llm::ChatModel::chat_stream`]. This is a
542    /// convenience handle so node factories built around an `Executor` can
543    /// fetch the sink via [`Executor::token_tx`] and capture a clone in each
544    /// node closure. When the same channel should also receive the
545    /// executor's own events, attach it to the run via
546    /// [`RunConfig::with_event_tx`] instead (or as well).
547    pub fn with_token_tx(mut self, token_tx: mpsc::Sender<GraphEvent>) -> Self {
548        self.token_tx = Some(token_tx);
549        self
550    }
551
552    /// The token broadcast sender, if one was attached via
553    /// [`Executor::with_token_tx`]. Clone it into node closures to stream
554    /// [`GraphEvent::Token`]s from within nodes.
555    pub fn token_tx(&self) -> Option<&mpsc::Sender<GraphEvent>> {
556        self.token_tx.as_ref()
557    }
558
559    /// Builder-style: install the governed-memory source this executor's
560    /// runs answer memory reads from (R0.8 wave 1; see
561    /// [`crate::memory::MemorySource`]).
562    ///
563    /// The executor never reads or writes memory itself this wave — the
564    /// full run-loop integration is a later wave. This is the seam's
565    /// installation point, mirroring [`Executor::with_token_tx`]: node
566    /// factories built around an `Executor` fetch the journaled handle via
567    /// [`Executor::memory`] once the run has started and capture a clone in
568    /// each node closure. Nodes may equally bind a source to an attached
569    /// journal directly ([`crate::journal::Journal::memory`]); both paths
570    /// journal into the same run evidence.
571    pub fn with_memory_source(mut self, source: crate::memory::MemorySource) -> Self {
572        self.memory_source = Some(source);
573        self
574    }
575
576    /// The configured memory source, if one was installed via
577    /// [`Executor::with_memory_source`].
578    pub fn memory_source(&self) -> Option<&crate::memory::MemorySource> {
579        self.memory_source.as_ref()
580    }
581
582    /// The journaled memory handle for the most recent run: the run's
583    /// journal (set at run start) bound to the configured source. `None`
584    /// before the first run or when no source is installed. The same
585    /// most-recent-run caveat as [`Executor::journal`] applies: driving
586    /// several runs concurrently through one `Executor` leaves this handle
587    /// pointing at whichever run started last — bind per-run handles via
588    /// [`crate::journal::Journal::memory`] when runs overlap.
589    pub fn memory(&self) -> Option<crate::memory::JournaledMemory> {
590        let journal = self.journal()?;
591        let source = self.memory_source.clone()?;
592        Some(crate::memory::JournaledMemory::new(&journal, source))
593    }
594
595    /// The configured checkpointer, if any. Shared (not consumed) so one
596    /// `Executor` can drive many concurrent runs over the same store.
597    pub fn checkpointer(&self) -> Option<&Arc<dyn Checkpointer>> {
598        self.checkpointer.as_ref()
599    }
600
601    /// The Flight Recorder journal of the most recent run started through
602    /// this executor — the one attached via [`RunConfig::with_journal`], or
603    /// the fresh journal the executor created for the run. `None` before the
604    /// first run.
605    ///
606    /// The journal is set at run start, so the evidence of a failed or
607    /// suspended run is retrievable exactly like a completed run's. Note the
608    /// executor is deliberately stateless across runs: driving several runs
609    /// concurrently through one `Executor` leaves this handle pointing at
610    /// whichever run started last — attach per-run journals via the config
611    /// when runs overlap.
612    pub fn journal(&self) -> Option<Journal> {
613        self.journal
614            .lock()
615            .unwrap_or_else(|e| e.into_inner())
616            .clone()
617    }
618
619    /// Run a compiled graph to completion (or interruption).
620    ///
621    /// - `graph`: the compiled, frozen graph topology.
622    /// - `spec`: the state schema (channels + reducers) used to merge node
623    ///   updates at each barrier.
624    /// - `initial_state`: the starting state. When `config.resume` is set and
625    ///   a checkpoint exists for `config.thread_id`, the checkpointed state
626    ///   and next-node set take precedence over this argument. When
627    ///   `config.checkpoint_id` is set, that specific checkpoint (rather than
628    ///   the latest) is restored — replay/time travel; forking into a fresh
629    ///   thread first via
630    ///   [`crate::checkpoint::Checkpointer::fork_thread`] is the safe pattern,
631    ///   since replaying on the same thread appends new history.
632    /// - `config`: run configuration (thread id, step limit, resume value,
633    ///   streaming sink).
634    ///
635    /// # Super-step semantics
636    ///
637    /// Each loop iteration runs one super-step as a transaction: the active
638    /// nodes execute in parallel over an immutable start-of-step snapshot;
639    /// the barrier discards the whole step's writes on any node failure and
640    /// suspends the run on an interrupt; only then are writes merged via the
641    /// channel reducers, routing computed against the post-barrier state,
642    /// and a boundary checkpoint persisted. The module-level docs enumerate
643    /// the six phases; `execute_super_step` is the implementation.
644    ///
645    /// The loop returns [`ExecutionOutcome::Done`] when routing yields an
646    /// empty next set, [`ExecutionOutcome::Interrupted`] when a node
647    /// interrupts, an [`RustyError::Graph`] error once `config.max_steps`
648    /// super-steps have run without termination, and
649    /// [`RustyError::Cancelled`] when [`RunConfig::cancellation`] is
650    /// observed cancelled at a super-step boundary (the boundary checkpoint
651    /// is intact; the run resumes from it).
652    pub async fn run(
653        &self,
654        graph: &Graph,
655        spec: &StateSpec,
656        initial_state: State,
657        config: RunConfig,
658    ) -> Result<ExecutionOutcome> {
659        // Run-level span: every super-step, node, and checkpoint trace in the
660        // run attaches to it. Attached via `.instrument()` (never `.enter()`)
661        // so no span guard is held across `.await` points and the returned
662        // future stays `Send`.
663        let run_span = tracing::info_span!(
664            "rusty.run",
665            thread_id = %config.thread_id,
666            max_steps = config.max_steps,
667            resume = config.resume.is_some(),
668            replay = config.checkpoint_id.is_some(),
669        );
670        self.run_inner(graph, spec, initial_state, config)
671            .instrument(run_span)
672            .await
673    }
674
675    /// The instrumented body of [`Executor::run`]; see that method's docs for
676    /// the super-step algorithm.
677    async fn run_inner(
678        &self,
679        graph: &Graph,
680        spec: &StateSpec,
681        initial_state: State,
682        config: RunConfig,
683    ) -> Result<ExecutionOutcome> {
684        let started = std::time::Instant::now();
685
686        // ---- flight recorder setup ----
687        //
688        // Resolve the determinism seams and the run's journal before any
689        // checkpoint work: every timestamp and id below flows through them.
690        // An attached journal carries its own identity and clock; otherwise
691        // the executor mints a run id from the (possibly seeded) RNG and a
692        // journal over the configured clock. The run reads time from the
693        // journal's clock when no explicit clock is configured, keeping one
694        // time source per run.
695        let rng = config.rng.clone().unwrap_or_default();
696        let journal = match &config.journal {
697            Some(attached) => attached.clone(),
698            None => Journal::new(
699                rng.uuid_string(),
700                config.thread_id.clone(),
701                config.clock.clone().unwrap_or_default(),
702            ),
703        };
704        let mut recorder = Recorder {
705            clock: config
706                .clock
707                .clone()
708                .unwrap_or_else(|| journal.clock().clone()),
709            rng,
710            graph_version: config
711                .graph_version
712                .clone()
713                .unwrap_or_else(|| "unversioned".to_owned()),
714            graph_hash: graph.topology_hash(),
715            policy_version: config.policy_version.clone().unwrap_or_default(),
716            manifest: config.manifest.clone(),
717            journal: journal.clone(),
718        };
719        // Publish before the loop: evidence of a failed or suspended run is
720        // retrievable exactly like a completed run's.
721        *self.journal.lock().unwrap_or_else(|e| e.into_inner()) = Some(journal);
722
723        // ---- initialization / resume ----
724        //
725        // On resume the checkpointed state and next-node set take precedence
726        // over `initial_state`; the resume value is delivered to the first
727        // super-step (whose active set is the checkpointed next-node set —
728        // after an interrupt, every node of the suspended step) via
729        // `NodeContext::resume_value()`.
730        //
731        // Time travel: when `config.checkpoint_id` is set, THAT checkpoint is
732        // restored instead of the latest — this is replay from an arbitrary
733        // history point. The two knobs compose: `checkpoint_id` selects WHERE
734        // the run restarts, `resume` (when also set) supplies the resume value
735        // for the first super-step.
736        let mut state = initial_state;
737        let mut active: Vec<ActiveTask>;
738        let mut step: usize = 0;
739        let mut pending_resume: Option<Value> = None;
740        // Causal parent for the first super-step: the resume event on a
741        // resumed run, nothing on a fresh one.
742        let mut step_parent: Option<String> = None;
743
744        if config.checkpoint_id.is_some() || config.resume.is_some() {
745            let checkpointer = self.checkpointer.as_ref().ok_or_else(|| {
746                RustyError::Checkpoint(
747                    "RunConfig.checkpoint_id/resume is set but no checkpointer is configured \
748                     on the executor"
749                        .into(),
750                )
751            })?;
752            let checkpoint = match &config.checkpoint_id {
753                Some(id) => checkpointer
754                    .get_by_id(&config.thread_id, id)
755                    .await?
756                    .ok_or_else(|| {
757                        RustyError::Checkpoint(format!(
758                            "cannot replay thread `{}`: checkpoint `{id}` not found",
759                            config.thread_id
760                        ))
761                    })?,
762                None => checkpointer
763                    .get_latest(&config.thread_id)
764                    .await?
765                    .ok_or_else(|| {
766                        RustyError::Checkpoint(format!(
767                            "cannot resume thread `{}`: no checkpoint found",
768                            config.thread_id
769                        ))
770                    })?,
771            };
772            // Policy-binding continuity (R0.8 wave 4): a resumed run keeps
773            // the policy version its checkpoint header pins, so a mid-run
774            // promotion never changes behavior under an in-flight execution.
775            // An explicit `RunConfig::with_policy_version` pin wins over
776            // inheritance — the caller asked for that version deliberately.
777            if config.policy_version.is_none() {
778                recorder.policy_version = checkpoint.header.policy_version.clone();
779            }
780            step_parent = Some(recorder.record(
781                EventDraft::new(RunEventKind::Resume, Effect::Pure).input(serde_json::json!({
782                    "checkpoint_id": checkpoint.id,
783                    "step": checkpoint.step,
784                    "resume": config.resume.clone().unwrap_or(Value::Null),
785                })),
786            ));
787            state = checkpoint.state;
788            step = checkpoint.step;
789            active = checkpoint
790                .next_nodes
791                .into_iter()
792                .map(|name| ActiveTask { name, scoped: None })
793                .collect();
794            pending_resume = config.resume.clone();
795            if active.is_empty() {
796                tracing::info!(
797                    steps = 0,
798                    duration_ms = started.elapsed().as_millis() as u64,
799                    "run complete"
800                );
801                return Ok(ExecutionOutcome::Done(state));
802            }
803        } else {
804            active = vec![ActiveTask {
805                name: graph.entry_point().to_owned(),
806                scoped: None,
807            }];
808        }
809
810        // One shared context for the whole run: clones handed to parallel
811        // nodes share its approval ledger, and later super-steps observe
812        // tokens already consumed by earlier calls. An explicitly injected
813        // context (R0.12 shadow runs) wins over the production one derived
814        // from the executor's compensation registry.
815        let effect_admission = config.effect_admission.clone().or_else(|| {
816            self.effect_compensations.as_ref().map(|compensations| {
817                EffectAdmissionContext::new(config.thread_id.clone())
818                    .with_approvals(config.effect_approvals.clone())
819                    .with_compensations(compensations.clone())
820            })
821        });
822
823        // ---- super-step loop ----
824        let mut steps_run: usize = 0;
825        loop {
826            if steps_run >= config.max_steps {
827                return Err(RustyError::Graph(format!(
828                    "max_steps ({}) exceeded: the graph did not terminate within the step \
829                     budget (possible infinite cycle; raise RunConfig::max_steps or add a \
830                     terminating route)",
831                    config.max_steps
832                )));
833            }
834            // Cooperative cancellation (drain): observed only here, at a
835            // super-step boundary. The last step's barrier merged cleanly
836            // and its boundary checkpoint is already persisted, so
837            // cancelling never tears a step — the run resumes from exactly
838            // this point. Nodes of the in-flight step always run to their
839            // barrier; cancellation granularity is one super-step.
840            if config
841                .cancellation
842                .as_ref()
843                .is_some_and(CancellationToken::is_cancelled)
844            {
845                tracing::warn!(
846                    steps_run,
847                    "run cancelled at a super-step boundary; resumable from the last checkpoint"
848                );
849                return Err(RustyError::Cancelled(format!(
850                    "cancelled after {steps_run} super-step(s); the boundary checkpoint of \
851                     thread `{}` is intact, so re-running the thread resumes from there",
852                    config.thread_id
853                )));
854            }
855
856            // The step body runs in a dedicated method so the whole body is
857            // one instrumented future under the per-step span.
858            let step_span =
859                tracing::debug_span!("rusty.super_step", step = step, active_nodes = active.len(),);
860
861            let transition = self
862                .execute_super_step(
863                    graph,
864                    spec,
865                    &config,
866                    &recorder,
867                    effect_admission.as_ref(),
868                    &mut state,
869                    &active,
870                    step,
871                    &mut pending_resume,
872                    step_parent.clone(),
873                )
874                .instrument(step_span)
875                .await?;
876
877            match transition {
878                StepTransition::Next(next, route_event) => {
879                    active = next;
880                    step += 1;
881                    steps_run += 1;
882                    step_parent = Some(route_event);
883                }
884                StepTransition::Finish(outcome) => {
885                    if !outcome.is_interrupted() {
886                        tracing::info!(
887                            steps = steps_run + 1,
888                            duration_ms = started.elapsed().as_millis() as u64,
889                            "run complete"
890                        );
891                    }
892                    return Ok(outcome);
893                }
894            }
895        }
896    }
897
898    /// Executes one super-step: plan -> compute -> barrier -> merge -> route
899    /// -> boundary checkpoint. Returns the next active set, or
900    /// [`StepTransition::Finish`] with the terminal outcome when the run ends
901    /// (`Done`) or suspends (`Interrupted`).
902    ///
903    /// Flight Recorder: every phase transition is journaled through
904    /// `recorder` — super-step start/end, one node input/output pair per
905    /// invocation (in deterministic active-set order, not finish order), the
906    /// routing decision, and the checkpoint write. Node events carry the
907    /// node's declared [`Effect`] classification.
908    #[allow(clippy::too_many_arguments)]
909    async fn execute_super_step(
910        &self,
911        graph: &Graph,
912        spec: &StateSpec,
913        config: &RunConfig,
914        recorder: &Recorder,
915        effect_admission: Option<&EffectAdmissionContext>,
916        state: &mut State,
917        active: &[ActiveTask],
918        step: usize,
919        pending_resume: &mut Option<Value>,
920        step_parent: Option<String>,
921    ) -> Result<StepTransition> {
922        // -- plan.
923        let active_names: Vec<String> = active.iter().map(|t| t.name.clone()).collect();
924        Self::emit(
925            config,
926            GraphEvent::SuperStep {
927                step,
928                active_nodes: active_names.clone(),
929            },
930        );
931        let mut plan_draft =
932            EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure).input(serde_json::json!({
933                "step": step,
934                "active_nodes": active_names,
935            }));
936        if let Some(parent) = step_parent {
937            plan_draft = plan_draft.parent(parent);
938        }
939        let step_start_event = recorder.record(plan_draft);
940
941        // -- compute. Scoped (Send) state is overlaid onto each invocation's
942        //    private copy of the start-of-step snapshot, so fan-out items
943        //    never collide in the shared state.
944        let snapshot = state.clone();
945        let effect_admission = effect_admission.cloned();
946        let mut join_set: JoinSet<(usize, String, Result<NodeOutput>, u64)> = JoinSet::new();
947        // Per-invocation journal metadata, aligned with `active`: the input
948        // event id (causal parent of the matching output) and the node's
949        // declared effect class. Fan-out invocations of the same node each
950        // get their own entry — index, not name, is the identity.
951        let mut input_events: Vec<String> = Vec::with_capacity(active.len());
952        let mut invocation_effects: Vec<Effect> = Vec::with_capacity(active.len());
953
954        for (index, task) in active.iter().enumerate() {
955            let node = graph.node(&task.name).ok_or_else(|| {
956                RustyError::Graph(format!("routing activated unknown node `{}`", task.name))
957            })?;
958
959            let mut node_state = snapshot.clone();
960            if let Some(scoped) = &task.scoped {
961                match scoped {
962                    Value::Object(map) => {
963                        for (channel, value) in map {
964                            node_state.insert(channel.clone(), value.clone());
965                        }
966                    }
967                    other => {
968                        return Err(RustyError::InvalidUpdate(format!(
969                            "Send scoped state for node `{}` must be a JSON object, \
970                             got {other}",
971                            task.name
972                        )));
973                    }
974                }
975            }
976
977            let input_event = recorder.record(
978                EventDraft::new(RunEventKind::NodeInput, node.effect())
979                    .node(task.name.clone())
980                    .input(node_state.to_value())
981                    .parent(step_start_event.clone()),
982            );
983            invocation_effects.push(node.effect());
984
985            let node_config = NodeConfig {
986                thread_id: config.thread_id.clone(),
987                step,
988                resume: pending_resume.clone(),
989                // Hand the invocation its own journal event id so node
990                // code can parent the effects it records (model/tool
991                // calls) to this invocation.
992                extra: HashMap::from([(
993                    crate::journal::PARENT_EVENT_KEY.to_owned(),
994                    Value::String(input_event.clone()),
995                )]),
996            };
997            input_events.push(input_event);
998            let name = task.name.clone();
999            Self::emit(
1000                config,
1001                GraphEvent::NodeStart {
1002                    node: name.clone(),
1003                    step,
1004                },
1005            );
1006            // A JoinSet polls tasks independently of the spawning task's
1007            // context, so the per-node span is attached to each spawned
1008            // future explicitly via `.instrument()`.
1009            let node_span = tracing::info_span!("rusty.node", node = %name, step = step);
1010            // Latency is read through the run's clock seam — under a logical
1011            // clock the value is reproducible; under the system clock it is
1012            // the real elapsed time.
1013            let clock = recorder.clock.clone();
1014            let chain = self.middleware.clone();
1015            let effect_admission = effect_admission.clone();
1016            join_set.spawn(
1017                async move {
1018                    let node_started = clock.now();
1019                    // Middleware chain: with layers attached, the invocation
1020                    // runs inside the onion — before-hooks may mutate the
1021                    // snapshot, reject the run, or short-circuit with a
1022                    // substitute output; after-hooks unwind over the result.
1023                    // No layers: the original dispatch, byte-identical.
1024                    let result = if chain.is_empty() {
1025                        node.run(
1026                            NodeContext::new(node_state, node_config)
1027                                .with_optional_effect_admission(effect_admission),
1028                        )
1029                        .await
1030                    } else {
1031                        let mut call = NodeCall::new(
1032                            node_config.thread_id.clone(),
1033                            name.clone(),
1034                            step,
1035                            node_state,
1036                        );
1037                        chain
1038                            .run_node(&mut call, |call| {
1039                                let ctx = NodeContext::new(call.state().clone(), node_config)
1040                                    .with_middleware(chain.clone())
1041                                    .with_optional_effect_admission(effect_admission);
1042                                let node = Arc::clone(&node);
1043                                async move { node.run(ctx).await }
1044                            })
1045                            .await
1046                    };
1047                    let latency_ms = (clock.now() - node_started).num_milliseconds().max(0) as u64;
1048                    (index, name, result, latency_ms)
1049                }
1050                .instrument(node_span),
1051            );
1052        }
1053        // The resume value is consumed by the first super-step after a resume.
1054        *pending_resume = None;
1055
1056        // -- barrier: collect every node result. The step is
1057        //    transactional: on any failure the JoinSet is dropped
1058        //    (aborting stragglers) and the step's writes are discarded.
1059        let mut writes: Vec<(String, HashMap<String, Value>)> = Vec::new();
1060        let mut commands: Vec<Command> = Vec::new();
1061        let mut ran_nodes: Vec<String> = Vec::new();
1062        let mut interrupted: Option<(usize, String, Value)> = None;
1063        // Journal payloads of finished invocations, in finish order here;
1064        // journaled in active-set order after the barrier.
1065        let mut completed: Vec<(usize, String, u64, Value)> = Vec::new();
1066
1067        while let Some(joined) = join_set.join_next().await {
1068            let (index, name, result, latency_ms) = joined.map_err(|e| {
1069                RustyError::Node(format!(
1070                    "node task failed to join (panic or cancellation): {e}"
1071                ))
1072            })?;
1073            match result {
1074                Ok(output) => {
1075                    Self::emit(
1076                        config,
1077                        GraphEvent::NodeEnd {
1078                            node: name.clone(),
1079                            step,
1080                        },
1081                    );
1082                    completed.push((
1083                        index,
1084                        name.clone(),
1085                        latency_ms,
1086                        serde_json::json!({
1087                            "updates": &output.updates,
1088                            "command": &output.command,
1089                        }),
1090                    ));
1091                    if let Some(command) = output.command {
1092                        if !command.goto.is_empty() {
1093                            commands.push(command);
1094                        }
1095                    }
1096                    ran_nodes.push(name.clone());
1097                    writes.push((name, output.updates));
1098                }
1099                Err(RustyError::Interrupt { value }) => {
1100                    // Record the suspension and stop the barrier loop; the
1101                    // JoinSet is dropped below to abort stragglers.
1102                    interrupted = Some((index, name, value));
1103                    break;
1104                }
1105                Err(e) => {
1106                    // A failed node is still evidence: journal the failure
1107                    // before the error unwinds the run.
1108                    recorder.record(
1109                        EventDraft::new(RunEventKind::NodeOutput, invocation_effects[index])
1110                            .node(name.clone())
1111                            .status(EventStatus::Error)
1112                            .output(serde_json::json!({ "error": e.to_string() }))
1113                            .latency_ms(latency_ms)
1114                            .parent(input_events[index].clone()),
1115                    );
1116                    // LLM and tool failures are the transient, retryable
1117                    // error classes; everything else is a hard failure.
1118                    let retryable = matches!(e, RustyError::Llm(_) | RustyError::Tool(_));
1119                    tracing::warn!(
1120                        node = %name,
1121                        step = step,
1122                        error = %e,
1123                        retryable = retryable,
1124                        "node failed; super-step aborted and its writes discarded"
1125                    );
1126                    return Err(RustyError::Node(format!(
1127                        "node `{name}` failed at super-step {step}: {e}"
1128                    )));
1129                }
1130            }
1131        }
1132
1133        if let Some((index, name, value)) = interrupted {
1134            // Suspend the run. The step is transactional, so no write of
1135            // this step survived — not even from siblings that completed
1136            // before the interrupt reached the barrier. The suspension
1137            // checkpoint therefore re-schedules the ENTIRE active set (the
1138            // interrupting node plus all siblings), otherwise completed
1139            // siblings' discarded writes would be silently lost and aborted
1140            // siblings would never re-run. Dropping the JoinSet first aborts
1141            // stragglers, preserving the transactional suspension point.
1142            drop(join_set);
1143            let interrupt_event = recorder.record(
1144                EventDraft::new(RunEventKind::Interrupt, invocation_effects[index])
1145                    .node(name.clone())
1146                    .input(value.clone())
1147                    .status(EventStatus::Interrupted)
1148                    .parent(input_events[index].clone()),
1149            );
1150            tracing::info!(
1151                node = %name,
1152                step = step,
1153                "node interrupted; run suspended (resumable via RunConfig::resume)"
1154            );
1155            let pending: Vec<String> = active.iter().map(|t| t.name.clone()).collect();
1156            let checkpoint =
1157                recorder.mint_checkpoint(config.thread_id.clone(), step, state.clone(), pending);
1158            let checkpoint_id = checkpoint.id.clone();
1159            if let Some(checkpointer) = &self.checkpointer {
1160                checkpointer.put(checkpoint).await?;
1161                recorder.record(
1162                    EventDraft::new(RunEventKind::CheckpointWritten, Effect::Idempotent)
1163                        .output(serde_json::json!({
1164                            "checkpoint_id": checkpoint_id,
1165                            "step": step,
1166                            "suspension": true,
1167                        }))
1168                        .parent(interrupt_event),
1169                );
1170                Self::emit(
1171                    config,
1172                    GraphEvent::CheckpointSaved {
1173                        checkpoint_id: checkpoint_id.clone(),
1174                        step,
1175                    },
1176                );
1177            }
1178            return Ok(StepTransition::Finish(ExecutionOutcome::Interrupted {
1179                value,
1180                state: state.clone(),
1181                checkpoint_id,
1182            }));
1183        }
1184
1185        // Journal node outputs in active-set order, not JoinSet finish
1186        // order: the finish order is scheduling-dependent, and the journal's
1187        // sequence is evidence — replay compares it.
1188        completed.sort_by_key(|(index, ..)| *index);
1189        let mut last_output_event: Option<String> = None;
1190        for (index, name, latency_ms, output_json) in completed {
1191            last_output_event = Some(
1192                recorder.record(
1193                    EventDraft::new(RunEventKind::NodeOutput, invocation_effects[index])
1194                        .node(name)
1195                        .output(output_json)
1196                        .latency_ms(latency_ms)
1197                        .parent(input_events[index].clone()),
1198                ),
1199            );
1200        }
1201
1202        // -- merge: reducers + LastValue single-write validation. On
1203        //    error the mutated state is dropped with the run
1204        //    (transactional super-step).
1205        //
1206        // The start-of-step snapshot's job ends at the barrier — every node
1207        // has reported (or the step has failed). Dropping it before the
1208        // merge releases its per-channel shared references, so channels no
1209        // checkpoint still shares reach the reducer with refcount 1 and
1210        // merge in place (W4 copy-on-write) instead of cloning.
1211        drop(snapshot);
1212        let written_channels: HashSet<String> = writes
1213            .iter()
1214            .flat_map(|(_, updates)| updates.keys().cloned())
1215            .collect();
1216        spec.apply_super_step(state, writes)?;
1217        // The event carries the post-reducer values read back out of the
1218        // merged state: when several nodes write the same channel in one
1219        // step (the normal Append fan-in case), reporting the raw partials
1220        // would keep only the last write and hide the rest.
1221        let mut merged_updates = serde_json::Map::new();
1222        for channel in &written_channels {
1223            if let Some(value) = state.get(channel) {
1224                merged_updates.insert(channel.clone(), value.clone());
1225            }
1226        }
1227        let channels_written: Vec<&str> = merged_updates.keys().map(String::as_str).collect();
1228        tracing::debug!(
1229            step = step,
1230            channels = ?channels_written,
1231            "merged node updates at super-step barrier"
1232        );
1233        // The super-step's reducer result is journaled even when empty —
1234        // "no channel changed" is evidence too.
1235        let step_end_event = recorder.record(
1236            EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1237                .output(Value::Object(merged_updates.clone()))
1238                .parent(last_output_event.unwrap_or_else(|| step_start_event.clone())),
1239        );
1240        if !merged_updates.is_empty() {
1241            Self::emit(
1242                config,
1243                GraphEvent::StateUpdate {
1244                    step,
1245                    updates: merged_updates,
1246                },
1247            );
1248        }
1249
1250        // -- route: Command::goto overrides the static edge set;
1251        //    otherwise evaluate outgoing edges of every node that ran
1252        //    against the post-barrier state.
1253        let mut next: Vec<ActiveTask> = Vec::new();
1254        let mut planned: HashSet<String> = HashSet::new();
1255
1256        if !commands.is_empty() {
1257            for command in &commands {
1258                for target in &command.goto {
1259                    if !graph.has_node(target) {
1260                        return Err(RustyError::Graph(format!(
1261                            "Command::goto references unknown node `{target}`"
1262                        )));
1263                    }
1264                    if planned.insert(target.clone()) {
1265                        next.push(ActiveTask {
1266                            name: target.clone(),
1267                            scoped: None,
1268                        });
1269                    }
1270                }
1271            }
1272        } else {
1273            let mut evaluated: HashSet<String> = HashSet::new();
1274            for name in &ran_nodes {
1275                // Fan-out invocations of the same node share one edge set;
1276                // evaluate it once.
1277                if !evaluated.insert(name.clone()) {
1278                    continue;
1279                }
1280                for edge in graph.outgoing_edges(name) {
1281                    match edge {
1282                        Edge::Direct { to, .. } => {
1283                            if planned.insert(to.clone()) {
1284                                next.push(ActiveTask {
1285                                    name: to.clone(),
1286                                    scoped: None,
1287                                });
1288                            }
1289                        }
1290                        Edge::Conditional { router, .. } => {
1291                            match router(state.clone()).await? {
1292                                Route::Node(target) => {
1293                                    if !graph.has_node(&target) {
1294                                        return Err(RustyError::Graph(format!(
1295                                            "conditional router from `{name}` returned \
1296                                             unknown node `{target}`"
1297                                        )));
1298                                    }
1299                                    if planned.insert(target.clone()) {
1300                                        next.push(ActiveTask {
1301                                            name: target,
1302                                            scoped: None,
1303                                        });
1304                                    }
1305                                }
1306                                Route::Send(sends) => {
1307                                    for send in sends {
1308                                        if !graph.has_node(&send.node) {
1309                                            return Err(RustyError::Graph(format!(
1310                                                "Route::Send from `{name}` targets unknown \
1311                                                 node `{}`",
1312                                                send.node
1313                                            )));
1314                                        }
1315                                        // Each Send is its own invocation with its own
1316                                        // scoped state, even when several target the
1317                                        // same node.
1318                                        next.push(ActiveTask {
1319                                            name: send.node,
1320                                            scoped: Some(send.state),
1321                                        });
1322                                    }
1323                                }
1324                                Route::End => {}
1325                            }
1326                        }
1327                    }
1328                }
1329            }
1330        }
1331
1332        let route_event = recorder.record(
1333            EventDraft::new(RunEventKind::RoutingDecision, Effect::Pure)
1334                .output(serde_json::json!({
1335                    "next": next
1336                        .iter()
1337                        .map(|t| serde_json::json!({
1338                            "node": &t.name,
1339                            "scoped": &t.scoped,
1340                        }))
1341                        .collect::<Vec<_>>(),
1342                    "goto": commands
1343                        .iter()
1344                        .flat_map(|command| command.goto.iter().cloned())
1345                        .collect::<Vec<String>>(),
1346                }))
1347                .parent(step_end_event.clone()),
1348        );
1349
1350        // -- checkpoint at the super-step boundary.
1351        if let Some(checkpointer) = &self.checkpointer {
1352            let next_names: Vec<String> = next.iter().map(|t| t.name.clone()).collect();
1353            let checkpoint =
1354                recorder.mint_checkpoint(config.thread_id.clone(), step, state.clone(), next_names);
1355            let checkpoint_id = checkpoint.id.clone();
1356            checkpointer.put(checkpoint).await?;
1357            recorder.record(
1358                EventDraft::new(RunEventKind::CheckpointWritten, Effect::Idempotent)
1359                    .output(serde_json::json!({
1360                        "checkpoint_id": checkpoint_id,
1361                        "step": step,
1362                        "suspension": false,
1363                    }))
1364                    .parent(route_event.clone()),
1365            );
1366            Self::emit(
1367                config,
1368                GraphEvent::CheckpointSaved {
1369                    checkpoint_id,
1370                    step,
1371                },
1372            );
1373        }
1374
1375        // -- terminate or schedule the next super-step.
1376        if next.is_empty() {
1377            return Ok(StepTransition::Finish(ExecutionOutcome::Done(
1378                state.clone(),
1379            )));
1380        }
1381        Ok(StepTransition::Next(next, route_event))
1382    }
1383
1384    /// Best-effort event emission: a full or closed channel never aborts a run.
1385    fn emit(config: &RunConfig, event: GraphEvent) {
1386        if let Some(tx) = &config.event_tx {
1387            let _ = tx.try_send(event);
1388        }
1389    }
1390}
1391
1392/// Per-run Flight Recorder state handed to every super-step: the journal,
1393/// the determinism seams, and the frozen provenance stamped into every
1394/// checkpoint of the run.
1395struct Recorder {
1396    journal: Journal,
1397    clock: Clock,
1398    rng: RngSource,
1399    graph_version: String,
1400    graph_hash: String,
1401    policy_version: PolicyVersion,
1402    manifest: Option<RunManifest>,
1403}
1404
1405impl Recorder {
1406    /// Append one event to the run's journal; returns the event id (the
1407    /// causal-parent handle for whatever the event causes).
1408    fn record(&self, draft: EventDraft) -> String {
1409        self.journal.record(draft)
1410    }
1411
1412    /// Mint a boundary checkpoint through the determinism seams: id from the
1413    /// run's RNG, timestamp from the run's clock, the frozen provenance
1414    /// header, and a journal reference pinning the evidence head as it stood
1415    /// *before* this checkpoint's own `checkpoint_written` event is
1416    /// recorded.
1417    fn mint_checkpoint(
1418        &self,
1419        thread_id: String,
1420        step: usize,
1421        state: State,
1422        next_nodes: Vec<String>,
1423    ) -> Checkpoint {
1424        Checkpoint {
1425            id: self.rng.uuid_string(),
1426            thread_id,
1427            step,
1428            state,
1429            next_nodes,
1430            created_at: self.clock.now(),
1431            header: CheckpointHeader {
1432                format_version: CURRENT_FORMAT_VERSION,
1433                graph_version: self.graph_version.clone(),
1434                graph_hash: self.graph_hash.clone(),
1435                policy_version: self.policy_version.clone(),
1436                logical_clock: self.clock.now_ms(),
1437                manifest: self.manifest.clone(),
1438            },
1439            journal_ref: Some(self.journal.head_ref()),
1440            // The executor always mints full snapshots; delta encoding is
1441            // decided inside the checkpointer (`checkpoint::encode_delta`).
1442            base: None,
1443        }
1444    }
1445}
1446
1447/// One scheduled node invocation within a super-step. `scoped` carries the
1448/// per-invocation input of a [`crate::graph::Send`] fan-out, overlaid onto
1449/// that invocation's private state snapshot before the node runs.
1450struct ActiveTask {
1451    name: String,
1452    scoped: Option<Value>,
1453}
1454
1455/// The control-flow result of a single super-step: either the next active
1456/// set (the loop continues) or the terminal run outcome (the loop breaks).
1457enum StepTransition {
1458    /// The next active set plus the routing-decision journal event that
1459    /// produced it (the causal parent of the next super-step's start event).
1460    Next(Vec<ActiveTask>, String),
1461    Finish(ExecutionOutcome),
1462}
1463
1464#[cfg(test)]
1465mod tests {
1466    use super::*;
1467    use crate::checkpoint::InMemoryCheckpointer;
1468    use crate::graph::GraphBuilder;
1469    use crate::llm::ChatModel;
1470    use crate::state::Reducer;
1471    use serde_json::json;
1472    use std::sync::Mutex;
1473
1474    #[tokio::test]
1475    async fn linear_two_node_graph_executes_in_order() {
1476        let spec = StateSpec::new().channel("log", Reducer::Append);
1477
1478        let mut builder = GraphBuilder::new();
1479        builder.add_node("first", |_ctx: NodeContext| async {
1480            Ok(NodeOutput::update("log", json!("first")))
1481        });
1482        builder.add_node("second", |ctx: NodeContext| async move {
1483            // The next super-step observes the previous step's merged writes.
1484            assert_eq!(ctx.state().get("log"), Some(&json!(["first"])));
1485            assert_eq!(ctx.step(), 1);
1486            Ok(NodeOutput::update("log", json!("second")))
1487        });
1488        builder.set_entry_point("first");
1489        builder.add_edge("first", "second");
1490        let graph = builder.compile().unwrap();
1491
1492        let outcome = Executor::new()
1493            .run(&graph, &spec, State::new(), RunConfig::new("t-linear"))
1494            .await
1495            .unwrap();
1496
1497        match outcome {
1498            ExecutionOutcome::Done(state) => {
1499                assert_eq!(state.get("log"), Some(&json!(["first", "second"])));
1500            }
1501            other => panic!("expected Done, got {other:?}"),
1502        }
1503    }
1504
1505    #[tokio::test]
1506    async fn parallel_fan_in_merges_via_reducer() {
1507        let spec = StateSpec::new().channel("results", Reducer::Append);
1508
1509        let mut builder = GraphBuilder::new();
1510        builder.add_node("start", |_ctx: NodeContext| async {
1511            Ok(NodeOutput::empty())
1512        });
1513        builder.add_node("worker_a", |ctx: NodeContext| async move {
1514            // Snapshot isolation: parallel workers cannot see each other.
1515            assert!(!ctx.state().contains("results"));
1516            Ok(NodeOutput::update("results", json!("a")))
1517        });
1518        builder.add_node("worker_b", |ctx: NodeContext| async move {
1519            assert!(!ctx.state().contains("results"));
1520            Ok(NodeOutput::update("results", json!("b")))
1521        });
1522        builder.set_entry_point("start");
1523        builder.add_edge("start", "worker_a");
1524        builder.add_edge("start", "worker_b");
1525        let graph = builder.compile().unwrap();
1526
1527        let outcome = Executor::new()
1528            .run(&graph, &spec, State::new(), RunConfig::new("t-fan-in"))
1529            .await
1530            .unwrap();
1531
1532        match outcome {
1533            ExecutionOutcome::Done(state) => {
1534                let results = state
1535                    .get("results")
1536                    .and_then(Value::as_array)
1537                    .expect("results channel must exist")
1538                    .clone();
1539                // Completion order across the JoinSet is nondeterministic.
1540                let mut items: Vec<&str> = results.iter().map(|v| v.as_str().unwrap()).collect();
1541                items.sort_unstable();
1542                assert_eq!(items, ["a", "b"]);
1543            }
1544            other => panic!("expected Done, got {other:?}"),
1545        }
1546    }
1547
1548    #[tokio::test]
1549    async fn interrupt_returns_interrupted_outcome_and_resume_completes() {
1550        let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
1551
1552        let mut builder = GraphBuilder::new();
1553        builder.add_node("gate", |ctx: NodeContext| async move {
1554            match ctx.resume_value() {
1555                Some(v) => Ok(NodeOutput::update("answer", v.clone())),
1556                None => Err(ctx.interrupt(json!({"question": "approve?"}))),
1557            }
1558        });
1559        builder.set_entry_point("gate");
1560        let graph = builder.compile().unwrap();
1561
1562        let checkpointer = Arc::new(InMemoryCheckpointer::new());
1563        let executor = Executor::with_checkpointer(checkpointer.clone());
1564
1565        // First run: the gate node interrupts and the run suspends.
1566        let outcome = executor
1567            .run(&graph, &spec, State::new(), RunConfig::new("t-hitl"))
1568            .await
1569            .unwrap();
1570
1571        let checkpoint_id = match outcome {
1572            ExecutionOutcome::Interrupted {
1573                value,
1574                checkpoint_id,
1575                ..
1576            } => {
1577                assert_eq!(value, json!({"question": "approve?"}));
1578                assert!(!checkpoint_id.is_empty());
1579                checkpoint_id
1580            }
1581            other => panic!("expected Interrupted, got {other:?}"),
1582        };
1583
1584        // The suspension point was persisted and schedules the gate node.
1585        let stored = checkpointer.get_latest("t-hitl").await.unwrap().unwrap();
1586        assert_eq!(stored.id, checkpoint_id);
1587        assert_eq!(stored.next_nodes, vec!["gate".to_string()]);
1588
1589        // Resume: the gate node re-runs with the resume value, writes its
1590        // answer, and the run terminates.
1591        let outcome = executor
1592            .run(
1593                &graph,
1594                &spec,
1595                State::new(),
1596                RunConfig::new("t-hitl").with_resume(json!(true)),
1597            )
1598            .await
1599            .unwrap();
1600
1601        match outcome {
1602            ExecutionOutcome::Done(state) => {
1603                assert_eq!(state.get("answer"), Some(&json!(true)));
1604            }
1605            other => panic!("expected Done after resume, got {other:?}"),
1606        }
1607    }
1608
1609    #[test]
1610    fn run_config_default_uses_default_step_limit() {
1611        // Regression: a derived `Default` would zero `max_steps`, making
1612        // every `RunConfig::default()` run fail immediately.
1613        let config = RunConfig::default();
1614        assert_eq!(config.max_steps, DEFAULT_MAX_STEPS);
1615        assert!(config.thread_id.is_empty());
1616        assert!(config.resume.is_none() && config.checkpoint_id.is_none());
1617    }
1618
1619    #[tokio::test]
1620    async fn interrupt_reschedules_entire_active_set() {
1621        // Regression: the suspension checkpoint used to schedule only the
1622        // interrupting node, silently dropping parallel siblings — including
1623        // ones that had already completed, whose writes are discarded with
1624        // the aborted step.
1625        let spec = StateSpec::new()
1626            .channel("log", Reducer::Append)
1627            .channel("answer", Reducer::Overwrite);
1628
1629        let mut builder = GraphBuilder::new();
1630        builder.add_node("start", |_ctx: NodeContext| async {
1631            Ok(NodeOutput::empty())
1632        });
1633        builder.add_node("gate", |ctx: NodeContext| async move {
1634            match ctx.resume_value() {
1635                Some(v) => Ok(NodeOutput::update("answer", v.clone())),
1636                None => Err(ctx.interrupt(json!({"question": "approve?"}))),
1637            }
1638        });
1639        // Completes immediately in the interrupted step; its write is
1640        // discarded with the step and must be recomputed on resume.
1641        builder.add_node("fast", |_ctx: NodeContext| async {
1642            Ok(NodeOutput::update("log", json!("fast")))
1643        });
1644        // Still in flight when the interrupt hits; aborted, re-run on resume.
1645        builder.add_node("slow", |ctx: NodeContext| async move {
1646            if ctx.resume_value().is_none() {
1647                tokio::time::sleep(std::time::Duration::from_secs(3600)).await;
1648            }
1649            Ok(NodeOutput::update("log", json!("slow")))
1650        });
1651        builder.set_entry_point("start");
1652        builder.add_edge("start", "gate");
1653        builder.add_edge("start", "slow");
1654        builder.add_edge("start", "fast");
1655        let graph = builder.compile().unwrap();
1656
1657        let checkpointer = Arc::new(InMemoryCheckpointer::new());
1658        let executor = Executor::with_checkpointer(checkpointer.clone());
1659
1660        let outcome = executor
1661            .run(&graph, &spec, State::new(), RunConfig::new("t-par-hitl"))
1662            .await
1663            .unwrap();
1664        match &outcome {
1665            ExecutionOutcome::Interrupted { state, .. } => {
1666                // Transactional suspension: fast's completed write was
1667                // discarded with the rest of the step.
1668                assert_eq!(state.get("log"), None);
1669            }
1670            other => panic!("expected Interrupted, got {other:?}"),
1671        }
1672
1673        // The suspension checkpoint re-schedules every node of the step.
1674        let stored = checkpointer
1675            .get_latest("t-par-hitl")
1676            .await
1677            .unwrap()
1678            .unwrap();
1679        let mut scheduled = stored.next_nodes.clone();
1680        scheduled.sort_unstable();
1681        assert_eq!(scheduled, ["fast", "gate", "slow"]);
1682
1683        // Resume: all three re-run (the resume value is broadcast to the
1684        // whole step); fast's write lands exactly once.
1685        let outcome = executor
1686            .run(
1687                &graph,
1688                &spec,
1689                State::new(),
1690                RunConfig::new("t-par-hitl").with_resume(json!(true)),
1691            )
1692            .await
1693            .unwrap();
1694        match outcome {
1695            ExecutionOutcome::Done(state) => {
1696                assert_eq!(state.get("answer"), Some(&json!(true)));
1697                let mut log: Vec<String> = state
1698                    .get("log")
1699                    .and_then(Value::as_array)
1700                    .expect("log channel must exist")
1701                    .iter()
1702                    .map(|v| v.as_str().unwrap().to_owned())
1703                    .collect();
1704                log.sort_unstable();
1705                assert_eq!(log, ["fast", "slow"]);
1706            }
1707            other => panic!("expected Done after resume, got {other:?}"),
1708        }
1709    }
1710
1711    #[tokio::test]
1712    async fn state_update_event_reports_post_reducer_values() {
1713        // Regression: with several writers on one channel, the event used to
1714        // carry raw per-node partials collapsed by last-write-wins, hiding
1715        // all but one write behind its documented "merged" contract.
1716        let spec = StateSpec::new().channel("results", Reducer::Append);
1717
1718        let mut builder = GraphBuilder::new();
1719        builder.add_node("start", |_ctx: NodeContext| async {
1720            Ok(NodeOutput::empty())
1721        });
1722        builder.add_node("worker_a", |_ctx: NodeContext| async {
1723            Ok(NodeOutput::update("results", json!("a")))
1724        });
1725        builder.add_node("worker_b", |_ctx: NodeContext| async {
1726            Ok(NodeOutput::update("results", json!("b")))
1727        });
1728        builder.set_entry_point("start");
1729        builder.add_edge("start", "worker_a");
1730        builder.add_edge("start", "worker_b");
1731        let graph = builder.compile().unwrap();
1732
1733        let (tx, mut rx) = mpsc::channel::<GraphEvent>(64);
1734        let outcome = Executor::new()
1735            .run(
1736                &graph,
1737                &spec,
1738                State::new(),
1739                RunConfig::new("t-event").with_event_tx(tx),
1740            )
1741            .await
1742            .unwrap();
1743        assert!(matches!(outcome, ExecutionOutcome::Done(_)));
1744
1745        let mut merged: Option<Vec<String>> = None;
1746        while let Ok(event) = rx.try_recv() {
1747            if let GraphEvent::StateUpdate { step: 1, updates } = event {
1748                let values = updates
1749                    .get("results")
1750                    .and_then(Value::as_array)
1751                    .expect("StateUpdate must carry the results channel")
1752                    .iter()
1753                    .map(|v| v.as_str().unwrap().to_owned())
1754                    .collect();
1755                merged = Some(values);
1756            }
1757        }
1758        let mut merged = merged.expect("expected a StateUpdate event for step 1");
1759        merged.sort_unstable();
1760        // Both partial writes are visible in the single post-reducer value.
1761        assert_eq!(merged, ["a", "b"]);
1762    }
1763
1764    #[tokio::test]
1765    async fn max_steps_guard_aborts_infinite_cycles() {
1766        let spec = StateSpec::new().channel("x", Reducer::Overwrite);
1767        let mut builder = GraphBuilder::new();
1768        builder.add_node("loop_node", |_ctx: NodeContext| async {
1769            Ok(NodeOutput::empty())
1770        });
1771        builder.set_entry_point("loop_node");
1772        builder.add_edge("loop_node", "loop_node");
1773        let graph = builder.compile().unwrap();
1774
1775        let err = Executor::new()
1776            .run(
1777                &graph,
1778                &spec,
1779                State::new(),
1780                RunConfig::new("t-loop").with_max_steps(5),
1781            )
1782            .await
1783            .unwrap_err();
1784        assert!(matches!(err, RustyError::Graph(_)));
1785    }
1786
1787    #[test]
1788    fn token_event_serde_roundtrip() {
1789        let event = GraphEvent::Token {
1790            node: "agent".into(),
1791            delta: "Hello".into(),
1792        };
1793        let wire = serde_json::to_string(&event).unwrap();
1794        // Internally tagged: the variant name travels on the wire.
1795        assert!(
1796            wire.contains("\"type\":\"token\""),
1797            "unexpected wire: {wire}"
1798        );
1799        let back: GraphEvent = serde_json::from_str(&wire).unwrap();
1800        assert_eq!(event, back);
1801
1802        // The other variants still roundtrip under the new serde derives.
1803        let step_event = GraphEvent::SuperStep {
1804            step: 3,
1805            active_nodes: vec!["a".into()],
1806        };
1807        let back: GraphEvent =
1808            serde_json::from_str(&serde_json::to_string(&step_event).unwrap()).unwrap();
1809        assert_eq!(step_event, back);
1810    }
1811
1812    /// A mock model whose `chat_stream` override emits real multi-chunk
1813    /// deltas, proving the accumulation contract outside any HTTP client.
1814    struct StreamingMock;
1815
1816    #[async_trait::async_trait]
1817    impl crate::llm::ChatModel for StreamingMock {
1818        async fn chat(
1819            &self,
1820            _messages: &[crate::llm::ChatMessage],
1821            _tools: &[Value],
1822        ) -> Result<crate::llm::ChatResponse> {
1823            Ok(crate::llm::ChatResponse {
1824                message: crate::llm::ChatMessage::assistant("Hello"),
1825                model: None,
1826                usage: None,
1827            })
1828        }
1829
1830        async fn chat_stream(
1831            &self,
1832            messages: &[crate::llm::ChatMessage],
1833            tools: &[Value],
1834            on_token: &mut (dyn FnMut(crate::llm::TokenChunk) + Send),
1835        ) -> Result<crate::llm::ChatResponse> {
1836            for piece in ["Hel", "lo"] {
1837                on_token(crate::llm::TokenChunk {
1838                    delta: piece.into(),
1839                    finish: false,
1840                    raw: None,
1841                });
1842            }
1843            on_token(crate::llm::TokenChunk {
1844                delta: String::new(),
1845                finish: true,
1846                raw: None,
1847            });
1848            self.chat(messages, tools).await
1849        }
1850    }
1851
1852    #[tokio::test]
1853    async fn node_streams_token_events_through_event_sink() {
1854        let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
1855
1856        let (tx, mut rx) = mpsc::channel::<GraphEvent>(64);
1857        // The wiring pattern from the ChatModel rustdoc: the node closure
1858        // captures a clone of the run's event sender and forwards chunks.
1859        let node_tx = tx.clone();
1860
1861        let mut builder = GraphBuilder::new();
1862        builder.add_node("agent", move |_ctx: NodeContext| {
1863            let node_tx = node_tx.clone();
1864            async move {
1865                let model = StreamingMock;
1866                let mut full = String::new();
1867                model
1868                    .chat_stream(&[], &[], &mut |chunk| {
1869                        if !chunk.delta.is_empty() {
1870                            full.push_str(&chunk.delta);
1871                            let _ = node_tx.try_send(GraphEvent::Token {
1872                                node: "agent".into(),
1873                                delta: chunk.delta,
1874                            });
1875                        }
1876                    })
1877                    .await
1878                    .unwrap();
1879                Ok(NodeOutput::update("answer", json!(full)))
1880            }
1881        });
1882        builder.set_entry_point("agent");
1883        let graph = builder.compile().unwrap();
1884
1885        let config = RunConfig::new("t-tokens").with_event_tx(tx);
1886        // The RunConfig helper hands out the same sender for node wiring.
1887        assert!(config.token_tx().is_some());
1888        // The Executor builder/accessor pair stores a broadcast handle.
1889        let executor = Executor::new().with_token_tx(config.token_tx().unwrap());
1890        assert!(executor.token_tx().is_some());
1891
1892        let outcome = executor
1893            .run(&graph, &spec, State::new(), config)
1894            .await
1895            .unwrap();
1896        match outcome {
1897            ExecutionOutcome::Done(state) => {
1898                assert_eq!(state.get("answer"), Some(&json!("Hello")))
1899            }
1900            other => panic!("expected Done, got {other:?}"),
1901        }
1902
1903        // Drain the event stream: token deltas interleave with executor events.
1904        let mut deltas = Vec::new();
1905        while let Ok(event) = rx.try_recv() {
1906            if let GraphEvent::Token { node, delta } = event {
1907                assert_eq!(node, "agent");
1908                deltas.push(delta);
1909            }
1910        }
1911        assert_eq!(deltas, ["Hel", "lo"]);
1912    }
1913
1914    /// A minimal `tracing::Subscriber` that records formatted event fields
1915    /// (`name=value` pairs) into a shared buffer, so tests can assert on the
1916    /// executor's instrumentation. Implemented directly against the
1917    /// `tracing` crate's own `Subscriber` trait (re-exported from
1918    /// `tracing-core`) — no `tracing-subscriber` dependency required.
1919    #[derive(Clone, Default)]
1920    struct EventCapture {
1921        events: Arc<Mutex<Vec<String>>>,
1922    }
1923
1924    /// Formats an event's fields as `"name=value "` pairs.
1925    struct FieldVisitor(String);
1926
1927    impl tracing::field::Visit for FieldVisitor {
1928        fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
1929            use std::fmt::Write as _;
1930            let _ = write!(self.0, "{}={:?} ", field.name(), value);
1931        }
1932    }
1933
1934    impl tracing::Subscriber for EventCapture {
1935        fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
1936            true
1937        }
1938
1939        fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
1940            // Spans are irrelevant to these assertions; one id serves all.
1941            tracing::span::Id::from_u64(1)
1942        }
1943
1944        fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
1945
1946        fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
1947
1948        fn event(&self, event: &tracing::Event<'_>) {
1949            let mut visitor = FieldVisitor(String::new());
1950            event.record(&mut visitor);
1951            self.events.lock().unwrap().push(visitor.0);
1952        }
1953
1954        fn enter(&self, _span: &tracing::span::Id) {}
1955
1956        fn exit(&self, _span: &tracing::span::Id) {}
1957    }
1958
1959    /// The instrumentation must be observability-only: installing a
1960    /// subscriber changes nothing about the run's outcome, and the expected
1961    /// telemetry (merge debug event, run-completion info event) is emitted.
1962    #[tokio::test]
1963    async fn instrumentation_emits_events_without_changing_outcome() {
1964        let capture = EventCapture::default();
1965        let events = capture.events.clone();
1966        // Global default subscriber, deliberately NOT a thread-local
1967        // `set_default`: callsite interest is cached process-wide and lazily
1968        // (re)registered from whichever thread first hits a callsite, so a
1969        // thread-local subscriber races with the other tests in this binary
1970        // that run graphs concurrently (they rebuild the cache against the
1971        // no-subscriber global default and our events get silently dropped).
1972        // A global default makes every thread's rebuild see this subscriber.
1973        // Setting it is additive — other tests neither set nor assert on
1974        // subscribers, and captured events from concurrent runs only help the
1975        // `any()` assertions below. `set_global_default` may only be called
1976        // once per process; this is the only test that installs a subscriber.
1977        tracing::subscriber::set_global_default(capture)
1978            .expect("no other test may install a global tracing subscriber");
1979
1980        let spec = StateSpec::new().channel("log", Reducer::Append);
1981        let mut builder = GraphBuilder::new();
1982        builder.add_node("only", |_ctx: NodeContext| async {
1983            Ok(NodeOutput::update("log", json!("x")))
1984        });
1985        builder.set_entry_point("only");
1986        let graph = builder.compile().unwrap();
1987
1988        let outcome = Executor::new()
1989            .run(&graph, &spec, State::new(), RunConfig::new("t-tracing"))
1990            .await
1991            .unwrap();
1992
1993        // Identical semantics: the run completes with the expected state.
1994        match outcome {
1995            ExecutionOutcome::Done(state) => {
1996                assert_eq!(state.get("log"), Some(&json!(["x"])));
1997            }
1998            other => panic!("expected Done, got {other:?}"),
1999        }
2000
2001        let captured = events.lock().unwrap();
2002        assert!(
2003            captured.iter().any(|e| e.contains("channels")),
2004            "expected a merge debug event listing written channels, got: {captured:?}"
2005        );
2006        assert!(
2007            captured
2008                .iter()
2009                .any(|e| e.contains("steps") && e.contains("duration_ms")),
2010            "expected a run-completion info event with steps and duration_ms, got: {captured:?}"
2011        );
2012    }
2013
2014    /// A 3-node linear graph (`a -> b -> c`) appending each node name to the
2015    /// `log` channel.
2016    fn linear_three_node_graph() -> (Graph, StateSpec) {
2017        let spec = StateSpec::new().channel("log", Reducer::Append);
2018        let mut builder = GraphBuilder::new();
2019        for name in ["a", "b", "c"] {
2020            builder.add_node(name, move |_ctx: NodeContext| async move {
2021                Ok(NodeOutput::update("log", json!(name)))
2022            });
2023        }
2024        builder.set_entry_point("a");
2025        builder.add_edge("a", "b");
2026        builder.add_edge("b", "c");
2027        (builder.compile().unwrap(), spec)
2028    }
2029
2030    #[tokio::test]
2031    async fn run_with_checkpoint_id_replays_from_earlier_state() {
2032        let (graph, spec) = linear_three_node_graph();
2033        let checkpointer = Arc::new(InMemoryCheckpointer::new());
2034        let executor = Executor::with_checkpointer(checkpointer.clone());
2035
2036        // Full run on the source thread: checkpoints at steps 0, 1, 2.
2037        let outcome = executor
2038            .run(&graph, &spec, State::new(), RunConfig::new("t-src"))
2039            .await
2040            .unwrap();
2041        match outcome {
2042            ExecutionOutcome::Done(state) => {
2043                assert_eq!(state.get("log"), Some(&json!(["a", "b", "c"])));
2044            }
2045            other => panic!("expected Done, got {other:?}"),
2046        }
2047
2048        let history = checkpointer.list("t-src").await.unwrap();
2049        assert_eq!(history.len(), 3);
2050        // The step-1 checkpoint: `a` and `b` have run, `c` is scheduled next.
2051        let step1 = history[1].clone();
2052        assert_eq!(step1.step, 1);
2053        assert_eq!(step1.state.get("log"), Some(&json!(["a", "b"])));
2054        assert_eq!(step1.next_nodes, vec!["c".to_string()]);
2055
2056        // Safe pattern: fork the thread at the step-1 checkpoint, then replay
2057        // the fork from that checkpoint (not the fork's latest — here the cut
2058        // point IS the latest, but `checkpoint_id` is what selects it).
2059        let copied = checkpointer
2060            .fork_thread("t-src", "t-fork", Some(&step1.id))
2061            .await
2062            .unwrap();
2063        assert_eq!(copied, 2);
2064
2065        let outcome = executor
2066            .run(
2067                &graph,
2068                &spec,
2069                State::new(),
2070                RunConfig::new("t-fork").with_checkpoint_id(step1.id.clone()),
2071            )
2072            .await
2073            .unwrap();
2074
2075        match outcome {
2076            ExecutionOutcome::Done(state) => {
2077                // Execution continued from the step-1 state: only `c` ran,
2078                // `b` was not re-executed.
2079                assert_eq!(state.get("log"), Some(&json!(["a", "b", "c"])));
2080            }
2081            other => panic!("expected Done, got {other:?}"),
2082        }
2083
2084        // The replay appended its own boundary checkpoint to the fork only.
2085        assert_eq!(checkpointer.list("t-src").await.unwrap().len(), 3);
2086        assert_eq!(checkpointer.list("t-fork").await.unwrap().len(), 3);
2087    }
2088
2089    #[tokio::test]
2090    async fn run_with_checkpoint_id_plus_resume_combined() {
2091        let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
2092
2093        let mut builder = GraphBuilder::new();
2094        builder.add_node("gate", |ctx: NodeContext| async move {
2095            match ctx.resume_value() {
2096                Some(v) => Ok(NodeOutput::update("answer", v.clone())),
2097                None => Err(ctx.interrupt(json!({"question": "approve?"}))),
2098            }
2099        });
2100        builder.set_entry_point("gate");
2101        let graph = builder.compile().unwrap();
2102
2103        let checkpointer = Arc::new(InMemoryCheckpointer::new());
2104        let executor = Executor::with_checkpointer(checkpointer);
2105
2106        // Suspend at the gate and capture the suspension checkpoint id.
2107        let outcome = executor
2108            .run(&graph, &spec, State::new(), RunConfig::new("t-hitl"))
2109            .await
2110            .unwrap();
2111        let checkpoint_id = match outcome {
2112            ExecutionOutcome::Interrupted { checkpoint_id, .. } => checkpoint_id,
2113            other => panic!("expected Interrupted, got {other:?}"),
2114        };
2115
2116        // checkpoint_id selects WHERE (the suspension checkpoint), resume
2117        // supplies the value delivered to the re-running gate node.
2118        let outcome = executor
2119            .run(
2120                &graph,
2121                &spec,
2122                State::new(),
2123                RunConfig::new("t-hitl")
2124                    .with_checkpoint_id(checkpoint_id)
2125                    .with_resume(json!(true)),
2126            )
2127            .await
2128            .unwrap();
2129
2130        match outcome {
2131            ExecutionOutcome::Done(state) => {
2132                assert_eq!(state.get("answer"), Some(&json!(true)));
2133            }
2134            other => panic!("expected Done after replay+resume, got {other:?}"),
2135        }
2136    }
2137
2138    #[tokio::test]
2139    async fn run_with_checkpoint_id_errors_without_checkpointer_or_unknown_id() {
2140        let (graph, spec) = linear_three_node_graph();
2141
2142        // No checkpointer configured: replay is impossible.
2143        let err = Executor::new()
2144            .run(
2145                &graph,
2146                &spec,
2147                State::new(),
2148                RunConfig::new("t-x").with_checkpoint_id("some-id"),
2149            )
2150            .await
2151            .unwrap_err();
2152        assert!(matches!(err, RustyError::Checkpoint(_)));
2153
2154        // Checkpointer present but the id does not exist on the thread.
2155        let checkpointer = Arc::new(InMemoryCheckpointer::new());
2156        let executor = Executor::with_checkpointer(checkpointer.clone());
2157        executor
2158            .run(&graph, &spec, State::new(), RunConfig::new("t-x"))
2159            .await
2160            .unwrap();
2161        let err = executor
2162            .run(
2163                &graph,
2164                &spec,
2165                State::new(),
2166                RunConfig::new("t-x").with_checkpoint_id("no-such-checkpoint"),
2167            )
2168            .await
2169            .unwrap_err();
2170        assert!(matches!(err, RustyError::Checkpoint(_)));
2171    }
2172
2173    /// A self-looping spinner: each super-step increments `n`; the router
2174    /// terminates the run once `n` reaches 5.
2175    fn spinner_graph() -> (Graph, StateSpec) {
2176        let spec = StateSpec::new().channel("n", Reducer::Overwrite);
2177        let mut builder = GraphBuilder::new();
2178        builder.add_node("spin", |ctx: NodeContext| async move {
2179            // A paced step: without it the loop would burn through the
2180            // default max_steps budget faster than a test can cancel it.
2181            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2182            let n = ctx.state().get("n").and_then(Value::as_i64).unwrap_or(0);
2183            Ok(NodeOutput::update("n", json!(n + 1)))
2184        });
2185        builder.set_entry_point("spin");
2186        builder.add_conditional_edges("spin", |state: State| async move {
2187            if state.get("n").and_then(Value::as_i64).unwrap_or(0) >= 5 {
2188                Ok(Route::End)
2189            } else {
2190                Ok(Route::Node("spin".into()))
2191            }
2192        });
2193        (builder.compile().unwrap(), spec)
2194    }
2195
2196    #[tokio::test]
2197    async fn pre_cancelled_token_stops_the_run_before_the_first_step() {
2198        let (graph, spec) = spinner_graph();
2199        let token = CancellationToken::new();
2200        token.cancel();
2201
2202        let err = Executor::new()
2203            .run(
2204                &graph,
2205                &spec,
2206                State::new(),
2207                RunConfig::new("t-cancel").with_cancellation(token),
2208            )
2209            .await
2210            .unwrap_err();
2211        assert!(matches!(err, RustyError::Cancelled(_)));
2212        assert!(
2213            err.to_string().contains("after 0 super-step(s)"),
2214            "a pre-cancelled run must not execute a single step: {err}"
2215        );
2216    }
2217
2218    #[tokio::test]
2219    async fn cancellation_stops_at_a_boundary_and_the_run_resumes_from_it() {
2220        let (graph, spec) = spinner_graph();
2221        let checkpointer = Arc::new(InMemoryCheckpointer::new());
2222        let executor = Executor::with_checkpointer(checkpointer.clone());
2223        let token = CancellationToken::new();
2224
2225        // Cancel as soon as the first boundary checkpoint lands: the stop
2226        // is then provably the token's doing, not the terminating route.
2227        let canceller = token.clone();
2228        let watcher = checkpointer.clone();
2229        tokio::spawn(async move {
2230            loop {
2231                if watcher.get_latest("t-cancel").await.unwrap().is_some() {
2232                    break;
2233                }
2234                tokio::time::sleep(std::time::Duration::from_millis(1)).await;
2235            }
2236            canceller.cancel();
2237        });
2238        let err = executor
2239            .run(
2240                &graph,
2241                &spec,
2242                State::new(),
2243                RunConfig::new("t-cancel").with_cancellation(token.clone()),
2244            )
2245            .await
2246            .unwrap_err();
2247        assert!(matches!(err, RustyError::Cancelled(_)));
2248
2249        // The boundary checkpoint is intact: n counts the completed steps —
2250        // strictly fewer than the terminating route's 5, proving the token
2251        // (not the route) ended the run.
2252        let checkpoint = checkpointer
2253            .get_latest("t-cancel")
2254            .await
2255            .unwrap()
2256            .expect("a boundary checkpoint must survive cancellation");
2257        let steps = checkpoint.state.get("n").and_then(Value::as_i64).unwrap();
2258        assert!(
2259            (1..5).contains(&steps),
2260            "unexpected stop point: n = {steps}"
2261        );
2262
2263        // Re-running the thread resumes the spinner from the checkpoint.
2264        // Drain tokens are one-shot (a cancelled token stays cancelled), so
2265        // the post-deploy process runs under a fresh one.
2266        let outcome = executor
2267            .run(
2268                &graph,
2269                &spec,
2270                State::new(),
2271                RunConfig::new("t-cancel")
2272                    .with_resume(json!(null))
2273                    .with_cancellation(CancellationToken::new()),
2274            )
2275            .await
2276            .unwrap();
2277        match outcome {
2278            ExecutionOutcome::Done(state) => {
2279                assert_eq!(state.get("n"), Some(&json!(5)));
2280            }
2281            other => panic!("expected Done after resume, got {other:?}"),
2282        }
2283    }
2284}