rusty_agent_runtime/executor.rs
1//! The executor: a Pregel/BSP-inspired super-step run loop.
2//!
3//! Execution proceeds in discrete **super-steps** (Google Pregel /
4//! Bulk-Synchronous-Parallel), each super-step being:
5//!
6//! 1. **Plan** — determine the active node set (entry point on step 0;
7//! afterwards the routing result of the previous step, including
8//! [`crate::node::Command::goto`] overrides and [`crate::graph::Send`]
9//! fan-outs).
10//! 2. **Compute** — run all active nodes concurrently in a
11//! `tokio::task::JoinSet`, each receiving an **immutable snapshot** of
12//! the state as of the start of the step. No node can observe another's
13//! in-progress writes.
14//! 3. **Barrier** — wait for all active nodes. The step is *transactional*:
15//! if any node fails, the step's writes are discarded. An
16//! [`RustyError::Interrupt`] suspends the whole run instead.
17//! 4. **Merge** — apply all node updates to the state via
18//! [`crate::state::StateSpec::apply_super_step`] (per-channel reducers +
19//! `LastValue` single-write validation).
20//! 5. **Route** — evaluate outgoing edges / commands against the
21//! post-barrier state to determine the next active set; `Route::End` (or
22//! an empty next set) terminates the run.
23//! 6. **Checkpoint** — persist a [`crate::checkpoint::Checkpoint`] recording
24//! step, state, and next nodes, and emit [`GraphEvent`]s for streaming.
25//!
26//! A graph *cycle* (e.g. the ReAct loop `agent → tools → agent`) is not
27//! call-stack recursion — it is nodes being re-scheduled across super-steps,
28//! which is why the guard is `max_steps`, not a stack limit.
29//!
30//! # Observability
31//!
32//! The executor emits `tracing` telemetry throughout a run (no subscriber is
33//! installed by the library — the application chooses one):
34//!
35//! - `rusty.run` (INFO span) — one per [`Executor::run`] call, carrying
36//! `thread_id` and `max_steps`; parent of everything below.
37//! - `rusty.super_step` (DEBUG span) — one per super-step, carrying
38//! `step` and `active_nodes`; covers plan → barrier → merge → route.
39//! - `rusty.node` (INFO span) — one per spawned node task, carrying
40//! `node` and `step`; attached to the `JoinSet` task via `.instrument()`.
41//! - DEBUG event on each barrier merge (channels written), INFO events on
42//! interrupt and run completion (`steps`, `duration_ms`), WARN events on
43//! node failure (with a `retryable` classification).
44
45use std::collections::{HashMap, HashSet};
46use std::sync::{Arc, Mutex};
47
48use serde::{Deserialize, Serialize};
49use serde_json::Value;
50use tokio::sync::mpsc;
51use tokio::task::JoinSet;
52use tokio_util::sync::CancellationToken;
53use tracing::Instrument;
54
55use crate::checkpoint::{Checkpoint, Checkpointer};
56use crate::effects::{CompensationRegistry, EffectAdmissionContext};
57use crate::error::{Result, RustyError};
58use crate::graph::{Edge, Graph, Route};
59use crate::journal::{Clock, EventDraft, Journal, RngSource};
60use crate::middleware::{Middleware, MiddlewareChain, NodeCall};
61use crate::node::{Command, NodeConfig, NodeContext, NodeOutput};
62use crate::record::{
63 CheckpointHeader, Effect, EventStatus, PolicyVersion, RunEventKind, RunManifest,
64 CURRENT_FORMAT_VERSION,
65};
66use crate::state::{State, StateSpec};
67
68/// How a run ended.
69#[derive(Debug)]
70pub enum ExecutionOutcome {
71 /// The run terminated normally (routing reached `Route::End` or no
72 /// nodes remained active). Carries the final state.
73 Done(State),
74
75 /// A node called `interrupt(payload)`: the run is suspended and
76 /// resumable. Carry on by calling [`Executor::run`] again with the same
77 /// `thread_id` and `RunConfig::resume` set.
78 ///
79 /// Suspension is run-wide, not node-local. The in-flight super-step is
80 /// transactional: every write of the step is discarded (including writes
81 /// from sibling nodes that completed before the interrupt was observed
82 /// at the barrier), still-running siblings are aborted, and the
83 /// suspension checkpoint re-schedules **every** node of the step. On
84 /// resume all of them re-execute from their start, so node logic must be
85 /// idempotent.
86 Interrupted {
87 /// The payload passed to `interrupt()` (surfaced to the caller,
88 /// e.g. a human-approval request).
89 value: Value,
90 /// The state as of the suspension point.
91 state: State,
92 /// The checkpoint persisted at the suspension point, for resuming
93 /// or time travel. When the executor has no checkpointer attached,
94 /// the run still suspends but nothing is persisted: the id is then
95 /// only an opaque handle and can never be replayed.
96 checkpoint_id: String,
97 },
98}
99
100impl ExecutionOutcome {
101 /// The final (or suspension-point) state, regardless of variant.
102 pub fn state(&self) -> &State {
103 match self {
104 ExecutionOutcome::Done(s) => s,
105 ExecutionOutcome::Interrupted { state, .. } => state,
106 }
107 }
108
109 /// Whether the run ended in [`ExecutionOutcome::Interrupted`] (suspended,
110 /// resumable) rather than [`ExecutionOutcome::Done`].
111 pub fn is_interrupted(&self) -> bool {
112 matches!(self, ExecutionOutcome::Interrupted { .. })
113 }
114}
115
116/// Per-run configuration (the LangGraph `RunnableConfig` analog).
117#[derive(Debug, Clone)]
118pub struct RunConfig {
119 /// Thread (session) id. Stable across interrupt/resume; namespaces all
120 /// checkpoints for this run. Required for persistence and resume.
121 pub thread_id: String,
122
123 /// Maximum number of super-steps before the run aborts with
124 /// [`crate::error::RustyError::Graph`] (the LangGraph
125 /// `recursion_limit` / `GraphRecursionError` guard). Default: 1000.
126 pub max_steps: usize,
127
128 /// Resume value for continuing an interrupted run. When set, the
129 /// executor restores the latest checkpoint for `thread_id` and
130 /// re-executes the checkpointed next-node set with
131 /// [`crate::node::NodeContext::resume_value`] returning this value.
132 ///
133 /// The value is **broadcast**: every node scheduled in the first
134 /// super-step after the resume observes it, not only the node that
135 /// originally interrupted (a suspension checkpoint re-schedules the
136 /// whole active set — see [`ExecutionOutcome::Interrupted`]). Nodes that
137 /// should react only when they themselves were resumed must key off
138 /// their own state, not the presence of a resume value.
139 pub resume: Option<Value>,
140
141 /// Replay/time-travel handle: the id of a specific checkpoint of
142 /// `thread_id` to resume from. When set, the executor loads **that**
143 /// checkpoint (not the latest) and continues the run from its state and
144 /// next-node set. Requires a checkpointer on the executor.
145 ///
146 /// Combines with `resume`: `checkpoint_id` selects **where** the run
147 /// restarts, `resume` (when also set) is delivered as the resume value to
148 /// the first super-step, exactly as in interrupt/resume.
149 ///
150 /// Safe pattern: replaying on the *same* thread appends new history on
151 /// top of the old timeline, so prefer forking first —
152 /// [`crate::checkpoint::Checkpointer::fork_thread`] the thread into a new
153 /// thread id, then run the fork with `checkpoint_id` set. Direct replay
154 /// on the original thread is supported for cases where appended history
155 /// is acceptable.
156 pub checkpoint_id: Option<String>,
157
158 /// Optional event sink for streaming: the executor emits [`GraphEvent`]s
159 /// as the run progresses (node start/end, state updates, checkpoints,
160 /// super-step boundaries). Consumers implement LangGraph's stream modes
161 /// (`values` / `updates` / `tasks` / ...) as filters over this stream.
162 pub event_tx: Option<mpsc::Sender<GraphEvent>>,
163
164 /// Flight Recorder determinism seam (R0.5): the run's time source. Every
165 /// executor timestamp — event `recorded_at`, node latencies, checkpoint
166 /// `created_at` — is read through it. `None` (the default) is the system
167 /// wall clock, byte-identical to pre-R0.5 behavior. Attach
168 /// [`Clock::Logical`] to make a recorded run re-drivable.
169 pub clock: Option<Clock>,
170
171 /// Flight Recorder determinism seam (R0.5): the run's randomness source.
172 /// Checkpoint ids (and the run id, when the executor creates the
173 /// journal) are minted through it. `None` (the default) is OS entropy,
174 /// byte-identical to pre-R0.5 behavior; [`RngSource::Seeded`] makes the
175 /// id stream reproducible.
176 pub rng: Option<RngSource>,
177
178 /// Flight Recorder (R0.5): attach a pre-built journal for this run. Node
179 /// closures capture a clone of the same [`Journal`] to record their own
180 /// model/tool/remote/WASM calls into the run's evidence (the journal
181 /// stamps sequence numbers and timestamps itself). When `None`, the
182 /// executor creates a fresh journal per run; either way the run's
183 /// journal is retrievable afterwards via [`Executor::journal`].
184 pub journal: Option<Journal>,
185
186 /// Flight Recorder (R0.5): the executor policy active for this run,
187 /// stamped into every checkpoint header. `None` records the static
188 /// default ([`PolicyVersion::STATIC_V0`]) — except on resume (R0.8 wave
189 /// 4), where the run inherits the version its checkpoint header pins so
190 /// an in-flight run is immune to mid-run promotions. An explicit pin
191 /// always wins over inheritance.
192 pub policy_version: Option<PolicyVersion>,
193
194 /// Flight Recorder (R0.5): the application's own graph version string,
195 /// stamped into every checkpoint header next to the topology hash.
196 /// `None` records `"unversioned"`. Bump it when node bodies change in
197 /// ways the topology hash cannot see.
198 pub graph_version: Option<String>,
199
200 /// Cooperative cancellation (R0.6 wave 2c, drain). When set, the
201 /// executor checks the token **at every super-step boundary** and, once
202 /// it is cancelled, stops the run before starting the next super-step,
203 /// returning [`RustyError::Cancelled`].
204 ///
205 /// The boundary is the only cancellation point, deliberately: a
206 /// super-step is transactional (its nodes run off an immutable snapshot
207 /// and its writes merge atomically), and its boundary checkpoint is
208 /// already persisted by the time the token is observed — so a cancelled
209 /// run resumes from exactly where it stopped, with no torn step and no
210 /// lost writes. Cancellation granularity is therefore one super-step;
211 /// work inside the in-flight step runs to its barrier. This is the hook
212 /// graceful shutdown uses: a draining server cancels the token and every
213 /// in-flight run parks itself at a resumable checkpoint instead of being
214 /// torn down mid-step.
215 pub cancellation: Option<CancellationToken>,
216
217 /// Effect kernel v2 (R0.7): the versioned run manifest — prompts, tool
218 /// schemas, model and parameters, memory schema, capsule versions the run
219 /// pins. Stamped into every checkpoint header so a resumed run keeps
220 /// executing against its pinned versions (see
221 /// [`crate::record::RunManifest`] for the upgrade-safety contract).
222 /// `None` (the default) pins nothing and leaves checkpoint bytes
223 /// byte-identical to R0.5/R0.6.
224 pub manifest: Option<RunManifest>,
225
226 /// Effect kernel v2 (R0.7): the approval tokens this run carries for its
227 /// irreversible effects (see [`crate::effects::ApprovalToken`]). The
228 /// executor consults them when [`Executor::with_effect_admission`] has
229 /// enabled the guarded tool path. Empty by default; an irreversible call
230 /// dispatched through that path then fails admission before its body
231 /// runs.
232 pub effect_approvals: Vec<crate::effects::ApprovalToken>,
233
234 /// Shadow runs (R0.12 wave 4): an explicit admission context for this
235 /// run, overriding the one the executor would otherwise build from
236 /// [`Executor::with_effect_admission`]. A shadow deployment builds a
237 /// context whose boundary refuses every effect above read-only and serves
238 /// refused calls from the recorded world (see
239 /// [`crate::effects::EffectAdmissionContext::shadow`]); injecting it here
240 /// is what makes the run execute against that boundary instead of the
241 /// production one. `None` (the default) is byte-identical to prior
242 /// behavior.
243 pub effect_admission: Option<EffectAdmissionContext>,
244}
245
246impl Default for RunConfig {
247 /// A config with an empty `thread_id` and the default step limit —
248 /// identical to `RunConfig::new("")`. Derived `Default` would zero
249 /// `max_steps`, so any `default()`-built run would instantly trip the
250 /// step guard; keep this impl in sync with [`RunConfig::new`].
251 fn default() -> Self {
252 Self::new(String::new())
253 }
254}
255
256impl RunConfig {
257 /// A config for `thread_id` with the default step limit.
258 pub fn new(thread_id: impl Into<String>) -> Self {
259 Self {
260 thread_id: thread_id.into(),
261 max_steps: DEFAULT_MAX_STEPS,
262 resume: None,
263 checkpoint_id: None,
264 event_tx: None,
265 clock: None,
266 rng: None,
267 journal: None,
268 policy_version: None,
269 graph_version: None,
270 cancellation: None,
271 manifest: None,
272 effect_approvals: Vec::new(),
273 effect_admission: None,
274 }
275 }
276
277 /// Builder-style: override the step limit.
278 pub fn with_max_steps(mut self, max_steps: usize) -> Self {
279 self.max_steps = max_steps;
280 self
281 }
282
283 /// Builder-style: set the resume value.
284 pub fn with_resume(mut self, value: Value) -> Self {
285 self.resume = Some(value);
286 self
287 }
288
289 /// Builder-style: replay from a specific checkpoint of `thread_id`
290 /// (time travel). See the [`RunConfig::checkpoint_id`] field docs for
291 /// semantics and the fork-first safe pattern.
292 pub fn with_checkpoint_id(mut self, checkpoint_id: impl Into<String>) -> Self {
293 self.checkpoint_id = Some(checkpoint_id.into());
294 self
295 }
296
297 /// Builder-style: attach a streaming event sink.
298 pub fn with_event_tx(mut self, tx: mpsc::Sender<GraphEvent>) -> Self {
299 self.event_tx = Some(tx);
300 self
301 }
302
303 /// Builder-style: source the run's time from `clock` (Flight Recorder
304 /// determinism seam; see the [`RunConfig::clock`] field docs).
305 pub fn with_clock(mut self, clock: Clock) -> Self {
306 self.clock = Some(clock);
307 self
308 }
309
310 /// Builder-style: source the run's randomness from `rng` (Flight
311 /// Recorder determinism seam; see the [`RunConfig::rng`] field docs).
312 pub fn with_rng(mut self, rng: RngSource) -> Self {
313 self.rng = Some(rng);
314 self
315 }
316
317 /// Builder-style: record the run into `journal` (Flight Recorder; see
318 /// the [`RunConfig::journal`] field docs). When the config carries no
319 /// explicit [`RunConfig::clock`], the executor also reads time from the
320 /// attached journal's clock, keeping one time source per run.
321 pub fn with_journal(mut self, journal: Journal) -> Self {
322 self.journal = Some(journal);
323 self
324 }
325
326 /// Builder-style: stamp the active executor policy version into every
327 /// checkpoint header (see the [`RunConfig::policy_version`] field docs).
328 pub fn with_policy_version(mut self, version: PolicyVersion) -> Self {
329 self.policy_version = Some(version);
330 self
331 }
332
333 /// Builder-style: stamp the application's graph version into every
334 /// checkpoint header (see the [`RunConfig::graph_version`] field docs).
335 pub fn with_graph_version(mut self, version: impl Into<String>) -> Self {
336 self.graph_version = Some(version.into());
337 self
338 }
339
340 /// Builder-style: make the run cooperatively cancellable (see the
341 /// [`RunConfig::cancellation`] field docs). The token is observed only at
342 /// super-step boundaries: cancelling it stops the run *after* the
343 /// in-flight step's barrier and boundary checkpoint, so the run is left
344 /// resumable from exactly that checkpoint with [`RustyError::Cancelled`].
345 pub fn with_cancellation(mut self, token: CancellationToken) -> Self {
346 self.cancellation = Some(token);
347 self
348 }
349
350 /// Builder-style: pin the run's versioned manifest into every checkpoint
351 /// header (see the [`RunConfig::manifest`] field docs).
352 pub fn with_manifest(mut self, manifest: RunManifest) -> Self {
353 self.manifest = Some(manifest);
354 self
355 }
356
357 /// Builder-style: carry approval tokens for the run's irreversible
358 /// effects (see the [`RunConfig::effect_approvals`] field docs).
359 pub fn with_effect_approvals(
360 mut self,
361 approvals: impl IntoIterator<Item = crate::effects::ApprovalToken>,
362 ) -> Self {
363 self.effect_approvals = approvals.into_iter().collect();
364 self
365 }
366
367 /// Builder-style: inject an explicit admission context for this run (see
368 /// the [`RunConfig::effect_admission`] field docs). Shadow deployments
369 /// use it to run the graph against the shadow boundary instead of the
370 /// production one.
371 pub fn with_effect_admission_context(mut self, context: EffectAdmissionContext) -> Self {
372 self.effect_admission = Some(context);
373 self
374 }
375
376 /// A clone of the event sink sender, for wiring into nodes that stream
377 /// [`GraphEvent::Token`] deltas (LangGraph's `messages` stream mode).
378 ///
379 /// Typical flow: create the channel, call `config.token_tx()` to obtain
380 /// the clone a node closure captures, then hand `config` to
381 /// [`Executor::run`]. `None` when no sink is attached. See the
382 /// [`crate::llm::ChatModel`] rustdoc for the full pattern.
383 pub fn token_tx(&self) -> Option<mpsc::Sender<GraphEvent>> {
384 self.event_tx.clone()
385 }
386}
387
388/// Default super-step limit. Deliberately far above LangGraph's default
389/// `recursion_limit` of 25: ReAct-style loops burn one super-step per
390/// agent/tool hop, so long tool chains legitimately exceed 25.
391pub const DEFAULT_MAX_STEPS: usize = 1000;
392
393/// Streaming events emitted by the executor during a run. All of LangGraph's
394/// stream modes are views over this single typed event stream.
395///
396/// The enum is serializable so event streams can cross process / FFI
397/// boundaries (e.g. a WebSocket bridge or a persisted event log).
398#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
399#[serde(tag = "type", rename_all = "snake_case")]
400pub enum GraphEvent {
401 /// A node began executing.
402 NodeStart {
403 /// Node name.
404 node: String,
405 /// Super-step index.
406 step: usize,
407 },
408 /// A node finished executing (successfully).
409 NodeEnd {
410 /// Node name.
411 node: String,
412 /// Super-step index.
413 step: usize,
414 },
415 /// A single LLM token delta (the LangGraph `messages` stream mode).
416 ///
417 /// The executor itself never emits this variant: tokens originate inside
418 /// nodes that call [`crate::llm::ChatModel::chat_stream`] and forward
419 /// each [`crate::llm::TokenChunk`] into the run's event channel (see the
420 /// `ChatModel` rustdoc for the wiring pattern and
421 /// [`RunConfig::token_tx`] / [`Executor::token_tx`] for sender handles).
422 Token {
423 /// Node that produced the token.
424 node: String,
425 /// Incremental text produced since the previous token.
426 delta: String,
427 },
428 /// State was updated at a super-step barrier (`updates` stream mode).
429 StateUpdate {
430 /// Super-step index at which the update was applied.
431 step: usize,
432 /// Every channel written in this step mapped to its **post-reducer**
433 /// value (e.g. the full appended list for an `Append` channel), read
434 /// back from the merged state — not the raw per-node partials.
435 updates: serde_json::Map<String, Value>,
436 },
437 /// A checkpoint was persisted at a super-step boundary.
438 CheckpointSaved {
439 /// The checkpoint id.
440 checkpoint_id: String,
441 /// Super-step index at the boundary.
442 step: usize,
443 },
444 /// A super-step began; lists the nodes activated in it.
445 SuperStep {
446 /// Super-step index.
447 step: usize,
448 /// Nodes active in this step.
449 active_nodes: Vec<String>,
450 },
451}
452
453/// The graph executor. Holds an optional checkpointer and an optional
454/// middleware chain; stateless with respect to individual runs, so one
455/// `Executor` can drive many concurrent runs (each with its own
456/// `thread_id`).
457#[derive(Default)]
458pub struct Executor {
459 checkpointer: Option<Arc<dyn Checkpointer>>,
460 token_tx: Option<mpsc::Sender<GraphEvent>>,
461 // The most recent run's Flight Recorder journal. Interior mutability
462 // keeps `run` taking `&self`; overwritten at the start of every run.
463 journal: Mutex<Option<Journal>>,
464 // Middleware/Interceptor SDK: ordered layers wrapping every node
465 // invocation (and, via `NodeContext::middleware`, the tool/model calls
466 // node code makes). An empty chain takes the original dispatch path.
467 middleware: MiddlewareChain,
468 // R0.8 wave 1: the governed-memory source this executor's runs answer
469 // memory reads from (live store or replay cursor). The executor never
470 // touches memory itself this wave — the full run-loop integration is a
471 // later wave; this is the installation point node factories use (see
472 // `Executor::memory`), mirroring `with_token_tx`.
473 memory_source: Option<crate::memory::MemorySource>,
474 // Present only when guarded tool effect admission is enabled. The run's
475 // thread id and approval tokens are combined with these rollback handlers
476 // into an EffectAdmissionContext for each node invocation.
477 effect_compensations: Option<CompensationRegistry>,
478}
479
480impl Executor {
481 /// An executor without persistence (runs cannot be interrupted/resumed
482 /// durably; interrupts will still surface but resume requires a
483 /// checkpointer).
484 pub fn new() -> Self {
485 Self::default()
486 }
487
488 /// An executor persisting checkpoints through `checkpointer`.
489 pub fn with_checkpointer(checkpointer: Arc<dyn Checkpointer>) -> Self {
490 Self {
491 checkpointer: Some(checkpointer),
492 ..Self::default()
493 }
494 }
495
496 /// Builder-style: attach a middleware layer (Middleware/Interceptor
497 /// SDK). Layers compose in registration order — before-hooks run in
498 /// `.layer()` order on the way into a node/model/tool call, after-hooks
499 /// in reverse order on the way out. See [`crate::middleware`].
500 pub fn layer<M: Middleware + 'static>(mut self, middleware: M) -> Self {
501 self.middleware.push(Arc::new(middleware));
502 self
503 }
504
505 /// Builder-style: attach a pre-shared middleware layer.
506 pub fn layer_shared(mut self, middleware: Arc<dyn Middleware>) -> Self {
507 self.middleware.push(middleware);
508 self
509 }
510
511 /// The attached middleware chain (empty when no layers were added).
512 pub fn middleware(&self) -> &MiddlewareChain {
513 &self.middleware
514 }
515
516 /// Builder-style: make guarded tool effect admission available to nodes.
517 ///
518 /// The supplied registry may be empty. Pure and read-only calls remain
519 /// automatic; idempotent calls require a key, compensatable calls require
520 /// a handler in this registry, and non-idempotent calls require a matching
521 /// token in [`RunConfig::effect_approvals`]. The prebuilt ReAct tools node
522 /// enforces this automatically. Custom nodes must pass
523 /// [`NodeContext::effect_admission`] to the [`crate::tool::ToolExecutor`]
524 /// they construct; direct [`crate::tool::Tool::call`] invocations are not
525 /// intercepted by the executor.
526 pub fn with_effect_admission(mut self, compensations: CompensationRegistry) -> Self {
527 self.effect_compensations = Some(compensations);
528 self
529 }
530
531 /// Whether this executor enforces the effect admission boundary.
532 pub fn effect_admission_enabled(&self) -> bool {
533 self.effect_compensations.is_some()
534 }
535
536 /// Builder-style: hold a token broadcast sender that nodes can clone to
537 /// publish [`GraphEvent::Token`] deltas (LangGraph's `messages` stream
538 /// mode).
539 ///
540 /// The executor never emits `Token` events itself — tokens originate in
541 /// nodes calling [`crate::llm::ChatModel::chat_stream`]. This is a
542 /// convenience handle so node factories built around an `Executor` can
543 /// fetch the sink via [`Executor::token_tx`] and capture a clone in each
544 /// node closure. When the same channel should also receive the
545 /// executor's own events, attach it to the run via
546 /// [`RunConfig::with_event_tx`] instead (or as well).
547 pub fn with_token_tx(mut self, token_tx: mpsc::Sender<GraphEvent>) -> Self {
548 self.token_tx = Some(token_tx);
549 self
550 }
551
552 /// The token broadcast sender, if one was attached via
553 /// [`Executor::with_token_tx`]. Clone it into node closures to stream
554 /// [`GraphEvent::Token`]s from within nodes.
555 pub fn token_tx(&self) -> Option<&mpsc::Sender<GraphEvent>> {
556 self.token_tx.as_ref()
557 }
558
559 /// Builder-style: install the governed-memory source this executor's
560 /// runs answer memory reads from (R0.8 wave 1; see
561 /// [`crate::memory::MemorySource`]).
562 ///
563 /// The executor never reads or writes memory itself this wave — the
564 /// full run-loop integration is a later wave. This is the seam's
565 /// installation point, mirroring [`Executor::with_token_tx`]: node
566 /// factories built around an `Executor` fetch the journaled handle via
567 /// [`Executor::memory`] once the run has started and capture a clone in
568 /// each node closure. Nodes may equally bind a source to an attached
569 /// journal directly ([`crate::journal::Journal::memory`]); both paths
570 /// journal into the same run evidence.
571 pub fn with_memory_source(mut self, source: crate::memory::MemorySource) -> Self {
572 self.memory_source = Some(source);
573 self
574 }
575
576 /// The configured memory source, if one was installed via
577 /// [`Executor::with_memory_source`].
578 pub fn memory_source(&self) -> Option<&crate::memory::MemorySource> {
579 self.memory_source.as_ref()
580 }
581
582 /// The journaled memory handle for the most recent run: the run's
583 /// journal (set at run start) bound to the configured source. `None`
584 /// before the first run or when no source is installed. The same
585 /// most-recent-run caveat as [`Executor::journal`] applies: driving
586 /// several runs concurrently through one `Executor` leaves this handle
587 /// pointing at whichever run started last — bind per-run handles via
588 /// [`crate::journal::Journal::memory`] when runs overlap.
589 pub fn memory(&self) -> Option<crate::memory::JournaledMemory> {
590 let journal = self.journal()?;
591 let source = self.memory_source.clone()?;
592 Some(crate::memory::JournaledMemory::new(&journal, source))
593 }
594
595 /// The configured checkpointer, if any. Shared (not consumed) so one
596 /// `Executor` can drive many concurrent runs over the same store.
597 pub fn checkpointer(&self) -> Option<&Arc<dyn Checkpointer>> {
598 self.checkpointer.as_ref()
599 }
600
601 /// The Flight Recorder journal of the most recent run started through
602 /// this executor — the one attached via [`RunConfig::with_journal`], or
603 /// the fresh journal the executor created for the run. `None` before the
604 /// first run.
605 ///
606 /// The journal is set at run start, so the evidence of a failed or
607 /// suspended run is retrievable exactly like a completed run's. Note the
608 /// executor is deliberately stateless across runs: driving several runs
609 /// concurrently through one `Executor` leaves this handle pointing at
610 /// whichever run started last — attach per-run journals via the config
611 /// when runs overlap.
612 pub fn journal(&self) -> Option<Journal> {
613 self.journal
614 .lock()
615 .unwrap_or_else(|e| e.into_inner())
616 .clone()
617 }
618
619 /// Run a compiled graph to completion (or interruption).
620 ///
621 /// - `graph`: the compiled, frozen graph topology.
622 /// - `spec`: the state schema (channels + reducers) used to merge node
623 /// updates at each barrier.
624 /// - `initial_state`: the starting state. When `config.resume` is set and
625 /// a checkpoint exists for `config.thread_id`, the checkpointed state
626 /// and next-node set take precedence over this argument. When
627 /// `config.checkpoint_id` is set, that specific checkpoint (rather than
628 /// the latest) is restored — replay/time travel; forking into a fresh
629 /// thread first via
630 /// [`crate::checkpoint::Checkpointer::fork_thread`] is the safe pattern,
631 /// since replaying on the same thread appends new history.
632 /// - `config`: run configuration (thread id, step limit, resume value,
633 /// streaming sink).
634 ///
635 /// # Super-step semantics
636 ///
637 /// Each loop iteration runs one super-step as a transaction: the active
638 /// nodes execute in parallel over an immutable start-of-step snapshot;
639 /// the barrier discards the whole step's writes on any node failure and
640 /// suspends the run on an interrupt; only then are writes merged via the
641 /// channel reducers, routing computed against the post-barrier state,
642 /// and a boundary checkpoint persisted. The module-level docs enumerate
643 /// the six phases; `execute_super_step` is the implementation.
644 ///
645 /// The loop returns [`ExecutionOutcome::Done`] when routing yields an
646 /// empty next set, [`ExecutionOutcome::Interrupted`] when a node
647 /// interrupts, an [`RustyError::Graph`] error once `config.max_steps`
648 /// super-steps have run without termination, and
649 /// [`RustyError::Cancelled`] when [`RunConfig::cancellation`] is
650 /// observed cancelled at a super-step boundary (the boundary checkpoint
651 /// is intact; the run resumes from it).
652 pub async fn run(
653 &self,
654 graph: &Graph,
655 spec: &StateSpec,
656 initial_state: State,
657 config: RunConfig,
658 ) -> Result<ExecutionOutcome> {
659 // Run-level span: every super-step, node, and checkpoint trace in the
660 // run attaches to it. Attached via `.instrument()` (never `.enter()`)
661 // so no span guard is held across `.await` points and the returned
662 // future stays `Send`.
663 let run_span = tracing::info_span!(
664 "rusty.run",
665 thread_id = %config.thread_id,
666 max_steps = config.max_steps,
667 resume = config.resume.is_some(),
668 replay = config.checkpoint_id.is_some(),
669 );
670 self.run_inner(graph, spec, initial_state, config)
671 .instrument(run_span)
672 .await
673 }
674
675 /// The instrumented body of [`Executor::run`]; see that method's docs for
676 /// the super-step algorithm.
677 async fn run_inner(
678 &self,
679 graph: &Graph,
680 spec: &StateSpec,
681 initial_state: State,
682 config: RunConfig,
683 ) -> Result<ExecutionOutcome> {
684 let started = std::time::Instant::now();
685
686 // ---- flight recorder setup ----
687 //
688 // Resolve the determinism seams and the run's journal before any
689 // checkpoint work: every timestamp and id below flows through them.
690 // An attached journal carries its own identity and clock; otherwise
691 // the executor mints a run id from the (possibly seeded) RNG and a
692 // journal over the configured clock. The run reads time from the
693 // journal's clock when no explicit clock is configured, keeping one
694 // time source per run.
695 let rng = config.rng.clone().unwrap_or_default();
696 let journal = match &config.journal {
697 Some(attached) => attached.clone(),
698 None => Journal::new(
699 rng.uuid_string(),
700 config.thread_id.clone(),
701 config.clock.clone().unwrap_or_default(),
702 ),
703 };
704 let mut recorder = Recorder {
705 clock: config
706 .clock
707 .clone()
708 .unwrap_or_else(|| journal.clock().clone()),
709 rng,
710 graph_version: config
711 .graph_version
712 .clone()
713 .unwrap_or_else(|| "unversioned".to_owned()),
714 graph_hash: graph.topology_hash(),
715 policy_version: config.policy_version.clone().unwrap_or_default(),
716 manifest: config.manifest.clone(),
717 journal: journal.clone(),
718 };
719 // Publish before the loop: evidence of a failed or suspended run is
720 // retrievable exactly like a completed run's.
721 *self.journal.lock().unwrap_or_else(|e| e.into_inner()) = Some(journal);
722
723 // ---- initialization / resume ----
724 //
725 // On resume the checkpointed state and next-node set take precedence
726 // over `initial_state`; the resume value is delivered to the first
727 // super-step (whose active set is the checkpointed next-node set —
728 // after an interrupt, every node of the suspended step) via
729 // `NodeContext::resume_value()`.
730 //
731 // Time travel: when `config.checkpoint_id` is set, THAT checkpoint is
732 // restored instead of the latest — this is replay from an arbitrary
733 // history point. The two knobs compose: `checkpoint_id` selects WHERE
734 // the run restarts, `resume` (when also set) supplies the resume value
735 // for the first super-step.
736 let mut state = initial_state;
737 let mut active: Vec<ActiveTask>;
738 let mut step: usize = 0;
739 let mut pending_resume: Option<Value> = None;
740 // Causal parent for the first super-step: the resume event on a
741 // resumed run, nothing on a fresh one.
742 let mut step_parent: Option<String> = None;
743
744 if config.checkpoint_id.is_some() || config.resume.is_some() {
745 let checkpointer = self.checkpointer.as_ref().ok_or_else(|| {
746 RustyError::Checkpoint(
747 "RunConfig.checkpoint_id/resume is set but no checkpointer is configured \
748 on the executor"
749 .into(),
750 )
751 })?;
752 let checkpoint = match &config.checkpoint_id {
753 Some(id) => checkpointer
754 .get_by_id(&config.thread_id, id)
755 .await?
756 .ok_or_else(|| {
757 RustyError::Checkpoint(format!(
758 "cannot replay thread `{}`: checkpoint `{id}` not found",
759 config.thread_id
760 ))
761 })?,
762 None => checkpointer
763 .get_latest(&config.thread_id)
764 .await?
765 .ok_or_else(|| {
766 RustyError::Checkpoint(format!(
767 "cannot resume thread `{}`: no checkpoint found",
768 config.thread_id
769 ))
770 })?,
771 };
772 // Policy-binding continuity (R0.8 wave 4): a resumed run keeps
773 // the policy version its checkpoint header pins, so a mid-run
774 // promotion never changes behavior under an in-flight execution.
775 // An explicit `RunConfig::with_policy_version` pin wins over
776 // inheritance — the caller asked for that version deliberately.
777 if config.policy_version.is_none() {
778 recorder.policy_version = checkpoint.header.policy_version.clone();
779 }
780 step_parent = Some(recorder.record(
781 EventDraft::new(RunEventKind::Resume, Effect::Pure).input(serde_json::json!({
782 "checkpoint_id": checkpoint.id,
783 "step": checkpoint.step,
784 "resume": config.resume.clone().unwrap_or(Value::Null),
785 })),
786 ));
787 state = checkpoint.state;
788 step = checkpoint.step;
789 active = checkpoint
790 .next_nodes
791 .into_iter()
792 .map(|name| ActiveTask { name, scoped: None })
793 .collect();
794 pending_resume = config.resume.clone();
795 if active.is_empty() {
796 tracing::info!(
797 steps = 0,
798 duration_ms = started.elapsed().as_millis() as u64,
799 "run complete"
800 );
801 return Ok(ExecutionOutcome::Done(state));
802 }
803 } else {
804 active = vec![ActiveTask {
805 name: graph.entry_point().to_owned(),
806 scoped: None,
807 }];
808 }
809
810 // One shared context for the whole run: clones handed to parallel
811 // nodes share its approval ledger, and later super-steps observe
812 // tokens already consumed by earlier calls. An explicitly injected
813 // context (R0.12 shadow runs) wins over the production one derived
814 // from the executor's compensation registry.
815 let effect_admission = config.effect_admission.clone().or_else(|| {
816 self.effect_compensations.as_ref().map(|compensations| {
817 EffectAdmissionContext::new(config.thread_id.clone())
818 .with_approvals(config.effect_approvals.clone())
819 .with_compensations(compensations.clone())
820 })
821 });
822
823 // ---- super-step loop ----
824 let mut steps_run: usize = 0;
825 loop {
826 if steps_run >= config.max_steps {
827 return Err(RustyError::Graph(format!(
828 "max_steps ({}) exceeded: the graph did not terminate within the step \
829 budget (possible infinite cycle; raise RunConfig::max_steps or add a \
830 terminating route)",
831 config.max_steps
832 )));
833 }
834 // Cooperative cancellation (drain): observed only here, at a
835 // super-step boundary. The last step's barrier merged cleanly
836 // and its boundary checkpoint is already persisted, so
837 // cancelling never tears a step — the run resumes from exactly
838 // this point. Nodes of the in-flight step always run to their
839 // barrier; cancellation granularity is one super-step.
840 if config
841 .cancellation
842 .as_ref()
843 .is_some_and(CancellationToken::is_cancelled)
844 {
845 tracing::warn!(
846 steps_run,
847 "run cancelled at a super-step boundary; resumable from the last checkpoint"
848 );
849 return Err(RustyError::Cancelled(format!(
850 "cancelled after {steps_run} super-step(s); the boundary checkpoint of \
851 thread `{}` is intact, so re-running the thread resumes from there",
852 config.thread_id
853 )));
854 }
855
856 // The step body runs in a dedicated method so the whole body is
857 // one instrumented future under the per-step span.
858 let step_span =
859 tracing::debug_span!("rusty.super_step", step = step, active_nodes = active.len(),);
860
861 let transition = self
862 .execute_super_step(
863 graph,
864 spec,
865 &config,
866 &recorder,
867 effect_admission.as_ref(),
868 &mut state,
869 &active,
870 step,
871 &mut pending_resume,
872 step_parent.clone(),
873 )
874 .instrument(step_span)
875 .await?;
876
877 match transition {
878 StepTransition::Next(next, route_event) => {
879 active = next;
880 step += 1;
881 steps_run += 1;
882 step_parent = Some(route_event);
883 }
884 StepTransition::Finish(outcome) => {
885 if !outcome.is_interrupted() {
886 tracing::info!(
887 steps = steps_run + 1,
888 duration_ms = started.elapsed().as_millis() as u64,
889 "run complete"
890 );
891 }
892 return Ok(outcome);
893 }
894 }
895 }
896 }
897
898 /// Executes one super-step: plan -> compute -> barrier -> merge -> route
899 /// -> boundary checkpoint. Returns the next active set, or
900 /// [`StepTransition::Finish`] with the terminal outcome when the run ends
901 /// (`Done`) or suspends (`Interrupted`).
902 ///
903 /// Flight Recorder: every phase transition is journaled through
904 /// `recorder` — super-step start/end, one node input/output pair per
905 /// invocation (in deterministic active-set order, not finish order), the
906 /// routing decision, and the checkpoint write. Node events carry the
907 /// node's declared [`Effect`] classification.
908 #[allow(clippy::too_many_arguments)]
909 async fn execute_super_step(
910 &self,
911 graph: &Graph,
912 spec: &StateSpec,
913 config: &RunConfig,
914 recorder: &Recorder,
915 effect_admission: Option<&EffectAdmissionContext>,
916 state: &mut State,
917 active: &[ActiveTask],
918 step: usize,
919 pending_resume: &mut Option<Value>,
920 step_parent: Option<String>,
921 ) -> Result<StepTransition> {
922 // -- plan.
923 let active_names: Vec<String> = active.iter().map(|t| t.name.clone()).collect();
924 Self::emit(
925 config,
926 GraphEvent::SuperStep {
927 step,
928 active_nodes: active_names.clone(),
929 },
930 );
931 let mut plan_draft =
932 EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure).input(serde_json::json!({
933 "step": step,
934 "active_nodes": active_names,
935 }));
936 if let Some(parent) = step_parent {
937 plan_draft = plan_draft.parent(parent);
938 }
939 let step_start_event = recorder.record(plan_draft);
940
941 // -- compute. Scoped (Send) state is overlaid onto each invocation's
942 // private copy of the start-of-step snapshot, so fan-out items
943 // never collide in the shared state.
944 let snapshot = state.clone();
945 let effect_admission = effect_admission.cloned();
946 let mut join_set: JoinSet<(usize, String, Result<NodeOutput>, u64)> = JoinSet::new();
947 // Per-invocation journal metadata, aligned with `active`: the input
948 // event id (causal parent of the matching output) and the node's
949 // declared effect class. Fan-out invocations of the same node each
950 // get their own entry — index, not name, is the identity.
951 let mut input_events: Vec<String> = Vec::with_capacity(active.len());
952 let mut invocation_effects: Vec<Effect> = Vec::with_capacity(active.len());
953
954 for (index, task) in active.iter().enumerate() {
955 let node = graph.node(&task.name).ok_or_else(|| {
956 RustyError::Graph(format!("routing activated unknown node `{}`", task.name))
957 })?;
958
959 let mut node_state = snapshot.clone();
960 if let Some(scoped) = &task.scoped {
961 match scoped {
962 Value::Object(map) => {
963 for (channel, value) in map {
964 node_state.insert(channel.clone(), value.clone());
965 }
966 }
967 other => {
968 return Err(RustyError::InvalidUpdate(format!(
969 "Send scoped state for node `{}` must be a JSON object, \
970 got {other}",
971 task.name
972 )));
973 }
974 }
975 }
976
977 let input_event = recorder.record(
978 EventDraft::new(RunEventKind::NodeInput, node.effect())
979 .node(task.name.clone())
980 .input(node_state.to_value())
981 .parent(step_start_event.clone()),
982 );
983 invocation_effects.push(node.effect());
984
985 let node_config = NodeConfig {
986 thread_id: config.thread_id.clone(),
987 step,
988 resume: pending_resume.clone(),
989 // Hand the invocation its own journal event id so node
990 // code can parent the effects it records (model/tool
991 // calls) to this invocation.
992 extra: HashMap::from([(
993 crate::journal::PARENT_EVENT_KEY.to_owned(),
994 Value::String(input_event.clone()),
995 )]),
996 };
997 input_events.push(input_event);
998 let name = task.name.clone();
999 Self::emit(
1000 config,
1001 GraphEvent::NodeStart {
1002 node: name.clone(),
1003 step,
1004 },
1005 );
1006 // A JoinSet polls tasks independently of the spawning task's
1007 // context, so the per-node span is attached to each spawned
1008 // future explicitly via `.instrument()`.
1009 let node_span = tracing::info_span!("rusty.node", node = %name, step = step);
1010 // Latency is read through the run's clock seam — under a logical
1011 // clock the value is reproducible; under the system clock it is
1012 // the real elapsed time.
1013 let clock = recorder.clock.clone();
1014 let chain = self.middleware.clone();
1015 let effect_admission = effect_admission.clone();
1016 join_set.spawn(
1017 async move {
1018 let node_started = clock.now();
1019 // Middleware chain: with layers attached, the invocation
1020 // runs inside the onion — before-hooks may mutate the
1021 // snapshot, reject the run, or short-circuit with a
1022 // substitute output; after-hooks unwind over the result.
1023 // No layers: the original dispatch, byte-identical.
1024 let result = if chain.is_empty() {
1025 node.run(
1026 NodeContext::new(node_state, node_config)
1027 .with_optional_effect_admission(effect_admission),
1028 )
1029 .await
1030 } else {
1031 let mut call = NodeCall::new(
1032 node_config.thread_id.clone(),
1033 name.clone(),
1034 step,
1035 node_state,
1036 );
1037 chain
1038 .run_node(&mut call, |call| {
1039 let ctx = NodeContext::new(call.state().clone(), node_config)
1040 .with_middleware(chain.clone())
1041 .with_optional_effect_admission(effect_admission);
1042 let node = Arc::clone(&node);
1043 async move { node.run(ctx).await }
1044 })
1045 .await
1046 };
1047 let latency_ms = (clock.now() - node_started).num_milliseconds().max(0) as u64;
1048 (index, name, result, latency_ms)
1049 }
1050 .instrument(node_span),
1051 );
1052 }
1053 // The resume value is consumed by the first super-step after a resume.
1054 *pending_resume = None;
1055
1056 // -- barrier: collect every node result. The step is
1057 // transactional: on any failure the JoinSet is dropped
1058 // (aborting stragglers) and the step's writes are discarded.
1059 let mut writes: Vec<(String, HashMap<String, Value>)> = Vec::new();
1060 let mut commands: Vec<Command> = Vec::new();
1061 let mut ran_nodes: Vec<String> = Vec::new();
1062 let mut interrupted: Option<(usize, String, Value)> = None;
1063 // Journal payloads of finished invocations, in finish order here;
1064 // journaled in active-set order after the barrier.
1065 let mut completed: Vec<(usize, String, u64, Value)> = Vec::new();
1066
1067 while let Some(joined) = join_set.join_next().await {
1068 let (index, name, result, latency_ms) = joined.map_err(|e| {
1069 RustyError::Node(format!(
1070 "node task failed to join (panic or cancellation): {e}"
1071 ))
1072 })?;
1073 match result {
1074 Ok(output) => {
1075 Self::emit(
1076 config,
1077 GraphEvent::NodeEnd {
1078 node: name.clone(),
1079 step,
1080 },
1081 );
1082 completed.push((
1083 index,
1084 name.clone(),
1085 latency_ms,
1086 serde_json::json!({
1087 "updates": &output.updates,
1088 "command": &output.command,
1089 }),
1090 ));
1091 if let Some(command) = output.command {
1092 if !command.goto.is_empty() {
1093 commands.push(command);
1094 }
1095 }
1096 ran_nodes.push(name.clone());
1097 writes.push((name, output.updates));
1098 }
1099 Err(RustyError::Interrupt { value }) => {
1100 // Record the suspension and stop the barrier loop; the
1101 // JoinSet is dropped below to abort stragglers.
1102 interrupted = Some((index, name, value));
1103 break;
1104 }
1105 Err(e) => {
1106 // A failed node is still evidence: journal the failure
1107 // before the error unwinds the run.
1108 recorder.record(
1109 EventDraft::new(RunEventKind::NodeOutput, invocation_effects[index])
1110 .node(name.clone())
1111 .status(EventStatus::Error)
1112 .output(serde_json::json!({ "error": e.to_string() }))
1113 .latency_ms(latency_ms)
1114 .parent(input_events[index].clone()),
1115 );
1116 // LLM and tool failures are the transient, retryable
1117 // error classes; everything else is a hard failure.
1118 let retryable = matches!(e, RustyError::Llm(_) | RustyError::Tool(_));
1119 tracing::warn!(
1120 node = %name,
1121 step = step,
1122 error = %e,
1123 retryable = retryable,
1124 "node failed; super-step aborted and its writes discarded"
1125 );
1126 return Err(RustyError::Node(format!(
1127 "node `{name}` failed at super-step {step}: {e}"
1128 )));
1129 }
1130 }
1131 }
1132
1133 if let Some((index, name, value)) = interrupted {
1134 // Suspend the run. The step is transactional, so no write of
1135 // this step survived — not even from siblings that completed
1136 // before the interrupt reached the barrier. The suspension
1137 // checkpoint therefore re-schedules the ENTIRE active set (the
1138 // interrupting node plus all siblings), otherwise completed
1139 // siblings' discarded writes would be silently lost and aborted
1140 // siblings would never re-run. Dropping the JoinSet first aborts
1141 // stragglers, preserving the transactional suspension point.
1142 drop(join_set);
1143 let interrupt_event = recorder.record(
1144 EventDraft::new(RunEventKind::Interrupt, invocation_effects[index])
1145 .node(name.clone())
1146 .input(value.clone())
1147 .status(EventStatus::Interrupted)
1148 .parent(input_events[index].clone()),
1149 );
1150 tracing::info!(
1151 node = %name,
1152 step = step,
1153 "node interrupted; run suspended (resumable via RunConfig::resume)"
1154 );
1155 let pending: Vec<String> = active.iter().map(|t| t.name.clone()).collect();
1156 let checkpoint =
1157 recorder.mint_checkpoint(config.thread_id.clone(), step, state.clone(), pending);
1158 let checkpoint_id = checkpoint.id.clone();
1159 if let Some(checkpointer) = &self.checkpointer {
1160 checkpointer.put(checkpoint).await?;
1161 recorder.record(
1162 EventDraft::new(RunEventKind::CheckpointWritten, Effect::Idempotent)
1163 .output(serde_json::json!({
1164 "checkpoint_id": checkpoint_id,
1165 "step": step,
1166 "suspension": true,
1167 }))
1168 .parent(interrupt_event),
1169 );
1170 Self::emit(
1171 config,
1172 GraphEvent::CheckpointSaved {
1173 checkpoint_id: checkpoint_id.clone(),
1174 step,
1175 },
1176 );
1177 }
1178 return Ok(StepTransition::Finish(ExecutionOutcome::Interrupted {
1179 value,
1180 state: state.clone(),
1181 checkpoint_id,
1182 }));
1183 }
1184
1185 // Journal node outputs in active-set order, not JoinSet finish
1186 // order: the finish order is scheduling-dependent, and the journal's
1187 // sequence is evidence — replay compares it.
1188 completed.sort_by_key(|(index, ..)| *index);
1189 let mut last_output_event: Option<String> = None;
1190 for (index, name, latency_ms, output_json) in completed {
1191 last_output_event = Some(
1192 recorder.record(
1193 EventDraft::new(RunEventKind::NodeOutput, invocation_effects[index])
1194 .node(name)
1195 .output(output_json)
1196 .latency_ms(latency_ms)
1197 .parent(input_events[index].clone()),
1198 ),
1199 );
1200 }
1201
1202 // -- merge: reducers + LastValue single-write validation. On
1203 // error the mutated state is dropped with the run
1204 // (transactional super-step).
1205 //
1206 // The start-of-step snapshot's job ends at the barrier — every node
1207 // has reported (or the step has failed). Dropping it before the
1208 // merge releases its per-channel shared references, so channels no
1209 // checkpoint still shares reach the reducer with refcount 1 and
1210 // merge in place (W4 copy-on-write) instead of cloning.
1211 drop(snapshot);
1212 let written_channels: HashSet<String> = writes
1213 .iter()
1214 .flat_map(|(_, updates)| updates.keys().cloned())
1215 .collect();
1216 spec.apply_super_step(state, writes)?;
1217 // The event carries the post-reducer values read back out of the
1218 // merged state: when several nodes write the same channel in one
1219 // step (the normal Append fan-in case), reporting the raw partials
1220 // would keep only the last write and hide the rest.
1221 let mut merged_updates = serde_json::Map::new();
1222 for channel in &written_channels {
1223 if let Some(value) = state.get(channel) {
1224 merged_updates.insert(channel.clone(), value.clone());
1225 }
1226 }
1227 let channels_written: Vec<&str> = merged_updates.keys().map(String::as_str).collect();
1228 tracing::debug!(
1229 step = step,
1230 channels = ?channels_written,
1231 "merged node updates at super-step barrier"
1232 );
1233 // The super-step's reducer result is journaled even when empty —
1234 // "no channel changed" is evidence too.
1235 let step_end_event = recorder.record(
1236 EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1237 .output(Value::Object(merged_updates.clone()))
1238 .parent(last_output_event.unwrap_or_else(|| step_start_event.clone())),
1239 );
1240 if !merged_updates.is_empty() {
1241 Self::emit(
1242 config,
1243 GraphEvent::StateUpdate {
1244 step,
1245 updates: merged_updates,
1246 },
1247 );
1248 }
1249
1250 // -- route: Command::goto overrides the static edge set;
1251 // otherwise evaluate outgoing edges of every node that ran
1252 // against the post-barrier state.
1253 let mut next: Vec<ActiveTask> = Vec::new();
1254 let mut planned: HashSet<String> = HashSet::new();
1255
1256 if !commands.is_empty() {
1257 for command in &commands {
1258 for target in &command.goto {
1259 if !graph.has_node(target) {
1260 return Err(RustyError::Graph(format!(
1261 "Command::goto references unknown node `{target}`"
1262 )));
1263 }
1264 if planned.insert(target.clone()) {
1265 next.push(ActiveTask {
1266 name: target.clone(),
1267 scoped: None,
1268 });
1269 }
1270 }
1271 }
1272 } else {
1273 let mut evaluated: HashSet<String> = HashSet::new();
1274 for name in &ran_nodes {
1275 // Fan-out invocations of the same node share one edge set;
1276 // evaluate it once.
1277 if !evaluated.insert(name.clone()) {
1278 continue;
1279 }
1280 for edge in graph.outgoing_edges(name) {
1281 match edge {
1282 Edge::Direct { to, .. } => {
1283 if planned.insert(to.clone()) {
1284 next.push(ActiveTask {
1285 name: to.clone(),
1286 scoped: None,
1287 });
1288 }
1289 }
1290 Edge::Conditional { router, .. } => {
1291 match router(state.clone()).await? {
1292 Route::Node(target) => {
1293 if !graph.has_node(&target) {
1294 return Err(RustyError::Graph(format!(
1295 "conditional router from `{name}` returned \
1296 unknown node `{target}`"
1297 )));
1298 }
1299 if planned.insert(target.clone()) {
1300 next.push(ActiveTask {
1301 name: target,
1302 scoped: None,
1303 });
1304 }
1305 }
1306 Route::Send(sends) => {
1307 for send in sends {
1308 if !graph.has_node(&send.node) {
1309 return Err(RustyError::Graph(format!(
1310 "Route::Send from `{name}` targets unknown \
1311 node `{}`",
1312 send.node
1313 )));
1314 }
1315 // Each Send is its own invocation with its own
1316 // scoped state, even when several target the
1317 // same node.
1318 next.push(ActiveTask {
1319 name: send.node,
1320 scoped: Some(send.state),
1321 });
1322 }
1323 }
1324 Route::End => {}
1325 }
1326 }
1327 }
1328 }
1329 }
1330 }
1331
1332 let route_event = recorder.record(
1333 EventDraft::new(RunEventKind::RoutingDecision, Effect::Pure)
1334 .output(serde_json::json!({
1335 "next": next
1336 .iter()
1337 .map(|t| serde_json::json!({
1338 "node": &t.name,
1339 "scoped": &t.scoped,
1340 }))
1341 .collect::<Vec<_>>(),
1342 "goto": commands
1343 .iter()
1344 .flat_map(|command| command.goto.iter().cloned())
1345 .collect::<Vec<String>>(),
1346 }))
1347 .parent(step_end_event.clone()),
1348 );
1349
1350 // -- checkpoint at the super-step boundary.
1351 if let Some(checkpointer) = &self.checkpointer {
1352 let next_names: Vec<String> = next.iter().map(|t| t.name.clone()).collect();
1353 let checkpoint =
1354 recorder.mint_checkpoint(config.thread_id.clone(), step, state.clone(), next_names);
1355 let checkpoint_id = checkpoint.id.clone();
1356 checkpointer.put(checkpoint).await?;
1357 recorder.record(
1358 EventDraft::new(RunEventKind::CheckpointWritten, Effect::Idempotent)
1359 .output(serde_json::json!({
1360 "checkpoint_id": checkpoint_id,
1361 "step": step,
1362 "suspension": false,
1363 }))
1364 .parent(route_event.clone()),
1365 );
1366 Self::emit(
1367 config,
1368 GraphEvent::CheckpointSaved {
1369 checkpoint_id,
1370 step,
1371 },
1372 );
1373 }
1374
1375 // -- terminate or schedule the next super-step.
1376 if next.is_empty() {
1377 return Ok(StepTransition::Finish(ExecutionOutcome::Done(
1378 state.clone(),
1379 )));
1380 }
1381 Ok(StepTransition::Next(next, route_event))
1382 }
1383
1384 /// Best-effort event emission: a full or closed channel never aborts a run.
1385 fn emit(config: &RunConfig, event: GraphEvent) {
1386 if let Some(tx) = &config.event_tx {
1387 let _ = tx.try_send(event);
1388 }
1389 }
1390}
1391
1392/// Per-run Flight Recorder state handed to every super-step: the journal,
1393/// the determinism seams, and the frozen provenance stamped into every
1394/// checkpoint of the run.
1395struct Recorder {
1396 journal: Journal,
1397 clock: Clock,
1398 rng: RngSource,
1399 graph_version: String,
1400 graph_hash: String,
1401 policy_version: PolicyVersion,
1402 manifest: Option<RunManifest>,
1403}
1404
1405impl Recorder {
1406 /// Append one event to the run's journal; returns the event id (the
1407 /// causal-parent handle for whatever the event causes).
1408 fn record(&self, draft: EventDraft) -> String {
1409 self.journal.record(draft)
1410 }
1411
1412 /// Mint a boundary checkpoint through the determinism seams: id from the
1413 /// run's RNG, timestamp from the run's clock, the frozen provenance
1414 /// header, and a journal reference pinning the evidence head as it stood
1415 /// *before* this checkpoint's own `checkpoint_written` event is
1416 /// recorded.
1417 fn mint_checkpoint(
1418 &self,
1419 thread_id: String,
1420 step: usize,
1421 state: State,
1422 next_nodes: Vec<String>,
1423 ) -> Checkpoint {
1424 Checkpoint {
1425 id: self.rng.uuid_string(),
1426 thread_id,
1427 step,
1428 state,
1429 next_nodes,
1430 created_at: self.clock.now(),
1431 header: CheckpointHeader {
1432 format_version: CURRENT_FORMAT_VERSION,
1433 graph_version: self.graph_version.clone(),
1434 graph_hash: self.graph_hash.clone(),
1435 policy_version: self.policy_version.clone(),
1436 logical_clock: self.clock.now_ms(),
1437 manifest: self.manifest.clone(),
1438 },
1439 journal_ref: Some(self.journal.head_ref()),
1440 // The executor always mints full snapshots; delta encoding is
1441 // decided inside the checkpointer (`checkpoint::encode_delta`).
1442 base: None,
1443 }
1444 }
1445}
1446
1447/// One scheduled node invocation within a super-step. `scoped` carries the
1448/// per-invocation input of a [`crate::graph::Send`] fan-out, overlaid onto
1449/// that invocation's private state snapshot before the node runs.
1450struct ActiveTask {
1451 name: String,
1452 scoped: Option<Value>,
1453}
1454
1455/// The control-flow result of a single super-step: either the next active
1456/// set (the loop continues) or the terminal run outcome (the loop breaks).
1457enum StepTransition {
1458 /// The next active set plus the routing-decision journal event that
1459 /// produced it (the causal parent of the next super-step's start event).
1460 Next(Vec<ActiveTask>, String),
1461 Finish(ExecutionOutcome),
1462}
1463
1464#[cfg(test)]
1465mod tests {
1466 use super::*;
1467 use crate::checkpoint::InMemoryCheckpointer;
1468 use crate::graph::GraphBuilder;
1469 use crate::llm::ChatModel;
1470 use crate::state::Reducer;
1471 use serde_json::json;
1472 use std::sync::Mutex;
1473
1474 #[tokio::test]
1475 async fn linear_two_node_graph_executes_in_order() {
1476 let spec = StateSpec::new().channel("log", Reducer::Append);
1477
1478 let mut builder = GraphBuilder::new();
1479 builder.add_node("first", |_ctx: NodeContext| async {
1480 Ok(NodeOutput::update("log", json!("first")))
1481 });
1482 builder.add_node("second", |ctx: NodeContext| async move {
1483 // The next super-step observes the previous step's merged writes.
1484 assert_eq!(ctx.state().get("log"), Some(&json!(["first"])));
1485 assert_eq!(ctx.step(), 1);
1486 Ok(NodeOutput::update("log", json!("second")))
1487 });
1488 builder.set_entry_point("first");
1489 builder.add_edge("first", "second");
1490 let graph = builder.compile().unwrap();
1491
1492 let outcome = Executor::new()
1493 .run(&graph, &spec, State::new(), RunConfig::new("t-linear"))
1494 .await
1495 .unwrap();
1496
1497 match outcome {
1498 ExecutionOutcome::Done(state) => {
1499 assert_eq!(state.get("log"), Some(&json!(["first", "second"])));
1500 }
1501 other => panic!("expected Done, got {other:?}"),
1502 }
1503 }
1504
1505 #[tokio::test]
1506 async fn parallel_fan_in_merges_via_reducer() {
1507 let spec = StateSpec::new().channel("results", Reducer::Append);
1508
1509 let mut builder = GraphBuilder::new();
1510 builder.add_node("start", |_ctx: NodeContext| async {
1511 Ok(NodeOutput::empty())
1512 });
1513 builder.add_node("worker_a", |ctx: NodeContext| async move {
1514 // Snapshot isolation: parallel workers cannot see each other.
1515 assert!(!ctx.state().contains("results"));
1516 Ok(NodeOutput::update("results", json!("a")))
1517 });
1518 builder.add_node("worker_b", |ctx: NodeContext| async move {
1519 assert!(!ctx.state().contains("results"));
1520 Ok(NodeOutput::update("results", json!("b")))
1521 });
1522 builder.set_entry_point("start");
1523 builder.add_edge("start", "worker_a");
1524 builder.add_edge("start", "worker_b");
1525 let graph = builder.compile().unwrap();
1526
1527 let outcome = Executor::new()
1528 .run(&graph, &spec, State::new(), RunConfig::new("t-fan-in"))
1529 .await
1530 .unwrap();
1531
1532 match outcome {
1533 ExecutionOutcome::Done(state) => {
1534 let results = state
1535 .get("results")
1536 .and_then(Value::as_array)
1537 .expect("results channel must exist")
1538 .clone();
1539 // Completion order across the JoinSet is nondeterministic.
1540 let mut items: Vec<&str> = results.iter().map(|v| v.as_str().unwrap()).collect();
1541 items.sort_unstable();
1542 assert_eq!(items, ["a", "b"]);
1543 }
1544 other => panic!("expected Done, got {other:?}"),
1545 }
1546 }
1547
1548 #[tokio::test]
1549 async fn interrupt_returns_interrupted_outcome_and_resume_completes() {
1550 let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
1551
1552 let mut builder = GraphBuilder::new();
1553 builder.add_node("gate", |ctx: NodeContext| async move {
1554 match ctx.resume_value() {
1555 Some(v) => Ok(NodeOutput::update("answer", v.clone())),
1556 None => Err(ctx.interrupt(json!({"question": "approve?"}))),
1557 }
1558 });
1559 builder.set_entry_point("gate");
1560 let graph = builder.compile().unwrap();
1561
1562 let checkpointer = Arc::new(InMemoryCheckpointer::new());
1563 let executor = Executor::with_checkpointer(checkpointer.clone());
1564
1565 // First run: the gate node interrupts and the run suspends.
1566 let outcome = executor
1567 .run(&graph, &spec, State::new(), RunConfig::new("t-hitl"))
1568 .await
1569 .unwrap();
1570
1571 let checkpoint_id = match outcome {
1572 ExecutionOutcome::Interrupted {
1573 value,
1574 checkpoint_id,
1575 ..
1576 } => {
1577 assert_eq!(value, json!({"question": "approve?"}));
1578 assert!(!checkpoint_id.is_empty());
1579 checkpoint_id
1580 }
1581 other => panic!("expected Interrupted, got {other:?}"),
1582 };
1583
1584 // The suspension point was persisted and schedules the gate node.
1585 let stored = checkpointer.get_latest("t-hitl").await.unwrap().unwrap();
1586 assert_eq!(stored.id, checkpoint_id);
1587 assert_eq!(stored.next_nodes, vec!["gate".to_string()]);
1588
1589 // Resume: the gate node re-runs with the resume value, writes its
1590 // answer, and the run terminates.
1591 let outcome = executor
1592 .run(
1593 &graph,
1594 &spec,
1595 State::new(),
1596 RunConfig::new("t-hitl").with_resume(json!(true)),
1597 )
1598 .await
1599 .unwrap();
1600
1601 match outcome {
1602 ExecutionOutcome::Done(state) => {
1603 assert_eq!(state.get("answer"), Some(&json!(true)));
1604 }
1605 other => panic!("expected Done after resume, got {other:?}"),
1606 }
1607 }
1608
1609 #[test]
1610 fn run_config_default_uses_default_step_limit() {
1611 // Regression: a derived `Default` would zero `max_steps`, making
1612 // every `RunConfig::default()` run fail immediately.
1613 let config = RunConfig::default();
1614 assert_eq!(config.max_steps, DEFAULT_MAX_STEPS);
1615 assert!(config.thread_id.is_empty());
1616 assert!(config.resume.is_none() && config.checkpoint_id.is_none());
1617 }
1618
1619 #[tokio::test]
1620 async fn interrupt_reschedules_entire_active_set() {
1621 // Regression: the suspension checkpoint used to schedule only the
1622 // interrupting node, silently dropping parallel siblings — including
1623 // ones that had already completed, whose writes are discarded with
1624 // the aborted step.
1625 let spec = StateSpec::new()
1626 .channel("log", Reducer::Append)
1627 .channel("answer", Reducer::Overwrite);
1628
1629 let mut builder = GraphBuilder::new();
1630 builder.add_node("start", |_ctx: NodeContext| async {
1631 Ok(NodeOutput::empty())
1632 });
1633 builder.add_node("gate", |ctx: NodeContext| async move {
1634 match ctx.resume_value() {
1635 Some(v) => Ok(NodeOutput::update("answer", v.clone())),
1636 None => Err(ctx.interrupt(json!({"question": "approve?"}))),
1637 }
1638 });
1639 // Completes immediately in the interrupted step; its write is
1640 // discarded with the step and must be recomputed on resume.
1641 builder.add_node("fast", |_ctx: NodeContext| async {
1642 Ok(NodeOutput::update("log", json!("fast")))
1643 });
1644 // Still in flight when the interrupt hits; aborted, re-run on resume.
1645 builder.add_node("slow", |ctx: NodeContext| async move {
1646 if ctx.resume_value().is_none() {
1647 tokio::time::sleep(std::time::Duration::from_secs(3600)).await;
1648 }
1649 Ok(NodeOutput::update("log", json!("slow")))
1650 });
1651 builder.set_entry_point("start");
1652 builder.add_edge("start", "gate");
1653 builder.add_edge("start", "slow");
1654 builder.add_edge("start", "fast");
1655 let graph = builder.compile().unwrap();
1656
1657 let checkpointer = Arc::new(InMemoryCheckpointer::new());
1658 let executor = Executor::with_checkpointer(checkpointer.clone());
1659
1660 let outcome = executor
1661 .run(&graph, &spec, State::new(), RunConfig::new("t-par-hitl"))
1662 .await
1663 .unwrap();
1664 match &outcome {
1665 ExecutionOutcome::Interrupted { state, .. } => {
1666 // Transactional suspension: fast's completed write was
1667 // discarded with the rest of the step.
1668 assert_eq!(state.get("log"), None);
1669 }
1670 other => panic!("expected Interrupted, got {other:?}"),
1671 }
1672
1673 // The suspension checkpoint re-schedules every node of the step.
1674 let stored = checkpointer
1675 .get_latest("t-par-hitl")
1676 .await
1677 .unwrap()
1678 .unwrap();
1679 let mut scheduled = stored.next_nodes.clone();
1680 scheduled.sort_unstable();
1681 assert_eq!(scheduled, ["fast", "gate", "slow"]);
1682
1683 // Resume: all three re-run (the resume value is broadcast to the
1684 // whole step); fast's write lands exactly once.
1685 let outcome = executor
1686 .run(
1687 &graph,
1688 &spec,
1689 State::new(),
1690 RunConfig::new("t-par-hitl").with_resume(json!(true)),
1691 )
1692 .await
1693 .unwrap();
1694 match outcome {
1695 ExecutionOutcome::Done(state) => {
1696 assert_eq!(state.get("answer"), Some(&json!(true)));
1697 let mut log: Vec<String> = state
1698 .get("log")
1699 .and_then(Value::as_array)
1700 .expect("log channel must exist")
1701 .iter()
1702 .map(|v| v.as_str().unwrap().to_owned())
1703 .collect();
1704 log.sort_unstable();
1705 assert_eq!(log, ["fast", "slow"]);
1706 }
1707 other => panic!("expected Done after resume, got {other:?}"),
1708 }
1709 }
1710
1711 #[tokio::test]
1712 async fn state_update_event_reports_post_reducer_values() {
1713 // Regression: with several writers on one channel, the event used to
1714 // carry raw per-node partials collapsed by last-write-wins, hiding
1715 // all but one write behind its documented "merged" contract.
1716 let spec = StateSpec::new().channel("results", Reducer::Append);
1717
1718 let mut builder = GraphBuilder::new();
1719 builder.add_node("start", |_ctx: NodeContext| async {
1720 Ok(NodeOutput::empty())
1721 });
1722 builder.add_node("worker_a", |_ctx: NodeContext| async {
1723 Ok(NodeOutput::update("results", json!("a")))
1724 });
1725 builder.add_node("worker_b", |_ctx: NodeContext| async {
1726 Ok(NodeOutput::update("results", json!("b")))
1727 });
1728 builder.set_entry_point("start");
1729 builder.add_edge("start", "worker_a");
1730 builder.add_edge("start", "worker_b");
1731 let graph = builder.compile().unwrap();
1732
1733 let (tx, mut rx) = mpsc::channel::<GraphEvent>(64);
1734 let outcome = Executor::new()
1735 .run(
1736 &graph,
1737 &spec,
1738 State::new(),
1739 RunConfig::new("t-event").with_event_tx(tx),
1740 )
1741 .await
1742 .unwrap();
1743 assert!(matches!(outcome, ExecutionOutcome::Done(_)));
1744
1745 let mut merged: Option<Vec<String>> = None;
1746 while let Ok(event) = rx.try_recv() {
1747 if let GraphEvent::StateUpdate { step: 1, updates } = event {
1748 let values = updates
1749 .get("results")
1750 .and_then(Value::as_array)
1751 .expect("StateUpdate must carry the results channel")
1752 .iter()
1753 .map(|v| v.as_str().unwrap().to_owned())
1754 .collect();
1755 merged = Some(values);
1756 }
1757 }
1758 let mut merged = merged.expect("expected a StateUpdate event for step 1");
1759 merged.sort_unstable();
1760 // Both partial writes are visible in the single post-reducer value.
1761 assert_eq!(merged, ["a", "b"]);
1762 }
1763
1764 #[tokio::test]
1765 async fn max_steps_guard_aborts_infinite_cycles() {
1766 let spec = StateSpec::new().channel("x", Reducer::Overwrite);
1767 let mut builder = GraphBuilder::new();
1768 builder.add_node("loop_node", |_ctx: NodeContext| async {
1769 Ok(NodeOutput::empty())
1770 });
1771 builder.set_entry_point("loop_node");
1772 builder.add_edge("loop_node", "loop_node");
1773 let graph = builder.compile().unwrap();
1774
1775 let err = Executor::new()
1776 .run(
1777 &graph,
1778 &spec,
1779 State::new(),
1780 RunConfig::new("t-loop").with_max_steps(5),
1781 )
1782 .await
1783 .unwrap_err();
1784 assert!(matches!(err, RustyError::Graph(_)));
1785 }
1786
1787 #[test]
1788 fn token_event_serde_roundtrip() {
1789 let event = GraphEvent::Token {
1790 node: "agent".into(),
1791 delta: "Hello".into(),
1792 };
1793 let wire = serde_json::to_string(&event).unwrap();
1794 // Internally tagged: the variant name travels on the wire.
1795 assert!(
1796 wire.contains("\"type\":\"token\""),
1797 "unexpected wire: {wire}"
1798 );
1799 let back: GraphEvent = serde_json::from_str(&wire).unwrap();
1800 assert_eq!(event, back);
1801
1802 // The other variants still roundtrip under the new serde derives.
1803 let step_event = GraphEvent::SuperStep {
1804 step: 3,
1805 active_nodes: vec!["a".into()],
1806 };
1807 let back: GraphEvent =
1808 serde_json::from_str(&serde_json::to_string(&step_event).unwrap()).unwrap();
1809 assert_eq!(step_event, back);
1810 }
1811
1812 /// A mock model whose `chat_stream` override emits real multi-chunk
1813 /// deltas, proving the accumulation contract outside any HTTP client.
1814 struct StreamingMock;
1815
1816 #[async_trait::async_trait]
1817 impl crate::llm::ChatModel for StreamingMock {
1818 async fn chat(
1819 &self,
1820 _messages: &[crate::llm::ChatMessage],
1821 _tools: &[Value],
1822 ) -> Result<crate::llm::ChatResponse> {
1823 Ok(crate::llm::ChatResponse {
1824 message: crate::llm::ChatMessage::assistant("Hello"),
1825 model: None,
1826 usage: None,
1827 })
1828 }
1829
1830 async fn chat_stream(
1831 &self,
1832 messages: &[crate::llm::ChatMessage],
1833 tools: &[Value],
1834 on_token: &mut (dyn FnMut(crate::llm::TokenChunk) + Send),
1835 ) -> Result<crate::llm::ChatResponse> {
1836 for piece in ["Hel", "lo"] {
1837 on_token(crate::llm::TokenChunk {
1838 delta: piece.into(),
1839 finish: false,
1840 raw: None,
1841 });
1842 }
1843 on_token(crate::llm::TokenChunk {
1844 delta: String::new(),
1845 finish: true,
1846 raw: None,
1847 });
1848 self.chat(messages, tools).await
1849 }
1850 }
1851
1852 #[tokio::test]
1853 async fn node_streams_token_events_through_event_sink() {
1854 let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
1855
1856 let (tx, mut rx) = mpsc::channel::<GraphEvent>(64);
1857 // The wiring pattern from the ChatModel rustdoc: the node closure
1858 // captures a clone of the run's event sender and forwards chunks.
1859 let node_tx = tx.clone();
1860
1861 let mut builder = GraphBuilder::new();
1862 builder.add_node("agent", move |_ctx: NodeContext| {
1863 let node_tx = node_tx.clone();
1864 async move {
1865 let model = StreamingMock;
1866 let mut full = String::new();
1867 model
1868 .chat_stream(&[], &[], &mut |chunk| {
1869 if !chunk.delta.is_empty() {
1870 full.push_str(&chunk.delta);
1871 let _ = node_tx.try_send(GraphEvent::Token {
1872 node: "agent".into(),
1873 delta: chunk.delta,
1874 });
1875 }
1876 })
1877 .await
1878 .unwrap();
1879 Ok(NodeOutput::update("answer", json!(full)))
1880 }
1881 });
1882 builder.set_entry_point("agent");
1883 let graph = builder.compile().unwrap();
1884
1885 let config = RunConfig::new("t-tokens").with_event_tx(tx);
1886 // The RunConfig helper hands out the same sender for node wiring.
1887 assert!(config.token_tx().is_some());
1888 // The Executor builder/accessor pair stores a broadcast handle.
1889 let executor = Executor::new().with_token_tx(config.token_tx().unwrap());
1890 assert!(executor.token_tx().is_some());
1891
1892 let outcome = executor
1893 .run(&graph, &spec, State::new(), config)
1894 .await
1895 .unwrap();
1896 match outcome {
1897 ExecutionOutcome::Done(state) => {
1898 assert_eq!(state.get("answer"), Some(&json!("Hello")))
1899 }
1900 other => panic!("expected Done, got {other:?}"),
1901 }
1902
1903 // Drain the event stream: token deltas interleave with executor events.
1904 let mut deltas = Vec::new();
1905 while let Ok(event) = rx.try_recv() {
1906 if let GraphEvent::Token { node, delta } = event {
1907 assert_eq!(node, "agent");
1908 deltas.push(delta);
1909 }
1910 }
1911 assert_eq!(deltas, ["Hel", "lo"]);
1912 }
1913
1914 /// A minimal `tracing::Subscriber` that records formatted event fields
1915 /// (`name=value` pairs) into a shared buffer, so tests can assert on the
1916 /// executor's instrumentation. Implemented directly against the
1917 /// `tracing` crate's own `Subscriber` trait (re-exported from
1918 /// `tracing-core`) — no `tracing-subscriber` dependency required.
1919 #[derive(Clone, Default)]
1920 struct EventCapture {
1921 events: Arc<Mutex<Vec<String>>>,
1922 }
1923
1924 /// Formats an event's fields as `"name=value "` pairs.
1925 struct FieldVisitor(String);
1926
1927 impl tracing::field::Visit for FieldVisitor {
1928 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
1929 use std::fmt::Write as _;
1930 let _ = write!(self.0, "{}={:?} ", field.name(), value);
1931 }
1932 }
1933
1934 impl tracing::Subscriber for EventCapture {
1935 fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
1936 true
1937 }
1938
1939 fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
1940 // Spans are irrelevant to these assertions; one id serves all.
1941 tracing::span::Id::from_u64(1)
1942 }
1943
1944 fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
1945
1946 fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
1947
1948 fn event(&self, event: &tracing::Event<'_>) {
1949 let mut visitor = FieldVisitor(String::new());
1950 event.record(&mut visitor);
1951 self.events.lock().unwrap().push(visitor.0);
1952 }
1953
1954 fn enter(&self, _span: &tracing::span::Id) {}
1955
1956 fn exit(&self, _span: &tracing::span::Id) {}
1957 }
1958
1959 /// The instrumentation must be observability-only: installing a
1960 /// subscriber changes nothing about the run's outcome, and the expected
1961 /// telemetry (merge debug event, run-completion info event) is emitted.
1962 #[tokio::test]
1963 async fn instrumentation_emits_events_without_changing_outcome() {
1964 let capture = EventCapture::default();
1965 let events = capture.events.clone();
1966 // Global default subscriber, deliberately NOT a thread-local
1967 // `set_default`: callsite interest is cached process-wide and lazily
1968 // (re)registered from whichever thread first hits a callsite, so a
1969 // thread-local subscriber races with the other tests in this binary
1970 // that run graphs concurrently (they rebuild the cache against the
1971 // no-subscriber global default and our events get silently dropped).
1972 // A global default makes every thread's rebuild see this subscriber.
1973 // Setting it is additive — other tests neither set nor assert on
1974 // subscribers, and captured events from concurrent runs only help the
1975 // `any()` assertions below. `set_global_default` may only be called
1976 // once per process; this is the only test that installs a subscriber.
1977 tracing::subscriber::set_global_default(capture)
1978 .expect("no other test may install a global tracing subscriber");
1979
1980 let spec = StateSpec::new().channel("log", Reducer::Append);
1981 let mut builder = GraphBuilder::new();
1982 builder.add_node("only", |_ctx: NodeContext| async {
1983 Ok(NodeOutput::update("log", json!("x")))
1984 });
1985 builder.set_entry_point("only");
1986 let graph = builder.compile().unwrap();
1987
1988 let outcome = Executor::new()
1989 .run(&graph, &spec, State::new(), RunConfig::new("t-tracing"))
1990 .await
1991 .unwrap();
1992
1993 // Identical semantics: the run completes with the expected state.
1994 match outcome {
1995 ExecutionOutcome::Done(state) => {
1996 assert_eq!(state.get("log"), Some(&json!(["x"])));
1997 }
1998 other => panic!("expected Done, got {other:?}"),
1999 }
2000
2001 let captured = events.lock().unwrap();
2002 assert!(
2003 captured.iter().any(|e| e.contains("channels")),
2004 "expected a merge debug event listing written channels, got: {captured:?}"
2005 );
2006 assert!(
2007 captured
2008 .iter()
2009 .any(|e| e.contains("steps") && e.contains("duration_ms")),
2010 "expected a run-completion info event with steps and duration_ms, got: {captured:?}"
2011 );
2012 }
2013
2014 /// A 3-node linear graph (`a -> b -> c`) appending each node name to the
2015 /// `log` channel.
2016 fn linear_three_node_graph() -> (Graph, StateSpec) {
2017 let spec = StateSpec::new().channel("log", Reducer::Append);
2018 let mut builder = GraphBuilder::new();
2019 for name in ["a", "b", "c"] {
2020 builder.add_node(name, move |_ctx: NodeContext| async move {
2021 Ok(NodeOutput::update("log", json!(name)))
2022 });
2023 }
2024 builder.set_entry_point("a");
2025 builder.add_edge("a", "b");
2026 builder.add_edge("b", "c");
2027 (builder.compile().unwrap(), spec)
2028 }
2029
2030 #[tokio::test]
2031 async fn run_with_checkpoint_id_replays_from_earlier_state() {
2032 let (graph, spec) = linear_three_node_graph();
2033 let checkpointer = Arc::new(InMemoryCheckpointer::new());
2034 let executor = Executor::with_checkpointer(checkpointer.clone());
2035
2036 // Full run on the source thread: checkpoints at steps 0, 1, 2.
2037 let outcome = executor
2038 .run(&graph, &spec, State::new(), RunConfig::new("t-src"))
2039 .await
2040 .unwrap();
2041 match outcome {
2042 ExecutionOutcome::Done(state) => {
2043 assert_eq!(state.get("log"), Some(&json!(["a", "b", "c"])));
2044 }
2045 other => panic!("expected Done, got {other:?}"),
2046 }
2047
2048 let history = checkpointer.list("t-src").await.unwrap();
2049 assert_eq!(history.len(), 3);
2050 // The step-1 checkpoint: `a` and `b` have run, `c` is scheduled next.
2051 let step1 = history[1].clone();
2052 assert_eq!(step1.step, 1);
2053 assert_eq!(step1.state.get("log"), Some(&json!(["a", "b"])));
2054 assert_eq!(step1.next_nodes, vec!["c".to_string()]);
2055
2056 // Safe pattern: fork the thread at the step-1 checkpoint, then replay
2057 // the fork from that checkpoint (not the fork's latest — here the cut
2058 // point IS the latest, but `checkpoint_id` is what selects it).
2059 let copied = checkpointer
2060 .fork_thread("t-src", "t-fork", Some(&step1.id))
2061 .await
2062 .unwrap();
2063 assert_eq!(copied, 2);
2064
2065 let outcome = executor
2066 .run(
2067 &graph,
2068 &spec,
2069 State::new(),
2070 RunConfig::new("t-fork").with_checkpoint_id(step1.id.clone()),
2071 )
2072 .await
2073 .unwrap();
2074
2075 match outcome {
2076 ExecutionOutcome::Done(state) => {
2077 // Execution continued from the step-1 state: only `c` ran,
2078 // `b` was not re-executed.
2079 assert_eq!(state.get("log"), Some(&json!(["a", "b", "c"])));
2080 }
2081 other => panic!("expected Done, got {other:?}"),
2082 }
2083
2084 // The replay appended its own boundary checkpoint to the fork only.
2085 assert_eq!(checkpointer.list("t-src").await.unwrap().len(), 3);
2086 assert_eq!(checkpointer.list("t-fork").await.unwrap().len(), 3);
2087 }
2088
2089 #[tokio::test]
2090 async fn run_with_checkpoint_id_plus_resume_combined() {
2091 let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
2092
2093 let mut builder = GraphBuilder::new();
2094 builder.add_node("gate", |ctx: NodeContext| async move {
2095 match ctx.resume_value() {
2096 Some(v) => Ok(NodeOutput::update("answer", v.clone())),
2097 None => Err(ctx.interrupt(json!({"question": "approve?"}))),
2098 }
2099 });
2100 builder.set_entry_point("gate");
2101 let graph = builder.compile().unwrap();
2102
2103 let checkpointer = Arc::new(InMemoryCheckpointer::new());
2104 let executor = Executor::with_checkpointer(checkpointer);
2105
2106 // Suspend at the gate and capture the suspension checkpoint id.
2107 let outcome = executor
2108 .run(&graph, &spec, State::new(), RunConfig::new("t-hitl"))
2109 .await
2110 .unwrap();
2111 let checkpoint_id = match outcome {
2112 ExecutionOutcome::Interrupted { checkpoint_id, .. } => checkpoint_id,
2113 other => panic!("expected Interrupted, got {other:?}"),
2114 };
2115
2116 // checkpoint_id selects WHERE (the suspension checkpoint), resume
2117 // supplies the value delivered to the re-running gate node.
2118 let outcome = executor
2119 .run(
2120 &graph,
2121 &spec,
2122 State::new(),
2123 RunConfig::new("t-hitl")
2124 .with_checkpoint_id(checkpoint_id)
2125 .with_resume(json!(true)),
2126 )
2127 .await
2128 .unwrap();
2129
2130 match outcome {
2131 ExecutionOutcome::Done(state) => {
2132 assert_eq!(state.get("answer"), Some(&json!(true)));
2133 }
2134 other => panic!("expected Done after replay+resume, got {other:?}"),
2135 }
2136 }
2137
2138 #[tokio::test]
2139 async fn run_with_checkpoint_id_errors_without_checkpointer_or_unknown_id() {
2140 let (graph, spec) = linear_three_node_graph();
2141
2142 // No checkpointer configured: replay is impossible.
2143 let err = Executor::new()
2144 .run(
2145 &graph,
2146 &spec,
2147 State::new(),
2148 RunConfig::new("t-x").with_checkpoint_id("some-id"),
2149 )
2150 .await
2151 .unwrap_err();
2152 assert!(matches!(err, RustyError::Checkpoint(_)));
2153
2154 // Checkpointer present but the id does not exist on the thread.
2155 let checkpointer = Arc::new(InMemoryCheckpointer::new());
2156 let executor = Executor::with_checkpointer(checkpointer.clone());
2157 executor
2158 .run(&graph, &spec, State::new(), RunConfig::new("t-x"))
2159 .await
2160 .unwrap();
2161 let err = executor
2162 .run(
2163 &graph,
2164 &spec,
2165 State::new(),
2166 RunConfig::new("t-x").with_checkpoint_id("no-such-checkpoint"),
2167 )
2168 .await
2169 .unwrap_err();
2170 assert!(matches!(err, RustyError::Checkpoint(_)));
2171 }
2172
2173 /// A self-looping spinner: each super-step increments `n`; the router
2174 /// terminates the run once `n` reaches 5.
2175 fn spinner_graph() -> (Graph, StateSpec) {
2176 let spec = StateSpec::new().channel("n", Reducer::Overwrite);
2177 let mut builder = GraphBuilder::new();
2178 builder.add_node("spin", |ctx: NodeContext| async move {
2179 // A paced step: without it the loop would burn through the
2180 // default max_steps budget faster than a test can cancel it.
2181 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2182 let n = ctx.state().get("n").and_then(Value::as_i64).unwrap_or(0);
2183 Ok(NodeOutput::update("n", json!(n + 1)))
2184 });
2185 builder.set_entry_point("spin");
2186 builder.add_conditional_edges("spin", |state: State| async move {
2187 if state.get("n").and_then(Value::as_i64).unwrap_or(0) >= 5 {
2188 Ok(Route::End)
2189 } else {
2190 Ok(Route::Node("spin".into()))
2191 }
2192 });
2193 (builder.compile().unwrap(), spec)
2194 }
2195
2196 #[tokio::test]
2197 async fn pre_cancelled_token_stops_the_run_before_the_first_step() {
2198 let (graph, spec) = spinner_graph();
2199 let token = CancellationToken::new();
2200 token.cancel();
2201
2202 let err = Executor::new()
2203 .run(
2204 &graph,
2205 &spec,
2206 State::new(),
2207 RunConfig::new("t-cancel").with_cancellation(token),
2208 )
2209 .await
2210 .unwrap_err();
2211 assert!(matches!(err, RustyError::Cancelled(_)));
2212 assert!(
2213 err.to_string().contains("after 0 super-step(s)"),
2214 "a pre-cancelled run must not execute a single step: {err}"
2215 );
2216 }
2217
2218 #[tokio::test]
2219 async fn cancellation_stops_at_a_boundary_and_the_run_resumes_from_it() {
2220 let (graph, spec) = spinner_graph();
2221 let checkpointer = Arc::new(InMemoryCheckpointer::new());
2222 let executor = Executor::with_checkpointer(checkpointer.clone());
2223 let token = CancellationToken::new();
2224
2225 // Cancel as soon as the first boundary checkpoint lands: the stop
2226 // is then provably the token's doing, not the terminating route.
2227 let canceller = token.clone();
2228 let watcher = checkpointer.clone();
2229 tokio::spawn(async move {
2230 loop {
2231 if watcher.get_latest("t-cancel").await.unwrap().is_some() {
2232 break;
2233 }
2234 tokio::time::sleep(std::time::Duration::from_millis(1)).await;
2235 }
2236 canceller.cancel();
2237 });
2238 let err = executor
2239 .run(
2240 &graph,
2241 &spec,
2242 State::new(),
2243 RunConfig::new("t-cancel").with_cancellation(token.clone()),
2244 )
2245 .await
2246 .unwrap_err();
2247 assert!(matches!(err, RustyError::Cancelled(_)));
2248
2249 // The boundary checkpoint is intact: n counts the completed steps —
2250 // strictly fewer than the terminating route's 5, proving the token
2251 // (not the route) ended the run.
2252 let checkpoint = checkpointer
2253 .get_latest("t-cancel")
2254 .await
2255 .unwrap()
2256 .expect("a boundary checkpoint must survive cancellation");
2257 let steps = checkpoint.state.get("n").and_then(Value::as_i64).unwrap();
2258 assert!(
2259 (1..5).contains(&steps),
2260 "unexpected stop point: n = {steps}"
2261 );
2262
2263 // Re-running the thread resumes the spinner from the checkpoint.
2264 // Drain tokens are one-shot (a cancelled token stays cancelled), so
2265 // the post-deploy process runs under a fresh one.
2266 let outcome = executor
2267 .run(
2268 &graph,
2269 &spec,
2270 State::new(),
2271 RunConfig::new("t-cancel")
2272 .with_resume(json!(null))
2273 .with_cancellation(CancellationToken::new()),
2274 )
2275 .await
2276 .unwrap();
2277 match outcome {
2278 ExecutionOutcome::Done(state) => {
2279 assert_eq!(state.get("n"), Some(&json!(5)));
2280 }
2281 other => panic!("expected Done after resume, got {other:?}"),
2282 }
2283 }
2284}