rusty_agent_runtime/executor.rs
1//! The executor: a Pregel/BSP-inspired super-step run loop.
2//!
3//! Execution proceeds in discrete **super-steps** (Google Pregel /
4//! Bulk-Synchronous-Parallel), each super-step being:
5//!
6//! 1. **Plan** — determine the active node set (entry point on step 0;
7//! afterwards the routing result of the previous step, including
8//! [`crate::node::Command::goto`] overrides and [`crate::graph::Send`]
9//! fan-outs).
10//! 2. **Compute** — run all active nodes concurrently in a
11//! `tokio::task::JoinSet`, each receiving an **immutable snapshot** of
12//! the state as of the start of the step. No node can observe another's
13//! in-progress writes.
14//! 3. **Barrier** — wait for all active nodes. The step is *transactional*:
15//! if any node fails, the step's writes are discarded. An
16//! [`RustyError::Interrupt`] suspends the whole run instead.
17//! 4. **Merge** — apply all node updates to the state via
18//! [`crate::state::StateSpec::apply_super_step`] (per-channel reducers +
19//! `LastValue` single-write validation).
20//! 5. **Route** — evaluate outgoing edges / commands against the
21//! post-barrier state to determine the next active set; `Route::End` (or
22//! an empty next set) terminates the run.
23//! 6. **Checkpoint** — persist a [`crate::checkpoint::Checkpoint`] recording
24//! step, state, and next nodes, and emit [`GraphEvent`]s for streaming.
25//!
26//! A graph *cycle* (e.g. the ReAct loop `agent → tools → agent`) is not
27//! call-stack recursion — it is nodes being re-scheduled across super-steps,
28//! which is why the guard is `max_steps`, not a stack limit.
29//!
30//! # Observability
31//!
32//! The executor emits `tracing` telemetry throughout a run (no subscriber is
33//! installed by the library — the application chooses one):
34//!
35//! - `rusty.run` (INFO span) — one per [`Executor::run`] call, carrying
36//! `thread_id` and `max_steps`; parent of everything below.
37//! - `rusty.super_step` (DEBUG span) — one per super-step, carrying
38//! `step` and `active_nodes`; covers plan → barrier → merge → route.
39//! - `rusty.node` (INFO span) — one per spawned node task, carrying
40//! `node` and `step`; attached to the `JoinSet` task via `.instrument()`.
41//! - DEBUG event on each barrier merge (channels written), INFO events on
42//! interrupt and run completion (`steps`, `duration_ms`), WARN events on
43//! node failure (with a `retryable` classification).
44
45use std::collections::{HashMap, HashSet};
46use std::sync::{Arc, Mutex};
47
48use serde::{Deserialize, Serialize};
49use serde_json::Value;
50use tokio::sync::mpsc;
51use tokio::task::JoinSet;
52use tokio_util::sync::CancellationToken;
53use tracing::Instrument;
54
55use crate::checkpoint::{Checkpoint, Checkpointer};
56use crate::effects::{CompensationRegistry, EffectAdmissionContext};
57use crate::error::{Result, RustyError};
58use crate::graph::{Edge, Graph, Route};
59use crate::journal::{Clock, EventDraft, Journal, RngSource};
60use crate::middleware::{Middleware, MiddlewareChain, NodeCall};
61use crate::node::{Command, NodeConfig, NodeContext, NodeOutput};
62use crate::record::{
63 CheckpointHeader, Effect, EventStatus, PolicyVersion, RunEventKind, RunManifest,
64 CURRENT_FORMAT_VERSION,
65};
66use crate::state::{State, StateSpec};
67
68/// How a run ended.
69#[derive(Debug)]
70pub enum ExecutionOutcome {
71 /// The run terminated normally (routing reached `Route::End` or no
72 /// nodes remained active). Carries the final state.
73 Done(State),
74
75 /// A node called `interrupt(payload)`: the run is suspended and
76 /// resumable. Carry on by calling [`Executor::run`] again with the same
77 /// `thread_id` and `RunConfig::resume` set.
78 ///
79 /// Suspension is run-wide, not node-local. The in-flight super-step is
80 /// transactional: every write of the step is discarded (including writes
81 /// from sibling nodes that completed before the interrupt was observed
82 /// at the barrier), still-running siblings are aborted, and the
83 /// suspension checkpoint re-schedules **every** node of the step. On
84 /// resume all of them re-execute from their start, so node logic must be
85 /// idempotent.
86 Interrupted {
87 /// The payload passed to `interrupt()` (surfaced to the caller,
88 /// e.g. a human-approval request).
89 value: Value,
90 /// The state as of the suspension point.
91 state: State,
92 /// The checkpoint persisted at the suspension point, for resuming
93 /// or time travel. When the executor has no checkpointer attached,
94 /// the run still suspends but nothing is persisted: the id is then
95 /// only an opaque handle and can never be replayed.
96 checkpoint_id: String,
97 },
98}
99
100impl ExecutionOutcome {
101 /// The final (or suspension-point) state, regardless of variant.
102 pub fn state(&self) -> &State {
103 match self {
104 ExecutionOutcome::Done(s) => s,
105 ExecutionOutcome::Interrupted { state, .. } => state,
106 }
107 }
108
109 /// Whether the run ended in [`ExecutionOutcome::Interrupted`] (suspended,
110 /// resumable) rather than [`ExecutionOutcome::Done`].
111 pub fn is_interrupted(&self) -> bool {
112 matches!(self, ExecutionOutcome::Interrupted { .. })
113 }
114}
115
116/// Per-run configuration (the LangGraph `RunnableConfig` analog).
117#[derive(Debug, Clone)]
118pub struct RunConfig {
119 /// Thread (session) id. Stable across interrupt/resume; namespaces all
120 /// checkpoints for this run. Required for persistence and resume.
121 pub thread_id: String,
122
123 /// Maximum number of super-steps before the run aborts with
124 /// [`crate::error::RustyError::Graph`] (the LangGraph
125 /// `recursion_limit` / `GraphRecursionError` guard). Default: 1000.
126 pub max_steps: usize,
127
128 /// Resume value for continuing an interrupted run. When set, the
129 /// executor restores the latest checkpoint for `thread_id` and
130 /// re-executes the checkpointed next-node set with
131 /// [`crate::node::NodeContext::resume_value`] returning this value.
132 ///
133 /// The value is **broadcast**: every node scheduled in the first
134 /// super-step after the resume observes it, not only the node that
135 /// originally interrupted (a suspension checkpoint re-schedules the
136 /// whole active set — see [`ExecutionOutcome::Interrupted`]). Nodes that
137 /// should react only when they themselves were resumed must key off
138 /// their own state, not the presence of a resume value.
139 pub resume: Option<Value>,
140
141 /// Replay/time-travel handle: the id of a specific checkpoint of
142 /// `thread_id` to resume from. When set, the executor loads **that**
143 /// checkpoint (not the latest) and continues the run from its state and
144 /// next-node set. Requires a checkpointer on the executor.
145 ///
146 /// Combines with `resume`: `checkpoint_id` selects **where** the run
147 /// restarts, `resume` (when also set) is delivered as the resume value to
148 /// the first super-step, exactly as in interrupt/resume.
149 ///
150 /// Safe pattern: replaying on the *same* thread appends new history on
151 /// top of the old timeline, so prefer forking first —
152 /// [`crate::checkpoint::Checkpointer::fork_thread`] the thread into a new
153 /// thread id, then run the fork with `checkpoint_id` set. Direct replay
154 /// on the original thread is supported for cases where appended history
155 /// is acceptable.
156 pub checkpoint_id: Option<String>,
157
158 /// Optional event sink for streaming: the executor emits [`GraphEvent`]s
159 /// as the run progresses (node start/end, state updates, checkpoints,
160 /// super-step boundaries). Consumers implement LangGraph's stream modes
161 /// (`values` / `updates` / `tasks` / ...) as filters over this stream.
162 pub event_tx: Option<mpsc::Sender<GraphEvent>>,
163
164 /// Flight Recorder determinism seam (R0.5): the run's time source. Every
165 /// executor timestamp — event `recorded_at`, node latencies, checkpoint
166 /// `created_at` — is read through it. `None` (the default) is the system
167 /// wall clock, byte-identical to pre-R0.5 behavior. Attach
168 /// [`Clock::Logical`] to make a recorded run re-drivable.
169 pub clock: Option<Clock>,
170
171 /// Flight Recorder determinism seam (R0.5): the run's randomness source.
172 /// Checkpoint ids (and the run id, when the executor creates the
173 /// journal) are minted through it. `None` (the default) is OS entropy,
174 /// byte-identical to pre-R0.5 behavior; [`RngSource::Seeded`] makes the
175 /// id stream reproducible.
176 pub rng: Option<RngSource>,
177
178 /// Flight Recorder (R0.5): attach a pre-built journal for this run. Node
179 /// closures capture a clone of the same [`Journal`] to record their own
180 /// model/tool/remote/WASM calls into the run's evidence (the journal
181 /// stamps sequence numbers and timestamps itself). When `None`, the
182 /// executor creates a fresh journal per run; either way the run's
183 /// journal is retrievable afterwards via [`Executor::journal`].
184 pub journal: Option<Journal>,
185
186 /// Flight Recorder (R0.5): the executor policy active for this run,
187 /// stamped into every checkpoint header. `None` records the static
188 /// default ([`PolicyVersion::STATIC_V0`]) — except on resume (R0.8 wave
189 /// 4), where the run inherits the version its checkpoint header pins so
190 /// an in-flight run is immune to mid-run promotions. An explicit pin
191 /// always wins over inheritance.
192 pub policy_version: Option<PolicyVersion>,
193
194 /// Flight Recorder (R0.5): the application's own graph version string,
195 /// stamped into every checkpoint header next to the topology hash.
196 /// `None` records `"unversioned"`. Bump it when node bodies change in
197 /// ways the topology hash cannot see.
198 pub graph_version: Option<String>,
199
200 /// Cooperative cancellation (R0.6 wave 2c, drain). When set, the
201 /// executor checks the token **at every super-step boundary** and, once
202 /// it is cancelled, stops the run before starting the next super-step,
203 /// returning [`RustyError::Cancelled`].
204 ///
205 /// The boundary is the only cancellation point, deliberately: a
206 /// super-step is transactional (its nodes run off an immutable snapshot
207 /// and its writes merge atomically), and its boundary checkpoint is
208 /// already persisted by the time the token is observed — so a cancelled
209 /// run resumes from exactly where it stopped, with no torn step and no
210 /// lost writes. Cancellation granularity is therefore one super-step;
211 /// work inside the in-flight step runs to its barrier. This is the hook
212 /// graceful shutdown uses: a draining server cancels the token and every
213 /// in-flight run parks itself at a resumable checkpoint instead of being
214 /// torn down mid-step.
215 pub cancellation: Option<CancellationToken>,
216
217 /// Effect kernel v2 (R0.7): the versioned run manifest — prompts, tool
218 /// schemas, model and parameters, memory schema, capsule versions the run
219 /// pins. Stamped into every checkpoint header so a resumed run keeps
220 /// executing against its pinned versions (see
221 /// [`crate::record::RunManifest`] for the upgrade-safety contract).
222 /// `None` (the default) pins nothing and leaves checkpoint bytes
223 /// byte-identical to R0.5/R0.6.
224 pub manifest: Option<RunManifest>,
225
226 /// Effect kernel v2 (R0.7): the approval tokens this run carries for its
227 /// irreversible effects (see [`crate::effects::ApprovalToken`]). The
228 /// executor consults them when [`Executor::with_effect_admission`] has
229 /// enabled the guarded tool path. Empty by default; an irreversible call
230 /// dispatched through that path then fails admission before its body
231 /// runs.
232 pub effect_approvals: Vec<crate::effects::ApprovalToken>,
233}
234
235impl Default for RunConfig {
236 /// A config with an empty `thread_id` and the default step limit —
237 /// identical to `RunConfig::new("")`. Derived `Default` would zero
238 /// `max_steps`, so any `default()`-built run would instantly trip the
239 /// step guard; keep this impl in sync with [`RunConfig::new`].
240 fn default() -> Self {
241 Self::new(String::new())
242 }
243}
244
245impl RunConfig {
246 /// A config for `thread_id` with the default step limit.
247 pub fn new(thread_id: impl Into<String>) -> Self {
248 Self {
249 thread_id: thread_id.into(),
250 max_steps: DEFAULT_MAX_STEPS,
251 resume: None,
252 checkpoint_id: None,
253 event_tx: None,
254 clock: None,
255 rng: None,
256 journal: None,
257 policy_version: None,
258 graph_version: None,
259 cancellation: None,
260 manifest: None,
261 effect_approvals: Vec::new(),
262 }
263 }
264
265 /// Builder-style: override the step limit.
266 pub fn with_max_steps(mut self, max_steps: usize) -> Self {
267 self.max_steps = max_steps;
268 self
269 }
270
271 /// Builder-style: set the resume value.
272 pub fn with_resume(mut self, value: Value) -> Self {
273 self.resume = Some(value);
274 self
275 }
276
277 /// Builder-style: replay from a specific checkpoint of `thread_id`
278 /// (time travel). See the [`RunConfig::checkpoint_id`] field docs for
279 /// semantics and the fork-first safe pattern.
280 pub fn with_checkpoint_id(mut self, checkpoint_id: impl Into<String>) -> Self {
281 self.checkpoint_id = Some(checkpoint_id.into());
282 self
283 }
284
285 /// Builder-style: attach a streaming event sink.
286 pub fn with_event_tx(mut self, tx: mpsc::Sender<GraphEvent>) -> Self {
287 self.event_tx = Some(tx);
288 self
289 }
290
291 /// Builder-style: source the run's time from `clock` (Flight Recorder
292 /// determinism seam; see the [`RunConfig::clock`] field docs).
293 pub fn with_clock(mut self, clock: Clock) -> Self {
294 self.clock = Some(clock);
295 self
296 }
297
298 /// Builder-style: source the run's randomness from `rng` (Flight
299 /// Recorder determinism seam; see the [`RunConfig::rng`] field docs).
300 pub fn with_rng(mut self, rng: RngSource) -> Self {
301 self.rng = Some(rng);
302 self
303 }
304
305 /// Builder-style: record the run into `journal` (Flight Recorder; see
306 /// the [`RunConfig::journal`] field docs). When the config carries no
307 /// explicit [`RunConfig::clock`], the executor also reads time from the
308 /// attached journal's clock, keeping one time source per run.
309 pub fn with_journal(mut self, journal: Journal) -> Self {
310 self.journal = Some(journal);
311 self
312 }
313
314 /// Builder-style: stamp the active executor policy version into every
315 /// checkpoint header (see the [`RunConfig::policy_version`] field docs).
316 pub fn with_policy_version(mut self, version: PolicyVersion) -> Self {
317 self.policy_version = Some(version);
318 self
319 }
320
321 /// Builder-style: stamp the application's graph version into every
322 /// checkpoint header (see the [`RunConfig::graph_version`] field docs).
323 pub fn with_graph_version(mut self, version: impl Into<String>) -> Self {
324 self.graph_version = Some(version.into());
325 self
326 }
327
328 /// Builder-style: make the run cooperatively cancellable (see the
329 /// [`RunConfig::cancellation`] field docs). The token is observed only at
330 /// super-step boundaries: cancelling it stops the run *after* the
331 /// in-flight step's barrier and boundary checkpoint, so the run is left
332 /// resumable from exactly that checkpoint with [`RustyError::Cancelled`].
333 pub fn with_cancellation(mut self, token: CancellationToken) -> Self {
334 self.cancellation = Some(token);
335 self
336 }
337
338 /// Builder-style: pin the run's versioned manifest into every checkpoint
339 /// header (see the [`RunConfig::manifest`] field docs).
340 pub fn with_manifest(mut self, manifest: RunManifest) -> Self {
341 self.manifest = Some(manifest);
342 self
343 }
344
345 /// Builder-style: carry approval tokens for the run's irreversible
346 /// effects (see the [`RunConfig::effect_approvals`] field docs).
347 pub fn with_effect_approvals(
348 mut self,
349 approvals: impl IntoIterator<Item = crate::effects::ApprovalToken>,
350 ) -> Self {
351 self.effect_approvals = approvals.into_iter().collect();
352 self
353 }
354
355 /// A clone of the event sink sender, for wiring into nodes that stream
356 /// [`GraphEvent::Token`] deltas (LangGraph's `messages` stream mode).
357 ///
358 /// Typical flow: create the channel, call `config.token_tx()` to obtain
359 /// the clone a node closure captures, then hand `config` to
360 /// [`Executor::run`]. `None` when no sink is attached. See the
361 /// [`crate::llm::ChatModel`] rustdoc for the full pattern.
362 pub fn token_tx(&self) -> Option<mpsc::Sender<GraphEvent>> {
363 self.event_tx.clone()
364 }
365}
366
367/// Default super-step limit. Deliberately far above LangGraph's default
368/// `recursion_limit` of 25: ReAct-style loops burn one super-step per
369/// agent/tool hop, so long tool chains legitimately exceed 25.
370pub const DEFAULT_MAX_STEPS: usize = 1000;
371
372/// Streaming events emitted by the executor during a run. All of LangGraph's
373/// stream modes are views over this single typed event stream.
374///
375/// The enum is serializable so event streams can cross process / FFI
376/// boundaries (e.g. a WebSocket bridge or a persisted event log).
377#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
378#[serde(tag = "type", rename_all = "snake_case")]
379pub enum GraphEvent {
380 /// A node began executing.
381 NodeStart {
382 /// Node name.
383 node: String,
384 /// Super-step index.
385 step: usize,
386 },
387 /// A node finished executing (successfully).
388 NodeEnd {
389 /// Node name.
390 node: String,
391 /// Super-step index.
392 step: usize,
393 },
394 /// A single LLM token delta (the LangGraph `messages` stream mode).
395 ///
396 /// The executor itself never emits this variant: tokens originate inside
397 /// nodes that call [`crate::llm::ChatModel::chat_stream`] and forward
398 /// each [`crate::llm::TokenChunk`] into the run's event channel (see the
399 /// `ChatModel` rustdoc for the wiring pattern and
400 /// [`RunConfig::token_tx`] / [`Executor::token_tx`] for sender handles).
401 Token {
402 /// Node that produced the token.
403 node: String,
404 /// Incremental text produced since the previous token.
405 delta: String,
406 },
407 /// State was updated at a super-step barrier (`updates` stream mode).
408 StateUpdate {
409 /// Super-step index at which the update was applied.
410 step: usize,
411 /// Every channel written in this step mapped to its **post-reducer**
412 /// value (e.g. the full appended list for an `Append` channel), read
413 /// back from the merged state — not the raw per-node partials.
414 updates: serde_json::Map<String, Value>,
415 },
416 /// A checkpoint was persisted at a super-step boundary.
417 CheckpointSaved {
418 /// The checkpoint id.
419 checkpoint_id: String,
420 /// Super-step index at the boundary.
421 step: usize,
422 },
423 /// A super-step began; lists the nodes activated in it.
424 SuperStep {
425 /// Super-step index.
426 step: usize,
427 /// Nodes active in this step.
428 active_nodes: Vec<String>,
429 },
430}
431
432/// The graph executor. Holds an optional checkpointer and an optional
433/// middleware chain; stateless with respect to individual runs, so one
434/// `Executor` can drive many concurrent runs (each with its own
435/// `thread_id`).
436#[derive(Default)]
437pub struct Executor {
438 checkpointer: Option<Arc<dyn Checkpointer>>,
439 token_tx: Option<mpsc::Sender<GraphEvent>>,
440 // The most recent run's Flight Recorder journal. Interior mutability
441 // keeps `run` taking `&self`; overwritten at the start of every run.
442 journal: Mutex<Option<Journal>>,
443 // Middleware/Interceptor SDK: ordered layers wrapping every node
444 // invocation (and, via `NodeContext::middleware`, the tool/model calls
445 // node code makes). An empty chain takes the original dispatch path.
446 middleware: MiddlewareChain,
447 // R0.8 wave 1: the governed-memory source this executor's runs answer
448 // memory reads from (live store or replay cursor). The executor never
449 // touches memory itself this wave — the full run-loop integration is a
450 // later wave; this is the installation point node factories use (see
451 // `Executor::memory`), mirroring `with_token_tx`.
452 memory_source: Option<crate::memory::MemorySource>,
453 // Present only when guarded tool effect admission is enabled. The run's
454 // thread id and approval tokens are combined with these rollback handlers
455 // into an EffectAdmissionContext for each node invocation.
456 effect_compensations: Option<CompensationRegistry>,
457}
458
459impl Executor {
460 /// An executor without persistence (runs cannot be interrupted/resumed
461 /// durably; interrupts will still surface but resume requires a
462 /// checkpointer).
463 pub fn new() -> Self {
464 Self::default()
465 }
466
467 /// An executor persisting checkpoints through `checkpointer`.
468 pub fn with_checkpointer(checkpointer: Arc<dyn Checkpointer>) -> Self {
469 Self {
470 checkpointer: Some(checkpointer),
471 ..Self::default()
472 }
473 }
474
475 /// Builder-style: attach a middleware layer (Middleware/Interceptor
476 /// SDK). Layers compose in registration order — before-hooks run in
477 /// `.layer()` order on the way into a node/model/tool call, after-hooks
478 /// in reverse order on the way out. See [`crate::middleware`].
479 pub fn layer<M: Middleware + 'static>(mut self, middleware: M) -> Self {
480 self.middleware.push(Arc::new(middleware));
481 self
482 }
483
484 /// Builder-style: attach a pre-shared middleware layer.
485 pub fn layer_shared(mut self, middleware: Arc<dyn Middleware>) -> Self {
486 self.middleware.push(middleware);
487 self
488 }
489
490 /// The attached middleware chain (empty when no layers were added).
491 pub fn middleware(&self) -> &MiddlewareChain {
492 &self.middleware
493 }
494
495 /// Builder-style: make guarded tool effect admission available to nodes.
496 ///
497 /// The supplied registry may be empty. Pure and read-only calls remain
498 /// automatic; idempotent calls require a key, compensatable calls require
499 /// a handler in this registry, and non-idempotent calls require a matching
500 /// token in [`RunConfig::effect_approvals`]. The prebuilt ReAct tools node
501 /// enforces this automatically. Custom nodes must pass
502 /// [`NodeContext::effect_admission`] to the [`crate::tool::ToolExecutor`]
503 /// they construct; direct [`crate::tool::Tool::call`] invocations are not
504 /// intercepted by the executor.
505 pub fn with_effect_admission(mut self, compensations: CompensationRegistry) -> Self {
506 self.effect_compensations = Some(compensations);
507 self
508 }
509
510 /// Whether this executor enforces the effect admission boundary.
511 pub fn effect_admission_enabled(&self) -> bool {
512 self.effect_compensations.is_some()
513 }
514
515 /// Builder-style: hold a token broadcast sender that nodes can clone to
516 /// publish [`GraphEvent::Token`] deltas (LangGraph's `messages` stream
517 /// mode).
518 ///
519 /// The executor never emits `Token` events itself — tokens originate in
520 /// nodes calling [`crate::llm::ChatModel::chat_stream`]. This is a
521 /// convenience handle so node factories built around an `Executor` can
522 /// fetch the sink via [`Executor::token_tx`] and capture a clone in each
523 /// node closure. When the same channel should also receive the
524 /// executor's own events, attach it to the run via
525 /// [`RunConfig::with_event_tx`] instead (or as well).
526 pub fn with_token_tx(mut self, token_tx: mpsc::Sender<GraphEvent>) -> Self {
527 self.token_tx = Some(token_tx);
528 self
529 }
530
531 /// The token broadcast sender, if one was attached via
532 /// [`Executor::with_token_tx`]. Clone it into node closures to stream
533 /// [`GraphEvent::Token`]s from within nodes.
534 pub fn token_tx(&self) -> Option<&mpsc::Sender<GraphEvent>> {
535 self.token_tx.as_ref()
536 }
537
538 /// Builder-style: install the governed-memory source this executor's
539 /// runs answer memory reads from (R0.8 wave 1; see
540 /// [`crate::memory::MemorySource`]).
541 ///
542 /// The executor never reads or writes memory itself this wave — the
543 /// full run-loop integration is a later wave. This is the seam's
544 /// installation point, mirroring [`Executor::with_token_tx`]: node
545 /// factories built around an `Executor` fetch the journaled handle via
546 /// [`Executor::memory`] once the run has started and capture a clone in
547 /// each node closure. Nodes may equally bind a source to an attached
548 /// journal directly ([`crate::journal::Journal::memory`]); both paths
549 /// journal into the same run evidence.
550 pub fn with_memory_source(mut self, source: crate::memory::MemorySource) -> Self {
551 self.memory_source = Some(source);
552 self
553 }
554
555 /// The configured memory source, if one was installed via
556 /// [`Executor::with_memory_source`].
557 pub fn memory_source(&self) -> Option<&crate::memory::MemorySource> {
558 self.memory_source.as_ref()
559 }
560
561 /// The journaled memory handle for the most recent run: the run's
562 /// journal (set at run start) bound to the configured source. `None`
563 /// before the first run or when no source is installed. The same
564 /// most-recent-run caveat as [`Executor::journal`] applies: driving
565 /// several runs concurrently through one `Executor` leaves this handle
566 /// pointing at whichever run started last — bind per-run handles via
567 /// [`crate::journal::Journal::memory`] when runs overlap.
568 pub fn memory(&self) -> Option<crate::memory::JournaledMemory> {
569 let journal = self.journal()?;
570 let source = self.memory_source.clone()?;
571 Some(crate::memory::JournaledMemory::new(&journal, source))
572 }
573
574 /// The configured checkpointer, if any. Shared (not consumed) so one
575 /// `Executor` can drive many concurrent runs over the same store.
576 pub fn checkpointer(&self) -> Option<&Arc<dyn Checkpointer>> {
577 self.checkpointer.as_ref()
578 }
579
580 /// The Flight Recorder journal of the most recent run started through
581 /// this executor — the one attached via [`RunConfig::with_journal`], or
582 /// the fresh journal the executor created for the run. `None` before the
583 /// first run.
584 ///
585 /// The journal is set at run start, so the evidence of a failed or
586 /// suspended run is retrievable exactly like a completed run's. Note the
587 /// executor is deliberately stateless across runs: driving several runs
588 /// concurrently through one `Executor` leaves this handle pointing at
589 /// whichever run started last — attach per-run journals via the config
590 /// when runs overlap.
591 pub fn journal(&self) -> Option<Journal> {
592 self.journal
593 .lock()
594 .unwrap_or_else(|e| e.into_inner())
595 .clone()
596 }
597
598 /// Run a compiled graph to completion (or interruption).
599 ///
600 /// - `graph`: the compiled, frozen graph topology.
601 /// - `spec`: the state schema (channels + reducers) used to merge node
602 /// updates at each barrier.
603 /// - `initial_state`: the starting state. When `config.resume` is set and
604 /// a checkpoint exists for `config.thread_id`, the checkpointed state
605 /// and next-node set take precedence over this argument. When
606 /// `config.checkpoint_id` is set, that specific checkpoint (rather than
607 /// the latest) is restored — replay/time travel; forking into a fresh
608 /// thread first via
609 /// [`crate::checkpoint::Checkpointer::fork_thread`] is the safe pattern,
610 /// since replaying on the same thread appends new history.
611 /// - `config`: run configuration (thread id, step limit, resume value,
612 /// streaming sink).
613 ///
614 /// # Super-step semantics
615 ///
616 /// Each loop iteration runs one super-step as a transaction: the active
617 /// nodes execute in parallel over an immutable start-of-step snapshot;
618 /// the barrier discards the whole step's writes on any node failure and
619 /// suspends the run on an interrupt; only then are writes merged via the
620 /// channel reducers, routing computed against the post-barrier state,
621 /// and a boundary checkpoint persisted. The module-level docs enumerate
622 /// the six phases; `execute_super_step` is the implementation.
623 ///
624 /// The loop returns [`ExecutionOutcome::Done`] when routing yields an
625 /// empty next set, [`ExecutionOutcome::Interrupted`] when a node
626 /// interrupts, an [`RustyError::Graph`] error once `config.max_steps`
627 /// super-steps have run without termination, and
628 /// [`RustyError::Cancelled`] when [`RunConfig::cancellation`] is
629 /// observed cancelled at a super-step boundary (the boundary checkpoint
630 /// is intact; the run resumes from it).
631 pub async fn run(
632 &self,
633 graph: &Graph,
634 spec: &StateSpec,
635 initial_state: State,
636 config: RunConfig,
637 ) -> Result<ExecutionOutcome> {
638 // Run-level span: every super-step, node, and checkpoint trace in the
639 // run attaches to it. Attached via `.instrument()` (never `.enter()`)
640 // so no span guard is held across `.await` points and the returned
641 // future stays `Send`.
642 let run_span = tracing::info_span!(
643 "rusty.run",
644 thread_id = %config.thread_id,
645 max_steps = config.max_steps,
646 resume = config.resume.is_some(),
647 replay = config.checkpoint_id.is_some(),
648 );
649 self.run_inner(graph, spec, initial_state, config)
650 .instrument(run_span)
651 .await
652 }
653
654 /// The instrumented body of [`Executor::run`]; see that method's docs for
655 /// the super-step algorithm.
656 async fn run_inner(
657 &self,
658 graph: &Graph,
659 spec: &StateSpec,
660 initial_state: State,
661 config: RunConfig,
662 ) -> Result<ExecutionOutcome> {
663 let started = std::time::Instant::now();
664
665 // ---- flight recorder setup ----
666 //
667 // Resolve the determinism seams and the run's journal before any
668 // checkpoint work: every timestamp and id below flows through them.
669 // An attached journal carries its own identity and clock; otherwise
670 // the executor mints a run id from the (possibly seeded) RNG and a
671 // journal over the configured clock. The run reads time from the
672 // journal's clock when no explicit clock is configured, keeping one
673 // time source per run.
674 let rng = config.rng.clone().unwrap_or_default();
675 let journal = match &config.journal {
676 Some(attached) => attached.clone(),
677 None => Journal::new(
678 rng.uuid_string(),
679 config.thread_id.clone(),
680 config.clock.clone().unwrap_or_default(),
681 ),
682 };
683 let mut recorder = Recorder {
684 clock: config
685 .clock
686 .clone()
687 .unwrap_or_else(|| journal.clock().clone()),
688 rng,
689 graph_version: config
690 .graph_version
691 .clone()
692 .unwrap_or_else(|| "unversioned".to_owned()),
693 graph_hash: graph.topology_hash(),
694 policy_version: config.policy_version.clone().unwrap_or_default(),
695 manifest: config.manifest.clone(),
696 journal: journal.clone(),
697 };
698 // Publish before the loop: evidence of a failed or suspended run is
699 // retrievable exactly like a completed run's.
700 *self.journal.lock().unwrap_or_else(|e| e.into_inner()) = Some(journal);
701
702 // ---- initialization / resume ----
703 //
704 // On resume the checkpointed state and next-node set take precedence
705 // over `initial_state`; the resume value is delivered to the first
706 // super-step (whose active set is the checkpointed next-node set —
707 // after an interrupt, every node of the suspended step) via
708 // `NodeContext::resume_value()`.
709 //
710 // Time travel: when `config.checkpoint_id` is set, THAT checkpoint is
711 // restored instead of the latest — this is replay from an arbitrary
712 // history point. The two knobs compose: `checkpoint_id` selects WHERE
713 // the run restarts, `resume` (when also set) supplies the resume value
714 // for the first super-step.
715 let mut state = initial_state;
716 let mut active: Vec<ActiveTask>;
717 let mut step: usize = 0;
718 let mut pending_resume: Option<Value> = None;
719 // Causal parent for the first super-step: the resume event on a
720 // resumed run, nothing on a fresh one.
721 let mut step_parent: Option<String> = None;
722
723 if config.checkpoint_id.is_some() || config.resume.is_some() {
724 let checkpointer = self.checkpointer.as_ref().ok_or_else(|| {
725 RustyError::Checkpoint(
726 "RunConfig.checkpoint_id/resume is set but no checkpointer is configured \
727 on the executor"
728 .into(),
729 )
730 })?;
731 let checkpoint = match &config.checkpoint_id {
732 Some(id) => checkpointer
733 .get_by_id(&config.thread_id, id)
734 .await?
735 .ok_or_else(|| {
736 RustyError::Checkpoint(format!(
737 "cannot replay thread `{}`: checkpoint `{id}` not found",
738 config.thread_id
739 ))
740 })?,
741 None => checkpointer
742 .get_latest(&config.thread_id)
743 .await?
744 .ok_or_else(|| {
745 RustyError::Checkpoint(format!(
746 "cannot resume thread `{}`: no checkpoint found",
747 config.thread_id
748 ))
749 })?,
750 };
751 // Policy-binding continuity (R0.8 wave 4): a resumed run keeps
752 // the policy version its checkpoint header pins, so a mid-run
753 // promotion never changes behavior under an in-flight execution.
754 // An explicit `RunConfig::with_policy_version` pin wins over
755 // inheritance — the caller asked for that version deliberately.
756 if config.policy_version.is_none() {
757 recorder.policy_version = checkpoint.header.policy_version.clone();
758 }
759 step_parent = Some(recorder.record(
760 EventDraft::new(RunEventKind::Resume, Effect::Pure).input(serde_json::json!({
761 "checkpoint_id": checkpoint.id,
762 "step": checkpoint.step,
763 "resume": config.resume.clone().unwrap_or(Value::Null),
764 })),
765 ));
766 state = checkpoint.state;
767 step = checkpoint.step;
768 active = checkpoint
769 .next_nodes
770 .into_iter()
771 .map(|name| ActiveTask { name, scoped: None })
772 .collect();
773 pending_resume = config.resume.clone();
774 if active.is_empty() {
775 tracing::info!(
776 steps = 0,
777 duration_ms = started.elapsed().as_millis() as u64,
778 "run complete"
779 );
780 return Ok(ExecutionOutcome::Done(state));
781 }
782 } else {
783 active = vec![ActiveTask {
784 name: graph.entry_point().to_owned(),
785 scoped: None,
786 }];
787 }
788
789 // One shared context for the whole run: clones handed to parallel
790 // nodes share its approval ledger, and later super-steps observe
791 // tokens already consumed by earlier calls.
792 let effect_admission = self.effect_compensations.as_ref().map(|compensations| {
793 EffectAdmissionContext::new(config.thread_id.clone())
794 .with_approvals(config.effect_approvals.clone())
795 .with_compensations(compensations.clone())
796 });
797
798 // ---- super-step loop ----
799 let mut steps_run: usize = 0;
800 loop {
801 if steps_run >= config.max_steps {
802 return Err(RustyError::Graph(format!(
803 "max_steps ({}) exceeded: the graph did not terminate within the step \
804 budget (possible infinite cycle; raise RunConfig::max_steps or add a \
805 terminating route)",
806 config.max_steps
807 )));
808 }
809 // Cooperative cancellation (drain): observed only here, at a
810 // super-step boundary. The last step's barrier merged cleanly
811 // and its boundary checkpoint is already persisted, so
812 // cancelling never tears a step — the run resumes from exactly
813 // this point. Nodes of the in-flight step always run to their
814 // barrier; cancellation granularity is one super-step.
815 if config
816 .cancellation
817 .as_ref()
818 .is_some_and(CancellationToken::is_cancelled)
819 {
820 tracing::warn!(
821 steps_run,
822 "run cancelled at a super-step boundary; resumable from the last checkpoint"
823 );
824 return Err(RustyError::Cancelled(format!(
825 "cancelled after {steps_run} super-step(s); the boundary checkpoint of \
826 thread `{}` is intact, so re-running the thread resumes from there",
827 config.thread_id
828 )));
829 }
830
831 // The step body runs in a dedicated method so the whole body is
832 // one instrumented future under the per-step span.
833 let step_span =
834 tracing::debug_span!("rusty.super_step", step = step, active_nodes = active.len(),);
835
836 let transition = self
837 .execute_super_step(
838 graph,
839 spec,
840 &config,
841 &recorder,
842 effect_admission.as_ref(),
843 &mut state,
844 &active,
845 step,
846 &mut pending_resume,
847 step_parent.clone(),
848 )
849 .instrument(step_span)
850 .await?;
851
852 match transition {
853 StepTransition::Next(next, route_event) => {
854 active = next;
855 step += 1;
856 steps_run += 1;
857 step_parent = Some(route_event);
858 }
859 StepTransition::Finish(outcome) => {
860 if !outcome.is_interrupted() {
861 tracing::info!(
862 steps = steps_run + 1,
863 duration_ms = started.elapsed().as_millis() as u64,
864 "run complete"
865 );
866 }
867 return Ok(outcome);
868 }
869 }
870 }
871 }
872
873 /// Executes one super-step: plan -> compute -> barrier -> merge -> route
874 /// -> boundary checkpoint. Returns the next active set, or
875 /// [`StepTransition::Finish`] with the terminal outcome when the run ends
876 /// (`Done`) or suspends (`Interrupted`).
877 ///
878 /// Flight Recorder: every phase transition is journaled through
879 /// `recorder` — super-step start/end, one node input/output pair per
880 /// invocation (in deterministic active-set order, not finish order), the
881 /// routing decision, and the checkpoint write. Node events carry the
882 /// node's declared [`Effect`] classification.
883 #[allow(clippy::too_many_arguments)]
884 async fn execute_super_step(
885 &self,
886 graph: &Graph,
887 spec: &StateSpec,
888 config: &RunConfig,
889 recorder: &Recorder,
890 effect_admission: Option<&EffectAdmissionContext>,
891 state: &mut State,
892 active: &[ActiveTask],
893 step: usize,
894 pending_resume: &mut Option<Value>,
895 step_parent: Option<String>,
896 ) -> Result<StepTransition> {
897 // -- plan.
898 let active_names: Vec<String> = active.iter().map(|t| t.name.clone()).collect();
899 Self::emit(
900 config,
901 GraphEvent::SuperStep {
902 step,
903 active_nodes: active_names.clone(),
904 },
905 );
906 let mut plan_draft =
907 EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure).input(serde_json::json!({
908 "step": step,
909 "active_nodes": active_names,
910 }));
911 if let Some(parent) = step_parent {
912 plan_draft = plan_draft.parent(parent);
913 }
914 let step_start_event = recorder.record(plan_draft);
915
916 // -- compute. Scoped (Send) state is overlaid onto each invocation's
917 // private copy of the start-of-step snapshot, so fan-out items
918 // never collide in the shared state.
919 let snapshot = state.clone();
920 let effect_admission = effect_admission.cloned();
921 let mut join_set: JoinSet<(usize, String, Result<NodeOutput>, u64)> = JoinSet::new();
922 // Per-invocation journal metadata, aligned with `active`: the input
923 // event id (causal parent of the matching output) and the node's
924 // declared effect class. Fan-out invocations of the same node each
925 // get their own entry — index, not name, is the identity.
926 let mut input_events: Vec<String> = Vec::with_capacity(active.len());
927 let mut invocation_effects: Vec<Effect> = Vec::with_capacity(active.len());
928
929 for (index, task) in active.iter().enumerate() {
930 let node = graph.node(&task.name).ok_or_else(|| {
931 RustyError::Graph(format!("routing activated unknown node `{}`", task.name))
932 })?;
933
934 let mut node_state = snapshot.clone();
935 if let Some(scoped) = &task.scoped {
936 match scoped {
937 Value::Object(map) => {
938 for (channel, value) in map {
939 node_state.insert(channel.clone(), value.clone());
940 }
941 }
942 other => {
943 return Err(RustyError::InvalidUpdate(format!(
944 "Send scoped state for node `{}` must be a JSON object, \
945 got {other}",
946 task.name
947 )));
948 }
949 }
950 }
951
952 let input_event = recorder.record(
953 EventDraft::new(RunEventKind::NodeInput, node.effect())
954 .node(task.name.clone())
955 .input(node_state.to_value())
956 .parent(step_start_event.clone()),
957 );
958 invocation_effects.push(node.effect());
959
960 let node_config = NodeConfig {
961 thread_id: config.thread_id.clone(),
962 step,
963 resume: pending_resume.clone(),
964 // Hand the invocation its own journal event id so node
965 // code can parent the effects it records (model/tool
966 // calls) to this invocation.
967 extra: HashMap::from([(
968 crate::journal::PARENT_EVENT_KEY.to_owned(),
969 Value::String(input_event.clone()),
970 )]),
971 };
972 input_events.push(input_event);
973 let name = task.name.clone();
974 Self::emit(
975 config,
976 GraphEvent::NodeStart {
977 node: name.clone(),
978 step,
979 },
980 );
981 // A JoinSet polls tasks independently of the spawning task's
982 // context, so the per-node span is attached to each spawned
983 // future explicitly via `.instrument()`.
984 let node_span = tracing::info_span!("rusty.node", node = %name, step = step);
985 // Latency is read through the run's clock seam — under a logical
986 // clock the value is reproducible; under the system clock it is
987 // the real elapsed time.
988 let clock = recorder.clock.clone();
989 let chain = self.middleware.clone();
990 let effect_admission = effect_admission.clone();
991 join_set.spawn(
992 async move {
993 let node_started = clock.now();
994 // Middleware chain: with layers attached, the invocation
995 // runs inside the onion — before-hooks may mutate the
996 // snapshot, reject the run, or short-circuit with a
997 // substitute output; after-hooks unwind over the result.
998 // No layers: the original dispatch, byte-identical.
999 let result = if chain.is_empty() {
1000 node.run(
1001 NodeContext::new(node_state, node_config)
1002 .with_optional_effect_admission(effect_admission),
1003 )
1004 .await
1005 } else {
1006 let mut call = NodeCall::new(
1007 node_config.thread_id.clone(),
1008 name.clone(),
1009 step,
1010 node_state,
1011 );
1012 chain
1013 .run_node(&mut call, |call| {
1014 let ctx = NodeContext::new(call.state().clone(), node_config)
1015 .with_middleware(chain.clone())
1016 .with_optional_effect_admission(effect_admission);
1017 let node = Arc::clone(&node);
1018 async move { node.run(ctx).await }
1019 })
1020 .await
1021 };
1022 let latency_ms = (clock.now() - node_started).num_milliseconds().max(0) as u64;
1023 (index, name, result, latency_ms)
1024 }
1025 .instrument(node_span),
1026 );
1027 }
1028 // The resume value is consumed by the first super-step after a resume.
1029 *pending_resume = None;
1030
1031 // -- barrier: collect every node result. The step is
1032 // transactional: on any failure the JoinSet is dropped
1033 // (aborting stragglers) and the step's writes are discarded.
1034 let mut writes: Vec<(String, HashMap<String, Value>)> = Vec::new();
1035 let mut commands: Vec<Command> = Vec::new();
1036 let mut ran_nodes: Vec<String> = Vec::new();
1037 let mut interrupted: Option<(usize, String, Value)> = None;
1038 // Journal payloads of finished invocations, in finish order here;
1039 // journaled in active-set order after the barrier.
1040 let mut completed: Vec<(usize, String, u64, Value)> = Vec::new();
1041
1042 while let Some(joined) = join_set.join_next().await {
1043 let (index, name, result, latency_ms) = joined.map_err(|e| {
1044 RustyError::Node(format!(
1045 "node task failed to join (panic or cancellation): {e}"
1046 ))
1047 })?;
1048 match result {
1049 Ok(output) => {
1050 Self::emit(
1051 config,
1052 GraphEvent::NodeEnd {
1053 node: name.clone(),
1054 step,
1055 },
1056 );
1057 completed.push((
1058 index,
1059 name.clone(),
1060 latency_ms,
1061 serde_json::json!({
1062 "updates": &output.updates,
1063 "command": &output.command,
1064 }),
1065 ));
1066 if let Some(command) = output.command {
1067 if !command.goto.is_empty() {
1068 commands.push(command);
1069 }
1070 }
1071 ran_nodes.push(name.clone());
1072 writes.push((name, output.updates));
1073 }
1074 Err(RustyError::Interrupt { value }) => {
1075 // Record the suspension and stop the barrier loop; the
1076 // JoinSet is dropped below to abort stragglers.
1077 interrupted = Some((index, name, value));
1078 break;
1079 }
1080 Err(e) => {
1081 // A failed node is still evidence: journal the failure
1082 // before the error unwinds the run.
1083 recorder.record(
1084 EventDraft::new(RunEventKind::NodeOutput, invocation_effects[index])
1085 .node(name.clone())
1086 .status(EventStatus::Error)
1087 .output(serde_json::json!({ "error": e.to_string() }))
1088 .latency_ms(latency_ms)
1089 .parent(input_events[index].clone()),
1090 );
1091 // LLM and tool failures are the transient, retryable
1092 // error classes; everything else is a hard failure.
1093 let retryable = matches!(e, RustyError::Llm(_) | RustyError::Tool(_));
1094 tracing::warn!(
1095 node = %name,
1096 step = step,
1097 error = %e,
1098 retryable = retryable,
1099 "node failed; super-step aborted and its writes discarded"
1100 );
1101 return Err(RustyError::Node(format!(
1102 "node `{name}` failed at super-step {step}: {e}"
1103 )));
1104 }
1105 }
1106 }
1107
1108 if let Some((index, name, value)) = interrupted {
1109 // Suspend the run. The step is transactional, so no write of
1110 // this step survived — not even from siblings that completed
1111 // before the interrupt reached the barrier. The suspension
1112 // checkpoint therefore re-schedules the ENTIRE active set (the
1113 // interrupting node plus all siblings), otherwise completed
1114 // siblings' discarded writes would be silently lost and aborted
1115 // siblings would never re-run. Dropping the JoinSet first aborts
1116 // stragglers, preserving the transactional suspension point.
1117 drop(join_set);
1118 let interrupt_event = recorder.record(
1119 EventDraft::new(RunEventKind::Interrupt, invocation_effects[index])
1120 .node(name.clone())
1121 .input(value.clone())
1122 .status(EventStatus::Interrupted)
1123 .parent(input_events[index].clone()),
1124 );
1125 tracing::info!(
1126 node = %name,
1127 step = step,
1128 "node interrupted; run suspended (resumable via RunConfig::resume)"
1129 );
1130 let pending: Vec<String> = active.iter().map(|t| t.name.clone()).collect();
1131 let checkpoint =
1132 recorder.mint_checkpoint(config.thread_id.clone(), step, state.clone(), pending);
1133 let checkpoint_id = checkpoint.id.clone();
1134 if let Some(checkpointer) = &self.checkpointer {
1135 checkpointer.put(checkpoint).await?;
1136 recorder.record(
1137 EventDraft::new(RunEventKind::CheckpointWritten, Effect::Idempotent)
1138 .output(serde_json::json!({
1139 "checkpoint_id": checkpoint_id,
1140 "step": step,
1141 "suspension": true,
1142 }))
1143 .parent(interrupt_event),
1144 );
1145 Self::emit(
1146 config,
1147 GraphEvent::CheckpointSaved {
1148 checkpoint_id: checkpoint_id.clone(),
1149 step,
1150 },
1151 );
1152 }
1153 return Ok(StepTransition::Finish(ExecutionOutcome::Interrupted {
1154 value,
1155 state: state.clone(),
1156 checkpoint_id,
1157 }));
1158 }
1159
1160 // Journal node outputs in active-set order, not JoinSet finish
1161 // order: the finish order is scheduling-dependent, and the journal's
1162 // sequence is evidence — replay compares it.
1163 completed.sort_by_key(|(index, ..)| *index);
1164 let mut last_output_event: Option<String> = None;
1165 for (index, name, latency_ms, output_json) in completed {
1166 last_output_event = Some(
1167 recorder.record(
1168 EventDraft::new(RunEventKind::NodeOutput, invocation_effects[index])
1169 .node(name)
1170 .output(output_json)
1171 .latency_ms(latency_ms)
1172 .parent(input_events[index].clone()),
1173 ),
1174 );
1175 }
1176
1177 // -- merge: reducers + LastValue single-write validation. On
1178 // error the mutated state is dropped with the run
1179 // (transactional super-step).
1180 //
1181 // The start-of-step snapshot's job ends at the barrier — every node
1182 // has reported (or the step has failed). Dropping it before the
1183 // merge releases its per-channel shared references, so channels no
1184 // checkpoint still shares reach the reducer with refcount 1 and
1185 // merge in place (W4 copy-on-write) instead of cloning.
1186 drop(snapshot);
1187 let written_channels: HashSet<String> = writes
1188 .iter()
1189 .flat_map(|(_, updates)| updates.keys().cloned())
1190 .collect();
1191 spec.apply_super_step(state, writes)?;
1192 // The event carries the post-reducer values read back out of the
1193 // merged state: when several nodes write the same channel in one
1194 // step (the normal Append fan-in case), reporting the raw partials
1195 // would keep only the last write and hide the rest.
1196 let mut merged_updates = serde_json::Map::new();
1197 for channel in &written_channels {
1198 if let Some(value) = state.get(channel) {
1199 merged_updates.insert(channel.clone(), value.clone());
1200 }
1201 }
1202 let channels_written: Vec<&str> = merged_updates.keys().map(String::as_str).collect();
1203 tracing::debug!(
1204 step = step,
1205 channels = ?channels_written,
1206 "merged node updates at super-step barrier"
1207 );
1208 // The super-step's reducer result is journaled even when empty —
1209 // "no channel changed" is evidence too.
1210 let step_end_event = recorder.record(
1211 EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
1212 .output(Value::Object(merged_updates.clone()))
1213 .parent(last_output_event.unwrap_or_else(|| step_start_event.clone())),
1214 );
1215 if !merged_updates.is_empty() {
1216 Self::emit(
1217 config,
1218 GraphEvent::StateUpdate {
1219 step,
1220 updates: merged_updates,
1221 },
1222 );
1223 }
1224
1225 // -- route: Command::goto overrides the static edge set;
1226 // otherwise evaluate outgoing edges of every node that ran
1227 // against the post-barrier state.
1228 let mut next: Vec<ActiveTask> = Vec::new();
1229 let mut planned: HashSet<String> = HashSet::new();
1230
1231 if !commands.is_empty() {
1232 for command in &commands {
1233 for target in &command.goto {
1234 if !graph.has_node(target) {
1235 return Err(RustyError::Graph(format!(
1236 "Command::goto references unknown node `{target}`"
1237 )));
1238 }
1239 if planned.insert(target.clone()) {
1240 next.push(ActiveTask {
1241 name: target.clone(),
1242 scoped: None,
1243 });
1244 }
1245 }
1246 }
1247 } else {
1248 let mut evaluated: HashSet<String> = HashSet::new();
1249 for name in &ran_nodes {
1250 // Fan-out invocations of the same node share one edge set;
1251 // evaluate it once.
1252 if !evaluated.insert(name.clone()) {
1253 continue;
1254 }
1255 for edge in graph.outgoing_edges(name) {
1256 match edge {
1257 Edge::Direct { to, .. } => {
1258 if planned.insert(to.clone()) {
1259 next.push(ActiveTask {
1260 name: to.clone(),
1261 scoped: None,
1262 });
1263 }
1264 }
1265 Edge::Conditional { router, .. } => {
1266 match router(state.clone()).await? {
1267 Route::Node(target) => {
1268 if !graph.has_node(&target) {
1269 return Err(RustyError::Graph(format!(
1270 "conditional router from `{name}` returned \
1271 unknown node `{target}`"
1272 )));
1273 }
1274 if planned.insert(target.clone()) {
1275 next.push(ActiveTask {
1276 name: target,
1277 scoped: None,
1278 });
1279 }
1280 }
1281 Route::Send(sends) => {
1282 for send in sends {
1283 if !graph.has_node(&send.node) {
1284 return Err(RustyError::Graph(format!(
1285 "Route::Send from `{name}` targets unknown \
1286 node `{}`",
1287 send.node
1288 )));
1289 }
1290 // Each Send is its own invocation with its own
1291 // scoped state, even when several target the
1292 // same node.
1293 next.push(ActiveTask {
1294 name: send.node,
1295 scoped: Some(send.state),
1296 });
1297 }
1298 }
1299 Route::End => {}
1300 }
1301 }
1302 }
1303 }
1304 }
1305 }
1306
1307 let route_event = recorder.record(
1308 EventDraft::new(RunEventKind::RoutingDecision, Effect::Pure)
1309 .output(serde_json::json!({
1310 "next": next
1311 .iter()
1312 .map(|t| serde_json::json!({
1313 "node": &t.name,
1314 "scoped": &t.scoped,
1315 }))
1316 .collect::<Vec<_>>(),
1317 "goto": commands
1318 .iter()
1319 .flat_map(|command| command.goto.iter().cloned())
1320 .collect::<Vec<String>>(),
1321 }))
1322 .parent(step_end_event.clone()),
1323 );
1324
1325 // -- checkpoint at the super-step boundary.
1326 if let Some(checkpointer) = &self.checkpointer {
1327 let next_names: Vec<String> = next.iter().map(|t| t.name.clone()).collect();
1328 let checkpoint =
1329 recorder.mint_checkpoint(config.thread_id.clone(), step, state.clone(), next_names);
1330 let checkpoint_id = checkpoint.id.clone();
1331 checkpointer.put(checkpoint).await?;
1332 recorder.record(
1333 EventDraft::new(RunEventKind::CheckpointWritten, Effect::Idempotent)
1334 .output(serde_json::json!({
1335 "checkpoint_id": checkpoint_id,
1336 "step": step,
1337 "suspension": false,
1338 }))
1339 .parent(route_event.clone()),
1340 );
1341 Self::emit(
1342 config,
1343 GraphEvent::CheckpointSaved {
1344 checkpoint_id,
1345 step,
1346 },
1347 );
1348 }
1349
1350 // -- terminate or schedule the next super-step.
1351 if next.is_empty() {
1352 return Ok(StepTransition::Finish(ExecutionOutcome::Done(
1353 state.clone(),
1354 )));
1355 }
1356 Ok(StepTransition::Next(next, route_event))
1357 }
1358
1359 /// Best-effort event emission: a full or closed channel never aborts a run.
1360 fn emit(config: &RunConfig, event: GraphEvent) {
1361 if let Some(tx) = &config.event_tx {
1362 let _ = tx.try_send(event);
1363 }
1364 }
1365}
1366
1367/// Per-run Flight Recorder state handed to every super-step: the journal,
1368/// the determinism seams, and the frozen provenance stamped into every
1369/// checkpoint of the run.
1370struct Recorder {
1371 journal: Journal,
1372 clock: Clock,
1373 rng: RngSource,
1374 graph_version: String,
1375 graph_hash: String,
1376 policy_version: PolicyVersion,
1377 manifest: Option<RunManifest>,
1378}
1379
1380impl Recorder {
1381 /// Append one event to the run's journal; returns the event id (the
1382 /// causal-parent handle for whatever the event causes).
1383 fn record(&self, draft: EventDraft) -> String {
1384 self.journal.record(draft)
1385 }
1386
1387 /// Mint a boundary checkpoint through the determinism seams: id from the
1388 /// run's RNG, timestamp from the run's clock, the frozen provenance
1389 /// header, and a journal reference pinning the evidence head as it stood
1390 /// *before* this checkpoint's own `checkpoint_written` event is
1391 /// recorded.
1392 fn mint_checkpoint(
1393 &self,
1394 thread_id: String,
1395 step: usize,
1396 state: State,
1397 next_nodes: Vec<String>,
1398 ) -> Checkpoint {
1399 Checkpoint {
1400 id: self.rng.uuid_string(),
1401 thread_id,
1402 step,
1403 state,
1404 next_nodes,
1405 created_at: self.clock.now(),
1406 header: CheckpointHeader {
1407 format_version: CURRENT_FORMAT_VERSION,
1408 graph_version: self.graph_version.clone(),
1409 graph_hash: self.graph_hash.clone(),
1410 policy_version: self.policy_version.clone(),
1411 logical_clock: self.clock.now_ms(),
1412 manifest: self.manifest.clone(),
1413 },
1414 journal_ref: Some(self.journal.head_ref()),
1415 // The executor always mints full snapshots; delta encoding is
1416 // decided inside the checkpointer (`checkpoint::encode_delta`).
1417 base: None,
1418 }
1419 }
1420}
1421
1422/// One scheduled node invocation within a super-step. `scoped` carries the
1423/// per-invocation input of a [`crate::graph::Send`] fan-out, overlaid onto
1424/// that invocation's private state snapshot before the node runs.
1425struct ActiveTask {
1426 name: String,
1427 scoped: Option<Value>,
1428}
1429
1430/// The control-flow result of a single super-step: either the next active
1431/// set (the loop continues) or the terminal run outcome (the loop breaks).
1432enum StepTransition {
1433 /// The next active set plus the routing-decision journal event that
1434 /// produced it (the causal parent of the next super-step's start event).
1435 Next(Vec<ActiveTask>, String),
1436 Finish(ExecutionOutcome),
1437}
1438
1439#[cfg(test)]
1440mod tests {
1441 use super::*;
1442 use crate::checkpoint::InMemoryCheckpointer;
1443 use crate::graph::GraphBuilder;
1444 use crate::llm::ChatModel;
1445 use crate::state::Reducer;
1446 use serde_json::json;
1447 use std::sync::Mutex;
1448
1449 #[tokio::test]
1450 async fn linear_two_node_graph_executes_in_order() {
1451 let spec = StateSpec::new().channel("log", Reducer::Append);
1452
1453 let mut builder = GraphBuilder::new();
1454 builder.add_node("first", |_ctx: NodeContext| async {
1455 Ok(NodeOutput::update("log", json!("first")))
1456 });
1457 builder.add_node("second", |ctx: NodeContext| async move {
1458 // The next super-step observes the previous step's merged writes.
1459 assert_eq!(ctx.state().get("log"), Some(&json!(["first"])));
1460 assert_eq!(ctx.step(), 1);
1461 Ok(NodeOutput::update("log", json!("second")))
1462 });
1463 builder.set_entry_point("first");
1464 builder.add_edge("first", "second");
1465 let graph = builder.compile().unwrap();
1466
1467 let outcome = Executor::new()
1468 .run(&graph, &spec, State::new(), RunConfig::new("t-linear"))
1469 .await
1470 .unwrap();
1471
1472 match outcome {
1473 ExecutionOutcome::Done(state) => {
1474 assert_eq!(state.get("log"), Some(&json!(["first", "second"])));
1475 }
1476 other => panic!("expected Done, got {other:?}"),
1477 }
1478 }
1479
1480 #[tokio::test]
1481 async fn parallel_fan_in_merges_via_reducer() {
1482 let spec = StateSpec::new().channel("results", Reducer::Append);
1483
1484 let mut builder = GraphBuilder::new();
1485 builder.add_node("start", |_ctx: NodeContext| async {
1486 Ok(NodeOutput::empty())
1487 });
1488 builder.add_node("worker_a", |ctx: NodeContext| async move {
1489 // Snapshot isolation: parallel workers cannot see each other.
1490 assert!(!ctx.state().contains("results"));
1491 Ok(NodeOutput::update("results", json!("a")))
1492 });
1493 builder.add_node("worker_b", |ctx: NodeContext| async move {
1494 assert!(!ctx.state().contains("results"));
1495 Ok(NodeOutput::update("results", json!("b")))
1496 });
1497 builder.set_entry_point("start");
1498 builder.add_edge("start", "worker_a");
1499 builder.add_edge("start", "worker_b");
1500 let graph = builder.compile().unwrap();
1501
1502 let outcome = Executor::new()
1503 .run(&graph, &spec, State::new(), RunConfig::new("t-fan-in"))
1504 .await
1505 .unwrap();
1506
1507 match outcome {
1508 ExecutionOutcome::Done(state) => {
1509 let results = state
1510 .get("results")
1511 .and_then(Value::as_array)
1512 .expect("results channel must exist")
1513 .clone();
1514 // Completion order across the JoinSet is nondeterministic.
1515 let mut items: Vec<&str> = results.iter().map(|v| v.as_str().unwrap()).collect();
1516 items.sort_unstable();
1517 assert_eq!(items, ["a", "b"]);
1518 }
1519 other => panic!("expected Done, got {other:?}"),
1520 }
1521 }
1522
1523 #[tokio::test]
1524 async fn interrupt_returns_interrupted_outcome_and_resume_completes() {
1525 let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
1526
1527 let mut builder = GraphBuilder::new();
1528 builder.add_node("gate", |ctx: NodeContext| async move {
1529 match ctx.resume_value() {
1530 Some(v) => Ok(NodeOutput::update("answer", v.clone())),
1531 None => Err(ctx.interrupt(json!({"question": "approve?"}))),
1532 }
1533 });
1534 builder.set_entry_point("gate");
1535 let graph = builder.compile().unwrap();
1536
1537 let checkpointer = Arc::new(InMemoryCheckpointer::new());
1538 let executor = Executor::with_checkpointer(checkpointer.clone());
1539
1540 // First run: the gate node interrupts and the run suspends.
1541 let outcome = executor
1542 .run(&graph, &spec, State::new(), RunConfig::new("t-hitl"))
1543 .await
1544 .unwrap();
1545
1546 let checkpoint_id = match outcome {
1547 ExecutionOutcome::Interrupted {
1548 value,
1549 checkpoint_id,
1550 ..
1551 } => {
1552 assert_eq!(value, json!({"question": "approve?"}));
1553 assert!(!checkpoint_id.is_empty());
1554 checkpoint_id
1555 }
1556 other => panic!("expected Interrupted, got {other:?}"),
1557 };
1558
1559 // The suspension point was persisted and schedules the gate node.
1560 let stored = checkpointer.get_latest("t-hitl").await.unwrap().unwrap();
1561 assert_eq!(stored.id, checkpoint_id);
1562 assert_eq!(stored.next_nodes, vec!["gate".to_string()]);
1563
1564 // Resume: the gate node re-runs with the resume value, writes its
1565 // answer, and the run terminates.
1566 let outcome = executor
1567 .run(
1568 &graph,
1569 &spec,
1570 State::new(),
1571 RunConfig::new("t-hitl").with_resume(json!(true)),
1572 )
1573 .await
1574 .unwrap();
1575
1576 match outcome {
1577 ExecutionOutcome::Done(state) => {
1578 assert_eq!(state.get("answer"), Some(&json!(true)));
1579 }
1580 other => panic!("expected Done after resume, got {other:?}"),
1581 }
1582 }
1583
1584 #[test]
1585 fn run_config_default_uses_default_step_limit() {
1586 // Regression: a derived `Default` would zero `max_steps`, making
1587 // every `RunConfig::default()` run fail immediately.
1588 let config = RunConfig::default();
1589 assert_eq!(config.max_steps, DEFAULT_MAX_STEPS);
1590 assert!(config.thread_id.is_empty());
1591 assert!(config.resume.is_none() && config.checkpoint_id.is_none());
1592 }
1593
1594 #[tokio::test]
1595 async fn interrupt_reschedules_entire_active_set() {
1596 // Regression: the suspension checkpoint used to schedule only the
1597 // interrupting node, silently dropping parallel siblings — including
1598 // ones that had already completed, whose writes are discarded with
1599 // the aborted step.
1600 let spec = StateSpec::new()
1601 .channel("log", Reducer::Append)
1602 .channel("answer", Reducer::Overwrite);
1603
1604 let mut builder = GraphBuilder::new();
1605 builder.add_node("start", |_ctx: NodeContext| async {
1606 Ok(NodeOutput::empty())
1607 });
1608 builder.add_node("gate", |ctx: NodeContext| async move {
1609 match ctx.resume_value() {
1610 Some(v) => Ok(NodeOutput::update("answer", v.clone())),
1611 None => Err(ctx.interrupt(json!({"question": "approve?"}))),
1612 }
1613 });
1614 // Completes immediately in the interrupted step; its write is
1615 // discarded with the step and must be recomputed on resume.
1616 builder.add_node("fast", |_ctx: NodeContext| async {
1617 Ok(NodeOutput::update("log", json!("fast")))
1618 });
1619 // Still in flight when the interrupt hits; aborted, re-run on resume.
1620 builder.add_node("slow", |ctx: NodeContext| async move {
1621 if ctx.resume_value().is_none() {
1622 tokio::time::sleep(std::time::Duration::from_secs(3600)).await;
1623 }
1624 Ok(NodeOutput::update("log", json!("slow")))
1625 });
1626 builder.set_entry_point("start");
1627 builder.add_edge("start", "gate");
1628 builder.add_edge("start", "slow");
1629 builder.add_edge("start", "fast");
1630 let graph = builder.compile().unwrap();
1631
1632 let checkpointer = Arc::new(InMemoryCheckpointer::new());
1633 let executor = Executor::with_checkpointer(checkpointer.clone());
1634
1635 let outcome = executor
1636 .run(&graph, &spec, State::new(), RunConfig::new("t-par-hitl"))
1637 .await
1638 .unwrap();
1639 match &outcome {
1640 ExecutionOutcome::Interrupted { state, .. } => {
1641 // Transactional suspension: fast's completed write was
1642 // discarded with the rest of the step.
1643 assert_eq!(state.get("log"), None);
1644 }
1645 other => panic!("expected Interrupted, got {other:?}"),
1646 }
1647
1648 // The suspension checkpoint re-schedules every node of the step.
1649 let stored = checkpointer
1650 .get_latest("t-par-hitl")
1651 .await
1652 .unwrap()
1653 .unwrap();
1654 let mut scheduled = stored.next_nodes.clone();
1655 scheduled.sort_unstable();
1656 assert_eq!(scheduled, ["fast", "gate", "slow"]);
1657
1658 // Resume: all three re-run (the resume value is broadcast to the
1659 // whole step); fast's write lands exactly once.
1660 let outcome = executor
1661 .run(
1662 &graph,
1663 &spec,
1664 State::new(),
1665 RunConfig::new("t-par-hitl").with_resume(json!(true)),
1666 )
1667 .await
1668 .unwrap();
1669 match outcome {
1670 ExecutionOutcome::Done(state) => {
1671 assert_eq!(state.get("answer"), Some(&json!(true)));
1672 let mut log: Vec<String> = state
1673 .get("log")
1674 .and_then(Value::as_array)
1675 .expect("log channel must exist")
1676 .iter()
1677 .map(|v| v.as_str().unwrap().to_owned())
1678 .collect();
1679 log.sort_unstable();
1680 assert_eq!(log, ["fast", "slow"]);
1681 }
1682 other => panic!("expected Done after resume, got {other:?}"),
1683 }
1684 }
1685
1686 #[tokio::test]
1687 async fn state_update_event_reports_post_reducer_values() {
1688 // Regression: with several writers on one channel, the event used to
1689 // carry raw per-node partials collapsed by last-write-wins, hiding
1690 // all but one write behind its documented "merged" contract.
1691 let spec = StateSpec::new().channel("results", Reducer::Append);
1692
1693 let mut builder = GraphBuilder::new();
1694 builder.add_node("start", |_ctx: NodeContext| async {
1695 Ok(NodeOutput::empty())
1696 });
1697 builder.add_node("worker_a", |_ctx: NodeContext| async {
1698 Ok(NodeOutput::update("results", json!("a")))
1699 });
1700 builder.add_node("worker_b", |_ctx: NodeContext| async {
1701 Ok(NodeOutput::update("results", json!("b")))
1702 });
1703 builder.set_entry_point("start");
1704 builder.add_edge("start", "worker_a");
1705 builder.add_edge("start", "worker_b");
1706 let graph = builder.compile().unwrap();
1707
1708 let (tx, mut rx) = mpsc::channel::<GraphEvent>(64);
1709 let outcome = Executor::new()
1710 .run(
1711 &graph,
1712 &spec,
1713 State::new(),
1714 RunConfig::new("t-event").with_event_tx(tx),
1715 )
1716 .await
1717 .unwrap();
1718 assert!(matches!(outcome, ExecutionOutcome::Done(_)));
1719
1720 let mut merged: Option<Vec<String>> = None;
1721 while let Ok(event) = rx.try_recv() {
1722 if let GraphEvent::StateUpdate { step: 1, updates } = event {
1723 let values = updates
1724 .get("results")
1725 .and_then(Value::as_array)
1726 .expect("StateUpdate must carry the results channel")
1727 .iter()
1728 .map(|v| v.as_str().unwrap().to_owned())
1729 .collect();
1730 merged = Some(values);
1731 }
1732 }
1733 let mut merged = merged.expect("expected a StateUpdate event for step 1");
1734 merged.sort_unstable();
1735 // Both partial writes are visible in the single post-reducer value.
1736 assert_eq!(merged, ["a", "b"]);
1737 }
1738
1739 #[tokio::test]
1740 async fn max_steps_guard_aborts_infinite_cycles() {
1741 let spec = StateSpec::new().channel("x", Reducer::Overwrite);
1742 let mut builder = GraphBuilder::new();
1743 builder.add_node("loop_node", |_ctx: NodeContext| async {
1744 Ok(NodeOutput::empty())
1745 });
1746 builder.set_entry_point("loop_node");
1747 builder.add_edge("loop_node", "loop_node");
1748 let graph = builder.compile().unwrap();
1749
1750 let err = Executor::new()
1751 .run(
1752 &graph,
1753 &spec,
1754 State::new(),
1755 RunConfig::new("t-loop").with_max_steps(5),
1756 )
1757 .await
1758 .unwrap_err();
1759 assert!(matches!(err, RustyError::Graph(_)));
1760 }
1761
1762 #[test]
1763 fn token_event_serde_roundtrip() {
1764 let event = GraphEvent::Token {
1765 node: "agent".into(),
1766 delta: "Hello".into(),
1767 };
1768 let wire = serde_json::to_string(&event).unwrap();
1769 // Internally tagged: the variant name travels on the wire.
1770 assert!(
1771 wire.contains("\"type\":\"token\""),
1772 "unexpected wire: {wire}"
1773 );
1774 let back: GraphEvent = serde_json::from_str(&wire).unwrap();
1775 assert_eq!(event, back);
1776
1777 // The other variants still roundtrip under the new serde derives.
1778 let step_event = GraphEvent::SuperStep {
1779 step: 3,
1780 active_nodes: vec!["a".into()],
1781 };
1782 let back: GraphEvent =
1783 serde_json::from_str(&serde_json::to_string(&step_event).unwrap()).unwrap();
1784 assert_eq!(step_event, back);
1785 }
1786
1787 /// A mock model whose `chat_stream` override emits real multi-chunk
1788 /// deltas, proving the accumulation contract outside any HTTP client.
1789 struct StreamingMock;
1790
1791 #[async_trait::async_trait]
1792 impl crate::llm::ChatModel for StreamingMock {
1793 async fn chat(
1794 &self,
1795 _messages: &[crate::llm::ChatMessage],
1796 _tools: &[Value],
1797 ) -> Result<crate::llm::ChatResponse> {
1798 Ok(crate::llm::ChatResponse {
1799 message: crate::llm::ChatMessage::assistant("Hello"),
1800 model: None,
1801 usage: None,
1802 })
1803 }
1804
1805 async fn chat_stream(
1806 &self,
1807 messages: &[crate::llm::ChatMessage],
1808 tools: &[Value],
1809 on_token: &mut (dyn FnMut(crate::llm::TokenChunk) + Send),
1810 ) -> Result<crate::llm::ChatResponse> {
1811 for piece in ["Hel", "lo"] {
1812 on_token(crate::llm::TokenChunk {
1813 delta: piece.into(),
1814 finish: false,
1815 raw: None,
1816 });
1817 }
1818 on_token(crate::llm::TokenChunk {
1819 delta: String::new(),
1820 finish: true,
1821 raw: None,
1822 });
1823 self.chat(messages, tools).await
1824 }
1825 }
1826
1827 #[tokio::test]
1828 async fn node_streams_token_events_through_event_sink() {
1829 let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
1830
1831 let (tx, mut rx) = mpsc::channel::<GraphEvent>(64);
1832 // The wiring pattern from the ChatModel rustdoc: the node closure
1833 // captures a clone of the run's event sender and forwards chunks.
1834 let node_tx = tx.clone();
1835
1836 let mut builder = GraphBuilder::new();
1837 builder.add_node("agent", move |_ctx: NodeContext| {
1838 let node_tx = node_tx.clone();
1839 async move {
1840 let model = StreamingMock;
1841 let mut full = String::new();
1842 model
1843 .chat_stream(&[], &[], &mut |chunk| {
1844 if !chunk.delta.is_empty() {
1845 full.push_str(&chunk.delta);
1846 let _ = node_tx.try_send(GraphEvent::Token {
1847 node: "agent".into(),
1848 delta: chunk.delta,
1849 });
1850 }
1851 })
1852 .await
1853 .unwrap();
1854 Ok(NodeOutput::update("answer", json!(full)))
1855 }
1856 });
1857 builder.set_entry_point("agent");
1858 let graph = builder.compile().unwrap();
1859
1860 let config = RunConfig::new("t-tokens").with_event_tx(tx);
1861 // The RunConfig helper hands out the same sender for node wiring.
1862 assert!(config.token_tx().is_some());
1863 // The Executor builder/accessor pair stores a broadcast handle.
1864 let executor = Executor::new().with_token_tx(config.token_tx().unwrap());
1865 assert!(executor.token_tx().is_some());
1866
1867 let outcome = executor
1868 .run(&graph, &spec, State::new(), config)
1869 .await
1870 .unwrap();
1871 match outcome {
1872 ExecutionOutcome::Done(state) => {
1873 assert_eq!(state.get("answer"), Some(&json!("Hello")))
1874 }
1875 other => panic!("expected Done, got {other:?}"),
1876 }
1877
1878 // Drain the event stream: token deltas interleave with executor events.
1879 let mut deltas = Vec::new();
1880 while let Ok(event) = rx.try_recv() {
1881 if let GraphEvent::Token { node, delta } = event {
1882 assert_eq!(node, "agent");
1883 deltas.push(delta);
1884 }
1885 }
1886 assert_eq!(deltas, ["Hel", "lo"]);
1887 }
1888
1889 /// A minimal `tracing::Subscriber` that records formatted event fields
1890 /// (`name=value` pairs) into a shared buffer, so tests can assert on the
1891 /// executor's instrumentation. Implemented directly against the
1892 /// `tracing` crate's own `Subscriber` trait (re-exported from
1893 /// `tracing-core`) — no `tracing-subscriber` dependency required.
1894 #[derive(Clone, Default)]
1895 struct EventCapture {
1896 events: Arc<Mutex<Vec<String>>>,
1897 }
1898
1899 /// Formats an event's fields as `"name=value "` pairs.
1900 struct FieldVisitor(String);
1901
1902 impl tracing::field::Visit for FieldVisitor {
1903 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
1904 use std::fmt::Write as _;
1905 let _ = write!(self.0, "{}={:?} ", field.name(), value);
1906 }
1907 }
1908
1909 impl tracing::Subscriber for EventCapture {
1910 fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool {
1911 true
1912 }
1913
1914 fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id {
1915 // Spans are irrelevant to these assertions; one id serves all.
1916 tracing::span::Id::from_u64(1)
1917 }
1918
1919 fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {}
1920
1921 fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {}
1922
1923 fn event(&self, event: &tracing::Event<'_>) {
1924 let mut visitor = FieldVisitor(String::new());
1925 event.record(&mut visitor);
1926 self.events.lock().unwrap().push(visitor.0);
1927 }
1928
1929 fn enter(&self, _span: &tracing::span::Id) {}
1930
1931 fn exit(&self, _span: &tracing::span::Id) {}
1932 }
1933
1934 /// The instrumentation must be observability-only: installing a
1935 /// subscriber changes nothing about the run's outcome, and the expected
1936 /// telemetry (merge debug event, run-completion info event) is emitted.
1937 #[tokio::test]
1938 async fn instrumentation_emits_events_without_changing_outcome() {
1939 let capture = EventCapture::default();
1940 let events = capture.events.clone();
1941 // Global default subscriber, deliberately NOT a thread-local
1942 // `set_default`: callsite interest is cached process-wide and lazily
1943 // (re)registered from whichever thread first hits a callsite, so a
1944 // thread-local subscriber races with the other tests in this binary
1945 // that run graphs concurrently (they rebuild the cache against the
1946 // no-subscriber global default and our events get silently dropped).
1947 // A global default makes every thread's rebuild see this subscriber.
1948 // Setting it is additive — other tests neither set nor assert on
1949 // subscribers, and captured events from concurrent runs only help the
1950 // `any()` assertions below. `set_global_default` may only be called
1951 // once per process; this is the only test that installs a subscriber.
1952 tracing::subscriber::set_global_default(capture)
1953 .expect("no other test may install a global tracing subscriber");
1954
1955 let spec = StateSpec::new().channel("log", Reducer::Append);
1956 let mut builder = GraphBuilder::new();
1957 builder.add_node("only", |_ctx: NodeContext| async {
1958 Ok(NodeOutput::update("log", json!("x")))
1959 });
1960 builder.set_entry_point("only");
1961 let graph = builder.compile().unwrap();
1962
1963 let outcome = Executor::new()
1964 .run(&graph, &spec, State::new(), RunConfig::new("t-tracing"))
1965 .await
1966 .unwrap();
1967
1968 // Identical semantics: the run completes with the expected state.
1969 match outcome {
1970 ExecutionOutcome::Done(state) => {
1971 assert_eq!(state.get("log"), Some(&json!(["x"])));
1972 }
1973 other => panic!("expected Done, got {other:?}"),
1974 }
1975
1976 let captured = events.lock().unwrap();
1977 assert!(
1978 captured.iter().any(|e| e.contains("channels")),
1979 "expected a merge debug event listing written channels, got: {captured:?}"
1980 );
1981 assert!(
1982 captured
1983 .iter()
1984 .any(|e| e.contains("steps") && e.contains("duration_ms")),
1985 "expected a run-completion info event with steps and duration_ms, got: {captured:?}"
1986 );
1987 }
1988
1989 /// A 3-node linear graph (`a -> b -> c`) appending each node name to the
1990 /// `log` channel.
1991 fn linear_three_node_graph() -> (Graph, StateSpec) {
1992 let spec = StateSpec::new().channel("log", Reducer::Append);
1993 let mut builder = GraphBuilder::new();
1994 for name in ["a", "b", "c"] {
1995 builder.add_node(name, move |_ctx: NodeContext| async move {
1996 Ok(NodeOutput::update("log", json!(name)))
1997 });
1998 }
1999 builder.set_entry_point("a");
2000 builder.add_edge("a", "b");
2001 builder.add_edge("b", "c");
2002 (builder.compile().unwrap(), spec)
2003 }
2004
2005 #[tokio::test]
2006 async fn run_with_checkpoint_id_replays_from_earlier_state() {
2007 let (graph, spec) = linear_three_node_graph();
2008 let checkpointer = Arc::new(InMemoryCheckpointer::new());
2009 let executor = Executor::with_checkpointer(checkpointer.clone());
2010
2011 // Full run on the source thread: checkpoints at steps 0, 1, 2.
2012 let outcome = executor
2013 .run(&graph, &spec, State::new(), RunConfig::new("t-src"))
2014 .await
2015 .unwrap();
2016 match outcome {
2017 ExecutionOutcome::Done(state) => {
2018 assert_eq!(state.get("log"), Some(&json!(["a", "b", "c"])));
2019 }
2020 other => panic!("expected Done, got {other:?}"),
2021 }
2022
2023 let history = checkpointer.list("t-src").await.unwrap();
2024 assert_eq!(history.len(), 3);
2025 // The step-1 checkpoint: `a` and `b` have run, `c` is scheduled next.
2026 let step1 = history[1].clone();
2027 assert_eq!(step1.step, 1);
2028 assert_eq!(step1.state.get("log"), Some(&json!(["a", "b"])));
2029 assert_eq!(step1.next_nodes, vec!["c".to_string()]);
2030
2031 // Safe pattern: fork the thread at the step-1 checkpoint, then replay
2032 // the fork from that checkpoint (not the fork's latest — here the cut
2033 // point IS the latest, but `checkpoint_id` is what selects it).
2034 let copied = checkpointer
2035 .fork_thread("t-src", "t-fork", Some(&step1.id))
2036 .await
2037 .unwrap();
2038 assert_eq!(copied, 2);
2039
2040 let outcome = executor
2041 .run(
2042 &graph,
2043 &spec,
2044 State::new(),
2045 RunConfig::new("t-fork").with_checkpoint_id(step1.id.clone()),
2046 )
2047 .await
2048 .unwrap();
2049
2050 match outcome {
2051 ExecutionOutcome::Done(state) => {
2052 // Execution continued from the step-1 state: only `c` ran,
2053 // `b` was not re-executed.
2054 assert_eq!(state.get("log"), Some(&json!(["a", "b", "c"])));
2055 }
2056 other => panic!("expected Done, got {other:?}"),
2057 }
2058
2059 // The replay appended its own boundary checkpoint to the fork only.
2060 assert_eq!(checkpointer.list("t-src").await.unwrap().len(), 3);
2061 assert_eq!(checkpointer.list("t-fork").await.unwrap().len(), 3);
2062 }
2063
2064 #[tokio::test]
2065 async fn run_with_checkpoint_id_plus_resume_combined() {
2066 let spec = StateSpec::new().channel("answer", Reducer::Overwrite);
2067
2068 let mut builder = GraphBuilder::new();
2069 builder.add_node("gate", |ctx: NodeContext| async move {
2070 match ctx.resume_value() {
2071 Some(v) => Ok(NodeOutput::update("answer", v.clone())),
2072 None => Err(ctx.interrupt(json!({"question": "approve?"}))),
2073 }
2074 });
2075 builder.set_entry_point("gate");
2076 let graph = builder.compile().unwrap();
2077
2078 let checkpointer = Arc::new(InMemoryCheckpointer::new());
2079 let executor = Executor::with_checkpointer(checkpointer);
2080
2081 // Suspend at the gate and capture the suspension checkpoint id.
2082 let outcome = executor
2083 .run(&graph, &spec, State::new(), RunConfig::new("t-hitl"))
2084 .await
2085 .unwrap();
2086 let checkpoint_id = match outcome {
2087 ExecutionOutcome::Interrupted { checkpoint_id, .. } => checkpoint_id,
2088 other => panic!("expected Interrupted, got {other:?}"),
2089 };
2090
2091 // checkpoint_id selects WHERE (the suspension checkpoint), resume
2092 // supplies the value delivered to the re-running gate node.
2093 let outcome = executor
2094 .run(
2095 &graph,
2096 &spec,
2097 State::new(),
2098 RunConfig::new("t-hitl")
2099 .with_checkpoint_id(checkpoint_id)
2100 .with_resume(json!(true)),
2101 )
2102 .await
2103 .unwrap();
2104
2105 match outcome {
2106 ExecutionOutcome::Done(state) => {
2107 assert_eq!(state.get("answer"), Some(&json!(true)));
2108 }
2109 other => panic!("expected Done after replay+resume, got {other:?}"),
2110 }
2111 }
2112
2113 #[tokio::test]
2114 async fn run_with_checkpoint_id_errors_without_checkpointer_or_unknown_id() {
2115 let (graph, spec) = linear_three_node_graph();
2116
2117 // No checkpointer configured: replay is impossible.
2118 let err = Executor::new()
2119 .run(
2120 &graph,
2121 &spec,
2122 State::new(),
2123 RunConfig::new("t-x").with_checkpoint_id("some-id"),
2124 )
2125 .await
2126 .unwrap_err();
2127 assert!(matches!(err, RustyError::Checkpoint(_)));
2128
2129 // Checkpointer present but the id does not exist on the thread.
2130 let checkpointer = Arc::new(InMemoryCheckpointer::new());
2131 let executor = Executor::with_checkpointer(checkpointer.clone());
2132 executor
2133 .run(&graph, &spec, State::new(), RunConfig::new("t-x"))
2134 .await
2135 .unwrap();
2136 let err = executor
2137 .run(
2138 &graph,
2139 &spec,
2140 State::new(),
2141 RunConfig::new("t-x").with_checkpoint_id("no-such-checkpoint"),
2142 )
2143 .await
2144 .unwrap_err();
2145 assert!(matches!(err, RustyError::Checkpoint(_)));
2146 }
2147
2148 /// A self-looping spinner: each super-step increments `n`; the router
2149 /// terminates the run once `n` reaches 5.
2150 fn spinner_graph() -> (Graph, StateSpec) {
2151 let spec = StateSpec::new().channel("n", Reducer::Overwrite);
2152 let mut builder = GraphBuilder::new();
2153 builder.add_node("spin", |ctx: NodeContext| async move {
2154 // A paced step: without it the loop would burn through the
2155 // default max_steps budget faster than a test can cancel it.
2156 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2157 let n = ctx.state().get("n").and_then(Value::as_i64).unwrap_or(0);
2158 Ok(NodeOutput::update("n", json!(n + 1)))
2159 });
2160 builder.set_entry_point("spin");
2161 builder.add_conditional_edges("spin", |state: State| async move {
2162 if state.get("n").and_then(Value::as_i64).unwrap_or(0) >= 5 {
2163 Ok(Route::End)
2164 } else {
2165 Ok(Route::Node("spin".into()))
2166 }
2167 });
2168 (builder.compile().unwrap(), spec)
2169 }
2170
2171 #[tokio::test]
2172 async fn pre_cancelled_token_stops_the_run_before_the_first_step() {
2173 let (graph, spec) = spinner_graph();
2174 let token = CancellationToken::new();
2175 token.cancel();
2176
2177 let err = Executor::new()
2178 .run(
2179 &graph,
2180 &spec,
2181 State::new(),
2182 RunConfig::new("t-cancel").with_cancellation(token),
2183 )
2184 .await
2185 .unwrap_err();
2186 assert!(matches!(err, RustyError::Cancelled(_)));
2187 assert!(
2188 err.to_string().contains("after 0 super-step(s)"),
2189 "a pre-cancelled run must not execute a single step: {err}"
2190 );
2191 }
2192
2193 #[tokio::test]
2194 async fn cancellation_stops_at_a_boundary_and_the_run_resumes_from_it() {
2195 let (graph, spec) = spinner_graph();
2196 let checkpointer = Arc::new(InMemoryCheckpointer::new());
2197 let executor = Executor::with_checkpointer(checkpointer.clone());
2198 let token = CancellationToken::new();
2199
2200 // Cancel as soon as the first boundary checkpoint lands: the stop
2201 // is then provably the token's doing, not the terminating route.
2202 let canceller = token.clone();
2203 let watcher = checkpointer.clone();
2204 tokio::spawn(async move {
2205 loop {
2206 if watcher.get_latest("t-cancel").await.unwrap().is_some() {
2207 break;
2208 }
2209 tokio::time::sleep(std::time::Duration::from_millis(1)).await;
2210 }
2211 canceller.cancel();
2212 });
2213 let err = executor
2214 .run(
2215 &graph,
2216 &spec,
2217 State::new(),
2218 RunConfig::new("t-cancel").with_cancellation(token.clone()),
2219 )
2220 .await
2221 .unwrap_err();
2222 assert!(matches!(err, RustyError::Cancelled(_)));
2223
2224 // The boundary checkpoint is intact: n counts the completed steps —
2225 // strictly fewer than the terminating route's 5, proving the token
2226 // (not the route) ended the run.
2227 let checkpoint = checkpointer
2228 .get_latest("t-cancel")
2229 .await
2230 .unwrap()
2231 .expect("a boundary checkpoint must survive cancellation");
2232 let steps = checkpoint.state.get("n").and_then(Value::as_i64).unwrap();
2233 assert!(
2234 (1..5).contains(&steps),
2235 "unexpected stop point: n = {steps}"
2236 );
2237
2238 // Re-running the thread resumes the spinner from the checkpoint.
2239 // Drain tokens are one-shot (a cancelled token stays cancelled), so
2240 // the post-deploy process runs under a fresh one.
2241 let outcome = executor
2242 .run(
2243 &graph,
2244 &spec,
2245 State::new(),
2246 RunConfig::new("t-cancel")
2247 .with_resume(json!(null))
2248 .with_cancellation(CancellationToken::new()),
2249 )
2250 .await
2251 .unwrap();
2252 match outcome {
2253 ExecutionOutcome::Done(state) => {
2254 assert_eq!(state.get("n"), Some(&json!(5)));
2255 }
2256 other => panic!("expected Done after resume, got {other:?}"),
2257 }
2258 }
2259}