rusty_agent_server/runs.rs
1//! Run scheduling, execution, and bookkeeping.
2//!
3//! A run goes: *schedule* (strategy check + handle insert) → *execute*
4//! (drive [`Executor`] in a spawned task, forwarding [`GraphEvent`]s to a
5//! per-run SSE frame log + broadcast channel) → *terminate* (terminal status
6//! + JSON recorded, waiters woken, next queued run for the thread spawned).
7//!
8//! Multitask: there is always at most one **active** run per thread. The
9//! `reject` strategy returns 409 when the thread is busy; `enqueue` appends
10//! to an in-memory per-thread FIFO queue (depth-capped by
11//! `ServerConfig::max_concurrent_runs_per_thread`) that drains automatically
12//! as runs finish.
13//!
14//! Retention: terminal runs are kept for `GET /runs/{id}` polling up to
15//! [`MAX_RETAINED_RUNS`] per process; the oldest terminal runs are evicted
16//! beyond that (active and queued runs are never evicted). Run history is
17//! in-memory by design — durability lives in the checkpoint log.
18//!
19//! Drain (R0.6 wave 2c): the server's shutdown token is threaded into every
20//! run's executor ([`RunConfig::with_cancellation`]). When it fires, each
21//! in-flight run stops at its next super-step boundary — a point where a
22//! checkpoint was just persisted — and ends terminal-[`RunStatus::Cancelled`],
23//! resumable by simply re-running the thread; new submissions answer 503 and
24//! queued runs are not promoted. Anything still mid-step when the grace
25//! window closes is abandoned, which is the crash case the checkpoint log
26//! already covers.
27//!
28//! Flight Recorder: every run is journaled. The journal is attached to the
29//! executor at run start, flushed to the server store at every checkpoint
30//! boundary (in [`forward_events`]) and once more at run completion, and
31//! served read-only by `GET /runs/{id}/events`.
32
33use std::collections::{HashMap, VecDeque};
34use std::panic::AssertUnwindSafe;
35use std::sync::atomic::{AtomicU64, Ordering};
36use std::sync::{Arc, Mutex as StdMutex};
37
38use futures::FutureExt;
39use rusty_agent_runtime::checkpoint::Checkpointer;
40use rusty_agent_runtime::error::RustyError;
41use rusty_agent_runtime::executor::{ExecutionOutcome, Executor, GraphEvent, RunConfig};
42use rusty_agent_runtime::journal::{Clock, EventDraft, Journal};
43use rusty_agent_runtime::record::{Effect, RunEventKind};
44use rusty_agent_runtime::state::State;
45use serde::Deserialize;
46use serde_json::{json, Value};
47use tokio::sync::{broadcast, mpsc, watch, Mutex};
48
49use crate::error::ApiError;
50use crate::server_store::ServerStore;
51use crate::GraphRegistry;
52
53// --------------------------------------------------------------------- //
54// Run payload (accepted by all three run endpoints)
55// --------------------------------------------------------------------- //
56
57/// The `command` field of a run payload: `{ "resume": <value> }` continues
58/// an interrupted thread via [`RunConfig::with_resume`].
59#[derive(Debug, Clone, Default, Deserialize)]
60pub struct CommandPayload {
61 /// Resume value delivered to the interrupted node.
62 #[serde(default)]
63 pub resume: Option<Value>,
64}
65
66/// The `config` field of a run payload.
67#[derive(Debug, Clone, Default, Deserialize)]
68pub struct RunConfigPayload {
69 /// Maps to [`RunConfig::with_max_steps`] (LangGraph `recursion_limit`).
70 #[serde(default)]
71 pub recursion_limit: Option<usize>,
72}
73
74/// The `checkpoint` field of a run payload: `{ "checkpoint_id": "…" }`
75/// replays the thread from that checkpoint (time travel) instead of the
76/// latest, via [`RunConfig::with_checkpoint_id`].
77#[derive(Debug, Clone, Default, Deserialize)]
78pub struct CheckpointPayload {
79 /// Id of a checkpoint of this thread (see `POST /threads/{id}/history`).
80 pub checkpoint_id: String,
81}
82
83/// The payload accepted by `POST /threads/{id}/runs{,/wait,/stream}`.
84#[derive(Debug, Clone, Default, Deserialize)]
85pub struct RunPayload {
86 /// Initial state (must be a JSON object). Ignored when resuming: the
87 /// checkpointed state takes precedence.
88 #[serde(default)]
89 pub input: Option<Value>,
90
91 /// `{ "resume": <value> }` — the human-in-the-loop channel.
92 #[serde(default)]
93 pub command: Option<CommandPayload>,
94
95 /// `{ "recursion_limit": n }`.
96 #[serde(default)]
97 pub config: Option<RunConfigPayload>,
98
99 /// `{ "checkpoint_id": "…" }` — time travel: replay the run from that
100 /// checkpoint of this thread instead of the latest (`404` when the
101 /// checkpoint is unknown). Prefer forking first
102 /// (`POST /threads/{id}/fork`) and replaying on the fork.
103 #[serde(default)]
104 pub checkpoint: Option<CheckpointPayload>,
105
106 /// Free-form run metadata (stored, not interpreted).
107 #[serde(default)]
108 pub metadata: Option<Value>,
109
110 /// Which frame families to emit on the SSE stream. Default:
111 /// `["values", "updates"]`. `metadata`, `error`, and `end` frames are
112 /// always emitted.
113 #[serde(default)]
114 pub stream_mode: Option<Vec<String>>,
115
116 /// `"reject"` (409 when the thread is busy) or `"enqueue"` (default:
117 /// queue onto the per-thread run queue).
118 #[serde(default)]
119 pub multitask_strategy: Option<String>,
120
121 /// Run through a named assistant (see `POST /assistants`). The
122 /// assistant must be bound to the same graph as the thread; its
123 /// `config.recursion_limit` applies when the payload does not set one.
124 #[serde(default)]
125 pub assistant_id: Option<String>,
126
127 /// The run's registry declaration (R0.11 Extension Plane, wave 2):
128 /// the named configuration artifacts the run uses and the
129 /// environment it targets. At admission each artifact resolves
130 /// through its environment-tagged version pointer and the resolved
131 /// content pins the run's manifest, with one `config_resolved` event
132 /// per artifact journaled ahead of the run's own events. Absent is
133 /// the pre-R0.11 behavior, byte-identically: no resolution, no
134 /// manifest, no new events.
135 #[serde(default)]
136 pub registry: Option<crate::registry::RegistryRunBinding>,
137
138 /// The run's deployment declaration (R0.12 Operations Plane, wave 3):
139 /// the environment the run is admitted to. At admission the
140 /// environment's deployment pointer binds a revision — identity and
141 /// topology checked against the registered graph — with one
142 /// `deployment_resolved` event journaled ahead of the run's own
143 /// events (chained after the registry resolutions, one causal unit).
144 /// Absent is the pre-R0.12 behavior, byte-identically: no
145 /// resolution, no new event.
146 #[serde(default)]
147 pub deployment: Option<crate::deploy::DeploymentRunBinding>,
148}
149
150/// How a second run on a busy thread is handled.
151#[derive(Debug, Clone, Copy, PartialEq, Eq)]
152pub enum MultitaskStrategy {
153 /// Queue behind the active run (default).
154 Enqueue,
155 /// Fail immediately with 409.
156 Reject,
157}
158
159impl MultitaskStrategy {
160 /// Parse the wire value (`None` defaults to `enqueue`).
161 pub fn parse(raw: Option<&str>) -> Result<Self, String> {
162 match raw {
163 None | Some("enqueue") => Ok(Self::Enqueue),
164 Some("reject") => Ok(Self::Reject),
165 Some(other) => Err(format!(
166 "unknown multitask_strategy `{other}` (expected `enqueue` or `reject`)"
167 )),
168 }
169 }
170}
171
172// --------------------------------------------------------------------- //
173// Run bookkeeping
174// --------------------------------------------------------------------- //
175
176/// Lifecycle status of a run.
177#[derive(Debug, Clone, Copy, PartialEq, Eq)]
178pub enum RunStatus {
179 /// Queued behind another run on the same thread.
180 Pending,
181 /// Currently executing.
182 Running,
183 /// Terminated normally.
184 Success,
185 /// Suspended on an interrupt; resumable via `command.resume`.
186 Interrupted,
187 /// Failed.
188 Error,
189 /// Stopped by the graceful-shutdown drain (R0.6 wave 2c) at a
190 /// super-step boundary. Control flow, not failure — the boundary
191 /// checkpoint is intact, so re-running the thread resumes the run from
192 /// exactly where it stopped.
193 Cancelled,
194}
195
196impl RunStatus {
197 /// The wire representation of the status.
198 pub fn as_str(&self) -> &'static str {
199 match self {
200 RunStatus::Pending => "pending",
201 RunStatus::Running => "running",
202 RunStatus::Success => "success",
203 RunStatus::Interrupted => "interrupted",
204 RunStatus::Error => "error",
205 RunStatus::Cancelled => "cancelled",
206 }
207 }
208
209 /// `true` once the run can no longer make progress in this process
210 /// (terminal statuses, including `Cancelled`: a drained run resumes in
211 /// a *new* run, never in place).
212 pub(crate) fn is_terminal(self) -> bool {
213 matches!(
214 self,
215 RunStatus::Success | RunStatus::Interrupted | RunStatus::Error | RunStatus::Cancelled
216 )
217 }
218}
219
220/// One SSE frame as recorded in the per-run event log and broadcast live.
221/// `id` follows the design doc's `{checkpoint_id}:{step}:{seq}` format.
222#[derive(Debug, Clone)]
223pub struct SseFrame {
224 /// Frame id: `{checkpoint_id}:{step}:{seq}`.
225 pub id: String,
226 /// SSE event name (`metadata`, `updates`, `values`, `error`, `end`).
227 pub event: String,
228 /// JSON payload.
229 pub data: Value,
230 /// Per-run monotonically increasing sequence number (1-based).
231 pub seq: u64,
232}
233
234/// Shared frame producer for one run: assigns sequence numbers, appends to
235/// the bounded event log, and fans out over the broadcast channel.
236#[derive(Clone)]
237pub(crate) struct FrameSink {
238 log: Arc<StdMutex<VecDeque<SseFrame>>>,
239 bcast: broadcast::Sender<SseFrame>,
240 seq: Arc<AtomicU64>,
241 last_checkpoint: Arc<StdMutex<String>>,
242 last_step: Arc<AtomicU64>,
243 capacity: usize,
244}
245
246/// Lock a std mutex, recovering from poisoning. Every guard obtained
247/// through this helper wraps a simple clone/push/assign critical section,
248/// so a panicked holder cannot leave the value structurally inconsistent —
249/// and unwinding a whole run (or wedging its thread slot) over a poisoned
250/// frame log is the worse outcome.
251pub(crate) fn lock_recover<T>(mutex: &StdMutex<T>) -> std::sync::MutexGuard<'_, T> {
252 mutex
253 .lock()
254 .unwrap_or_else(|poisoned| poisoned.into_inner())
255}
256
257impl FrameSink {
258 fn new(capacity: usize, bcast: broadcast::Sender<SseFrame>) -> Self {
259 Self {
260 log: Arc::new(StdMutex::new(VecDeque::new())),
261 bcast,
262 seq: Arc::new(AtomicU64::new(0)),
263 last_checkpoint: Arc::new(StdMutex::new("-".to_string())),
264 last_step: Arc::new(AtomicU64::new(0)),
265 capacity,
266 }
267 }
268
269 /// Record and broadcast one frame.
270 pub(crate) fn push(&self, event: &str, step: usize, data: Value) {
271 let seq = self.seq.fetch_add(1, Ordering::Relaxed) + 1;
272 self.last_step.store(step as u64, Ordering::Relaxed);
273 let checkpoint = lock_recover(&self.last_checkpoint).clone();
274 let frame = SseFrame {
275 id: format!("{checkpoint}:{step}:{seq}"),
276 event: event.to_string(),
277 data,
278 seq,
279 };
280 {
281 let mut log = lock_recover(&self.log);
282 if log.len() >= self.capacity {
283 log.pop_front();
284 }
285 log.push_back(frame.clone());
286 }
287 // No live subscribers is normal (background runs); not an error.
288 let _ = self.bcast.send(frame);
289 }
290
291 /// Point subsequent frame ids at a freshly persisted checkpoint.
292 pub(crate) fn note_checkpoint(&self, checkpoint_id: &str) {
293 *lock_recover(&self.last_checkpoint) = checkpoint_id.to_string();
294 }
295
296 /// The super-step of the most recently pushed frame.
297 pub(crate) fn current_step(&self) -> usize {
298 self.last_step.load(Ordering::Relaxed) as usize
299 }
300}
301
302/// Everything the executor task needs, snapshotted from a [`RunHandle`].
303pub(crate) struct RunSnapshot {
304 /// Internal (tenant-scoped) thread id: used for the checkpointer, the
305 /// executor config, and RunManager bookkeeping.
306 pub thread_id: String,
307 /// External thread id as the client knows it — the only form that may
308 /// appear on the wire (SSE frames, terminal JSON).
309 pub wire_thread_id: String,
310 pub graph: String,
311 pub attempt: usize,
312 pub payload: RunPayload,
313 /// The registry binding resolved at admission (R0.11 wave 2).
314 pub admission: Option<crate::registry::RegistryAdmission>,
315 /// The deployment binding resolved at admission (R0.12 wave 3).
316 pub deployment: Option<crate::deploy::DeploymentAdmission>,
317 pub sink: FrameSink,
318 pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
319 /// This run's own cancellation token (R0.7 wave 2): a child of the
320 /// server's drain token, so a run-level cancel ([`RunManager::cancel_run`]
321 /// — the cancellation tree's run half) stops this run at its next
322 /// super-step boundary without touching any other run, while a server
323 /// drain still stops them all. Observed by the executor exactly where
324 /// the drain token always was.
325 pub cancel: tokio_util::sync::CancellationToken,
326}
327
328/// Read-only view of a run (used by the rollback and status endpoints).
329pub(crate) struct RunInfo {
330 /// Internal (tenant-scoped) thread id — handlers check tenant ownership
331 /// against it before revealing anything about the run.
332 pub thread_id: String,
333 /// External thread id for wire responses.
334 pub wire_thread_id: String,
335 pub graph: String,
336 pub attempt: usize,
337 pub status: RunStatus,
338 /// The terminal JSON once the run has finished (`None` while active).
339 pub terminal: Option<Value>,
340 pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
341}
342
343/// Handle for one scheduled run, owned by the [`RunManager`]. Crate-private
344/// surface: external users interact with runs over HTTP, not this type.
345pub struct RunHandle {
346 /// Run id (UUID v4).
347 pub(crate) run_id: String,
348 /// Internal (tenant-scoped) thread id this run executes against.
349 pub(crate) thread_id: String,
350 /// External thread id reported on the wire.
351 pub(crate) wire_thread_id: String,
352 /// Registered graph name.
353 pub(crate) graph: String,
354 /// 1-based attempt counter for the thread.
355 pub(crate) attempt: usize,
356 /// Lifecycle status.
357 pub(crate) status: RunStatus,
358 /// Original run payload.
359 pub(crate) payload: RunPayload,
360 /// The registry binding resolved at admission (R0.11 wave 2) — `None`
361 /// for an unbound run, which behaves byte-identically to before.
362 pub(crate) admission: Option<crate::registry::RegistryAdmission>,
363 /// The deployment binding resolved at admission (R0.12 wave 3) —
364 /// `None` for an undeclared run, byte-identical to before.
365 pub(crate) deployment: Option<crate::deploy::DeploymentAdmission>,
366 sink: FrameSink,
367 terminal: watch::Sender<Option<Value>>,
368 checkpoint_ids: Arc<StdMutex<Vec<String>>>,
369 /// This run's own cancellation token (see [`RunSnapshot::cancel`]).
370 /// Firing it is the run-level half of the R0.7 cancellation tree; the
371 /// executor observes it at super-step boundaries, after the boundary
372 /// checkpoint has landed.
373 cancel: tokio_util::sync::CancellationToken,
374}
375
376impl RunHandle {
377 /// Subscribe to the live frame stream.
378 pub(crate) fn subscribe(&self) -> broadcast::Receiver<SseFrame> {
379 self.sink.bcast.subscribe()
380 }
381
382 /// A point-in-time copy of the event log (for replay).
383 pub(crate) fn log_snapshot(&self) -> Vec<SseFrame> {
384 lock_recover(&self.sink.log).iter().cloned().collect()
385 }
386}
387
388/// What [`RunManager::insert`] decided for a freshly scheduled run.
389pub(crate) enum ScheduleDecision {
390 /// The thread slot was free; the run must be spawned now.
391 Started,
392 /// The run was queued behind the active run.
393 Queued,
394}
395
396/// What [`RunManager::cancel_run`] did (R0.7 wave 2).
397#[derive(Debug, Clone, Copy, PartialEq, Eq)]
398pub(crate) enum RunCancel {
399 /// The run was executing; its cancellation token fired. The terminal
400 /// transition lands when the executor observes it at the next
401 /// super-step boundary.
402 Signalled,
403 /// The run was queued behind another; it was dequeued and finished
404 /// terminal-`cancelled` without ever starting.
405 CancelledQueued,
406 /// The run was already terminal; nothing changed.
407 Terminal,
408 /// No such run.
409 Unknown,
410}
411
412/// What [`RunManager::cancel_thread_runs`] did to one thread's runs
413/// (R0.7 wave 2), split by how each run's cancellation lands — mirroring
414/// the task queue's [`crate::tasks::RunCancellation`] shape.
415#[derive(Debug, Default)]
416pub(crate) struct ThreadCancellation {
417 /// Running runs whose cancellation tokens fired (terminal at their
418 /// next boundary).
419 pub signalled: Vec<String>,
420 /// Queued runs dequeued into terminal-`cancelled` immediately.
421 pub cancelled: Vec<String>,
422}
423
424impl ThreadCancellation {
425 /// `true` when no run was touched (the thread had no active or queued
426 /// runs) — the cancel route journals an `AgentExit` only when the
427 /// cancellation actually landed somewhere.
428 pub(crate) fn is_empty(&self) -> bool {
429 self.signalled.is_empty() && self.cancelled.is_empty()
430 }
431}
432
433#[derive(Default)]
434struct RunManagerInner {
435 runs: HashMap<String, RunHandle>,
436 active_by_thread: HashMap<String, String>,
437 queues: HashMap<String, VecDeque<String>>,
438 attempts: HashMap<String, usize>,
439 /// Insertion order of run ids, feeding terminal-run eviction.
440 order: VecDeque<String>,
441}
442
443/// Cap on retained runs (see the module docs' retention note). Without a
444/// cap, `runs` would grow by one record — payload clone, terminal JSON, and
445/// up to `event_log_capacity` SSE frames — per run for the process
446/// lifetime: a steady memory leak on any busy cron schedule.
447const MAX_RETAINED_RUNS: usize = 1024;
448
449/// Registry of all runs, plus per-thread scheduling state. Cheap to clone
450/// (shared inner).
451#[derive(Default, Clone)]
452pub struct RunManager {
453 inner: Arc<Mutex<RunManagerInner>>,
454}
455
456impl RunManager {
457 /// An empty manager.
458 pub fn new() -> Self {
459 Self::default()
460 }
461
462 /// Insert a new run under the given multitask strategy, assigning its
463 /// per-thread attempt number.
464 pub(crate) async fn insert(
465 &self,
466 mut handle: RunHandle,
467 strategy: MultitaskStrategy,
468 queue_cap: usize,
469 ) -> Result<ScheduleDecision, ApiError> {
470 let mut inner = self.inner.lock().await;
471 let busy = inner.active_by_thread.contains_key(&handle.thread_id);
472 let attempt = {
473 let counter = inner.attempts.entry(handle.thread_id.clone()).or_insert(0);
474 *counter += 1;
475 *counter
476 };
477 handle.attempt = attempt;
478 inner.order.push_back(handle.run_id.clone());
479
480 match strategy {
481 MultitaskStrategy::Reject if busy => Err(ApiError::conflict(format!(
482 "thread `{}` already has an active run",
483 handle.thread_id
484 ))),
485 _ if busy => {
486 let queue = inner.queues.entry(handle.thread_id.clone()).or_default();
487 if queue.len() >= queue_cap {
488 return Err(ApiError::conflict(format!(
489 "thread `{}` run queue is full (cap {queue_cap})",
490 handle.thread_id
491 )));
492 }
493 queue.push_back(handle.run_id.clone());
494 inner.runs.insert(handle.run_id.clone(), handle);
495 Ok(ScheduleDecision::Queued)
496 }
497 _ => {
498 inner
499 .active_by_thread
500 .insert(handle.thread_id.clone(), handle.run_id.clone());
501 handle.status = RunStatus::Running;
502 inner.runs.insert(handle.run_id.clone(), handle);
503 Ok(ScheduleDecision::Started)
504 }
505 }
506 }
507
508 /// Snapshot everything the executor task needs for `run_id`.
509 pub(crate) async fn snapshot(&self, run_id: &str) -> Option<RunSnapshot> {
510 let inner = self.inner.lock().await;
511 inner.runs.get(run_id).map(|h| RunSnapshot {
512 thread_id: h.thread_id.clone(),
513 wire_thread_id: h.wire_thread_id.clone(),
514 graph: h.graph.clone(),
515 attempt: h.attempt,
516 payload: h.payload.clone(),
517 admission: h.admission.clone(),
518 deployment: h.deployment.clone(),
519 sink: h.sink.clone(),
520 checkpoint_ids: Arc::clone(&h.checkpoint_ids),
521 cancel: h.cancel.clone(),
522 })
523 }
524
525 /// Read-only run info for API endpoints.
526 pub(crate) async fn info(&self, run_id: &str) -> Option<RunInfo> {
527 let inner = self.inner.lock().await;
528 inner.runs.get(run_id).map(|h| RunInfo {
529 thread_id: h.thread_id.clone(),
530 wire_thread_id: h.wire_thread_id.clone(),
531 graph: h.graph.clone(),
532 attempt: h.attempt,
533 status: h.status,
534 terminal: h.terminal.borrow().clone(),
535 checkpoint_ids: Arc::clone(&h.checkpoint_ids),
536 })
537 }
538
539 /// Replay log + live subscription + internal thread id for the
540 /// SSE attach endpoint (`GET /runs/{id}/stream`).
541 pub(crate) async fn stream_parts(
542 &self,
543 run_id: &str,
544 ) -> Option<(Vec<SseFrame>, broadcast::Receiver<SseFrame>, String)> {
545 let inner = self.inner.lock().await;
546 inner
547 .runs
548 .get(run_id)
549 .map(|h| (h.log_snapshot(), h.subscribe(), h.thread_id.clone()))
550 }
551
552 /// `true` while the thread has an active run or a non-empty queue —
553 /// rollback refuses to delete checkpoints out from under them.
554 pub(crate) async fn thread_busy(&self, thread_id: &str) -> bool {
555 let inner = self.inner.lock().await;
556 inner.active_by_thread.contains_key(thread_id)
557 || inner
558 .queues
559 .get(thread_id)
560 .is_some_and(|queue| !queue.is_empty())
561 }
562
563 /// Cancel one run (R0.7 wave 2 — the run-level half of the
564 /// cancellation tree). A running run is *signalled*: its own
565 /// cancellation token fires and the executor stops it at the next
566 /// super-step boundary — after the boundary checkpoint has landed —
567 /// ending terminal-`cancelled` and resumable by re-running the thread,
568 /// exactly like the server drain. A queued (pending) run never started,
569 /// so it is dequeued and finished terminal-`cancelled` immediately —
570 /// leaving it queued would let a dead run promote and execute.
571 pub(crate) async fn cancel_run(&self, run_id: &str) -> RunCancel {
572 let mut inner = self.inner.lock().await;
573 let Some(handle) = inner.runs.get_mut(run_id) else {
574 return RunCancel::Unknown;
575 };
576 match handle.status {
577 RunStatus::Running => {
578 handle.cancel.cancel();
579 RunCancel::Signalled
580 }
581 RunStatus::Pending => {
582 let thread_id = handle.thread_id.clone();
583 let wire_thread_id = handle.wire_thread_id.clone();
584 if let Some(queue) = inner.queues.get_mut(&thread_id) {
585 queue.retain(|queued| queued != run_id);
586 }
587 let handle = inner
588 .runs
589 .get_mut(run_id)
590 .expect("the handle was resolved above");
591 handle.status = RunStatus::Cancelled;
592 let terminal = json!({
593 "run_id": run_id,
594 "thread_id": wire_thread_id,
595 "status": "cancelled",
596 "message": "cancelled while queued, before its first step",
597 });
598 handle.terminal.send_replace(Some(terminal));
599 RunCancel::CancelledQueued
600 }
601 // Terminal runs (including an already-cancelled one) are
602 // untouched — cancellation is control flow, idempotent by
603 // no-op, never a second terminal transition.
604 _ => RunCancel::Terminal,
605 }
606 }
607
608 /// Cancel every run of one thread (R0.7 wave 2): the active run is
609 /// signalled, every queued run is dequeued-cancelled — the whole
610 /// per-thread run state, so cancelling an agent's thread leaves no
611 /// pending run that would re-drive it.
612 pub(crate) async fn cancel_thread_runs(&self, thread_id: &str) -> ThreadCancellation {
613 let (active, queued) = {
614 let inner = self.inner.lock().await;
615 (
616 inner.active_by_thread.get(thread_id).cloned(),
617 inner
618 .queues
619 .get(thread_id)
620 .map(|q| q.iter().cloned().collect::<Vec<_>>())
621 .unwrap_or_default(),
622 )
623 };
624 let mut outcome = ThreadCancellation::default();
625 if let Some(run_id) = active {
626 if matches!(self.cancel_run(&run_id).await, RunCancel::Signalled) {
627 outcome.signalled.push(run_id);
628 }
629 }
630 for run_id in queued {
631 if matches!(self.cancel_run(&run_id).await, RunCancel::CancelledQueued) {
632 outcome.cancelled.push(run_id);
633 }
634 }
635 outcome
636 }
637
638 /// Record the terminal status + JSON, wake waiters, release the thread
639 /// slot, and return the next queued run id for the thread (if any), now
640 /// marked active.
641 ///
642 /// `draining` (the server's shutdown drain) suppresses queue promotion:
643 /// the slot frees but nothing is returned — a run promoted into a
644 /// shutting-down process would only be cancelled at its first boundary.
645 /// Queued runs stay `Pending`; their threads' checkpoints are intact
646 /// for the next process to re-drive.
647 pub(crate) async fn finish(
648 &self,
649 run_id: &str,
650 status: RunStatus,
651 terminal: Value,
652 draining: bool,
653 ) -> Option<String> {
654 let mut inner = self.inner.lock().await;
655 let handle = inner.runs.get_mut(run_id)?;
656 handle.status = status;
657 // `send_replace` (not `send`) so the terminal JSON is stored even
658 // when no waiter holds a receiver (background runs); status polling
659 // via `info` reads it back through `watch::Sender::borrow`.
660 handle.terminal.send_replace(Some(terminal));
661 let thread_id = handle.thread_id.clone();
662
663 if inner
664 .active_by_thread
665 .get(&thread_id)
666 .is_some_and(|active| active == run_id)
667 {
668 inner.active_by_thread.remove(&thread_id);
669 }
670
671 let next = if draining {
672 None
673 } else {
674 inner
675 .queues
676 .get_mut(&thread_id)
677 .and_then(VecDeque::pop_front)
678 };
679 if let Some(next_id) = &next {
680 if let Some(h) = inner.runs.get_mut(next_id) {
681 h.status = RunStatus::Running;
682 }
683 inner.active_by_thread.insert(thread_id, next_id.clone());
684 }
685
686 // Evict the oldest terminal runs beyond the retention cap; active
687 // and queued runs keep their slots in `order`.
688 let mut excess = inner.runs.len().saturating_sub(MAX_RETAINED_RUNS);
689 let mut skipped = Vec::new();
690 while excess > 0 {
691 let Some(candidate) = inner.order.pop_front() else {
692 break;
693 };
694 let evictable = inner
695 .runs
696 .get(&candidate)
697 .is_some_and(|h| h.status.is_terminal());
698 if evictable {
699 inner.runs.remove(&candidate);
700 excess -= 1;
701 } else {
702 skipped.push(candidate);
703 }
704 }
705 inner.order.extend(skipped);
706
707 next
708 }
709}
710
711/// Everything the run machinery needs from the application: registry,
712/// checkpointer, manager, and caps. Cheap to clone.
713#[derive(Clone)]
714pub(crate) struct RunDeps {
715 pub registry: GraphRegistry,
716 pub checkpointer: Arc<dyn Checkpointer>,
717 pub manager: RunManager,
718 /// Flight Recorder journal persistence (`GET /runs/{id}/events`).
719 pub server_store: Arc<dyn ServerStore>,
720 pub queue_cap: usize,
721 pub log_capacity: usize,
722 /// The server's drain control (R0.6 wave 2c): threaded into every run's
723 /// executor, which observes it at super-step boundaries.
724 pub shutdown: tokio_util::sync::CancellationToken,
725 /// The deployment's default environment tag (R0.11 wave 2): the
726 /// promotion target a registry-bound run resolves against when its
727 /// binding names no environment (`None`: the untagged surface).
728 pub default_environment_tag: Option<rusty_agent_runtime::learn::EnvironmentTag>,
729}
730
731/// The result of successfully scheduling a run: everything an endpoint
732/// needs to answer (background ack, wait, or stream).
733pub(crate) struct Scheduled {
734 pub run_id: String,
735 pub status: RunStatus,
736 pub terminal: watch::Receiver<Option<Value>>,
737 pub broadcast: broadcast::Receiver<SseFrame>,
738 pub replay: Vec<SseFrame>,
739}
740
741/// Create a run handle, apply the multitask strategy, and spawn execution
742/// immediately when the thread slot is free.
743///
744/// `thread_id` is the internal (tenant-scoped) id used for the checkpointer,
745/// executor, and RunManager bookkeeping; `wire_thread_id` is the external id
746/// reported in SSE frames and terminal JSON.
747pub(crate) async fn schedule(
748 deps: &RunDeps,
749 thread_id: &str,
750 wire_thread_id: &str,
751 graph: &str,
752 payload: RunPayload,
753 strategy: MultitaskStrategy,
754) -> Result<Scheduled, ApiError> {
755 // A draining server must not take new runs: the token would cancel
756 // them at their first boundary anyway, and a 503 lets the caller (or
757 // its load balancer) retry against a pod that is still serving.
758 if deps.shutdown.is_cancelled() {
759 return Err(ApiError::shutting_down(format!(
760 "server is draining; resubmit run on thread `{wire_thread_id}` against a running instance"
761 )));
762 }
763 let run_id = uuid::Uuid::new_v4().to_string();
764 // Registry admission (R0.11 wave 2): the binding resolves now, at
765 // admission — a promotion landing afterwards never reaches this run
766 // (the conservatism checkpoint pinning has kept since R0.7), and a
767 // queued run binds at admission, not at dequeue. The tenant comes
768 // from the internal thread id, so every entry point (HTTP, cron,
769 // trigger, bridge) resolves in the submitter's namespace without
770 // threading the request context through. A resolution failure is an
771 // admission failure: the run never enters the manager.
772 let admission = match &payload.registry {
773 Some(binding) => Some(
774 crate::registry::resolve_admission(
775 &deps.server_store,
776 crate::auth::tenant_of_internal(thread_id),
777 deps.default_environment_tag.as_ref(),
778 &run_id,
779 binding,
780 )
781 .await?,
782 ),
783 None => None,
784 };
785 // Deployment admission (R0.12 wave 3): the environment's pointer
786 // binds a revision now, at admission — a promotion landing afterwards
787 // never reaches this run (the registry admission's conservatism,
788 // lifted to deployments). The revision's identity checks against the
789 // registered graph (name and current topology hash), so a build the
790 // revision no longer describes is refused, never run. A resolution
791 // failure is an admission failure: the run never enters the manager.
792 let deployment = match &payload.deployment {
793 Some(binding) => {
794 let (graph_obj, _spec) = deps.registry.get(graph).ok_or_else(|| {
795 ApiError::internal(format!(
796 "graph `{graph}` left the registry between route validation and admission"
797 ))
798 })?;
799 Some(
800 crate::deploy::resolve_admission(
801 &deps.server_store,
802 crate::auth::tenant_of_internal(thread_id),
803 &run_id,
804 binding,
805 graph,
806 &graph_obj.topology_hash(),
807 )
808 .await?,
809 )
810 }
811 None => None,
812 };
813 let (bcast_tx, _bcast_rx) = broadcast::channel(256);
814 let (terminal_tx, terminal_rx) = watch::channel(None);
815 let handle = RunHandle {
816 run_id: run_id.clone(),
817 thread_id: thread_id.to_string(),
818 wire_thread_id: wire_thread_id.to_string(),
819 graph: graph.to_string(),
820 attempt: 0, // assigned by RunManager::insert
821 status: RunStatus::Pending,
822 payload,
823 admission,
824 deployment,
825 sink: FrameSink::new(deps.log_capacity, bcast_tx),
826 terminal: terminal_tx,
827 checkpoint_ids: Arc::new(StdMutex::new(Vec::new())),
828 // A child of the server drain token: the drain still stops every
829 // run, and a run-level cancel (R0.7 wave 2) stops only this one.
830 cancel: deps.shutdown.child_token(),
831 };
832 // Subscribe/snapshot before any execution can emit frames.
833 let replay = handle.log_snapshot();
834 let broadcast = handle.subscribe();
835
836 let decision = deps
837 .manager
838 .insert(handle, strategy, deps.queue_cap)
839 .await?;
840 let status = match decision {
841 ScheduleDecision::Started => {
842 spawn_execute(deps.clone(), run_id.clone());
843 RunStatus::Running
844 }
845 ScheduleDecision::Queued => RunStatus::Pending,
846 };
847
848 Ok(Scheduled {
849 run_id,
850 status,
851 terminal: terminal_rx,
852 broadcast,
853 replay,
854 })
855}
856
857/// Drive one run to its terminal state and chain the next queued run.
858async fn execute(deps: RunDeps, run_id: String) {
859 let Some(snap) = deps.manager.snapshot(&run_id).await else {
860 tracing::warn!(%run_id, "scheduled run vanished before execution");
861 return;
862 };
863 let sink = snap.sink.clone();
864 sink.push(
865 "metadata",
866 0,
867 json!({
868 "run_id": run_id,
869 "thread_id": snap.wire_thread_id,
870 "graph": snap.graph,
871 "attempt": snap.attempt,
872 "metadata": snap.payload.metadata,
873 }),
874 );
875
876 let Some((graph, spec)) = deps.registry.get(&snap.graph) else {
877 let message = format!("graph `{}` is no longer registered", snap.graph);
878 tracing::error!(%run_id, %message);
879 sink.push(
880 "error",
881 0,
882 json!({"error": "unknown_graph", "message": message}),
883 );
884 sink.push("end", 0, json!({"status": "error"}));
885 let terminal = json!({
886 "run_id": run_id,
887 "thread_id": snap.wire_thread_id,
888 "status": "error",
889 "error": "unknown_graph",
890 "message": message,
891 });
892 terminate(&deps, &run_id, RunStatus::Error, terminal).await;
893 return;
894 };
895
896 let modes: Vec<String> = snap
897 .payload
898 .stream_mode
899 .clone()
900 .unwrap_or_else(|| vec!["values".to_string(), "updates".to_string()]);
901 // Flight Recorder: one journal per run, keyed by the server-minted run
902 // id. Events carry the external (wire) thread id — the internal
903 // tenant-scoped id must never appear in served evidence. The journal's
904 // clock is the default system clock, so timestamps match pre-R0.5
905 // behavior; attaching it makes the executor read time through it.
906 let journal = Journal::new(run_id.clone(), snap.wire_thread_id.clone(), Clock::System);
907 let (evt_tx, evt_rx) = mpsc::channel::<GraphEvent>(256);
908 let forwarder = tokio::spawn(forward_events(
909 evt_rx,
910 sink.clone(),
911 ForwardDeps {
912 checkpointer: Arc::clone(&deps.checkpointer),
913 server_store: Arc::clone(&deps.server_store),
914 journal: journal.clone(),
915 thread_id: snap.thread_id.clone(),
916 checkpoint_ids: Arc::clone(&snap.checkpoint_ids),
917 modes,
918 },
919 ));
920
921 let mut config = RunConfig::new(snap.thread_id.clone())
922 .with_event_tx(evt_tx)
923 .with_journal(journal.clone())
924 // Cancellation hook: this run's own token (a child of the server
925 // drain token). When either fires, the run stops at its next
926 // super-step boundary — a point where a checkpoint was just
927 // persisted — instead of being torn down mid-step.
928 .with_cancellation(snap.cancel.clone());
929 // Registry admission (R0.11 wave 2): the binding resolved at schedule
930 // time becomes evidence now, ahead of the run's own events — one
931 // `config_resolved` per artifact (chained: each resolution's parent
932 // is the previous, so the admission reads as one causal unit) — and
933 // the resolved manifest stamps every checkpoint header, which is how
934 // the receipt reads it back. Resolution decides nothing about what
935 // will run, so the events are read-only (the `CapsuleResolved`
936 // precedent); a serialization failure here is a bug, not a runtime
937 // condition — the payload type is the server's own.
938 let mut parent = None;
939 if let Some(admission) = &snap.admission {
940 for resolution in &admission.resolutions {
941 let output =
942 serde_json::to_value(resolution).expect("ConfigResolution always serializes");
943 let mut draft =
944 EventDraft::new(RunEventKind::ConfigResolved, Effect::ReadOnly).output(output);
945 if let Some(parent) = parent {
946 draft = draft.parent(parent);
947 }
948 parent = Some(journal.record(draft));
949 }
950 config = config.with_manifest(admission.manifest.clone());
951 }
952 // Deployment admission (R0.12 wave 3): the revision the environment's
953 // pointer bound becomes evidence the same way — one
954 // `deployment_resolved`, chained after the registry resolutions, so
955 // the receipt's walk reads journal head → this event → the bound
956 // revision → its frozen pins. Read-only, like the resolutions above.
957 if let Some(deployment) = &snap.deployment {
958 let output =
959 serde_json::to_value(&deployment.resolution).expect("DeploymentResolved serializes");
960 let mut draft =
961 EventDraft::new(RunEventKind::DeploymentResolved, Effect::ReadOnly).output(output);
962 if let Some(parent) = parent {
963 draft = draft.parent(parent);
964 }
965 journal.record(draft);
966 }
967 if let Some(command) = &snap.payload.command {
968 if let Some(resume) = &command.resume {
969 config = config.with_resume(resume.clone());
970 }
971 }
972 if let Some(checkpoint) = &snap.payload.checkpoint {
973 config = config.with_checkpoint_id(checkpoint.checkpoint_id.clone());
974 }
975 if let Some(run_cfg) = &snap.payload.config {
976 if let Some(limit) = run_cfg.recursion_limit {
977 config = config.with_max_steps(limit);
978 }
979 }
980 let initial = snap
981 .payload
982 .input
983 .clone()
984 .and_then(|v| State::from_value(v).ok())
985 .unwrap_or_default();
986
987 let mut executor = Executor::with_checkpointer(Arc::clone(&deps.checkpointer));
988 // The resolved middleware chain (R0.11 wave 4) attaches in journaled
989 // order — the same layers the manifest's `middleware` digest pins and
990 // the admission resolution's `layers` field names. Attached after the
991 // admission journal writes above, so the evidence of *what* serves
992 // precedes the run it serves.
993 if let Some(admission) = &snap.admission {
994 if let Some(chain) = &admission.middleware {
995 for layer in chain.layers() {
996 executor = executor.layer_shared(Arc::clone(layer));
997 }
998 }
999 }
1000 let result = executor.run(&graph, &spec, initial, config).await;
1001 // `config` (holding the only sender) is dropped with the run; the
1002 // forwarder drains what remains and exits.
1003 let _ = forwarder.await;
1004
1005 // Final journal write: the complete evidence of the run, including the
1006 // events recorded after the last checkpoint boundary. Persisted before
1007 // the run goes terminal so `complete: true` on the events endpoint never
1008 // races ahead of the snapshot it serves. Evidence of a failed run is
1009 // still evidence — this write happens on every outcome.
1010 persist_journal(&deps.server_store, &journal).await;
1011
1012 let step = sink.current_step();
1013 let (status, terminal) = match result {
1014 Ok(ExecutionOutcome::Done(state)) => {
1015 sink.push("end", step, json!({"status": "success"}));
1016 let terminal = json!({
1017 "run_id": run_id,
1018 "thread_id": snap.wire_thread_id,
1019 "status": "success",
1020 "output": state.to_value(),
1021 });
1022 (RunStatus::Success, terminal)
1023 }
1024 Ok(ExecutionOutcome::Interrupted {
1025 value,
1026 state,
1027 checkpoint_id,
1028 }) => {
1029 sink.push(
1030 "end",
1031 step,
1032 json!({"status": "interrupted", "interrupt": value}),
1033 );
1034 let terminal = json!({
1035 "run_id": run_id,
1036 "thread_id": snap.wire_thread_id,
1037 "status": "interrupted",
1038 "interrupt": value,
1039 "checkpoint_id": checkpoint_id,
1040 "state": state.to_value(),
1041 });
1042 (RunStatus::Interrupted, terminal)
1043 }
1044 Err(error @ RustyError::Cancelled(_)) => {
1045 // Drain, not failure: the executor stopped at a super-step
1046 // boundary, so the run's last checkpoint is intact and a fresh
1047 // run on the thread resumes from it. The wire status is
1048 // `cancelled` — matching the task queue's treatment of
1049 // cancellation as control flow, never an error.
1050 let message = error.to_string();
1051 tracing::info!(%run_id, "run drained at a checkpoint boundary; resumable");
1052 sink.push("end", step, json!({"status": "cancelled"}));
1053 let terminal = json!({
1054 "run_id": run_id,
1055 "thread_id": snap.wire_thread_id,
1056 "status": "cancelled",
1057 "message": message,
1058 });
1059 (RunStatus::Cancelled, terminal)
1060 }
1061 Err(error) => {
1062 let kind = error_kind(&error);
1063 let message = error.to_string();
1064 tracing::warn!(%run_id, %error, "run failed");
1065 sink.push("error", step, json!({"error": kind, "message": message}));
1066 sink.push("end", step, json!({"status": "error"}));
1067 let terminal = json!({
1068 "run_id": run_id,
1069 "thread_id": snap.wire_thread_id,
1070 "status": "error",
1071 "error": kind,
1072 "message": message,
1073 });
1074 (RunStatus::Error, terminal)
1075 }
1076 };
1077 terminate(&deps, &run_id, status, terminal).await;
1078}
1079
1080/// Everything [`forward_events`] needs beyond the frame sink, bundled to
1081/// keep the task's argument list readable.
1082struct ForwardDeps {
1083 checkpointer: Arc<dyn Checkpointer>,
1084 server_store: Arc<dyn ServerStore>,
1085 journal: Journal,
1086 /// Internal (tenant-scoped) thread id, for checkpoint read-backs.
1087 thread_id: String,
1088 checkpoint_ids: Arc<StdMutex<Vec<String>>>,
1089 modes: Vec<String>,
1090}
1091
1092/// Map executor events to SSE frames per the design doc's §4 table. Also the
1093/// Flight Recorder's checkpoint-boundary persistence point: every
1094/// `CheckpointSaved` event flushes the journal's current snapshot to the
1095/// server store, so the stored evidence trails the live journal by at most
1096/// one super-step.
1097async fn forward_events(mut rx: mpsc::Receiver<GraphEvent>, sink: FrameSink, deps: ForwardDeps) {
1098 let ForwardDeps {
1099 checkpointer,
1100 server_store,
1101 journal,
1102 thread_id,
1103 checkpoint_ids,
1104 modes,
1105 } = deps;
1106 while let Some(event) = rx.recv().await {
1107 match event {
1108 GraphEvent::StateUpdate { step, updates } => {
1109 if modes.iter().any(|m| m == "updates") {
1110 sink.push("updates", step, json!({"step": step, "updates": updates}));
1111 }
1112 }
1113 GraphEvent::Token { node, delta } => {
1114 if modes.iter().any(|m| m == "messages") {
1115 let step = sink.current_step();
1116 sink.push("messages", step, json!({"node": node, "delta": delta}));
1117 }
1118 }
1119 GraphEvent::CheckpointSaved {
1120 checkpoint_id,
1121 step,
1122 } => {
1123 lock_recover(&checkpoint_ids).push(checkpoint_id.clone());
1124 sink.note_checkpoint(&checkpoint_id);
1125 persist_journal(&server_store, &journal).await;
1126 if modes.iter().any(|m| m == "values") {
1127 match read_back_state(&*checkpointer, &thread_id, &checkpoint_id).await {
1128 Ok(Some(values)) => sink.push("values", step, values),
1129 Ok(None) => {
1130 tracing::debug!(%checkpoint_id, "checkpoint not found for values frame")
1131 }
1132 Err(error) => {
1133 tracing::warn!(%checkpoint_id, %error, "values frame read-back failed")
1134 }
1135 }
1136 }
1137 }
1138 // Reserved for the future `tasks` / `debug` stream modes.
1139 GraphEvent::SuperStep { .. }
1140 | GraphEvent::NodeStart { .. }
1141 | GraphEvent::NodeEnd { .. } => {}
1142 }
1143 }
1144}
1145
1146/// Flush the journal's current snapshot to the server store. A persistence
1147/// failure is logged, not raised: the run's execution must not fail because
1148/// its evidence could not be written, and the next checkpoint boundary (or
1149/// the completion write) retries.
1150async fn persist_journal(server_store: &Arc<dyn ServerStore>, journal: &Journal) {
1151 if let Err(error) = server_store.put_journal(&journal.snapshot()).await {
1152 tracing::warn!(run_id = %journal.run_id(), %error, "journal persistence failed");
1153 }
1154}
1155
1156/// `values` frames carry the full state persisted at a super-step boundary,
1157/// read back from the checkpoint log (design doc §4). A point lookup, not a
1158/// full `list()` scan — that would be O(history) per super-step, O(n²) per
1159/// run.
1160async fn read_back_state(
1161 checkpointer: &dyn Checkpointer,
1162 thread_id: &str,
1163 checkpoint_id: &str,
1164) -> rusty_agent_runtime::error::Result<Option<Value>> {
1165 Ok(checkpointer
1166 .get_by_id(thread_id, checkpoint_id)
1167 .await?
1168 .map(|cp| cp.state.to_value()))
1169}
1170
1171/// Spawn `execute` for a run, guarding the thread's scheduling slot: if the
1172/// task panics (executor bug — the poison-prone lock sites recover via
1173/// [`lock_recover`]), the run is force-finished as `error` so
1174/// `active_by_thread` releases the slot and queued runs drain instead of
1175/// wedging behind a ghost.
1176///
1177/// The future is boxed behind a trait object to break the
1178/// `execute → terminate → spawn(execute)` type cycle, which would otherwise
1179/// make `Send` inference recursive and fail.
1180fn spawn_execute(deps: RunDeps, run_id: String) {
1181 let fut: std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> = Box::pin({
1182 let deps = deps.clone();
1183 let run_id = run_id.clone();
1184 async move { execute(deps, run_id).await }
1185 });
1186 tokio::spawn(async move {
1187 if AssertUnwindSafe(fut).catch_unwind().await.is_ok() {
1188 return;
1189 }
1190 tracing::error!(%run_id, "run task panicked; force-finishing as error");
1191 let Some(snap) = deps.manager.snapshot(&run_id).await else {
1192 return;
1193 };
1194 // If the panic happened after `terminate` completed, the slot is
1195 // already released — finishing again would double-promote the queue.
1196 if matches!(deps.manager.info(&run_id).await, Some(info) if info.status.is_terminal()) {
1197 return;
1198 }
1199 let step = snap.sink.current_step();
1200 snap.sink.push(
1201 "error",
1202 step,
1203 json!({"error": "internal_panic", "message": "run task panicked"}),
1204 );
1205 snap.sink.push("end", step, json!({"status": "error"}));
1206 let terminal = json!({
1207 "run_id": run_id,
1208 "thread_id": snap.wire_thread_id,
1209 "status": "error",
1210 "error": "internal_panic",
1211 "message": "run task panicked",
1212 });
1213 terminate(&deps, &run_id, RunStatus::Error, terminal).await;
1214 });
1215}
1216
1217/// Record the terminal state and spawn the next queued run, if any. While
1218/// the server drains, the queue does not advance (see
1219/// [`RunManager::finish`]'s `draining` flag).
1220async fn terminate(deps: &RunDeps, run_id: &str, status: RunStatus, terminal: Value) {
1221 let draining = deps.shutdown.is_cancelled();
1222 if let Some(next) = deps
1223 .manager
1224 .finish(run_id, status, terminal, draining)
1225 .await
1226 {
1227 spawn_execute(deps.clone(), next);
1228 }
1229}
1230
1231/// Stable error-kind labels for the wire.
1232fn error_kind(error: &RustyError) -> &'static str {
1233 match error {
1234 RustyError::Graph(_) => "graph_error",
1235 RustyError::Node(_) => "node_error",
1236 RustyError::Interrupt { .. } => "interrupted",
1237 RustyError::Checkpoint(_) => "checkpoint_error",
1238 RustyError::Llm(_) => "llm_error",
1239 RustyError::Tool(_) => "tool_error",
1240 RustyError::Serialization(_) => "serialization_error",
1241 RustyError::InvalidUpdate(_) => "invalid_update",
1242 RustyError::Replay(_) => "replay_error",
1243 // Drain cancellation is control flow and takes its own terminal
1244 // path in `execute`; this arm exists for exhaustiveness only.
1245 RustyError::Cancelled(_) => "cancelled",
1246 }
1247}