Skip to main content

rusty_agent_server/
runs.rs

1//! Run scheduling, execution, and bookkeeping.
2//!
3//! A run goes: *schedule* (strategy check + handle insert) → *execute*
4//! (drive [`Executor`] in a spawned task, forwarding [`GraphEvent`]s to a
5//! per-run SSE frame log + broadcast channel) → *terminate* (terminal status
6//! + JSON recorded, waiters woken, next queued run for the thread spawned).
7//!
8//! Multitask: there is always at most one **active** run per thread. The
9//! `reject` strategy returns 409 when the thread is busy; `enqueue` appends
10//! to an in-memory per-thread FIFO queue (depth-capped by
11//! `ServerConfig::max_concurrent_runs_per_thread`) that drains automatically
12//! as runs finish.
13//!
14//! Retention: terminal runs are kept for `GET /runs/{id}` polling up to
15//! [`MAX_RETAINED_RUNS`] per process; the oldest terminal runs are evicted
16//! beyond that (active and queued runs are never evicted). Run history is
17//! in-memory by design — durability lives in the checkpoint log.
18//!
19//! Drain (R0.6 wave 2c): the server's shutdown token is threaded into every
20//! run's executor ([`RunConfig::with_cancellation`]). When it fires, each
21//! in-flight run stops at its next super-step boundary — a point where a
22//! checkpoint was just persisted — and ends terminal-[`RunStatus::Cancelled`],
23//! resumable by simply re-running the thread; new submissions answer 503 and
24//! queued runs are not promoted. Anything still mid-step when the grace
25//! window closes is abandoned, which is the crash case the checkpoint log
26//! already covers.
27//!
28//! Flight Recorder: every run is journaled. The journal is attached to the
29//! executor at run start, flushed to the server store at every checkpoint
30//! boundary (in [`forward_events`]) and once more at run completion, and
31//! served read-only by `GET /runs/{id}/events`.
32
33use std::collections::{HashMap, VecDeque};
34use std::panic::AssertUnwindSafe;
35use std::sync::atomic::{AtomicU64, Ordering};
36use std::sync::{Arc, Mutex as StdMutex};
37
38use futures::FutureExt;
39use rusty_agent_runtime::checkpoint::Checkpointer;
40use rusty_agent_runtime::error::RustyError;
41use rusty_agent_runtime::executor::{ExecutionOutcome, Executor, GraphEvent, RunConfig};
42use rusty_agent_runtime::journal::{Clock, EventDraft, Journal};
43use rusty_agent_runtime::record::{Effect, RunEventKind};
44use rusty_agent_runtime::state::State;
45use serde::Deserialize;
46use serde_json::{json, Value};
47use tokio::sync::{broadcast, mpsc, watch, Mutex};
48
49use crate::error::ApiError;
50use crate::server_store::ServerStore;
51use crate::GraphRegistry;
52
53// --------------------------------------------------------------------- //
54// Run payload (accepted by all three run endpoints)
55// --------------------------------------------------------------------- //
56
57/// The `command` field of a run payload: `{ "resume": <value> }` continues
58/// an interrupted thread via [`RunConfig::with_resume`].
59#[derive(Debug, Clone, Default, Deserialize)]
60pub struct CommandPayload {
61    /// Resume value delivered to the interrupted node.
62    #[serde(default)]
63    pub resume: Option<Value>,
64}
65
66/// The `config` field of a run payload.
67#[derive(Debug, Clone, Default, Deserialize)]
68pub struct RunConfigPayload {
69    /// Maps to [`RunConfig::with_max_steps`] (LangGraph `recursion_limit`).
70    #[serde(default)]
71    pub recursion_limit: Option<usize>,
72}
73
74/// The `checkpoint` field of a run payload: `{ "checkpoint_id": "…" }`
75/// replays the thread from that checkpoint (time travel) instead of the
76/// latest, via [`RunConfig::with_checkpoint_id`].
77#[derive(Debug, Clone, Default, Deserialize)]
78pub struct CheckpointPayload {
79    /// Id of a checkpoint of this thread (see `POST /threads/{id}/history`).
80    pub checkpoint_id: String,
81}
82
83/// The payload accepted by `POST /threads/{id}/runs{,/wait,/stream}`.
84#[derive(Debug, Clone, Default, Deserialize)]
85pub struct RunPayload {
86    /// Initial state (must be a JSON object). Ignored when resuming: the
87    /// checkpointed state takes precedence.
88    #[serde(default)]
89    pub input: Option<Value>,
90
91    /// `{ "resume": <value> }` — the human-in-the-loop channel.
92    #[serde(default)]
93    pub command: Option<CommandPayload>,
94
95    /// `{ "recursion_limit": n }`.
96    #[serde(default)]
97    pub config: Option<RunConfigPayload>,
98
99    /// `{ "checkpoint_id": "…" }` — time travel: replay the run from that
100    /// checkpoint of this thread instead of the latest (`404` when the
101    /// checkpoint is unknown). Prefer forking first
102    /// (`POST /threads/{id}/fork`) and replaying on the fork.
103    #[serde(default)]
104    pub checkpoint: Option<CheckpointPayload>,
105
106    /// Free-form run metadata (stored, not interpreted).
107    #[serde(default)]
108    pub metadata: Option<Value>,
109
110    /// Which frame families to emit on the SSE stream. Default:
111    /// `["values", "updates"]`. `metadata`, `error`, and `end` frames are
112    /// always emitted.
113    #[serde(default)]
114    pub stream_mode: Option<Vec<String>>,
115
116    /// `"reject"` (409 when the thread is busy) or `"enqueue"` (default:
117    /// queue onto the per-thread run queue).
118    #[serde(default)]
119    pub multitask_strategy: Option<String>,
120
121    /// Run through a named assistant (see `POST /assistants`). The
122    /// assistant must be bound to the same graph as the thread; its
123    /// `config.recursion_limit` applies when the payload does not set one.
124    #[serde(default)]
125    pub assistant_id: Option<String>,
126
127    /// The run's registry declaration (R0.11 Extension Plane, wave 2):
128    /// the named configuration artifacts the run uses and the
129    /// environment it targets. At admission each artifact resolves
130    /// through its environment-tagged version pointer and the resolved
131    /// content pins the run's manifest, with one `config_resolved` event
132    /// per artifact journaled ahead of the run's own events. Absent is
133    /// the pre-R0.11 behavior, byte-identically: no resolution, no
134    /// manifest, no new events.
135    #[serde(default)]
136    pub registry: Option<crate::registry::RegistryRunBinding>,
137
138    /// The run's deployment declaration (R0.12 Operations Plane, wave 3):
139    /// the environment the run is admitted to. At admission the
140    /// environment's deployment pointer binds a revision — identity and
141    /// topology checked against the registered graph — with one
142    /// `deployment_resolved` event journaled ahead of the run's own
143    /// events (chained after the registry resolutions, one causal unit).
144    /// Absent is the pre-R0.12 behavior, byte-identically: no
145    /// resolution, no new event.
146    #[serde(default)]
147    pub deployment: Option<crate::deploy::DeploymentRunBinding>,
148}
149
150/// How a second run on a busy thread is handled.
151#[derive(Debug, Clone, Copy, PartialEq, Eq)]
152pub enum MultitaskStrategy {
153    /// Queue behind the active run (default).
154    Enqueue,
155    /// Fail immediately with 409.
156    Reject,
157}
158
159impl MultitaskStrategy {
160    /// Parse the wire value (`None` defaults to `enqueue`).
161    pub fn parse(raw: Option<&str>) -> Result<Self, String> {
162        match raw {
163            None | Some("enqueue") => Ok(Self::Enqueue),
164            Some("reject") => Ok(Self::Reject),
165            Some(other) => Err(format!(
166                "unknown multitask_strategy `{other}` (expected `enqueue` or `reject`)"
167            )),
168        }
169    }
170}
171
172// --------------------------------------------------------------------- //
173// Run bookkeeping
174// --------------------------------------------------------------------- //
175
176/// Lifecycle status of a run.
177#[derive(Debug, Clone, Copy, PartialEq, Eq)]
178pub enum RunStatus {
179    /// Queued behind another run on the same thread.
180    Pending,
181    /// Currently executing.
182    Running,
183    /// Terminated normally.
184    Success,
185    /// Suspended on an interrupt; resumable via `command.resume`.
186    Interrupted,
187    /// Failed.
188    Error,
189    /// Stopped by the graceful-shutdown drain (R0.6 wave 2c) at a
190    /// super-step boundary. Control flow, not failure — the boundary
191    /// checkpoint is intact, so re-running the thread resumes the run from
192    /// exactly where it stopped.
193    Cancelled,
194}
195
196impl RunStatus {
197    /// The wire representation of the status.
198    pub fn as_str(&self) -> &'static str {
199        match self {
200            RunStatus::Pending => "pending",
201            RunStatus::Running => "running",
202            RunStatus::Success => "success",
203            RunStatus::Interrupted => "interrupted",
204            RunStatus::Error => "error",
205            RunStatus::Cancelled => "cancelled",
206        }
207    }
208
209    /// `true` once the run can no longer make progress in this process
210    /// (terminal statuses, including `Cancelled`: a drained run resumes in
211    /// a *new* run, never in place).
212    pub(crate) fn is_terminal(self) -> bool {
213        matches!(
214            self,
215            RunStatus::Success | RunStatus::Interrupted | RunStatus::Error | RunStatus::Cancelled
216        )
217    }
218}
219
220/// One SSE frame as recorded in the per-run event log and broadcast live.
221/// `id` follows the design doc's `{checkpoint_id}:{step}:{seq}` format.
222#[derive(Debug, Clone)]
223pub struct SseFrame {
224    /// Frame id: `{checkpoint_id}:{step}:{seq}`.
225    pub id: String,
226    /// SSE event name (`metadata`, `updates`, `values`, `error`, `end`).
227    pub event: String,
228    /// JSON payload.
229    pub data: Value,
230    /// Per-run monotonically increasing sequence number (1-based).
231    pub seq: u64,
232}
233
234/// Shared frame producer for one run: assigns sequence numbers, appends to
235/// the bounded event log, and fans out over the broadcast channel.
236#[derive(Clone)]
237pub(crate) struct FrameSink {
238    log: Arc<StdMutex<VecDeque<SseFrame>>>,
239    bcast: broadcast::Sender<SseFrame>,
240    seq: Arc<AtomicU64>,
241    last_checkpoint: Arc<StdMutex<String>>,
242    last_step: Arc<AtomicU64>,
243    capacity: usize,
244}
245
246/// Lock a std mutex, recovering from poisoning. Every guard obtained
247/// through this helper wraps a simple clone/push/assign critical section,
248/// so a panicked holder cannot leave the value structurally inconsistent —
249/// and unwinding a whole run (or wedging its thread slot) over a poisoned
250/// frame log is the worse outcome.
251pub(crate) fn lock_recover<T>(mutex: &StdMutex<T>) -> std::sync::MutexGuard<'_, T> {
252    mutex
253        .lock()
254        .unwrap_or_else(|poisoned| poisoned.into_inner())
255}
256
257impl FrameSink {
258    fn new(capacity: usize, bcast: broadcast::Sender<SseFrame>) -> Self {
259        Self {
260            log: Arc::new(StdMutex::new(VecDeque::new())),
261            bcast,
262            seq: Arc::new(AtomicU64::new(0)),
263            last_checkpoint: Arc::new(StdMutex::new("-".to_string())),
264            last_step: Arc::new(AtomicU64::new(0)),
265            capacity,
266        }
267    }
268
269    /// Record and broadcast one frame.
270    pub(crate) fn push(&self, event: &str, step: usize, data: Value) {
271        let seq = self.seq.fetch_add(1, Ordering::Relaxed) + 1;
272        self.last_step.store(step as u64, Ordering::Relaxed);
273        let checkpoint = lock_recover(&self.last_checkpoint).clone();
274        let frame = SseFrame {
275            id: format!("{checkpoint}:{step}:{seq}"),
276            event: event.to_string(),
277            data,
278            seq,
279        };
280        {
281            let mut log = lock_recover(&self.log);
282            if log.len() >= self.capacity {
283                log.pop_front();
284            }
285            log.push_back(frame.clone());
286        }
287        // No live subscribers is normal (background runs); not an error.
288        let _ = self.bcast.send(frame);
289    }
290
291    /// Point subsequent frame ids at a freshly persisted checkpoint.
292    pub(crate) fn note_checkpoint(&self, checkpoint_id: &str) {
293        *lock_recover(&self.last_checkpoint) = checkpoint_id.to_string();
294    }
295
296    /// The super-step of the most recently pushed frame.
297    pub(crate) fn current_step(&self) -> usize {
298        self.last_step.load(Ordering::Relaxed) as usize
299    }
300}
301
302/// Everything the executor task needs, snapshotted from a [`RunHandle`].
303pub(crate) struct RunSnapshot {
304    /// Internal (tenant-scoped) thread id: used for the checkpointer, the
305    /// executor config, and RunManager bookkeeping.
306    pub thread_id: String,
307    /// External thread id as the client knows it — the only form that may
308    /// appear on the wire (SSE frames, terminal JSON).
309    pub wire_thread_id: String,
310    pub graph: String,
311    pub attempt: usize,
312    pub payload: RunPayload,
313    /// The registry binding resolved at admission (R0.11 wave 2).
314    pub admission: Option<crate::registry::RegistryAdmission>,
315    /// The deployment binding resolved at admission (R0.12 wave 3).
316    pub deployment: Option<crate::deploy::DeploymentAdmission>,
317    pub sink: FrameSink,
318    pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
319    /// This run's own cancellation token (R0.7 wave 2): a child of the
320    /// server's drain token, so a run-level cancel ([`RunManager::cancel_run`]
321    /// — the cancellation tree's run half) stops this run at its next
322    /// super-step boundary without touching any other run, while a server
323    /// drain still stops them all. Observed by the executor exactly where
324    /// the drain token always was.
325    pub cancel: tokio_util::sync::CancellationToken,
326}
327
328/// Read-only view of a run (used by the rollback and status endpoints).
329pub(crate) struct RunInfo {
330    /// Internal (tenant-scoped) thread id — handlers check tenant ownership
331    /// against it before revealing anything about the run.
332    pub thread_id: String,
333    /// External thread id for wire responses.
334    pub wire_thread_id: String,
335    pub graph: String,
336    pub attempt: usize,
337    pub status: RunStatus,
338    /// The terminal JSON once the run has finished (`None` while active).
339    pub terminal: Option<Value>,
340    pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
341}
342
343/// Handle for one scheduled run, owned by the [`RunManager`]. Crate-private
344/// surface: external users interact with runs over HTTP, not this type.
345pub struct RunHandle {
346    /// Run id (UUID v4).
347    pub(crate) run_id: String,
348    /// Internal (tenant-scoped) thread id this run executes against.
349    pub(crate) thread_id: String,
350    /// External thread id reported on the wire.
351    pub(crate) wire_thread_id: String,
352    /// Registered graph name.
353    pub(crate) graph: String,
354    /// 1-based attempt counter for the thread.
355    pub(crate) attempt: usize,
356    /// Lifecycle status.
357    pub(crate) status: RunStatus,
358    /// Original run payload.
359    pub(crate) payload: RunPayload,
360    /// The registry binding resolved at admission (R0.11 wave 2) — `None`
361    /// for an unbound run, which behaves byte-identically to before.
362    pub(crate) admission: Option<crate::registry::RegistryAdmission>,
363    /// The deployment binding resolved at admission (R0.12 wave 3) —
364    /// `None` for an undeclared run, byte-identical to before.
365    pub(crate) deployment: Option<crate::deploy::DeploymentAdmission>,
366    sink: FrameSink,
367    terminal: watch::Sender<Option<Value>>,
368    checkpoint_ids: Arc<StdMutex<Vec<String>>>,
369    /// This run's own cancellation token (see [`RunSnapshot::cancel`]).
370    /// Firing it is the run-level half of the R0.7 cancellation tree; the
371    /// executor observes it at super-step boundaries, after the boundary
372    /// checkpoint has landed.
373    cancel: tokio_util::sync::CancellationToken,
374}
375
376impl RunHandle {
377    /// Subscribe to the live frame stream.
378    pub(crate) fn subscribe(&self) -> broadcast::Receiver<SseFrame> {
379        self.sink.bcast.subscribe()
380    }
381
382    /// A point-in-time copy of the event log (for replay).
383    pub(crate) fn log_snapshot(&self) -> Vec<SseFrame> {
384        lock_recover(&self.sink.log).iter().cloned().collect()
385    }
386}
387
388/// What [`RunManager::insert`] decided for a freshly scheduled run.
389pub(crate) enum ScheduleDecision {
390    /// The thread slot was free; the run must be spawned now.
391    Started,
392    /// The run was queued behind the active run.
393    Queued,
394}
395
396/// What [`RunManager::cancel_run`] did (R0.7 wave 2).
397#[derive(Debug, Clone, Copy, PartialEq, Eq)]
398pub(crate) enum RunCancel {
399    /// The run was executing; its cancellation token fired. The terminal
400    /// transition lands when the executor observes it at the next
401    /// super-step boundary.
402    Signalled,
403    /// The run was queued behind another; it was dequeued and finished
404    /// terminal-`cancelled` without ever starting.
405    CancelledQueued,
406    /// The run was already terminal; nothing changed.
407    Terminal,
408    /// No such run.
409    Unknown,
410}
411
412/// What [`RunManager::cancel_thread_runs`] did to one thread's runs
413/// (R0.7 wave 2), split by how each run's cancellation lands — mirroring
414/// the task queue's [`crate::tasks::RunCancellation`] shape.
415#[derive(Debug, Default)]
416pub(crate) struct ThreadCancellation {
417    /// Running runs whose cancellation tokens fired (terminal at their
418    /// next boundary).
419    pub signalled: Vec<String>,
420    /// Queued runs dequeued into terminal-`cancelled` immediately.
421    pub cancelled: Vec<String>,
422}
423
424impl ThreadCancellation {
425    /// `true` when no run was touched (the thread had no active or queued
426    /// runs) — the cancel route journals an `AgentExit` only when the
427    /// cancellation actually landed somewhere.
428    pub(crate) fn is_empty(&self) -> bool {
429        self.signalled.is_empty() && self.cancelled.is_empty()
430    }
431}
432
433#[derive(Default)]
434struct RunManagerInner {
435    runs: HashMap<String, RunHandle>,
436    active_by_thread: HashMap<String, String>,
437    queues: HashMap<String, VecDeque<String>>,
438    attempts: HashMap<String, usize>,
439    /// Insertion order of run ids, feeding terminal-run eviction.
440    order: VecDeque<String>,
441}
442
443/// Cap on retained runs (see the module docs' retention note). Without a
444/// cap, `runs` would grow by one record — payload clone, terminal JSON, and
445/// up to `event_log_capacity` SSE frames — per run for the process
446/// lifetime: a steady memory leak on any busy cron schedule.
447const MAX_RETAINED_RUNS: usize = 1024;
448
449/// Registry of all runs, plus per-thread scheduling state. Cheap to clone
450/// (shared inner).
451#[derive(Default, Clone)]
452pub struct RunManager {
453    inner: Arc<Mutex<RunManagerInner>>,
454}
455
456impl RunManager {
457    /// An empty manager.
458    pub fn new() -> Self {
459        Self::default()
460    }
461
462    /// Insert a new run under the given multitask strategy, assigning its
463    /// per-thread attempt number.
464    pub(crate) async fn insert(
465        &self,
466        mut handle: RunHandle,
467        strategy: MultitaskStrategy,
468        queue_cap: usize,
469    ) -> Result<ScheduleDecision, ApiError> {
470        let mut inner = self.inner.lock().await;
471        let busy = inner.active_by_thread.contains_key(&handle.thread_id);
472        let attempt = {
473            let counter = inner.attempts.entry(handle.thread_id.clone()).or_insert(0);
474            *counter += 1;
475            *counter
476        };
477        handle.attempt = attempt;
478        inner.order.push_back(handle.run_id.clone());
479
480        match strategy {
481            MultitaskStrategy::Reject if busy => Err(ApiError::conflict(format!(
482                "thread `{}` already has an active run",
483                handle.thread_id
484            ))),
485            _ if busy => {
486                let queue = inner.queues.entry(handle.thread_id.clone()).or_default();
487                if queue.len() >= queue_cap {
488                    return Err(ApiError::conflict(format!(
489                        "thread `{}` run queue is full (cap {queue_cap})",
490                        handle.thread_id
491                    )));
492                }
493                queue.push_back(handle.run_id.clone());
494                inner.runs.insert(handle.run_id.clone(), handle);
495                Ok(ScheduleDecision::Queued)
496            }
497            _ => {
498                inner
499                    .active_by_thread
500                    .insert(handle.thread_id.clone(), handle.run_id.clone());
501                handle.status = RunStatus::Running;
502                inner.runs.insert(handle.run_id.clone(), handle);
503                Ok(ScheduleDecision::Started)
504            }
505        }
506    }
507
508    /// Snapshot everything the executor task needs for `run_id`.
509    pub(crate) async fn snapshot(&self, run_id: &str) -> Option<RunSnapshot> {
510        let inner = self.inner.lock().await;
511        inner.runs.get(run_id).map(|h| RunSnapshot {
512            thread_id: h.thread_id.clone(),
513            wire_thread_id: h.wire_thread_id.clone(),
514            graph: h.graph.clone(),
515            attempt: h.attempt,
516            payload: h.payload.clone(),
517            admission: h.admission.clone(),
518            deployment: h.deployment.clone(),
519            sink: h.sink.clone(),
520            checkpoint_ids: Arc::clone(&h.checkpoint_ids),
521            cancel: h.cancel.clone(),
522        })
523    }
524
525    /// Read-only run info for API endpoints.
526    pub(crate) async fn info(&self, run_id: &str) -> Option<RunInfo> {
527        let inner = self.inner.lock().await;
528        inner.runs.get(run_id).map(|h| RunInfo {
529            thread_id: h.thread_id.clone(),
530            wire_thread_id: h.wire_thread_id.clone(),
531            graph: h.graph.clone(),
532            attempt: h.attempt,
533            status: h.status,
534            terminal: h.terminal.borrow().clone(),
535            checkpoint_ids: Arc::clone(&h.checkpoint_ids),
536        })
537    }
538
539    /// Replay log + live subscription + internal thread id for the
540    /// SSE attach endpoint (`GET /runs/{id}/stream`).
541    pub(crate) async fn stream_parts(
542        &self,
543        run_id: &str,
544    ) -> Option<(Vec<SseFrame>, broadcast::Receiver<SseFrame>, String)> {
545        let inner = self.inner.lock().await;
546        inner
547            .runs
548            .get(run_id)
549            .map(|h| (h.log_snapshot(), h.subscribe(), h.thread_id.clone()))
550    }
551
552    /// `true` while the thread has an active run or a non-empty queue —
553    /// rollback refuses to delete checkpoints out from under them.
554    pub(crate) async fn thread_busy(&self, thread_id: &str) -> bool {
555        let inner = self.inner.lock().await;
556        inner.active_by_thread.contains_key(thread_id)
557            || inner
558                .queues
559                .get(thread_id)
560                .is_some_and(|queue| !queue.is_empty())
561    }
562
563    /// Cancel one run (R0.7 wave 2 — the run-level half of the
564    /// cancellation tree). A running run is *signalled*: its own
565    /// cancellation token fires and the executor stops it at the next
566    /// super-step boundary — after the boundary checkpoint has landed —
567    /// ending terminal-`cancelled` and resumable by re-running the thread,
568    /// exactly like the server drain. A queued (pending) run never started,
569    /// so it is dequeued and finished terminal-`cancelled` immediately —
570    /// leaving it queued would let a dead run promote and execute.
571    pub(crate) async fn cancel_run(&self, run_id: &str) -> RunCancel {
572        let mut inner = self.inner.lock().await;
573        let Some(handle) = inner.runs.get_mut(run_id) else {
574            return RunCancel::Unknown;
575        };
576        match handle.status {
577            RunStatus::Running => {
578                handle.cancel.cancel();
579                RunCancel::Signalled
580            }
581            RunStatus::Pending => {
582                let thread_id = handle.thread_id.clone();
583                let wire_thread_id = handle.wire_thread_id.clone();
584                if let Some(queue) = inner.queues.get_mut(&thread_id) {
585                    queue.retain(|queued| queued != run_id);
586                }
587                let handle = inner
588                    .runs
589                    .get_mut(run_id)
590                    .expect("the handle was resolved above");
591                handle.status = RunStatus::Cancelled;
592                let terminal = json!({
593                    "run_id": run_id,
594                    "thread_id": wire_thread_id,
595                    "status": "cancelled",
596                    "message": "cancelled while queued, before its first step",
597                });
598                handle.terminal.send_replace(Some(terminal));
599                RunCancel::CancelledQueued
600            }
601            // Terminal runs (including an already-cancelled one) are
602            // untouched — cancellation is control flow, idempotent by
603            // no-op, never a second terminal transition.
604            _ => RunCancel::Terminal,
605        }
606    }
607
608    /// Cancel every run of one thread (R0.7 wave 2): the active run is
609    /// signalled, every queued run is dequeued-cancelled — the whole
610    /// per-thread run state, so cancelling an agent's thread leaves no
611    /// pending run that would re-drive it.
612    pub(crate) async fn cancel_thread_runs(&self, thread_id: &str) -> ThreadCancellation {
613        let (active, queued) = {
614            let inner = self.inner.lock().await;
615            (
616                inner.active_by_thread.get(thread_id).cloned(),
617                inner
618                    .queues
619                    .get(thread_id)
620                    .map(|q| q.iter().cloned().collect::<Vec<_>>())
621                    .unwrap_or_default(),
622            )
623        };
624        let mut outcome = ThreadCancellation::default();
625        if let Some(run_id) = active {
626            if matches!(self.cancel_run(&run_id).await, RunCancel::Signalled) {
627                outcome.signalled.push(run_id);
628            }
629        }
630        for run_id in queued {
631            if matches!(self.cancel_run(&run_id).await, RunCancel::CancelledQueued) {
632                outcome.cancelled.push(run_id);
633            }
634        }
635        outcome
636    }
637
638    /// Record the terminal status + JSON, wake waiters, release the thread
639    /// slot, and return the next queued run id for the thread (if any), now
640    /// marked active.
641    ///
642    /// `draining` (the server's shutdown drain) suppresses queue promotion:
643    /// the slot frees but nothing is returned — a run promoted into a
644    /// shutting-down process would only be cancelled at its first boundary.
645    /// Queued runs stay `Pending`; their threads' checkpoints are intact
646    /// for the next process to re-drive.
647    pub(crate) async fn finish(
648        &self,
649        run_id: &str,
650        status: RunStatus,
651        terminal: Value,
652        draining: bool,
653    ) -> Option<String> {
654        let mut inner = self.inner.lock().await;
655        let handle = inner.runs.get_mut(run_id)?;
656        handle.status = status;
657        // `send_replace` (not `send`) so the terminal JSON is stored even
658        // when no waiter holds a receiver (background runs); status polling
659        // via `info` reads it back through `watch::Sender::borrow`.
660        handle.terminal.send_replace(Some(terminal));
661        let thread_id = handle.thread_id.clone();
662
663        if inner
664            .active_by_thread
665            .get(&thread_id)
666            .is_some_and(|active| active == run_id)
667        {
668            inner.active_by_thread.remove(&thread_id);
669        }
670
671        let next = if draining {
672            None
673        } else {
674            inner
675                .queues
676                .get_mut(&thread_id)
677                .and_then(VecDeque::pop_front)
678        };
679        if let Some(next_id) = &next {
680            if let Some(h) = inner.runs.get_mut(next_id) {
681                h.status = RunStatus::Running;
682            }
683            inner.active_by_thread.insert(thread_id, next_id.clone());
684        }
685
686        // Evict the oldest terminal runs beyond the retention cap; active
687        // and queued runs keep their slots in `order`.
688        let mut excess = inner.runs.len().saturating_sub(MAX_RETAINED_RUNS);
689        let mut skipped = Vec::new();
690        while excess > 0 {
691            let Some(candidate) = inner.order.pop_front() else {
692                break;
693            };
694            let evictable = inner
695                .runs
696                .get(&candidate)
697                .is_some_and(|h| h.status.is_terminal());
698            if evictable {
699                inner.runs.remove(&candidate);
700                excess -= 1;
701            } else {
702                skipped.push(candidate);
703            }
704        }
705        inner.order.extend(skipped);
706
707        next
708    }
709}
710
711/// Everything the run machinery needs from the application: registry,
712/// checkpointer, manager, and caps. Cheap to clone.
713#[derive(Clone)]
714pub(crate) struct RunDeps {
715    pub registry: GraphRegistry,
716    pub checkpointer: Arc<dyn Checkpointer>,
717    pub manager: RunManager,
718    /// Flight Recorder journal persistence (`GET /runs/{id}/events`).
719    pub server_store: Arc<dyn ServerStore>,
720    pub queue_cap: usize,
721    pub log_capacity: usize,
722    /// The server's drain control (R0.6 wave 2c): threaded into every run's
723    /// executor, which observes it at super-step boundaries.
724    pub shutdown: tokio_util::sync::CancellationToken,
725    /// The deployment's default environment tag (R0.11 wave 2): the
726    /// promotion target a registry-bound run resolves against when its
727    /// binding names no environment (`None`: the untagged surface).
728    pub default_environment_tag: Option<rusty_agent_runtime::learn::EnvironmentTag>,
729}
730
731/// The result of successfully scheduling a run: everything an endpoint
732/// needs to answer (background ack, wait, or stream).
733pub(crate) struct Scheduled {
734    pub run_id: String,
735    pub status: RunStatus,
736    pub terminal: watch::Receiver<Option<Value>>,
737    pub broadcast: broadcast::Receiver<SseFrame>,
738    pub replay: Vec<SseFrame>,
739}
740
741/// Create a run handle, apply the multitask strategy, and spawn execution
742/// immediately when the thread slot is free.
743///
744/// `thread_id` is the internal (tenant-scoped) id used for the checkpointer,
745/// executor, and RunManager bookkeeping; `wire_thread_id` is the external id
746/// reported in SSE frames and terminal JSON.
747pub(crate) async fn schedule(
748    deps: &RunDeps,
749    thread_id: &str,
750    wire_thread_id: &str,
751    graph: &str,
752    payload: RunPayload,
753    strategy: MultitaskStrategy,
754) -> Result<Scheduled, ApiError> {
755    // A draining server must not take new runs: the token would cancel
756    // them at their first boundary anyway, and a 503 lets the caller (or
757    // its load balancer) retry against a pod that is still serving.
758    if deps.shutdown.is_cancelled() {
759        return Err(ApiError::shutting_down(format!(
760            "server is draining; resubmit run on thread `{wire_thread_id}` against a running instance"
761        )));
762    }
763    let run_id = uuid::Uuid::new_v4().to_string();
764    // Registry admission (R0.11 wave 2): the binding resolves now, at
765    // admission — a promotion landing afterwards never reaches this run
766    // (the conservatism checkpoint pinning has kept since R0.7), and a
767    // queued run binds at admission, not at dequeue. The tenant comes
768    // from the internal thread id, so every entry point (HTTP, cron,
769    // trigger, bridge) resolves in the submitter's namespace without
770    // threading the request context through. A resolution failure is an
771    // admission failure: the run never enters the manager.
772    let admission = match &payload.registry {
773        Some(binding) => Some(
774            crate::registry::resolve_admission(
775                &deps.server_store,
776                crate::auth::tenant_of_internal(thread_id),
777                deps.default_environment_tag.as_ref(),
778                &run_id,
779                binding,
780            )
781            .await?,
782        ),
783        None => None,
784    };
785    // Deployment admission (R0.12 wave 3): the environment's pointer
786    // binds a revision now, at admission — a promotion landing afterwards
787    // never reaches this run (the registry admission's conservatism,
788    // lifted to deployments). The revision's identity checks against the
789    // registered graph (name and current topology hash), so a build the
790    // revision no longer describes is refused, never run. A resolution
791    // failure is an admission failure: the run never enters the manager.
792    let deployment = match &payload.deployment {
793        Some(binding) => {
794            let (graph_obj, _spec) = deps.registry.get(graph).ok_or_else(|| {
795                ApiError::internal(format!(
796                    "graph `{graph}` left the registry between route validation and admission"
797                ))
798            })?;
799            Some(
800                crate::deploy::resolve_admission(
801                    &deps.server_store,
802                    crate::auth::tenant_of_internal(thread_id),
803                    &run_id,
804                    binding,
805                    graph,
806                    &graph_obj.topology_hash(),
807                )
808                .await?,
809            )
810        }
811        None => None,
812    };
813    let (bcast_tx, _bcast_rx) = broadcast::channel(256);
814    let (terminal_tx, terminal_rx) = watch::channel(None);
815    let handle = RunHandle {
816        run_id: run_id.clone(),
817        thread_id: thread_id.to_string(),
818        wire_thread_id: wire_thread_id.to_string(),
819        graph: graph.to_string(),
820        attempt: 0, // assigned by RunManager::insert
821        status: RunStatus::Pending,
822        payload,
823        admission,
824        deployment,
825        sink: FrameSink::new(deps.log_capacity, bcast_tx),
826        terminal: terminal_tx,
827        checkpoint_ids: Arc::new(StdMutex::new(Vec::new())),
828        // A child of the server drain token: the drain still stops every
829        // run, and a run-level cancel (R0.7 wave 2) stops only this one.
830        cancel: deps.shutdown.child_token(),
831    };
832    // Subscribe/snapshot before any execution can emit frames.
833    let replay = handle.log_snapshot();
834    let broadcast = handle.subscribe();
835
836    let decision = deps
837        .manager
838        .insert(handle, strategy, deps.queue_cap)
839        .await?;
840    let status = match decision {
841        ScheduleDecision::Started => {
842            spawn_execute(deps.clone(), run_id.clone());
843            RunStatus::Running
844        }
845        ScheduleDecision::Queued => RunStatus::Pending,
846    };
847
848    Ok(Scheduled {
849        run_id,
850        status,
851        terminal: terminal_rx,
852        broadcast,
853        replay,
854    })
855}
856
857/// Drive one run to its terminal state and chain the next queued run.
858async fn execute(deps: RunDeps, run_id: String) {
859    let Some(snap) = deps.manager.snapshot(&run_id).await else {
860        tracing::warn!(%run_id, "scheduled run vanished before execution");
861        return;
862    };
863    let sink = snap.sink.clone();
864    sink.push(
865        "metadata",
866        0,
867        json!({
868            "run_id": run_id,
869            "thread_id": snap.wire_thread_id,
870            "graph": snap.graph,
871            "attempt": snap.attempt,
872            "metadata": snap.payload.metadata,
873        }),
874    );
875
876    let Some((graph, spec)) = deps.registry.get(&snap.graph) else {
877        let message = format!("graph `{}` is no longer registered", snap.graph);
878        tracing::error!(%run_id, %message);
879        sink.push(
880            "error",
881            0,
882            json!({"error": "unknown_graph", "message": message}),
883        );
884        sink.push("end", 0, json!({"status": "error"}));
885        let terminal = json!({
886            "run_id": run_id,
887            "thread_id": snap.wire_thread_id,
888            "status": "error",
889            "error": "unknown_graph",
890            "message": message,
891        });
892        terminate(&deps, &run_id, RunStatus::Error, terminal).await;
893        return;
894    };
895
896    let modes: Vec<String> = snap
897        .payload
898        .stream_mode
899        .clone()
900        .unwrap_or_else(|| vec!["values".to_string(), "updates".to_string()]);
901    // Flight Recorder: one journal per run, keyed by the server-minted run
902    // id. Events carry the external (wire) thread id — the internal
903    // tenant-scoped id must never appear in served evidence. The journal's
904    // clock is the default system clock, so timestamps match pre-R0.5
905    // behavior; attaching it makes the executor read time through it.
906    let journal = Journal::new(run_id.clone(), snap.wire_thread_id.clone(), Clock::System);
907    let (evt_tx, evt_rx) = mpsc::channel::<GraphEvent>(256);
908    let forwarder = tokio::spawn(forward_events(
909        evt_rx,
910        sink.clone(),
911        ForwardDeps {
912            checkpointer: Arc::clone(&deps.checkpointer),
913            server_store: Arc::clone(&deps.server_store),
914            journal: journal.clone(),
915            thread_id: snap.thread_id.clone(),
916            checkpoint_ids: Arc::clone(&snap.checkpoint_ids),
917            modes,
918        },
919    ));
920
921    let mut config = RunConfig::new(snap.thread_id.clone())
922        .with_event_tx(evt_tx)
923        .with_journal(journal.clone())
924        // Cancellation hook: this run's own token (a child of the server
925        // drain token). When either fires, the run stops at its next
926        // super-step boundary — a point where a checkpoint was just
927        // persisted — instead of being torn down mid-step.
928        .with_cancellation(snap.cancel.clone());
929    // Registry admission (R0.11 wave 2): the binding resolved at schedule
930    // time becomes evidence now, ahead of the run's own events — one
931    // `config_resolved` per artifact (chained: each resolution's parent
932    // is the previous, so the admission reads as one causal unit) — and
933    // the resolved manifest stamps every checkpoint header, which is how
934    // the receipt reads it back. Resolution decides nothing about what
935    // will run, so the events are read-only (the `CapsuleResolved`
936    // precedent); a serialization failure here is a bug, not a runtime
937    // condition — the payload type is the server's own.
938    let mut parent = None;
939    if let Some(admission) = &snap.admission {
940        for resolution in &admission.resolutions {
941            let output =
942                serde_json::to_value(resolution).expect("ConfigResolution always serializes");
943            let mut draft =
944                EventDraft::new(RunEventKind::ConfigResolved, Effect::ReadOnly).output(output);
945            if let Some(parent) = parent {
946                draft = draft.parent(parent);
947            }
948            parent = Some(journal.record(draft));
949        }
950        config = config.with_manifest(admission.manifest.clone());
951    }
952    // Deployment admission (R0.12 wave 3): the revision the environment's
953    // pointer bound becomes evidence the same way — one
954    // `deployment_resolved`, chained after the registry resolutions, so
955    // the receipt's walk reads journal head → this event → the bound
956    // revision → its frozen pins. Read-only, like the resolutions above.
957    if let Some(deployment) = &snap.deployment {
958        let output =
959            serde_json::to_value(&deployment.resolution).expect("DeploymentResolved serializes");
960        let mut draft =
961            EventDraft::new(RunEventKind::DeploymentResolved, Effect::ReadOnly).output(output);
962        if let Some(parent) = parent {
963            draft = draft.parent(parent);
964        }
965        journal.record(draft);
966    }
967    if let Some(command) = &snap.payload.command {
968        if let Some(resume) = &command.resume {
969            config = config.with_resume(resume.clone());
970        }
971    }
972    if let Some(checkpoint) = &snap.payload.checkpoint {
973        config = config.with_checkpoint_id(checkpoint.checkpoint_id.clone());
974    }
975    if let Some(run_cfg) = &snap.payload.config {
976        if let Some(limit) = run_cfg.recursion_limit {
977            config = config.with_max_steps(limit);
978        }
979    }
980    let initial = snap
981        .payload
982        .input
983        .clone()
984        .and_then(|v| State::from_value(v).ok())
985        .unwrap_or_default();
986
987    let mut executor = Executor::with_checkpointer(Arc::clone(&deps.checkpointer));
988    // The resolved middleware chain (R0.11 wave 4) attaches in journaled
989    // order — the same layers the manifest's `middleware` digest pins and
990    // the admission resolution's `layers` field names. Attached after the
991    // admission journal writes above, so the evidence of *what* serves
992    // precedes the run it serves.
993    if let Some(admission) = &snap.admission {
994        if let Some(chain) = &admission.middleware {
995            for layer in chain.layers() {
996                executor = executor.layer_shared(Arc::clone(layer));
997            }
998        }
999    }
1000    let result = executor.run(&graph, &spec, initial, config).await;
1001    // `config` (holding the only sender) is dropped with the run; the
1002    // forwarder drains what remains and exits.
1003    let _ = forwarder.await;
1004
1005    // Final journal write: the complete evidence of the run, including the
1006    // events recorded after the last checkpoint boundary. Persisted before
1007    // the run goes terminal so `complete: true` on the events endpoint never
1008    // races ahead of the snapshot it serves. Evidence of a failed run is
1009    // still evidence — this write happens on every outcome.
1010    persist_journal(&deps.server_store, &journal).await;
1011
1012    let step = sink.current_step();
1013    let (status, terminal) = match result {
1014        Ok(ExecutionOutcome::Done(state)) => {
1015            sink.push("end", step, json!({"status": "success"}));
1016            let terminal = json!({
1017                "run_id": run_id,
1018                "thread_id": snap.wire_thread_id,
1019                "status": "success",
1020                "output": state.to_value(),
1021            });
1022            (RunStatus::Success, terminal)
1023        }
1024        Ok(ExecutionOutcome::Interrupted {
1025            value,
1026            state,
1027            checkpoint_id,
1028        }) => {
1029            sink.push(
1030                "end",
1031                step,
1032                json!({"status": "interrupted", "interrupt": value}),
1033            );
1034            let terminal = json!({
1035                "run_id": run_id,
1036                "thread_id": snap.wire_thread_id,
1037                "status": "interrupted",
1038                "interrupt": value,
1039                "checkpoint_id": checkpoint_id,
1040                "state": state.to_value(),
1041            });
1042            (RunStatus::Interrupted, terminal)
1043        }
1044        Err(error @ RustyError::Cancelled(_)) => {
1045            // Drain, not failure: the executor stopped at a super-step
1046            // boundary, so the run's last checkpoint is intact and a fresh
1047            // run on the thread resumes from it. The wire status is
1048            // `cancelled` — matching the task queue's treatment of
1049            // cancellation as control flow, never an error.
1050            let message = error.to_string();
1051            tracing::info!(%run_id, "run drained at a checkpoint boundary; resumable");
1052            sink.push("end", step, json!({"status": "cancelled"}));
1053            let terminal = json!({
1054                "run_id": run_id,
1055                "thread_id": snap.wire_thread_id,
1056                "status": "cancelled",
1057                "message": message,
1058            });
1059            (RunStatus::Cancelled, terminal)
1060        }
1061        Err(error) => {
1062            let kind = error_kind(&error);
1063            let message = error.to_string();
1064            tracing::warn!(%run_id, %error, "run failed");
1065            sink.push("error", step, json!({"error": kind, "message": message}));
1066            sink.push("end", step, json!({"status": "error"}));
1067            let terminal = json!({
1068                "run_id": run_id,
1069                "thread_id": snap.wire_thread_id,
1070                "status": "error",
1071                "error": kind,
1072                "message": message,
1073            });
1074            (RunStatus::Error, terminal)
1075        }
1076    };
1077    terminate(&deps, &run_id, status, terminal).await;
1078}
1079
1080/// Everything [`forward_events`] needs beyond the frame sink, bundled to
1081/// keep the task's argument list readable.
1082struct ForwardDeps {
1083    checkpointer: Arc<dyn Checkpointer>,
1084    server_store: Arc<dyn ServerStore>,
1085    journal: Journal,
1086    /// Internal (tenant-scoped) thread id, for checkpoint read-backs.
1087    thread_id: String,
1088    checkpoint_ids: Arc<StdMutex<Vec<String>>>,
1089    modes: Vec<String>,
1090}
1091
1092/// Map executor events to SSE frames per the design doc's §4 table. Also the
1093/// Flight Recorder's checkpoint-boundary persistence point: every
1094/// `CheckpointSaved` event flushes the journal's current snapshot to the
1095/// server store, so the stored evidence trails the live journal by at most
1096/// one super-step.
1097async fn forward_events(mut rx: mpsc::Receiver<GraphEvent>, sink: FrameSink, deps: ForwardDeps) {
1098    let ForwardDeps {
1099        checkpointer,
1100        server_store,
1101        journal,
1102        thread_id,
1103        checkpoint_ids,
1104        modes,
1105    } = deps;
1106    while let Some(event) = rx.recv().await {
1107        match event {
1108            GraphEvent::StateUpdate { step, updates } => {
1109                if modes.iter().any(|m| m == "updates") {
1110                    sink.push("updates", step, json!({"step": step, "updates": updates}));
1111                }
1112            }
1113            GraphEvent::Token { node, delta } => {
1114                if modes.iter().any(|m| m == "messages") {
1115                    let step = sink.current_step();
1116                    sink.push("messages", step, json!({"node": node, "delta": delta}));
1117                }
1118            }
1119            GraphEvent::CheckpointSaved {
1120                checkpoint_id,
1121                step,
1122            } => {
1123                lock_recover(&checkpoint_ids).push(checkpoint_id.clone());
1124                sink.note_checkpoint(&checkpoint_id);
1125                persist_journal(&server_store, &journal).await;
1126                if modes.iter().any(|m| m == "values") {
1127                    match read_back_state(&*checkpointer, &thread_id, &checkpoint_id).await {
1128                        Ok(Some(values)) => sink.push("values", step, values),
1129                        Ok(None) => {
1130                            tracing::debug!(%checkpoint_id, "checkpoint not found for values frame")
1131                        }
1132                        Err(error) => {
1133                            tracing::warn!(%checkpoint_id, %error, "values frame read-back failed")
1134                        }
1135                    }
1136                }
1137            }
1138            // Reserved for the future `tasks` / `debug` stream modes.
1139            GraphEvent::SuperStep { .. }
1140            | GraphEvent::NodeStart { .. }
1141            | GraphEvent::NodeEnd { .. } => {}
1142        }
1143    }
1144}
1145
1146/// Flush the journal's current snapshot to the server store. A persistence
1147/// failure is logged, not raised: the run's execution must not fail because
1148/// its evidence could not be written, and the next checkpoint boundary (or
1149/// the completion write) retries.
1150async fn persist_journal(server_store: &Arc<dyn ServerStore>, journal: &Journal) {
1151    if let Err(error) = server_store.put_journal(&journal.snapshot()).await {
1152        tracing::warn!(run_id = %journal.run_id(), %error, "journal persistence failed");
1153    }
1154}
1155
1156/// `values` frames carry the full state persisted at a super-step boundary,
1157/// read back from the checkpoint log (design doc §4). A point lookup, not a
1158/// full `list()` scan — that would be O(history) per super-step, O(n²) per
1159/// run.
1160async fn read_back_state(
1161    checkpointer: &dyn Checkpointer,
1162    thread_id: &str,
1163    checkpoint_id: &str,
1164) -> rusty_agent_runtime::error::Result<Option<Value>> {
1165    Ok(checkpointer
1166        .get_by_id(thread_id, checkpoint_id)
1167        .await?
1168        .map(|cp| cp.state.to_value()))
1169}
1170
1171/// Spawn `execute` for a run, guarding the thread's scheduling slot: if the
1172/// task panics (executor bug — the poison-prone lock sites recover via
1173/// [`lock_recover`]), the run is force-finished as `error` so
1174/// `active_by_thread` releases the slot and queued runs drain instead of
1175/// wedging behind a ghost.
1176///
1177/// The future is boxed behind a trait object to break the
1178/// `execute → terminate → spawn(execute)` type cycle, which would otherwise
1179/// make `Send` inference recursive and fail.
1180fn spawn_execute(deps: RunDeps, run_id: String) {
1181    let fut: std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> = Box::pin({
1182        let deps = deps.clone();
1183        let run_id = run_id.clone();
1184        async move { execute(deps, run_id).await }
1185    });
1186    tokio::spawn(async move {
1187        if AssertUnwindSafe(fut).catch_unwind().await.is_ok() {
1188            return;
1189        }
1190        tracing::error!(%run_id, "run task panicked; force-finishing as error");
1191        let Some(snap) = deps.manager.snapshot(&run_id).await else {
1192            return;
1193        };
1194        // If the panic happened after `terminate` completed, the slot is
1195        // already released — finishing again would double-promote the queue.
1196        if matches!(deps.manager.info(&run_id).await, Some(info) if info.status.is_terminal()) {
1197            return;
1198        }
1199        let step = snap.sink.current_step();
1200        snap.sink.push(
1201            "error",
1202            step,
1203            json!({"error": "internal_panic", "message": "run task panicked"}),
1204        );
1205        snap.sink.push("end", step, json!({"status": "error"}));
1206        let terminal = json!({
1207            "run_id": run_id,
1208            "thread_id": snap.wire_thread_id,
1209            "status": "error",
1210            "error": "internal_panic",
1211            "message": "run task panicked",
1212        });
1213        terminate(&deps, &run_id, RunStatus::Error, terminal).await;
1214    });
1215}
1216
1217/// Record the terminal state and spawn the next queued run, if any. While
1218/// the server drains, the queue does not advance (see
1219/// [`RunManager::finish`]'s `draining` flag).
1220async fn terminate(deps: &RunDeps, run_id: &str, status: RunStatus, terminal: Value) {
1221    let draining = deps.shutdown.is_cancelled();
1222    if let Some(next) = deps
1223        .manager
1224        .finish(run_id, status, terminal, draining)
1225        .await
1226    {
1227        spawn_execute(deps.clone(), next);
1228    }
1229}
1230
1231/// Stable error-kind labels for the wire.
1232fn error_kind(error: &RustyError) -> &'static str {
1233    match error {
1234        RustyError::Graph(_) => "graph_error",
1235        RustyError::Node(_) => "node_error",
1236        RustyError::Interrupt { .. } => "interrupted",
1237        RustyError::Checkpoint(_) => "checkpoint_error",
1238        RustyError::Llm(_) => "llm_error",
1239        RustyError::Tool(_) => "tool_error",
1240        RustyError::Serialization(_) => "serialization_error",
1241        RustyError::InvalidUpdate(_) => "invalid_update",
1242        RustyError::Replay(_) => "replay_error",
1243        // Drain cancellation is control flow and takes its own terminal
1244        // path in `execute`; this arm exists for exhaustiveness only.
1245        RustyError::Cancelled(_) => "cancelled",
1246    }
1247}