Skip to main content

rusty_agent_server/
runs.rs

1//! Run scheduling, execution, and bookkeeping.
2//!
3//! A run goes: *schedule* (strategy check + handle insert) → *execute*
4//! (drive [`Executor`] in a spawned task, forwarding [`GraphEvent`]s to a
5//! per-run SSE frame log + broadcast channel) → *terminate* (terminal status
6//! + JSON recorded, waiters woken, next queued run for the thread spawned).
7//!
8//! Multitask: there is always at most one **active** run per thread. The
9//! `reject` strategy returns 409 when the thread is busy; `enqueue` appends
10//! to an in-memory per-thread FIFO queue (depth-capped by
11//! `ServerConfig::max_concurrent_runs_per_thread`) that drains automatically
12//! as runs finish.
13//!
14//! Retention: terminal runs are kept for `GET /runs/{id}` polling up to
15//! [`MAX_RETAINED_RUNS`] per process; the oldest terminal runs are evicted
16//! beyond that (active and queued runs are never evicted). Run history is
17//! in-memory by design — durability lives in the checkpoint log.
18//!
19//! Drain (R0.6 wave 2c): the server's shutdown token is threaded into every
20//! run's executor ([`RunConfig::with_cancellation`]). When it fires, each
21//! in-flight run stops at its next super-step boundary — a point where a
22//! checkpoint was just persisted — and ends terminal-[`RunStatus::Cancelled`],
23//! resumable by simply re-running the thread; new submissions answer 503 and
24//! queued runs are not promoted. Anything still mid-step when the grace
25//! window closes is abandoned, which is the crash case the checkpoint log
26//! already covers.
27//!
28//! Flight Recorder: every run is journaled. The journal is attached to the
29//! executor at run start, flushed to the server store at every checkpoint
30//! boundary (in [`forward_events`]) and once more at run completion, and
31//! served read-only by `GET /runs/{id}/events`.
32
33use std::collections::{HashMap, VecDeque};
34use std::panic::AssertUnwindSafe;
35use std::sync::atomic::{AtomicU64, Ordering};
36use std::sync::{Arc, Mutex as StdMutex};
37
38use futures::FutureExt;
39use rusty_agent_runtime::checkpoint::Checkpointer;
40use rusty_agent_runtime::error::RustyError;
41use rusty_agent_runtime::executor::{ExecutionOutcome, Executor, GraphEvent, RunConfig};
42use rusty_agent_runtime::journal::{Clock, Journal};
43use rusty_agent_runtime::state::State;
44use serde::Deserialize;
45use serde_json::{json, Value};
46use tokio::sync::{broadcast, mpsc, watch, Mutex};
47
48use crate::error::ApiError;
49use crate::server_store::ServerStore;
50use crate::GraphRegistry;
51
52// --------------------------------------------------------------------- //
53// Run payload (accepted by all three run endpoints)
54// --------------------------------------------------------------------- //
55
56/// The `command` field of a run payload: `{ "resume": <value> }` continues
57/// an interrupted thread via [`RunConfig::with_resume`].
58#[derive(Debug, Clone, Default, Deserialize)]
59pub struct CommandPayload {
60    /// Resume value delivered to the interrupted node.
61    #[serde(default)]
62    pub resume: Option<Value>,
63}
64
65/// The `config` field of a run payload.
66#[derive(Debug, Clone, Default, Deserialize)]
67pub struct RunConfigPayload {
68    /// Maps to [`RunConfig::with_max_steps`] (LangGraph `recursion_limit`).
69    #[serde(default)]
70    pub recursion_limit: Option<usize>,
71}
72
73/// The `checkpoint` field of a run payload: `{ "checkpoint_id": "…" }`
74/// replays the thread from that checkpoint (time travel) instead of the
75/// latest, via [`RunConfig::with_checkpoint_id`].
76#[derive(Debug, Clone, Default, Deserialize)]
77pub struct CheckpointPayload {
78    /// Id of a checkpoint of this thread (see `POST /threads/{id}/history`).
79    pub checkpoint_id: String,
80}
81
82/// The payload accepted by `POST /threads/{id}/runs{,/wait,/stream}`.
83#[derive(Debug, Clone, Default, Deserialize)]
84pub struct RunPayload {
85    /// Initial state (must be a JSON object). Ignored when resuming: the
86    /// checkpointed state takes precedence.
87    #[serde(default)]
88    pub input: Option<Value>,
89
90    /// `{ "resume": <value> }` — the human-in-the-loop channel.
91    #[serde(default)]
92    pub command: Option<CommandPayload>,
93
94    /// `{ "recursion_limit": n }`.
95    #[serde(default)]
96    pub config: Option<RunConfigPayload>,
97
98    /// `{ "checkpoint_id": "…" }` — time travel: replay the run from that
99    /// checkpoint of this thread instead of the latest (`404` when the
100    /// checkpoint is unknown). Prefer forking first
101    /// (`POST /threads/{id}/fork`) and replaying on the fork.
102    #[serde(default)]
103    pub checkpoint: Option<CheckpointPayload>,
104
105    /// Free-form run metadata (stored, not interpreted).
106    #[serde(default)]
107    pub metadata: Option<Value>,
108
109    /// Which frame families to emit on the SSE stream. Default:
110    /// `["values", "updates"]`. `metadata`, `error`, and `end` frames are
111    /// always emitted.
112    #[serde(default)]
113    pub stream_mode: Option<Vec<String>>,
114
115    /// `"reject"` (409 when the thread is busy) or `"enqueue"` (default:
116    /// queue onto the per-thread run queue).
117    #[serde(default)]
118    pub multitask_strategy: Option<String>,
119
120    /// Run through a named assistant (see `POST /assistants`). The
121    /// assistant must be bound to the same graph as the thread; its
122    /// `config.recursion_limit` applies when the payload does not set one.
123    #[serde(default)]
124    pub assistant_id: Option<String>,
125}
126
127/// How a second run on a busy thread is handled.
128#[derive(Debug, Clone, Copy, PartialEq, Eq)]
129pub enum MultitaskStrategy {
130    /// Queue behind the active run (default).
131    Enqueue,
132    /// Fail immediately with 409.
133    Reject,
134}
135
136impl MultitaskStrategy {
137    /// Parse the wire value (`None` defaults to `enqueue`).
138    pub fn parse(raw: Option<&str>) -> Result<Self, String> {
139        match raw {
140            None | Some("enqueue") => Ok(Self::Enqueue),
141            Some("reject") => Ok(Self::Reject),
142            Some(other) => Err(format!(
143                "unknown multitask_strategy `{other}` (expected `enqueue` or `reject`)"
144            )),
145        }
146    }
147}
148
149// --------------------------------------------------------------------- //
150// Run bookkeeping
151// --------------------------------------------------------------------- //
152
153/// Lifecycle status of a run.
154#[derive(Debug, Clone, Copy, PartialEq, Eq)]
155pub enum RunStatus {
156    /// Queued behind another run on the same thread.
157    Pending,
158    /// Currently executing.
159    Running,
160    /// Terminated normally.
161    Success,
162    /// Suspended on an interrupt; resumable via `command.resume`.
163    Interrupted,
164    /// Failed.
165    Error,
166    /// Stopped by the graceful-shutdown drain (R0.6 wave 2c) at a
167    /// super-step boundary. Control flow, not failure — the boundary
168    /// checkpoint is intact, so re-running the thread resumes the run from
169    /// exactly where it stopped.
170    Cancelled,
171}
172
173impl RunStatus {
174    /// The wire representation of the status.
175    pub fn as_str(&self) -> &'static str {
176        match self {
177            RunStatus::Pending => "pending",
178            RunStatus::Running => "running",
179            RunStatus::Success => "success",
180            RunStatus::Interrupted => "interrupted",
181            RunStatus::Error => "error",
182            RunStatus::Cancelled => "cancelled",
183        }
184    }
185
186    /// `true` once the run can no longer make progress in this process
187    /// (terminal statuses, including `Cancelled`: a drained run resumes in
188    /// a *new* run, never in place).
189    pub(crate) fn is_terminal(self) -> bool {
190        matches!(
191            self,
192            RunStatus::Success | RunStatus::Interrupted | RunStatus::Error | RunStatus::Cancelled
193        )
194    }
195}
196
197/// One SSE frame as recorded in the per-run event log and broadcast live.
198/// `id` follows the design doc's `{checkpoint_id}:{step}:{seq}` format.
199#[derive(Debug, Clone)]
200pub struct SseFrame {
201    /// Frame id: `{checkpoint_id}:{step}:{seq}`.
202    pub id: String,
203    /// SSE event name (`metadata`, `updates`, `values`, `error`, `end`).
204    pub event: String,
205    /// JSON payload.
206    pub data: Value,
207    /// Per-run monotonically increasing sequence number (1-based).
208    pub seq: u64,
209}
210
211/// Shared frame producer for one run: assigns sequence numbers, appends to
212/// the bounded event log, and fans out over the broadcast channel.
213#[derive(Clone)]
214pub(crate) struct FrameSink {
215    log: Arc<StdMutex<VecDeque<SseFrame>>>,
216    bcast: broadcast::Sender<SseFrame>,
217    seq: Arc<AtomicU64>,
218    last_checkpoint: Arc<StdMutex<String>>,
219    last_step: Arc<AtomicU64>,
220    capacity: usize,
221}
222
223/// Lock a std mutex, recovering from poisoning. Every guard obtained
224/// through this helper wraps a simple clone/push/assign critical section,
225/// so a panicked holder cannot leave the value structurally inconsistent —
226/// and unwinding a whole run (or wedging its thread slot) over a poisoned
227/// frame log is the worse outcome.
228pub(crate) fn lock_recover<T>(mutex: &StdMutex<T>) -> std::sync::MutexGuard<'_, T> {
229    mutex
230        .lock()
231        .unwrap_or_else(|poisoned| poisoned.into_inner())
232}
233
234impl FrameSink {
235    fn new(capacity: usize, bcast: broadcast::Sender<SseFrame>) -> Self {
236        Self {
237            log: Arc::new(StdMutex::new(VecDeque::new())),
238            bcast,
239            seq: Arc::new(AtomicU64::new(0)),
240            last_checkpoint: Arc::new(StdMutex::new("-".to_string())),
241            last_step: Arc::new(AtomicU64::new(0)),
242            capacity,
243        }
244    }
245
246    /// Record and broadcast one frame.
247    pub(crate) fn push(&self, event: &str, step: usize, data: Value) {
248        let seq = self.seq.fetch_add(1, Ordering::Relaxed) + 1;
249        self.last_step.store(step as u64, Ordering::Relaxed);
250        let checkpoint = lock_recover(&self.last_checkpoint).clone();
251        let frame = SseFrame {
252            id: format!("{checkpoint}:{step}:{seq}"),
253            event: event.to_string(),
254            data,
255            seq,
256        };
257        {
258            let mut log = lock_recover(&self.log);
259            if log.len() >= self.capacity {
260                log.pop_front();
261            }
262            log.push_back(frame.clone());
263        }
264        // No live subscribers is normal (background runs); not an error.
265        let _ = self.bcast.send(frame);
266    }
267
268    /// Point subsequent frame ids at a freshly persisted checkpoint.
269    pub(crate) fn note_checkpoint(&self, checkpoint_id: &str) {
270        *lock_recover(&self.last_checkpoint) = checkpoint_id.to_string();
271    }
272
273    /// The super-step of the most recently pushed frame.
274    pub(crate) fn current_step(&self) -> usize {
275        self.last_step.load(Ordering::Relaxed) as usize
276    }
277}
278
279/// Everything the executor task needs, snapshotted from a [`RunHandle`].
280pub(crate) struct RunSnapshot {
281    /// Internal (tenant-scoped) thread id: used for the checkpointer, the
282    /// executor config, and RunManager bookkeeping.
283    pub thread_id: String,
284    /// External thread id as the client knows it — the only form that may
285    /// appear on the wire (SSE frames, terminal JSON).
286    pub wire_thread_id: String,
287    pub graph: String,
288    pub attempt: usize,
289    pub payload: RunPayload,
290    pub sink: FrameSink,
291    pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
292    /// This run's own cancellation token (R0.7 wave 2): a child of the
293    /// server's drain token, so a run-level cancel ([`RunManager::cancel_run`]
294    /// — the cancellation tree's run half) stops this run at its next
295    /// super-step boundary without touching any other run, while a server
296    /// drain still stops them all. Observed by the executor exactly where
297    /// the drain token always was.
298    pub cancel: tokio_util::sync::CancellationToken,
299}
300
301/// Read-only view of a run (used by the rollback and status endpoints).
302pub(crate) struct RunInfo {
303    /// Internal (tenant-scoped) thread id — handlers check tenant ownership
304    /// against it before revealing anything about the run.
305    pub thread_id: String,
306    /// External thread id for wire responses.
307    pub wire_thread_id: String,
308    pub graph: String,
309    pub attempt: usize,
310    pub status: RunStatus,
311    /// The terminal JSON once the run has finished (`None` while active).
312    pub terminal: Option<Value>,
313    pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
314}
315
316/// Handle for one scheduled run, owned by the [`RunManager`]. Crate-private
317/// surface: external users interact with runs over HTTP, not this type.
318pub struct RunHandle {
319    /// Run id (UUID v4).
320    pub(crate) run_id: String,
321    /// Internal (tenant-scoped) thread id this run executes against.
322    pub(crate) thread_id: String,
323    /// External thread id reported on the wire.
324    pub(crate) wire_thread_id: String,
325    /// Registered graph name.
326    pub(crate) graph: String,
327    /// 1-based attempt counter for the thread.
328    pub(crate) attempt: usize,
329    /// Lifecycle status.
330    pub(crate) status: RunStatus,
331    /// Original run payload.
332    pub(crate) payload: RunPayload,
333    sink: FrameSink,
334    terminal: watch::Sender<Option<Value>>,
335    checkpoint_ids: Arc<StdMutex<Vec<String>>>,
336    /// This run's own cancellation token (see [`RunSnapshot::cancel`]).
337    /// Firing it is the run-level half of the R0.7 cancellation tree; the
338    /// executor observes it at super-step boundaries, after the boundary
339    /// checkpoint has landed.
340    cancel: tokio_util::sync::CancellationToken,
341}
342
343impl RunHandle {
344    /// Subscribe to the live frame stream.
345    pub(crate) fn subscribe(&self) -> broadcast::Receiver<SseFrame> {
346        self.sink.bcast.subscribe()
347    }
348
349    /// A point-in-time copy of the event log (for replay).
350    pub(crate) fn log_snapshot(&self) -> Vec<SseFrame> {
351        lock_recover(&self.sink.log).iter().cloned().collect()
352    }
353}
354
355/// What [`RunManager::insert`] decided for a freshly scheduled run.
356pub(crate) enum ScheduleDecision {
357    /// The thread slot was free; the run must be spawned now.
358    Started,
359    /// The run was queued behind the active run.
360    Queued,
361}
362
363/// What [`RunManager::cancel_run`] did (R0.7 wave 2).
364#[derive(Debug, Clone, Copy, PartialEq, Eq)]
365pub(crate) enum RunCancel {
366    /// The run was executing; its cancellation token fired. The terminal
367    /// transition lands when the executor observes it at the next
368    /// super-step boundary.
369    Signalled,
370    /// The run was queued behind another; it was dequeued and finished
371    /// terminal-`cancelled` without ever starting.
372    CancelledQueued,
373    /// The run was already terminal; nothing changed.
374    Terminal,
375    /// No such run.
376    Unknown,
377}
378
379/// What [`RunManager::cancel_thread_runs`] did to one thread's runs
380/// (R0.7 wave 2), split by how each run's cancellation lands — mirroring
381/// the task queue's [`crate::tasks::RunCancellation`] shape.
382#[derive(Debug, Default)]
383pub(crate) struct ThreadCancellation {
384    /// Running runs whose cancellation tokens fired (terminal at their
385    /// next boundary).
386    pub signalled: Vec<String>,
387    /// Queued runs dequeued into terminal-`cancelled` immediately.
388    pub cancelled: Vec<String>,
389}
390
391impl ThreadCancellation {
392    /// `true` when no run was touched (the thread had no active or queued
393    /// runs) — the cancel route journals an `AgentExit` only when the
394    /// cancellation actually landed somewhere.
395    pub(crate) fn is_empty(&self) -> bool {
396        self.signalled.is_empty() && self.cancelled.is_empty()
397    }
398}
399
400#[derive(Default)]
401struct RunManagerInner {
402    runs: HashMap<String, RunHandle>,
403    active_by_thread: HashMap<String, String>,
404    queues: HashMap<String, VecDeque<String>>,
405    attempts: HashMap<String, usize>,
406    /// Insertion order of run ids, feeding terminal-run eviction.
407    order: VecDeque<String>,
408}
409
410/// Cap on retained runs (see the module docs' retention note). Without a
411/// cap, `runs` would grow by one record — payload clone, terminal JSON, and
412/// up to `event_log_capacity` SSE frames — per run for the process
413/// lifetime: a steady memory leak on any busy cron schedule.
414const MAX_RETAINED_RUNS: usize = 1024;
415
416/// Registry of all runs, plus per-thread scheduling state. Cheap to clone
417/// (shared inner).
418#[derive(Default, Clone)]
419pub struct RunManager {
420    inner: Arc<Mutex<RunManagerInner>>,
421}
422
423impl RunManager {
424    /// An empty manager.
425    pub fn new() -> Self {
426        Self::default()
427    }
428
429    /// Insert a new run under the given multitask strategy, assigning its
430    /// per-thread attempt number.
431    pub(crate) async fn insert(
432        &self,
433        mut handle: RunHandle,
434        strategy: MultitaskStrategy,
435        queue_cap: usize,
436    ) -> Result<ScheduleDecision, ApiError> {
437        let mut inner = self.inner.lock().await;
438        let busy = inner.active_by_thread.contains_key(&handle.thread_id);
439        let attempt = {
440            let counter = inner.attempts.entry(handle.thread_id.clone()).or_insert(0);
441            *counter += 1;
442            *counter
443        };
444        handle.attempt = attempt;
445        inner.order.push_back(handle.run_id.clone());
446
447        match strategy {
448            MultitaskStrategy::Reject if busy => Err(ApiError::conflict(format!(
449                "thread `{}` already has an active run",
450                handle.thread_id
451            ))),
452            _ if busy => {
453                let queue = inner.queues.entry(handle.thread_id.clone()).or_default();
454                if queue.len() >= queue_cap {
455                    return Err(ApiError::conflict(format!(
456                        "thread `{}` run queue is full (cap {queue_cap})",
457                        handle.thread_id
458                    )));
459                }
460                queue.push_back(handle.run_id.clone());
461                inner.runs.insert(handle.run_id.clone(), handle);
462                Ok(ScheduleDecision::Queued)
463            }
464            _ => {
465                inner
466                    .active_by_thread
467                    .insert(handle.thread_id.clone(), handle.run_id.clone());
468                handle.status = RunStatus::Running;
469                inner.runs.insert(handle.run_id.clone(), handle);
470                Ok(ScheduleDecision::Started)
471            }
472        }
473    }
474
475    /// Snapshot everything the executor task needs for `run_id`.
476    pub(crate) async fn snapshot(&self, run_id: &str) -> Option<RunSnapshot> {
477        let inner = self.inner.lock().await;
478        inner.runs.get(run_id).map(|h| RunSnapshot {
479            thread_id: h.thread_id.clone(),
480            wire_thread_id: h.wire_thread_id.clone(),
481            graph: h.graph.clone(),
482            attempt: h.attempt,
483            payload: h.payload.clone(),
484            sink: h.sink.clone(),
485            checkpoint_ids: Arc::clone(&h.checkpoint_ids),
486            cancel: h.cancel.clone(),
487        })
488    }
489
490    /// Read-only run info for API endpoints.
491    pub(crate) async fn info(&self, run_id: &str) -> Option<RunInfo> {
492        let inner = self.inner.lock().await;
493        inner.runs.get(run_id).map(|h| RunInfo {
494            thread_id: h.thread_id.clone(),
495            wire_thread_id: h.wire_thread_id.clone(),
496            graph: h.graph.clone(),
497            attempt: h.attempt,
498            status: h.status,
499            terminal: h.terminal.borrow().clone(),
500            checkpoint_ids: Arc::clone(&h.checkpoint_ids),
501        })
502    }
503
504    /// Replay log + live subscription + internal thread id for the
505    /// SSE attach endpoint (`GET /runs/{id}/stream`).
506    pub(crate) async fn stream_parts(
507        &self,
508        run_id: &str,
509    ) -> Option<(Vec<SseFrame>, broadcast::Receiver<SseFrame>, String)> {
510        let inner = self.inner.lock().await;
511        inner
512            .runs
513            .get(run_id)
514            .map(|h| (h.log_snapshot(), h.subscribe(), h.thread_id.clone()))
515    }
516
517    /// `true` while the thread has an active run or a non-empty queue —
518    /// rollback refuses to delete checkpoints out from under them.
519    pub(crate) async fn thread_busy(&self, thread_id: &str) -> bool {
520        let inner = self.inner.lock().await;
521        inner.active_by_thread.contains_key(thread_id)
522            || inner
523                .queues
524                .get(thread_id)
525                .is_some_and(|queue| !queue.is_empty())
526    }
527
528    /// Cancel one run (R0.7 wave 2 — the run-level half of the
529    /// cancellation tree). A running run is *signalled*: its own
530    /// cancellation token fires and the executor stops it at the next
531    /// super-step boundary — after the boundary checkpoint has landed —
532    /// ending terminal-`cancelled` and resumable by re-running the thread,
533    /// exactly like the server drain. A queued (pending) run never started,
534    /// so it is dequeued and finished terminal-`cancelled` immediately —
535    /// leaving it queued would let a dead run promote and execute.
536    pub(crate) async fn cancel_run(&self, run_id: &str) -> RunCancel {
537        let mut inner = self.inner.lock().await;
538        let Some(handle) = inner.runs.get_mut(run_id) else {
539            return RunCancel::Unknown;
540        };
541        match handle.status {
542            RunStatus::Running => {
543                handle.cancel.cancel();
544                RunCancel::Signalled
545            }
546            RunStatus::Pending => {
547                let thread_id = handle.thread_id.clone();
548                let wire_thread_id = handle.wire_thread_id.clone();
549                if let Some(queue) = inner.queues.get_mut(&thread_id) {
550                    queue.retain(|queued| queued != run_id);
551                }
552                let handle = inner
553                    .runs
554                    .get_mut(run_id)
555                    .expect("the handle was resolved above");
556                handle.status = RunStatus::Cancelled;
557                let terminal = json!({
558                    "run_id": run_id,
559                    "thread_id": wire_thread_id,
560                    "status": "cancelled",
561                    "message": "cancelled while queued, before its first step",
562                });
563                handle.terminal.send_replace(Some(terminal));
564                RunCancel::CancelledQueued
565            }
566            // Terminal runs (including an already-cancelled one) are
567            // untouched — cancellation is control flow, idempotent by
568            // no-op, never a second terminal transition.
569            _ => RunCancel::Terminal,
570        }
571    }
572
573    /// Cancel every run of one thread (R0.7 wave 2): the active run is
574    /// signalled, every queued run is dequeued-cancelled — the whole
575    /// per-thread run state, so cancelling an agent's thread leaves no
576    /// pending run that would re-drive it.
577    pub(crate) async fn cancel_thread_runs(&self, thread_id: &str) -> ThreadCancellation {
578        let (active, queued) = {
579            let inner = self.inner.lock().await;
580            (
581                inner.active_by_thread.get(thread_id).cloned(),
582                inner
583                    .queues
584                    .get(thread_id)
585                    .map(|q| q.iter().cloned().collect::<Vec<_>>())
586                    .unwrap_or_default(),
587            )
588        };
589        let mut outcome = ThreadCancellation::default();
590        if let Some(run_id) = active {
591            if matches!(self.cancel_run(&run_id).await, RunCancel::Signalled) {
592                outcome.signalled.push(run_id);
593            }
594        }
595        for run_id in queued {
596            if matches!(self.cancel_run(&run_id).await, RunCancel::CancelledQueued) {
597                outcome.cancelled.push(run_id);
598            }
599        }
600        outcome
601    }
602
603    /// Record the terminal status + JSON, wake waiters, release the thread
604    /// slot, and return the next queued run id for the thread (if any), now
605    /// marked active.
606    ///
607    /// `draining` (the server's shutdown drain) suppresses queue promotion:
608    /// the slot frees but nothing is returned — a run promoted into a
609    /// shutting-down process would only be cancelled at its first boundary.
610    /// Queued runs stay `Pending`; their threads' checkpoints are intact
611    /// for the next process to re-drive.
612    pub(crate) async fn finish(
613        &self,
614        run_id: &str,
615        status: RunStatus,
616        terminal: Value,
617        draining: bool,
618    ) -> Option<String> {
619        let mut inner = self.inner.lock().await;
620        let handle = inner.runs.get_mut(run_id)?;
621        handle.status = status;
622        // `send_replace` (not `send`) so the terminal JSON is stored even
623        // when no waiter holds a receiver (background runs); status polling
624        // via `info` reads it back through `watch::Sender::borrow`.
625        handle.terminal.send_replace(Some(terminal));
626        let thread_id = handle.thread_id.clone();
627
628        if inner
629            .active_by_thread
630            .get(&thread_id)
631            .is_some_and(|active| active == run_id)
632        {
633            inner.active_by_thread.remove(&thread_id);
634        }
635
636        let next = if draining {
637            None
638        } else {
639            inner
640                .queues
641                .get_mut(&thread_id)
642                .and_then(VecDeque::pop_front)
643        };
644        if let Some(next_id) = &next {
645            if let Some(h) = inner.runs.get_mut(next_id) {
646                h.status = RunStatus::Running;
647            }
648            inner.active_by_thread.insert(thread_id, next_id.clone());
649        }
650
651        // Evict the oldest terminal runs beyond the retention cap; active
652        // and queued runs keep their slots in `order`.
653        let mut excess = inner.runs.len().saturating_sub(MAX_RETAINED_RUNS);
654        let mut skipped = Vec::new();
655        while excess > 0 {
656            let Some(candidate) = inner.order.pop_front() else {
657                break;
658            };
659            let evictable = inner
660                .runs
661                .get(&candidate)
662                .is_some_and(|h| h.status.is_terminal());
663            if evictable {
664                inner.runs.remove(&candidate);
665                excess -= 1;
666            } else {
667                skipped.push(candidate);
668            }
669        }
670        inner.order.extend(skipped);
671
672        next
673    }
674}
675
676/// Everything the run machinery needs from the application: registry,
677/// checkpointer, manager, and caps. Cheap to clone.
678#[derive(Clone)]
679pub(crate) struct RunDeps {
680    pub registry: GraphRegistry,
681    pub checkpointer: Arc<dyn Checkpointer>,
682    pub manager: RunManager,
683    /// Flight Recorder journal persistence (`GET /runs/{id}/events`).
684    pub server_store: Arc<dyn ServerStore>,
685    pub queue_cap: usize,
686    pub log_capacity: usize,
687    /// The server's drain control (R0.6 wave 2c): threaded into every run's
688    /// executor, which observes it at super-step boundaries.
689    pub shutdown: tokio_util::sync::CancellationToken,
690}
691
692/// The result of successfully scheduling a run: everything an endpoint
693/// needs to answer (background ack, wait, or stream).
694pub(crate) struct Scheduled {
695    pub run_id: String,
696    pub status: RunStatus,
697    pub terminal: watch::Receiver<Option<Value>>,
698    pub broadcast: broadcast::Receiver<SseFrame>,
699    pub replay: Vec<SseFrame>,
700}
701
702/// Create a run handle, apply the multitask strategy, and spawn execution
703/// immediately when the thread slot is free.
704///
705/// `thread_id` is the internal (tenant-scoped) id used for the checkpointer,
706/// executor, and RunManager bookkeeping; `wire_thread_id` is the external id
707/// reported in SSE frames and terminal JSON.
708pub(crate) async fn schedule(
709    deps: &RunDeps,
710    thread_id: &str,
711    wire_thread_id: &str,
712    graph: &str,
713    payload: RunPayload,
714    strategy: MultitaskStrategy,
715) -> Result<Scheduled, ApiError> {
716    // A draining server must not take new runs: the token would cancel
717    // them at their first boundary anyway, and a 503 lets the caller (or
718    // its load balancer) retry against a pod that is still serving.
719    if deps.shutdown.is_cancelled() {
720        return Err(ApiError::shutting_down(format!(
721            "server is draining; resubmit run on thread `{wire_thread_id}` against a running instance"
722        )));
723    }
724    let run_id = uuid::Uuid::new_v4().to_string();
725    let (bcast_tx, _bcast_rx) = broadcast::channel(256);
726    let (terminal_tx, terminal_rx) = watch::channel(None);
727    let handle = RunHandle {
728        run_id: run_id.clone(),
729        thread_id: thread_id.to_string(),
730        wire_thread_id: wire_thread_id.to_string(),
731        graph: graph.to_string(),
732        attempt: 0, // assigned by RunManager::insert
733        status: RunStatus::Pending,
734        payload,
735        sink: FrameSink::new(deps.log_capacity, bcast_tx),
736        terminal: terminal_tx,
737        checkpoint_ids: Arc::new(StdMutex::new(Vec::new())),
738        // A child of the server drain token: the drain still stops every
739        // run, and a run-level cancel (R0.7 wave 2) stops only this one.
740        cancel: deps.shutdown.child_token(),
741    };
742    // Subscribe/snapshot before any execution can emit frames.
743    let replay = handle.log_snapshot();
744    let broadcast = handle.subscribe();
745
746    let decision = deps
747        .manager
748        .insert(handle, strategy, deps.queue_cap)
749        .await?;
750    let status = match decision {
751        ScheduleDecision::Started => {
752            spawn_execute(deps.clone(), run_id.clone());
753            RunStatus::Running
754        }
755        ScheduleDecision::Queued => RunStatus::Pending,
756    };
757
758    Ok(Scheduled {
759        run_id,
760        status,
761        terminal: terminal_rx,
762        broadcast,
763        replay,
764    })
765}
766
767/// Drive one run to its terminal state and chain the next queued run.
768async fn execute(deps: RunDeps, run_id: String) {
769    let Some(snap) = deps.manager.snapshot(&run_id).await else {
770        tracing::warn!(%run_id, "scheduled run vanished before execution");
771        return;
772    };
773    let sink = snap.sink.clone();
774    sink.push(
775        "metadata",
776        0,
777        json!({
778            "run_id": run_id,
779            "thread_id": snap.wire_thread_id,
780            "graph": snap.graph,
781            "attempt": snap.attempt,
782            "metadata": snap.payload.metadata,
783        }),
784    );
785
786    let Some((graph, spec)) = deps.registry.get(&snap.graph) else {
787        let message = format!("graph `{}` is no longer registered", snap.graph);
788        tracing::error!(%run_id, %message);
789        sink.push(
790            "error",
791            0,
792            json!({"error": "unknown_graph", "message": message}),
793        );
794        sink.push("end", 0, json!({"status": "error"}));
795        let terminal = json!({
796            "run_id": run_id,
797            "thread_id": snap.wire_thread_id,
798            "status": "error",
799            "error": "unknown_graph",
800            "message": message,
801        });
802        terminate(&deps, &run_id, RunStatus::Error, terminal).await;
803        return;
804    };
805
806    let modes: Vec<String> = snap
807        .payload
808        .stream_mode
809        .clone()
810        .unwrap_or_else(|| vec!["values".to_string(), "updates".to_string()]);
811    // Flight Recorder: one journal per run, keyed by the server-minted run
812    // id. Events carry the external (wire) thread id — the internal
813    // tenant-scoped id must never appear in served evidence. The journal's
814    // clock is the default system clock, so timestamps match pre-R0.5
815    // behavior; attaching it makes the executor read time through it.
816    let journal = Journal::new(run_id.clone(), snap.wire_thread_id.clone(), Clock::System);
817    let (evt_tx, evt_rx) = mpsc::channel::<GraphEvent>(256);
818    let forwarder = tokio::spawn(forward_events(
819        evt_rx,
820        sink.clone(),
821        ForwardDeps {
822            checkpointer: Arc::clone(&deps.checkpointer),
823            server_store: Arc::clone(&deps.server_store),
824            journal: journal.clone(),
825            thread_id: snap.thread_id.clone(),
826            checkpoint_ids: Arc::clone(&snap.checkpoint_ids),
827            modes,
828        },
829    ));
830
831    let mut config = RunConfig::new(snap.thread_id.clone())
832        .with_event_tx(evt_tx)
833        .with_journal(journal.clone())
834        // Cancellation hook: this run's own token (a child of the server
835        // drain token). When either fires, the run stops at its next
836        // super-step boundary — a point where a checkpoint was just
837        // persisted — instead of being torn down mid-step.
838        .with_cancellation(snap.cancel.clone());
839    if let Some(command) = &snap.payload.command {
840        if let Some(resume) = &command.resume {
841            config = config.with_resume(resume.clone());
842        }
843    }
844    if let Some(checkpoint) = &snap.payload.checkpoint {
845        config = config.with_checkpoint_id(checkpoint.checkpoint_id.clone());
846    }
847    if let Some(run_cfg) = &snap.payload.config {
848        if let Some(limit) = run_cfg.recursion_limit {
849            config = config.with_max_steps(limit);
850        }
851    }
852    let initial = snap
853        .payload
854        .input
855        .clone()
856        .and_then(|v| State::from_value(v).ok())
857        .unwrap_or_default();
858
859    let result = Executor::with_checkpointer(Arc::clone(&deps.checkpointer))
860        .run(&graph, &spec, initial, config)
861        .await;
862    // `config` (holding the only sender) is dropped with the run; the
863    // forwarder drains what remains and exits.
864    let _ = forwarder.await;
865
866    // Final journal write: the complete evidence of the run, including the
867    // events recorded after the last checkpoint boundary. Persisted before
868    // the run goes terminal so `complete: true` on the events endpoint never
869    // races ahead of the snapshot it serves. Evidence of a failed run is
870    // still evidence — this write happens on every outcome.
871    persist_journal(&deps.server_store, &journal).await;
872
873    let step = sink.current_step();
874    let (status, terminal) = match result {
875        Ok(ExecutionOutcome::Done(state)) => {
876            sink.push("end", step, json!({"status": "success"}));
877            let terminal = json!({
878                "run_id": run_id,
879                "thread_id": snap.wire_thread_id,
880                "status": "success",
881                "output": state.to_value(),
882            });
883            (RunStatus::Success, terminal)
884        }
885        Ok(ExecutionOutcome::Interrupted {
886            value,
887            state,
888            checkpoint_id,
889        }) => {
890            sink.push(
891                "end",
892                step,
893                json!({"status": "interrupted", "interrupt": value}),
894            );
895            let terminal = json!({
896                "run_id": run_id,
897                "thread_id": snap.wire_thread_id,
898                "status": "interrupted",
899                "interrupt": value,
900                "checkpoint_id": checkpoint_id,
901                "state": state.to_value(),
902            });
903            (RunStatus::Interrupted, terminal)
904        }
905        Err(error @ RustyError::Cancelled(_)) => {
906            // Drain, not failure: the executor stopped at a super-step
907            // boundary, so the run's last checkpoint is intact and a fresh
908            // run on the thread resumes from it. The wire status is
909            // `cancelled` — matching the task queue's treatment of
910            // cancellation as control flow, never an error.
911            let message = error.to_string();
912            tracing::info!(%run_id, "run drained at a checkpoint boundary; resumable");
913            sink.push("end", step, json!({"status": "cancelled"}));
914            let terminal = json!({
915                "run_id": run_id,
916                "thread_id": snap.wire_thread_id,
917                "status": "cancelled",
918                "message": message,
919            });
920            (RunStatus::Cancelled, terminal)
921        }
922        Err(error) => {
923            let kind = error_kind(&error);
924            let message = error.to_string();
925            tracing::warn!(%run_id, %error, "run failed");
926            sink.push("error", step, json!({"error": kind, "message": message}));
927            sink.push("end", step, json!({"status": "error"}));
928            let terminal = json!({
929                "run_id": run_id,
930                "thread_id": snap.wire_thread_id,
931                "status": "error",
932                "error": kind,
933                "message": message,
934            });
935            (RunStatus::Error, terminal)
936        }
937    };
938    terminate(&deps, &run_id, status, terminal).await;
939}
940
941/// Everything [`forward_events`] needs beyond the frame sink, bundled to
942/// keep the task's argument list readable.
943struct ForwardDeps {
944    checkpointer: Arc<dyn Checkpointer>,
945    server_store: Arc<dyn ServerStore>,
946    journal: Journal,
947    /// Internal (tenant-scoped) thread id, for checkpoint read-backs.
948    thread_id: String,
949    checkpoint_ids: Arc<StdMutex<Vec<String>>>,
950    modes: Vec<String>,
951}
952
953/// Map executor events to SSE frames per the design doc's §4 table. Also the
954/// Flight Recorder's checkpoint-boundary persistence point: every
955/// `CheckpointSaved` event flushes the journal's current snapshot to the
956/// server store, so the stored evidence trails the live journal by at most
957/// one super-step.
958async fn forward_events(mut rx: mpsc::Receiver<GraphEvent>, sink: FrameSink, deps: ForwardDeps) {
959    let ForwardDeps {
960        checkpointer,
961        server_store,
962        journal,
963        thread_id,
964        checkpoint_ids,
965        modes,
966    } = deps;
967    while let Some(event) = rx.recv().await {
968        match event {
969            GraphEvent::StateUpdate { step, updates } => {
970                if modes.iter().any(|m| m == "updates") {
971                    sink.push("updates", step, json!({"step": step, "updates": updates}));
972                }
973            }
974            GraphEvent::Token { node, delta } => {
975                if modes.iter().any(|m| m == "messages") {
976                    let step = sink.current_step();
977                    sink.push("messages", step, json!({"node": node, "delta": delta}));
978                }
979            }
980            GraphEvent::CheckpointSaved {
981                checkpoint_id,
982                step,
983            } => {
984                lock_recover(&checkpoint_ids).push(checkpoint_id.clone());
985                sink.note_checkpoint(&checkpoint_id);
986                persist_journal(&server_store, &journal).await;
987                if modes.iter().any(|m| m == "values") {
988                    match read_back_state(&*checkpointer, &thread_id, &checkpoint_id).await {
989                        Ok(Some(values)) => sink.push("values", step, values),
990                        Ok(None) => {
991                            tracing::debug!(%checkpoint_id, "checkpoint not found for values frame")
992                        }
993                        Err(error) => {
994                            tracing::warn!(%checkpoint_id, %error, "values frame read-back failed")
995                        }
996                    }
997                }
998            }
999            // Reserved for the future `tasks` / `debug` stream modes.
1000            GraphEvent::SuperStep { .. }
1001            | GraphEvent::NodeStart { .. }
1002            | GraphEvent::NodeEnd { .. } => {}
1003        }
1004    }
1005}
1006
1007/// Flush the journal's current snapshot to the server store. A persistence
1008/// failure is logged, not raised: the run's execution must not fail because
1009/// its evidence could not be written, and the next checkpoint boundary (or
1010/// the completion write) retries.
1011async fn persist_journal(server_store: &Arc<dyn ServerStore>, journal: &Journal) {
1012    if let Err(error) = server_store.put_journal(&journal.snapshot()).await {
1013        tracing::warn!(run_id = %journal.run_id(), %error, "journal persistence failed");
1014    }
1015}
1016
1017/// `values` frames carry the full state persisted at a super-step boundary,
1018/// read back from the checkpoint log (design doc §4). A point lookup, not a
1019/// full `list()` scan — that would be O(history) per super-step, O(n²) per
1020/// run.
1021async fn read_back_state(
1022    checkpointer: &dyn Checkpointer,
1023    thread_id: &str,
1024    checkpoint_id: &str,
1025) -> rusty_agent_runtime::error::Result<Option<Value>> {
1026    Ok(checkpointer
1027        .get_by_id(thread_id, checkpoint_id)
1028        .await?
1029        .map(|cp| cp.state.to_value()))
1030}
1031
1032/// Spawn `execute` for a run, guarding the thread's scheduling slot: if the
1033/// task panics (executor bug — the poison-prone lock sites recover via
1034/// [`lock_recover`]), the run is force-finished as `error` so
1035/// `active_by_thread` releases the slot and queued runs drain instead of
1036/// wedging behind a ghost.
1037///
1038/// The future is boxed behind a trait object to break the
1039/// `execute → terminate → spawn(execute)` type cycle, which would otherwise
1040/// make `Send` inference recursive and fail.
1041fn spawn_execute(deps: RunDeps, run_id: String) {
1042    let fut: std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> = Box::pin({
1043        let deps = deps.clone();
1044        let run_id = run_id.clone();
1045        async move { execute(deps, run_id).await }
1046    });
1047    tokio::spawn(async move {
1048        if AssertUnwindSafe(fut).catch_unwind().await.is_ok() {
1049            return;
1050        }
1051        tracing::error!(%run_id, "run task panicked; force-finishing as error");
1052        let Some(snap) = deps.manager.snapshot(&run_id).await else {
1053            return;
1054        };
1055        // If the panic happened after `terminate` completed, the slot is
1056        // already released — finishing again would double-promote the queue.
1057        if matches!(deps.manager.info(&run_id).await, Some(info) if info.status.is_terminal()) {
1058            return;
1059        }
1060        let step = snap.sink.current_step();
1061        snap.sink.push(
1062            "error",
1063            step,
1064            json!({"error": "internal_panic", "message": "run task panicked"}),
1065        );
1066        snap.sink.push("end", step, json!({"status": "error"}));
1067        let terminal = json!({
1068            "run_id": run_id,
1069            "thread_id": snap.wire_thread_id,
1070            "status": "error",
1071            "error": "internal_panic",
1072            "message": "run task panicked",
1073        });
1074        terminate(&deps, &run_id, RunStatus::Error, terminal).await;
1075    });
1076}
1077
1078/// Record the terminal state and spawn the next queued run, if any. While
1079/// the server drains, the queue does not advance (see
1080/// [`RunManager::finish`]'s `draining` flag).
1081async fn terminate(deps: &RunDeps, run_id: &str, status: RunStatus, terminal: Value) {
1082    let draining = deps.shutdown.is_cancelled();
1083    if let Some(next) = deps
1084        .manager
1085        .finish(run_id, status, terminal, draining)
1086        .await
1087    {
1088        spawn_execute(deps.clone(), next);
1089    }
1090}
1091
1092/// Stable error-kind labels for the wire.
1093fn error_kind(error: &RustyError) -> &'static str {
1094    match error {
1095        RustyError::Graph(_) => "graph_error",
1096        RustyError::Node(_) => "node_error",
1097        RustyError::Interrupt { .. } => "interrupted",
1098        RustyError::Checkpoint(_) => "checkpoint_error",
1099        RustyError::Llm(_) => "llm_error",
1100        RustyError::Tool(_) => "tool_error",
1101        RustyError::Serialization(_) => "serialization_error",
1102        RustyError::InvalidUpdate(_) => "invalid_update",
1103        RustyError::Replay(_) => "replay_error",
1104        // Drain cancellation is control flow and takes its own terminal
1105        // path in `execute`; this arm exists for exhaustiveness only.
1106        RustyError::Cancelled(_) => "cancelled",
1107    }
1108}