Skip to main content

rusty_agent_server/
runs.rs

1//! Run scheduling, execution, and bookkeeping.
2//!
3//! A run goes: *schedule* (strategy check + handle insert) → *execute*
4//! (drive [`Executor`] in a spawned task, forwarding [`GraphEvent`]s to a
5//! per-run SSE frame log + broadcast channel) → *terminate* (terminal status
6//! + JSON recorded, waiters woken, next queued run for the thread spawned).
7//!
8//! Multitask: there is always at most one **active** run per thread. The
9//! `reject` strategy returns 409 when the thread is busy; `enqueue` appends
10//! to an in-memory per-thread FIFO queue (depth-capped by
11//! `ServerConfig::max_concurrent_runs_per_thread`) that drains automatically
12//! as runs finish.
13//!
14//! Retention: terminal runs are kept for `GET /runs/{id}` polling up to
15//! [`MAX_RETAINED_RUNS`] per process; the oldest terminal runs are evicted
16//! beyond that (active and queued runs are never evicted). Run history is
17//! in-memory by design — durability lives in the checkpoint log.
18//!
19//! Drain (R0.6 wave 2c): the server's shutdown token is threaded into every
20//! run's executor ([`RunConfig::with_cancellation`]). When it fires, each
21//! in-flight run stops at its next super-step boundary — a point where a
22//! checkpoint was just persisted — and ends terminal-[`RunStatus::Cancelled`],
23//! resumable by simply re-running the thread; new submissions answer 503 and
24//! queued runs are not promoted. Anything still mid-step when the grace
25//! window closes is abandoned, which is the crash case the checkpoint log
26//! already covers.
27//!
28//! Flight Recorder: every run is journaled. The journal is attached to the
29//! executor at run start, flushed to the server store at every checkpoint
30//! boundary (in [`forward_events`]) and once more at run completion, and
31//! served read-only by `GET /runs/{id}/events`.
32
33use std::collections::{HashMap, VecDeque};
34use std::panic::AssertUnwindSafe;
35use std::sync::atomic::{AtomicU64, Ordering};
36use std::sync::{Arc, Mutex as StdMutex};
37
38use futures::FutureExt;
39use rusty_agent_runtime::checkpoint::Checkpointer;
40use rusty_agent_runtime::error::RustyError;
41use rusty_agent_runtime::executor::{ExecutionOutcome, Executor, GraphEvent, RunConfig};
42use rusty_agent_runtime::journal::{Clock, EventDraft, Journal};
43use rusty_agent_runtime::record::{Effect, RunEventKind};
44use rusty_agent_runtime::state::State;
45use serde::Deserialize;
46use serde_json::{json, Value};
47use tokio::sync::{broadcast, mpsc, watch, Mutex};
48
49use crate::error::ApiError;
50use crate::server_store::ServerStore;
51use crate::GraphRegistry;
52
53// --------------------------------------------------------------------- //
54// Run payload (accepted by all three run endpoints)
55// --------------------------------------------------------------------- //
56
57/// The `command` field of a run payload: `{ "resume": <value> }` continues
58/// an interrupted thread via [`RunConfig::with_resume`].
59#[derive(Debug, Clone, Default, Deserialize)]
60pub struct CommandPayload {
61    /// Resume value delivered to the interrupted node.
62    #[serde(default)]
63    pub resume: Option<Value>,
64}
65
66/// The `config` field of a run payload.
67#[derive(Debug, Clone, Default, Deserialize)]
68pub struct RunConfigPayload {
69    /// Maps to [`RunConfig::with_max_steps`] (LangGraph `recursion_limit`).
70    #[serde(default)]
71    pub recursion_limit: Option<usize>,
72}
73
74/// The `checkpoint` field of a run payload: `{ "checkpoint_id": "…" }`
75/// replays the thread from that checkpoint (time travel) instead of the
76/// latest, via [`RunConfig::with_checkpoint_id`].
77#[derive(Debug, Clone, Default, Deserialize)]
78pub struct CheckpointPayload {
79    /// Id of a checkpoint of this thread (see `POST /threads/{id}/history`).
80    pub checkpoint_id: String,
81}
82
83/// The payload accepted by `POST /threads/{id}/runs{,/wait,/stream}`.
84#[derive(Debug, Clone, Default, Deserialize)]
85pub struct RunPayload {
86    /// Initial state (must be a JSON object). Ignored when resuming: the
87    /// checkpointed state takes precedence.
88    #[serde(default)]
89    pub input: Option<Value>,
90
91    /// `{ "resume": <value> }` — the human-in-the-loop channel.
92    #[serde(default)]
93    pub command: Option<CommandPayload>,
94
95    /// `{ "recursion_limit": n }`.
96    #[serde(default)]
97    pub config: Option<RunConfigPayload>,
98
99    /// `{ "checkpoint_id": "…" }` — time travel: replay the run from that
100    /// checkpoint of this thread instead of the latest (`404` when the
101    /// checkpoint is unknown). Prefer forking first
102    /// (`POST /threads/{id}/fork`) and replaying on the fork.
103    #[serde(default)]
104    pub checkpoint: Option<CheckpointPayload>,
105
106    /// Free-form run metadata (stored, not interpreted).
107    #[serde(default)]
108    pub metadata: Option<Value>,
109
110    /// Which frame families to emit on the SSE stream. Default:
111    /// `["values", "updates"]`. `metadata`, `error`, and `end` frames are
112    /// always emitted.
113    #[serde(default)]
114    pub stream_mode: Option<Vec<String>>,
115
116    /// `"reject"` (409 when the thread is busy) or `"enqueue"` (default:
117    /// queue onto the per-thread run queue).
118    #[serde(default)]
119    pub multitask_strategy: Option<String>,
120
121    /// Run through a named assistant (see `POST /assistants`). The
122    /// assistant must be bound to the same graph as the thread; its
123    /// `config.recursion_limit` applies when the payload does not set one.
124    #[serde(default)]
125    pub assistant_id: Option<String>,
126
127    /// The run's registry declaration (R0.11 Extension Plane, wave 2):
128    /// the named configuration artifacts the run uses and the
129    /// environment it targets. At admission each artifact resolves
130    /// through its environment-tagged version pointer and the resolved
131    /// content pins the run's manifest, with one `config_resolved` event
132    /// per artifact journaled ahead of the run's own events. Absent is
133    /// the pre-R0.11 behavior, byte-identically: no resolution, no
134    /// manifest, no new events.
135    #[serde(default)]
136    pub registry: Option<crate::registry::RegistryRunBinding>,
137}
138
139/// How a second run on a busy thread is handled.
140#[derive(Debug, Clone, Copy, PartialEq, Eq)]
141pub enum MultitaskStrategy {
142    /// Queue behind the active run (default).
143    Enqueue,
144    /// Fail immediately with 409.
145    Reject,
146}
147
148impl MultitaskStrategy {
149    /// Parse the wire value (`None` defaults to `enqueue`).
150    pub fn parse(raw: Option<&str>) -> Result<Self, String> {
151        match raw {
152            None | Some("enqueue") => Ok(Self::Enqueue),
153            Some("reject") => Ok(Self::Reject),
154            Some(other) => Err(format!(
155                "unknown multitask_strategy `{other}` (expected `enqueue` or `reject`)"
156            )),
157        }
158    }
159}
160
161// --------------------------------------------------------------------- //
162// Run bookkeeping
163// --------------------------------------------------------------------- //
164
165/// Lifecycle status of a run.
166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub enum RunStatus {
168    /// Queued behind another run on the same thread.
169    Pending,
170    /// Currently executing.
171    Running,
172    /// Terminated normally.
173    Success,
174    /// Suspended on an interrupt; resumable via `command.resume`.
175    Interrupted,
176    /// Failed.
177    Error,
178    /// Stopped by the graceful-shutdown drain (R0.6 wave 2c) at a
179    /// super-step boundary. Control flow, not failure — the boundary
180    /// checkpoint is intact, so re-running the thread resumes the run from
181    /// exactly where it stopped.
182    Cancelled,
183}
184
185impl RunStatus {
186    /// The wire representation of the status.
187    pub fn as_str(&self) -> &'static str {
188        match self {
189            RunStatus::Pending => "pending",
190            RunStatus::Running => "running",
191            RunStatus::Success => "success",
192            RunStatus::Interrupted => "interrupted",
193            RunStatus::Error => "error",
194            RunStatus::Cancelled => "cancelled",
195        }
196    }
197
198    /// `true` once the run can no longer make progress in this process
199    /// (terminal statuses, including `Cancelled`: a drained run resumes in
200    /// a *new* run, never in place).
201    pub(crate) fn is_terminal(self) -> bool {
202        matches!(
203            self,
204            RunStatus::Success | RunStatus::Interrupted | RunStatus::Error | RunStatus::Cancelled
205        )
206    }
207}
208
209/// One SSE frame as recorded in the per-run event log and broadcast live.
210/// `id` follows the design doc's `{checkpoint_id}:{step}:{seq}` format.
211#[derive(Debug, Clone)]
212pub struct SseFrame {
213    /// Frame id: `{checkpoint_id}:{step}:{seq}`.
214    pub id: String,
215    /// SSE event name (`metadata`, `updates`, `values`, `error`, `end`).
216    pub event: String,
217    /// JSON payload.
218    pub data: Value,
219    /// Per-run monotonically increasing sequence number (1-based).
220    pub seq: u64,
221}
222
223/// Shared frame producer for one run: assigns sequence numbers, appends to
224/// the bounded event log, and fans out over the broadcast channel.
225#[derive(Clone)]
226pub(crate) struct FrameSink {
227    log: Arc<StdMutex<VecDeque<SseFrame>>>,
228    bcast: broadcast::Sender<SseFrame>,
229    seq: Arc<AtomicU64>,
230    last_checkpoint: Arc<StdMutex<String>>,
231    last_step: Arc<AtomicU64>,
232    capacity: usize,
233}
234
235/// Lock a std mutex, recovering from poisoning. Every guard obtained
236/// through this helper wraps a simple clone/push/assign critical section,
237/// so a panicked holder cannot leave the value structurally inconsistent —
238/// and unwinding a whole run (or wedging its thread slot) over a poisoned
239/// frame log is the worse outcome.
240pub(crate) fn lock_recover<T>(mutex: &StdMutex<T>) -> std::sync::MutexGuard<'_, T> {
241    mutex
242        .lock()
243        .unwrap_or_else(|poisoned| poisoned.into_inner())
244}
245
246impl FrameSink {
247    fn new(capacity: usize, bcast: broadcast::Sender<SseFrame>) -> Self {
248        Self {
249            log: Arc::new(StdMutex::new(VecDeque::new())),
250            bcast,
251            seq: Arc::new(AtomicU64::new(0)),
252            last_checkpoint: Arc::new(StdMutex::new("-".to_string())),
253            last_step: Arc::new(AtomicU64::new(0)),
254            capacity,
255        }
256    }
257
258    /// Record and broadcast one frame.
259    pub(crate) fn push(&self, event: &str, step: usize, data: Value) {
260        let seq = self.seq.fetch_add(1, Ordering::Relaxed) + 1;
261        self.last_step.store(step as u64, Ordering::Relaxed);
262        let checkpoint = lock_recover(&self.last_checkpoint).clone();
263        let frame = SseFrame {
264            id: format!("{checkpoint}:{step}:{seq}"),
265            event: event.to_string(),
266            data,
267            seq,
268        };
269        {
270            let mut log = lock_recover(&self.log);
271            if log.len() >= self.capacity {
272                log.pop_front();
273            }
274            log.push_back(frame.clone());
275        }
276        // No live subscribers is normal (background runs); not an error.
277        let _ = self.bcast.send(frame);
278    }
279
280    /// Point subsequent frame ids at a freshly persisted checkpoint.
281    pub(crate) fn note_checkpoint(&self, checkpoint_id: &str) {
282        *lock_recover(&self.last_checkpoint) = checkpoint_id.to_string();
283    }
284
285    /// The super-step of the most recently pushed frame.
286    pub(crate) fn current_step(&self) -> usize {
287        self.last_step.load(Ordering::Relaxed) as usize
288    }
289}
290
291/// Everything the executor task needs, snapshotted from a [`RunHandle`].
292pub(crate) struct RunSnapshot {
293    /// Internal (tenant-scoped) thread id: used for the checkpointer, the
294    /// executor config, and RunManager bookkeeping.
295    pub thread_id: String,
296    /// External thread id as the client knows it — the only form that may
297    /// appear on the wire (SSE frames, terminal JSON).
298    pub wire_thread_id: String,
299    pub graph: String,
300    pub attempt: usize,
301    pub payload: RunPayload,
302    /// The registry binding resolved at admission (R0.11 wave 2).
303    pub admission: Option<crate::registry::RegistryAdmission>,
304    pub sink: FrameSink,
305    pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
306    /// This run's own cancellation token (R0.7 wave 2): a child of the
307    /// server's drain token, so a run-level cancel ([`RunManager::cancel_run`]
308    /// — the cancellation tree's run half) stops this run at its next
309    /// super-step boundary without touching any other run, while a server
310    /// drain still stops them all. Observed by the executor exactly where
311    /// the drain token always was.
312    pub cancel: tokio_util::sync::CancellationToken,
313}
314
315/// Read-only view of a run (used by the rollback and status endpoints).
316pub(crate) struct RunInfo {
317    /// Internal (tenant-scoped) thread id — handlers check tenant ownership
318    /// against it before revealing anything about the run.
319    pub thread_id: String,
320    /// External thread id for wire responses.
321    pub wire_thread_id: String,
322    pub graph: String,
323    pub attempt: usize,
324    pub status: RunStatus,
325    /// The terminal JSON once the run has finished (`None` while active).
326    pub terminal: Option<Value>,
327    pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
328}
329
330/// Handle for one scheduled run, owned by the [`RunManager`]. Crate-private
331/// surface: external users interact with runs over HTTP, not this type.
332pub struct RunHandle {
333    /// Run id (UUID v4).
334    pub(crate) run_id: String,
335    /// Internal (tenant-scoped) thread id this run executes against.
336    pub(crate) thread_id: String,
337    /// External thread id reported on the wire.
338    pub(crate) wire_thread_id: String,
339    /// Registered graph name.
340    pub(crate) graph: String,
341    /// 1-based attempt counter for the thread.
342    pub(crate) attempt: usize,
343    /// Lifecycle status.
344    pub(crate) status: RunStatus,
345    /// Original run payload.
346    pub(crate) payload: RunPayload,
347    /// The registry binding resolved at admission (R0.11 wave 2) — `None`
348    /// for an unbound run, which behaves byte-identically to before.
349    pub(crate) admission: Option<crate::registry::RegistryAdmission>,
350    sink: FrameSink,
351    terminal: watch::Sender<Option<Value>>,
352    checkpoint_ids: Arc<StdMutex<Vec<String>>>,
353    /// This run's own cancellation token (see [`RunSnapshot::cancel`]).
354    /// Firing it is the run-level half of the R0.7 cancellation tree; the
355    /// executor observes it at super-step boundaries, after the boundary
356    /// checkpoint has landed.
357    cancel: tokio_util::sync::CancellationToken,
358}
359
360impl RunHandle {
361    /// Subscribe to the live frame stream.
362    pub(crate) fn subscribe(&self) -> broadcast::Receiver<SseFrame> {
363        self.sink.bcast.subscribe()
364    }
365
366    /// A point-in-time copy of the event log (for replay).
367    pub(crate) fn log_snapshot(&self) -> Vec<SseFrame> {
368        lock_recover(&self.sink.log).iter().cloned().collect()
369    }
370}
371
372/// What [`RunManager::insert`] decided for a freshly scheduled run.
373pub(crate) enum ScheduleDecision {
374    /// The thread slot was free; the run must be spawned now.
375    Started,
376    /// The run was queued behind the active run.
377    Queued,
378}
379
380/// What [`RunManager::cancel_run`] did (R0.7 wave 2).
381#[derive(Debug, Clone, Copy, PartialEq, Eq)]
382pub(crate) enum RunCancel {
383    /// The run was executing; its cancellation token fired. The terminal
384    /// transition lands when the executor observes it at the next
385    /// super-step boundary.
386    Signalled,
387    /// The run was queued behind another; it was dequeued and finished
388    /// terminal-`cancelled` without ever starting.
389    CancelledQueued,
390    /// The run was already terminal; nothing changed.
391    Terminal,
392    /// No such run.
393    Unknown,
394}
395
396/// What [`RunManager::cancel_thread_runs`] did to one thread's runs
397/// (R0.7 wave 2), split by how each run's cancellation lands — mirroring
398/// the task queue's [`crate::tasks::RunCancellation`] shape.
399#[derive(Debug, Default)]
400pub(crate) struct ThreadCancellation {
401    /// Running runs whose cancellation tokens fired (terminal at their
402    /// next boundary).
403    pub signalled: Vec<String>,
404    /// Queued runs dequeued into terminal-`cancelled` immediately.
405    pub cancelled: Vec<String>,
406}
407
408impl ThreadCancellation {
409    /// `true` when no run was touched (the thread had no active or queued
410    /// runs) — the cancel route journals an `AgentExit` only when the
411    /// cancellation actually landed somewhere.
412    pub(crate) fn is_empty(&self) -> bool {
413        self.signalled.is_empty() && self.cancelled.is_empty()
414    }
415}
416
417#[derive(Default)]
418struct RunManagerInner {
419    runs: HashMap<String, RunHandle>,
420    active_by_thread: HashMap<String, String>,
421    queues: HashMap<String, VecDeque<String>>,
422    attempts: HashMap<String, usize>,
423    /// Insertion order of run ids, feeding terminal-run eviction.
424    order: VecDeque<String>,
425}
426
427/// Cap on retained runs (see the module docs' retention note). Without a
428/// cap, `runs` would grow by one record — payload clone, terminal JSON, and
429/// up to `event_log_capacity` SSE frames — per run for the process
430/// lifetime: a steady memory leak on any busy cron schedule.
431const MAX_RETAINED_RUNS: usize = 1024;
432
433/// Registry of all runs, plus per-thread scheduling state. Cheap to clone
434/// (shared inner).
435#[derive(Default, Clone)]
436pub struct RunManager {
437    inner: Arc<Mutex<RunManagerInner>>,
438}
439
440impl RunManager {
441    /// An empty manager.
442    pub fn new() -> Self {
443        Self::default()
444    }
445
446    /// Insert a new run under the given multitask strategy, assigning its
447    /// per-thread attempt number.
448    pub(crate) async fn insert(
449        &self,
450        mut handle: RunHandle,
451        strategy: MultitaskStrategy,
452        queue_cap: usize,
453    ) -> Result<ScheduleDecision, ApiError> {
454        let mut inner = self.inner.lock().await;
455        let busy = inner.active_by_thread.contains_key(&handle.thread_id);
456        let attempt = {
457            let counter = inner.attempts.entry(handle.thread_id.clone()).or_insert(0);
458            *counter += 1;
459            *counter
460        };
461        handle.attempt = attempt;
462        inner.order.push_back(handle.run_id.clone());
463
464        match strategy {
465            MultitaskStrategy::Reject if busy => Err(ApiError::conflict(format!(
466                "thread `{}` already has an active run",
467                handle.thread_id
468            ))),
469            _ if busy => {
470                let queue = inner.queues.entry(handle.thread_id.clone()).or_default();
471                if queue.len() >= queue_cap {
472                    return Err(ApiError::conflict(format!(
473                        "thread `{}` run queue is full (cap {queue_cap})",
474                        handle.thread_id
475                    )));
476                }
477                queue.push_back(handle.run_id.clone());
478                inner.runs.insert(handle.run_id.clone(), handle);
479                Ok(ScheduleDecision::Queued)
480            }
481            _ => {
482                inner
483                    .active_by_thread
484                    .insert(handle.thread_id.clone(), handle.run_id.clone());
485                handle.status = RunStatus::Running;
486                inner.runs.insert(handle.run_id.clone(), handle);
487                Ok(ScheduleDecision::Started)
488            }
489        }
490    }
491
492    /// Snapshot everything the executor task needs for `run_id`.
493    pub(crate) async fn snapshot(&self, run_id: &str) -> Option<RunSnapshot> {
494        let inner = self.inner.lock().await;
495        inner.runs.get(run_id).map(|h| RunSnapshot {
496            thread_id: h.thread_id.clone(),
497            wire_thread_id: h.wire_thread_id.clone(),
498            graph: h.graph.clone(),
499            attempt: h.attempt,
500            payload: h.payload.clone(),
501            admission: h.admission.clone(),
502            sink: h.sink.clone(),
503            checkpoint_ids: Arc::clone(&h.checkpoint_ids),
504            cancel: h.cancel.clone(),
505        })
506    }
507
508    /// Read-only run info for API endpoints.
509    pub(crate) async fn info(&self, run_id: &str) -> Option<RunInfo> {
510        let inner = self.inner.lock().await;
511        inner.runs.get(run_id).map(|h| RunInfo {
512            thread_id: h.thread_id.clone(),
513            wire_thread_id: h.wire_thread_id.clone(),
514            graph: h.graph.clone(),
515            attempt: h.attempt,
516            status: h.status,
517            terminal: h.terminal.borrow().clone(),
518            checkpoint_ids: Arc::clone(&h.checkpoint_ids),
519        })
520    }
521
522    /// Replay log + live subscription + internal thread id for the
523    /// SSE attach endpoint (`GET /runs/{id}/stream`).
524    pub(crate) async fn stream_parts(
525        &self,
526        run_id: &str,
527    ) -> Option<(Vec<SseFrame>, broadcast::Receiver<SseFrame>, String)> {
528        let inner = self.inner.lock().await;
529        inner
530            .runs
531            .get(run_id)
532            .map(|h| (h.log_snapshot(), h.subscribe(), h.thread_id.clone()))
533    }
534
535    /// `true` while the thread has an active run or a non-empty queue —
536    /// rollback refuses to delete checkpoints out from under them.
537    pub(crate) async fn thread_busy(&self, thread_id: &str) -> bool {
538        let inner = self.inner.lock().await;
539        inner.active_by_thread.contains_key(thread_id)
540            || inner
541                .queues
542                .get(thread_id)
543                .is_some_and(|queue| !queue.is_empty())
544    }
545
546    /// Cancel one run (R0.7 wave 2 — the run-level half of the
547    /// cancellation tree). A running run is *signalled*: its own
548    /// cancellation token fires and the executor stops it at the next
549    /// super-step boundary — after the boundary checkpoint has landed —
550    /// ending terminal-`cancelled` and resumable by re-running the thread,
551    /// exactly like the server drain. A queued (pending) run never started,
552    /// so it is dequeued and finished terminal-`cancelled` immediately —
553    /// leaving it queued would let a dead run promote and execute.
554    pub(crate) async fn cancel_run(&self, run_id: &str) -> RunCancel {
555        let mut inner = self.inner.lock().await;
556        let Some(handle) = inner.runs.get_mut(run_id) else {
557            return RunCancel::Unknown;
558        };
559        match handle.status {
560            RunStatus::Running => {
561                handle.cancel.cancel();
562                RunCancel::Signalled
563            }
564            RunStatus::Pending => {
565                let thread_id = handle.thread_id.clone();
566                let wire_thread_id = handle.wire_thread_id.clone();
567                if let Some(queue) = inner.queues.get_mut(&thread_id) {
568                    queue.retain(|queued| queued != run_id);
569                }
570                let handle = inner
571                    .runs
572                    .get_mut(run_id)
573                    .expect("the handle was resolved above");
574                handle.status = RunStatus::Cancelled;
575                let terminal = json!({
576                    "run_id": run_id,
577                    "thread_id": wire_thread_id,
578                    "status": "cancelled",
579                    "message": "cancelled while queued, before its first step",
580                });
581                handle.terminal.send_replace(Some(terminal));
582                RunCancel::CancelledQueued
583            }
584            // Terminal runs (including an already-cancelled one) are
585            // untouched — cancellation is control flow, idempotent by
586            // no-op, never a second terminal transition.
587            _ => RunCancel::Terminal,
588        }
589    }
590
591    /// Cancel every run of one thread (R0.7 wave 2): the active run is
592    /// signalled, every queued run is dequeued-cancelled — the whole
593    /// per-thread run state, so cancelling an agent's thread leaves no
594    /// pending run that would re-drive it.
595    pub(crate) async fn cancel_thread_runs(&self, thread_id: &str) -> ThreadCancellation {
596        let (active, queued) = {
597            let inner = self.inner.lock().await;
598            (
599                inner.active_by_thread.get(thread_id).cloned(),
600                inner
601                    .queues
602                    .get(thread_id)
603                    .map(|q| q.iter().cloned().collect::<Vec<_>>())
604                    .unwrap_or_default(),
605            )
606        };
607        let mut outcome = ThreadCancellation::default();
608        if let Some(run_id) = active {
609            if matches!(self.cancel_run(&run_id).await, RunCancel::Signalled) {
610                outcome.signalled.push(run_id);
611            }
612        }
613        for run_id in queued {
614            if matches!(self.cancel_run(&run_id).await, RunCancel::CancelledQueued) {
615                outcome.cancelled.push(run_id);
616            }
617        }
618        outcome
619    }
620
621    /// Record the terminal status + JSON, wake waiters, release the thread
622    /// slot, and return the next queued run id for the thread (if any), now
623    /// marked active.
624    ///
625    /// `draining` (the server's shutdown drain) suppresses queue promotion:
626    /// the slot frees but nothing is returned — a run promoted into a
627    /// shutting-down process would only be cancelled at its first boundary.
628    /// Queued runs stay `Pending`; their threads' checkpoints are intact
629    /// for the next process to re-drive.
630    pub(crate) async fn finish(
631        &self,
632        run_id: &str,
633        status: RunStatus,
634        terminal: Value,
635        draining: bool,
636    ) -> Option<String> {
637        let mut inner = self.inner.lock().await;
638        let handle = inner.runs.get_mut(run_id)?;
639        handle.status = status;
640        // `send_replace` (not `send`) so the terminal JSON is stored even
641        // when no waiter holds a receiver (background runs); status polling
642        // via `info` reads it back through `watch::Sender::borrow`.
643        handle.terminal.send_replace(Some(terminal));
644        let thread_id = handle.thread_id.clone();
645
646        if inner
647            .active_by_thread
648            .get(&thread_id)
649            .is_some_and(|active| active == run_id)
650        {
651            inner.active_by_thread.remove(&thread_id);
652        }
653
654        let next = if draining {
655            None
656        } else {
657            inner
658                .queues
659                .get_mut(&thread_id)
660                .and_then(VecDeque::pop_front)
661        };
662        if let Some(next_id) = &next {
663            if let Some(h) = inner.runs.get_mut(next_id) {
664                h.status = RunStatus::Running;
665            }
666            inner.active_by_thread.insert(thread_id, next_id.clone());
667        }
668
669        // Evict the oldest terminal runs beyond the retention cap; active
670        // and queued runs keep their slots in `order`.
671        let mut excess = inner.runs.len().saturating_sub(MAX_RETAINED_RUNS);
672        let mut skipped = Vec::new();
673        while excess > 0 {
674            let Some(candidate) = inner.order.pop_front() else {
675                break;
676            };
677            let evictable = inner
678                .runs
679                .get(&candidate)
680                .is_some_and(|h| h.status.is_terminal());
681            if evictable {
682                inner.runs.remove(&candidate);
683                excess -= 1;
684            } else {
685                skipped.push(candidate);
686            }
687        }
688        inner.order.extend(skipped);
689
690        next
691    }
692}
693
694/// Everything the run machinery needs from the application: registry,
695/// checkpointer, manager, and caps. Cheap to clone.
696#[derive(Clone)]
697pub(crate) struct RunDeps {
698    pub registry: GraphRegistry,
699    pub checkpointer: Arc<dyn Checkpointer>,
700    pub manager: RunManager,
701    /// Flight Recorder journal persistence (`GET /runs/{id}/events`).
702    pub server_store: Arc<dyn ServerStore>,
703    pub queue_cap: usize,
704    pub log_capacity: usize,
705    /// The server's drain control (R0.6 wave 2c): threaded into every run's
706    /// executor, which observes it at super-step boundaries.
707    pub shutdown: tokio_util::sync::CancellationToken,
708    /// The deployment's default environment tag (R0.11 wave 2): the
709    /// promotion target a registry-bound run resolves against when its
710    /// binding names no environment (`None`: the untagged surface).
711    pub default_environment_tag: Option<rusty_agent_runtime::learn::EnvironmentTag>,
712}
713
714/// The result of successfully scheduling a run: everything an endpoint
715/// needs to answer (background ack, wait, or stream).
716pub(crate) struct Scheduled {
717    pub run_id: String,
718    pub status: RunStatus,
719    pub terminal: watch::Receiver<Option<Value>>,
720    pub broadcast: broadcast::Receiver<SseFrame>,
721    pub replay: Vec<SseFrame>,
722}
723
724/// Create a run handle, apply the multitask strategy, and spawn execution
725/// immediately when the thread slot is free.
726///
727/// `thread_id` is the internal (tenant-scoped) id used for the checkpointer,
728/// executor, and RunManager bookkeeping; `wire_thread_id` is the external id
729/// reported in SSE frames and terminal JSON.
730pub(crate) async fn schedule(
731    deps: &RunDeps,
732    thread_id: &str,
733    wire_thread_id: &str,
734    graph: &str,
735    payload: RunPayload,
736    strategy: MultitaskStrategy,
737) -> Result<Scheduled, ApiError> {
738    // A draining server must not take new runs: the token would cancel
739    // them at their first boundary anyway, and a 503 lets the caller (or
740    // its load balancer) retry against a pod that is still serving.
741    if deps.shutdown.is_cancelled() {
742        return Err(ApiError::shutting_down(format!(
743            "server is draining; resubmit run on thread `{wire_thread_id}` against a running instance"
744        )));
745    }
746    let run_id = uuid::Uuid::new_v4().to_string();
747    // Registry admission (R0.11 wave 2): the binding resolves now, at
748    // admission — a promotion landing afterwards never reaches this run
749    // (the conservatism checkpoint pinning has kept since R0.7), and a
750    // queued run binds at admission, not at dequeue. The tenant comes
751    // from the internal thread id, so every entry point (HTTP, cron,
752    // trigger, bridge) resolves in the submitter's namespace without
753    // threading the request context through. A resolution failure is an
754    // admission failure: the run never enters the manager.
755    let admission = match &payload.registry {
756        Some(binding) => Some(
757            crate::registry::resolve_admission(
758                &deps.server_store,
759                crate::auth::tenant_of_internal(thread_id),
760                deps.default_environment_tag.as_ref(),
761                &run_id,
762                binding,
763            )
764            .await?,
765        ),
766        None => None,
767    };
768    let (bcast_tx, _bcast_rx) = broadcast::channel(256);
769    let (terminal_tx, terminal_rx) = watch::channel(None);
770    let handle = RunHandle {
771        run_id: run_id.clone(),
772        thread_id: thread_id.to_string(),
773        wire_thread_id: wire_thread_id.to_string(),
774        graph: graph.to_string(),
775        attempt: 0, // assigned by RunManager::insert
776        status: RunStatus::Pending,
777        payload,
778        admission,
779        sink: FrameSink::new(deps.log_capacity, bcast_tx),
780        terminal: terminal_tx,
781        checkpoint_ids: Arc::new(StdMutex::new(Vec::new())),
782        // A child of the server drain token: the drain still stops every
783        // run, and a run-level cancel (R0.7 wave 2) stops only this one.
784        cancel: deps.shutdown.child_token(),
785    };
786    // Subscribe/snapshot before any execution can emit frames.
787    let replay = handle.log_snapshot();
788    let broadcast = handle.subscribe();
789
790    let decision = deps
791        .manager
792        .insert(handle, strategy, deps.queue_cap)
793        .await?;
794    let status = match decision {
795        ScheduleDecision::Started => {
796            spawn_execute(deps.clone(), run_id.clone());
797            RunStatus::Running
798        }
799        ScheduleDecision::Queued => RunStatus::Pending,
800    };
801
802    Ok(Scheduled {
803        run_id,
804        status,
805        terminal: terminal_rx,
806        broadcast,
807        replay,
808    })
809}
810
811/// Drive one run to its terminal state and chain the next queued run.
812async fn execute(deps: RunDeps, run_id: String) {
813    let Some(snap) = deps.manager.snapshot(&run_id).await else {
814        tracing::warn!(%run_id, "scheduled run vanished before execution");
815        return;
816    };
817    let sink = snap.sink.clone();
818    sink.push(
819        "metadata",
820        0,
821        json!({
822            "run_id": run_id,
823            "thread_id": snap.wire_thread_id,
824            "graph": snap.graph,
825            "attempt": snap.attempt,
826            "metadata": snap.payload.metadata,
827        }),
828    );
829
830    let Some((graph, spec)) = deps.registry.get(&snap.graph) else {
831        let message = format!("graph `{}` is no longer registered", snap.graph);
832        tracing::error!(%run_id, %message);
833        sink.push(
834            "error",
835            0,
836            json!({"error": "unknown_graph", "message": message}),
837        );
838        sink.push("end", 0, json!({"status": "error"}));
839        let terminal = json!({
840            "run_id": run_id,
841            "thread_id": snap.wire_thread_id,
842            "status": "error",
843            "error": "unknown_graph",
844            "message": message,
845        });
846        terminate(&deps, &run_id, RunStatus::Error, terminal).await;
847        return;
848    };
849
850    let modes: Vec<String> = snap
851        .payload
852        .stream_mode
853        .clone()
854        .unwrap_or_else(|| vec!["values".to_string(), "updates".to_string()]);
855    // Flight Recorder: one journal per run, keyed by the server-minted run
856    // id. Events carry the external (wire) thread id — the internal
857    // tenant-scoped id must never appear in served evidence. The journal's
858    // clock is the default system clock, so timestamps match pre-R0.5
859    // behavior; attaching it makes the executor read time through it.
860    let journal = Journal::new(run_id.clone(), snap.wire_thread_id.clone(), Clock::System);
861    let (evt_tx, evt_rx) = mpsc::channel::<GraphEvent>(256);
862    let forwarder = tokio::spawn(forward_events(
863        evt_rx,
864        sink.clone(),
865        ForwardDeps {
866            checkpointer: Arc::clone(&deps.checkpointer),
867            server_store: Arc::clone(&deps.server_store),
868            journal: journal.clone(),
869            thread_id: snap.thread_id.clone(),
870            checkpoint_ids: Arc::clone(&snap.checkpoint_ids),
871            modes,
872        },
873    ));
874
875    let mut config = RunConfig::new(snap.thread_id.clone())
876        .with_event_tx(evt_tx)
877        .with_journal(journal.clone())
878        // Cancellation hook: this run's own token (a child of the server
879        // drain token). When either fires, the run stops at its next
880        // super-step boundary — a point where a checkpoint was just
881        // persisted — instead of being torn down mid-step.
882        .with_cancellation(snap.cancel.clone());
883    // Registry admission (R0.11 wave 2): the binding resolved at schedule
884    // time becomes evidence now, ahead of the run's own events — one
885    // `config_resolved` per artifact (chained: each resolution's parent
886    // is the previous, so the admission reads as one causal unit) — and
887    // the resolved manifest stamps every checkpoint header, which is how
888    // the receipt reads it back. Resolution decides nothing about what
889    // will run, so the events are read-only (the `CapsuleResolved`
890    // precedent); a serialization failure here is a bug, not a runtime
891    // condition — the payload type is the server's own.
892    if let Some(admission) = &snap.admission {
893        let mut parent = None;
894        for resolution in &admission.resolutions {
895            let output =
896                serde_json::to_value(resolution).expect("ConfigResolution always serializes");
897            let mut draft =
898                EventDraft::new(RunEventKind::ConfigResolved, Effect::ReadOnly).output(output);
899            if let Some(parent) = parent {
900                draft = draft.parent(parent);
901            }
902            parent = Some(journal.record(draft));
903        }
904        config = config.with_manifest(admission.manifest.clone());
905    }
906    if let Some(command) = &snap.payload.command {
907        if let Some(resume) = &command.resume {
908            config = config.with_resume(resume.clone());
909        }
910    }
911    if let Some(checkpoint) = &snap.payload.checkpoint {
912        config = config.with_checkpoint_id(checkpoint.checkpoint_id.clone());
913    }
914    if let Some(run_cfg) = &snap.payload.config {
915        if let Some(limit) = run_cfg.recursion_limit {
916            config = config.with_max_steps(limit);
917        }
918    }
919    let initial = snap
920        .payload
921        .input
922        .clone()
923        .and_then(|v| State::from_value(v).ok())
924        .unwrap_or_default();
925
926    let mut executor = Executor::with_checkpointer(Arc::clone(&deps.checkpointer));
927    // The resolved middleware chain (R0.11 wave 4) attaches in journaled
928    // order — the same layers the manifest's `middleware` digest pins and
929    // the admission resolution's `layers` field names. Attached after the
930    // admission journal writes above, so the evidence of *what* serves
931    // precedes the run it serves.
932    if let Some(admission) = &snap.admission {
933        if let Some(chain) = &admission.middleware {
934            for layer in chain.layers() {
935                executor = executor.layer_shared(Arc::clone(layer));
936            }
937        }
938    }
939    let result = executor.run(&graph, &spec, initial, config).await;
940    // `config` (holding the only sender) is dropped with the run; the
941    // forwarder drains what remains and exits.
942    let _ = forwarder.await;
943
944    // Final journal write: the complete evidence of the run, including the
945    // events recorded after the last checkpoint boundary. Persisted before
946    // the run goes terminal so `complete: true` on the events endpoint never
947    // races ahead of the snapshot it serves. Evidence of a failed run is
948    // still evidence — this write happens on every outcome.
949    persist_journal(&deps.server_store, &journal).await;
950
951    let step = sink.current_step();
952    let (status, terminal) = match result {
953        Ok(ExecutionOutcome::Done(state)) => {
954            sink.push("end", step, json!({"status": "success"}));
955            let terminal = json!({
956                "run_id": run_id,
957                "thread_id": snap.wire_thread_id,
958                "status": "success",
959                "output": state.to_value(),
960            });
961            (RunStatus::Success, terminal)
962        }
963        Ok(ExecutionOutcome::Interrupted {
964            value,
965            state,
966            checkpoint_id,
967        }) => {
968            sink.push(
969                "end",
970                step,
971                json!({"status": "interrupted", "interrupt": value}),
972            );
973            let terminal = json!({
974                "run_id": run_id,
975                "thread_id": snap.wire_thread_id,
976                "status": "interrupted",
977                "interrupt": value,
978                "checkpoint_id": checkpoint_id,
979                "state": state.to_value(),
980            });
981            (RunStatus::Interrupted, terminal)
982        }
983        Err(error @ RustyError::Cancelled(_)) => {
984            // Drain, not failure: the executor stopped at a super-step
985            // boundary, so the run's last checkpoint is intact and a fresh
986            // run on the thread resumes from it. The wire status is
987            // `cancelled` — matching the task queue's treatment of
988            // cancellation as control flow, never an error.
989            let message = error.to_string();
990            tracing::info!(%run_id, "run drained at a checkpoint boundary; resumable");
991            sink.push("end", step, json!({"status": "cancelled"}));
992            let terminal = json!({
993                "run_id": run_id,
994                "thread_id": snap.wire_thread_id,
995                "status": "cancelled",
996                "message": message,
997            });
998            (RunStatus::Cancelled, terminal)
999        }
1000        Err(error) => {
1001            let kind = error_kind(&error);
1002            let message = error.to_string();
1003            tracing::warn!(%run_id, %error, "run failed");
1004            sink.push("error", step, json!({"error": kind, "message": message}));
1005            sink.push("end", step, json!({"status": "error"}));
1006            let terminal = json!({
1007                "run_id": run_id,
1008                "thread_id": snap.wire_thread_id,
1009                "status": "error",
1010                "error": kind,
1011                "message": message,
1012            });
1013            (RunStatus::Error, terminal)
1014        }
1015    };
1016    terminate(&deps, &run_id, status, terminal).await;
1017}
1018
1019/// Everything [`forward_events`] needs beyond the frame sink, bundled to
1020/// keep the task's argument list readable.
1021struct ForwardDeps {
1022    checkpointer: Arc<dyn Checkpointer>,
1023    server_store: Arc<dyn ServerStore>,
1024    journal: Journal,
1025    /// Internal (tenant-scoped) thread id, for checkpoint read-backs.
1026    thread_id: String,
1027    checkpoint_ids: Arc<StdMutex<Vec<String>>>,
1028    modes: Vec<String>,
1029}
1030
1031/// Map executor events to SSE frames per the design doc's §4 table. Also the
1032/// Flight Recorder's checkpoint-boundary persistence point: every
1033/// `CheckpointSaved` event flushes the journal's current snapshot to the
1034/// server store, so the stored evidence trails the live journal by at most
1035/// one super-step.
1036async fn forward_events(mut rx: mpsc::Receiver<GraphEvent>, sink: FrameSink, deps: ForwardDeps) {
1037    let ForwardDeps {
1038        checkpointer,
1039        server_store,
1040        journal,
1041        thread_id,
1042        checkpoint_ids,
1043        modes,
1044    } = deps;
1045    while let Some(event) = rx.recv().await {
1046        match event {
1047            GraphEvent::StateUpdate { step, updates } => {
1048                if modes.iter().any(|m| m == "updates") {
1049                    sink.push("updates", step, json!({"step": step, "updates": updates}));
1050                }
1051            }
1052            GraphEvent::Token { node, delta } => {
1053                if modes.iter().any(|m| m == "messages") {
1054                    let step = sink.current_step();
1055                    sink.push("messages", step, json!({"node": node, "delta": delta}));
1056                }
1057            }
1058            GraphEvent::CheckpointSaved {
1059                checkpoint_id,
1060                step,
1061            } => {
1062                lock_recover(&checkpoint_ids).push(checkpoint_id.clone());
1063                sink.note_checkpoint(&checkpoint_id);
1064                persist_journal(&server_store, &journal).await;
1065                if modes.iter().any(|m| m == "values") {
1066                    match read_back_state(&*checkpointer, &thread_id, &checkpoint_id).await {
1067                        Ok(Some(values)) => sink.push("values", step, values),
1068                        Ok(None) => {
1069                            tracing::debug!(%checkpoint_id, "checkpoint not found for values frame")
1070                        }
1071                        Err(error) => {
1072                            tracing::warn!(%checkpoint_id, %error, "values frame read-back failed")
1073                        }
1074                    }
1075                }
1076            }
1077            // Reserved for the future `tasks` / `debug` stream modes.
1078            GraphEvent::SuperStep { .. }
1079            | GraphEvent::NodeStart { .. }
1080            | GraphEvent::NodeEnd { .. } => {}
1081        }
1082    }
1083}
1084
1085/// Flush the journal's current snapshot to the server store. A persistence
1086/// failure is logged, not raised: the run's execution must not fail because
1087/// its evidence could not be written, and the next checkpoint boundary (or
1088/// the completion write) retries.
1089async fn persist_journal(server_store: &Arc<dyn ServerStore>, journal: &Journal) {
1090    if let Err(error) = server_store.put_journal(&journal.snapshot()).await {
1091        tracing::warn!(run_id = %journal.run_id(), %error, "journal persistence failed");
1092    }
1093}
1094
1095/// `values` frames carry the full state persisted at a super-step boundary,
1096/// read back from the checkpoint log (design doc §4). A point lookup, not a
1097/// full `list()` scan — that would be O(history) per super-step, O(n²) per
1098/// run.
1099async fn read_back_state(
1100    checkpointer: &dyn Checkpointer,
1101    thread_id: &str,
1102    checkpoint_id: &str,
1103) -> rusty_agent_runtime::error::Result<Option<Value>> {
1104    Ok(checkpointer
1105        .get_by_id(thread_id, checkpoint_id)
1106        .await?
1107        .map(|cp| cp.state.to_value()))
1108}
1109
1110/// Spawn `execute` for a run, guarding the thread's scheduling slot: if the
1111/// task panics (executor bug — the poison-prone lock sites recover via
1112/// [`lock_recover`]), the run is force-finished as `error` so
1113/// `active_by_thread` releases the slot and queued runs drain instead of
1114/// wedging behind a ghost.
1115///
1116/// The future is boxed behind a trait object to break the
1117/// `execute → terminate → spawn(execute)` type cycle, which would otherwise
1118/// make `Send` inference recursive and fail.
1119fn spawn_execute(deps: RunDeps, run_id: String) {
1120    let fut: std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> = Box::pin({
1121        let deps = deps.clone();
1122        let run_id = run_id.clone();
1123        async move { execute(deps, run_id).await }
1124    });
1125    tokio::spawn(async move {
1126        if AssertUnwindSafe(fut).catch_unwind().await.is_ok() {
1127            return;
1128        }
1129        tracing::error!(%run_id, "run task panicked; force-finishing as error");
1130        let Some(snap) = deps.manager.snapshot(&run_id).await else {
1131            return;
1132        };
1133        // If the panic happened after `terminate` completed, the slot is
1134        // already released — finishing again would double-promote the queue.
1135        if matches!(deps.manager.info(&run_id).await, Some(info) if info.status.is_terminal()) {
1136            return;
1137        }
1138        let step = snap.sink.current_step();
1139        snap.sink.push(
1140            "error",
1141            step,
1142            json!({"error": "internal_panic", "message": "run task panicked"}),
1143        );
1144        snap.sink.push("end", step, json!({"status": "error"}));
1145        let terminal = json!({
1146            "run_id": run_id,
1147            "thread_id": snap.wire_thread_id,
1148            "status": "error",
1149            "error": "internal_panic",
1150            "message": "run task panicked",
1151        });
1152        terminate(&deps, &run_id, RunStatus::Error, terminal).await;
1153    });
1154}
1155
1156/// Record the terminal state and spawn the next queued run, if any. While
1157/// the server drains, the queue does not advance (see
1158/// [`RunManager::finish`]'s `draining` flag).
1159async fn terminate(deps: &RunDeps, run_id: &str, status: RunStatus, terminal: Value) {
1160    let draining = deps.shutdown.is_cancelled();
1161    if let Some(next) = deps
1162        .manager
1163        .finish(run_id, status, terminal, draining)
1164        .await
1165    {
1166        spawn_execute(deps.clone(), next);
1167    }
1168}
1169
1170/// Stable error-kind labels for the wire.
1171fn error_kind(error: &RustyError) -> &'static str {
1172    match error {
1173        RustyError::Graph(_) => "graph_error",
1174        RustyError::Node(_) => "node_error",
1175        RustyError::Interrupt { .. } => "interrupted",
1176        RustyError::Checkpoint(_) => "checkpoint_error",
1177        RustyError::Llm(_) => "llm_error",
1178        RustyError::Tool(_) => "tool_error",
1179        RustyError::Serialization(_) => "serialization_error",
1180        RustyError::InvalidUpdate(_) => "invalid_update",
1181        RustyError::Replay(_) => "replay_error",
1182        // Drain cancellation is control flow and takes its own terminal
1183        // path in `execute`; this arm exists for exhaustiveness only.
1184        RustyError::Cancelled(_) => "cancelled",
1185    }
1186}