rusty_agent_server/runs.rs
1//! Run scheduling, execution, and bookkeeping.
2//!
3//! A run goes: *schedule* (strategy check + handle insert) → *execute*
4//! (drive [`Executor`] in a spawned task, forwarding [`GraphEvent`]s to a
5//! per-run SSE frame log + broadcast channel) → *terminate* (terminal status
6//! + JSON recorded, waiters woken, next queued run for the thread spawned).
7//!
8//! Multitask: there is always at most one **active** run per thread. The
9//! `reject` strategy returns 409 when the thread is busy; `enqueue` appends
10//! to an in-memory per-thread FIFO queue (depth-capped by
11//! `ServerConfig::max_concurrent_runs_per_thread`) that drains automatically
12//! as runs finish.
13//!
14//! Retention: terminal runs are kept for `GET /runs/{id}` polling up to
15//! [`MAX_RETAINED_RUNS`] per process; the oldest terminal runs are evicted
16//! beyond that (active and queued runs are never evicted). Run history is
17//! in-memory by design — durability lives in the checkpoint log.
18//!
19//! Drain (R0.6 wave 2c): the server's shutdown token is threaded into every
20//! run's executor ([`RunConfig::with_cancellation`]). When it fires, each
21//! in-flight run stops at its next super-step boundary — a point where a
22//! checkpoint was just persisted — and ends terminal-[`RunStatus::Cancelled`],
23//! resumable by simply re-running the thread; new submissions answer 503 and
24//! queued runs are not promoted. Anything still mid-step when the grace
25//! window closes is abandoned, which is the crash case the checkpoint log
26//! already covers.
27//!
28//! Flight Recorder: every run is journaled. The journal is attached to the
29//! executor at run start, flushed to the server store at every checkpoint
30//! boundary (in [`forward_events`]) and once more at run completion, and
31//! served read-only by `GET /runs/{id}/events`.
32
33use std::collections::{HashMap, VecDeque};
34use std::panic::AssertUnwindSafe;
35use std::sync::atomic::{AtomicU64, Ordering};
36use std::sync::{Arc, Mutex as StdMutex};
37
38use futures::FutureExt;
39use rusty_agent_runtime::checkpoint::Checkpointer;
40use rusty_agent_runtime::error::RustyError;
41use rusty_agent_runtime::executor::{ExecutionOutcome, Executor, GraphEvent, RunConfig};
42use rusty_agent_runtime::journal::{Clock, EventDraft, Journal};
43use rusty_agent_runtime::record::{Effect, RunEventKind};
44use rusty_agent_runtime::state::State;
45use serde::Deserialize;
46use serde_json::{json, Value};
47use tokio::sync::{broadcast, mpsc, watch, Mutex};
48
49use crate::error::ApiError;
50use crate::server_store::ServerStore;
51use crate::GraphRegistry;
52
53// --------------------------------------------------------------------- //
54// Run payload (accepted by all three run endpoints)
55// --------------------------------------------------------------------- //
56
57/// The `command` field of a run payload: `{ "resume": <value> }` continues
58/// an interrupted thread via [`RunConfig::with_resume`].
59#[derive(Debug, Clone, Default, Deserialize)]
60pub struct CommandPayload {
61 /// Resume value delivered to the interrupted node.
62 #[serde(default)]
63 pub resume: Option<Value>,
64}
65
66/// The `config` field of a run payload.
67#[derive(Debug, Clone, Default, Deserialize)]
68pub struct RunConfigPayload {
69 /// Maps to [`RunConfig::with_max_steps`] (LangGraph `recursion_limit`).
70 #[serde(default)]
71 pub recursion_limit: Option<usize>,
72}
73
74/// The `checkpoint` field of a run payload: `{ "checkpoint_id": "…" }`
75/// replays the thread from that checkpoint (time travel) instead of the
76/// latest, via [`RunConfig::with_checkpoint_id`].
77#[derive(Debug, Clone, Default, Deserialize)]
78pub struct CheckpointPayload {
79 /// Id of a checkpoint of this thread (see `POST /threads/{id}/history`).
80 pub checkpoint_id: String,
81}
82
83/// The payload accepted by `POST /threads/{id}/runs{,/wait,/stream}`.
84#[derive(Debug, Clone, Default, Deserialize)]
85pub struct RunPayload {
86 /// Initial state (must be a JSON object). Ignored when resuming: the
87 /// checkpointed state takes precedence.
88 #[serde(default)]
89 pub input: Option<Value>,
90
91 /// `{ "resume": <value> }` — the human-in-the-loop channel.
92 #[serde(default)]
93 pub command: Option<CommandPayload>,
94
95 /// `{ "recursion_limit": n }`.
96 #[serde(default)]
97 pub config: Option<RunConfigPayload>,
98
99 /// `{ "checkpoint_id": "…" }` — time travel: replay the run from that
100 /// checkpoint of this thread instead of the latest (`404` when the
101 /// checkpoint is unknown). Prefer forking first
102 /// (`POST /threads/{id}/fork`) and replaying on the fork.
103 #[serde(default)]
104 pub checkpoint: Option<CheckpointPayload>,
105
106 /// Free-form run metadata (stored, not interpreted).
107 #[serde(default)]
108 pub metadata: Option<Value>,
109
110 /// Which frame families to emit on the SSE stream. Default:
111 /// `["values", "updates"]`. `metadata`, `error`, and `end` frames are
112 /// always emitted.
113 #[serde(default)]
114 pub stream_mode: Option<Vec<String>>,
115
116 /// `"reject"` (409 when the thread is busy) or `"enqueue"` (default:
117 /// queue onto the per-thread run queue).
118 #[serde(default)]
119 pub multitask_strategy: Option<String>,
120
121 /// Run through a named assistant (see `POST /assistants`). The
122 /// assistant must be bound to the same graph as the thread; its
123 /// `config.recursion_limit` applies when the payload does not set one.
124 #[serde(default)]
125 pub assistant_id: Option<String>,
126
127 /// The run's registry declaration (R0.11 Extension Plane, wave 2):
128 /// the named configuration artifacts the run uses and the
129 /// environment it targets. At admission each artifact resolves
130 /// through its environment-tagged version pointer and the resolved
131 /// content pins the run's manifest, with one `config_resolved` event
132 /// per artifact journaled ahead of the run's own events. Absent is
133 /// the pre-R0.11 behavior, byte-identically: no resolution, no
134 /// manifest, no new events.
135 #[serde(default)]
136 pub registry: Option<crate::registry::RegistryRunBinding>,
137}
138
139/// How a second run on a busy thread is handled.
140#[derive(Debug, Clone, Copy, PartialEq, Eq)]
141pub enum MultitaskStrategy {
142 /// Queue behind the active run (default).
143 Enqueue,
144 /// Fail immediately with 409.
145 Reject,
146}
147
148impl MultitaskStrategy {
149 /// Parse the wire value (`None` defaults to `enqueue`).
150 pub fn parse(raw: Option<&str>) -> Result<Self, String> {
151 match raw {
152 None | Some("enqueue") => Ok(Self::Enqueue),
153 Some("reject") => Ok(Self::Reject),
154 Some(other) => Err(format!(
155 "unknown multitask_strategy `{other}` (expected `enqueue` or `reject`)"
156 )),
157 }
158 }
159}
160
161// --------------------------------------------------------------------- //
162// Run bookkeeping
163// --------------------------------------------------------------------- //
164
165/// Lifecycle status of a run.
166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub enum RunStatus {
168 /// Queued behind another run on the same thread.
169 Pending,
170 /// Currently executing.
171 Running,
172 /// Terminated normally.
173 Success,
174 /// Suspended on an interrupt; resumable via `command.resume`.
175 Interrupted,
176 /// Failed.
177 Error,
178 /// Stopped by the graceful-shutdown drain (R0.6 wave 2c) at a
179 /// super-step boundary. Control flow, not failure — the boundary
180 /// checkpoint is intact, so re-running the thread resumes the run from
181 /// exactly where it stopped.
182 Cancelled,
183}
184
185impl RunStatus {
186 /// The wire representation of the status.
187 pub fn as_str(&self) -> &'static str {
188 match self {
189 RunStatus::Pending => "pending",
190 RunStatus::Running => "running",
191 RunStatus::Success => "success",
192 RunStatus::Interrupted => "interrupted",
193 RunStatus::Error => "error",
194 RunStatus::Cancelled => "cancelled",
195 }
196 }
197
198 /// `true` once the run can no longer make progress in this process
199 /// (terminal statuses, including `Cancelled`: a drained run resumes in
200 /// a *new* run, never in place).
201 pub(crate) fn is_terminal(self) -> bool {
202 matches!(
203 self,
204 RunStatus::Success | RunStatus::Interrupted | RunStatus::Error | RunStatus::Cancelled
205 )
206 }
207}
208
209/// One SSE frame as recorded in the per-run event log and broadcast live.
210/// `id` follows the design doc's `{checkpoint_id}:{step}:{seq}` format.
211#[derive(Debug, Clone)]
212pub struct SseFrame {
213 /// Frame id: `{checkpoint_id}:{step}:{seq}`.
214 pub id: String,
215 /// SSE event name (`metadata`, `updates`, `values`, `error`, `end`).
216 pub event: String,
217 /// JSON payload.
218 pub data: Value,
219 /// Per-run monotonically increasing sequence number (1-based).
220 pub seq: u64,
221}
222
223/// Shared frame producer for one run: assigns sequence numbers, appends to
224/// the bounded event log, and fans out over the broadcast channel.
225#[derive(Clone)]
226pub(crate) struct FrameSink {
227 log: Arc<StdMutex<VecDeque<SseFrame>>>,
228 bcast: broadcast::Sender<SseFrame>,
229 seq: Arc<AtomicU64>,
230 last_checkpoint: Arc<StdMutex<String>>,
231 last_step: Arc<AtomicU64>,
232 capacity: usize,
233}
234
235/// Lock a std mutex, recovering from poisoning. Every guard obtained
236/// through this helper wraps a simple clone/push/assign critical section,
237/// so a panicked holder cannot leave the value structurally inconsistent —
238/// and unwinding a whole run (or wedging its thread slot) over a poisoned
239/// frame log is the worse outcome.
240pub(crate) fn lock_recover<T>(mutex: &StdMutex<T>) -> std::sync::MutexGuard<'_, T> {
241 mutex
242 .lock()
243 .unwrap_or_else(|poisoned| poisoned.into_inner())
244}
245
246impl FrameSink {
247 fn new(capacity: usize, bcast: broadcast::Sender<SseFrame>) -> Self {
248 Self {
249 log: Arc::new(StdMutex::new(VecDeque::new())),
250 bcast,
251 seq: Arc::new(AtomicU64::new(0)),
252 last_checkpoint: Arc::new(StdMutex::new("-".to_string())),
253 last_step: Arc::new(AtomicU64::new(0)),
254 capacity,
255 }
256 }
257
258 /// Record and broadcast one frame.
259 pub(crate) fn push(&self, event: &str, step: usize, data: Value) {
260 let seq = self.seq.fetch_add(1, Ordering::Relaxed) + 1;
261 self.last_step.store(step as u64, Ordering::Relaxed);
262 let checkpoint = lock_recover(&self.last_checkpoint).clone();
263 let frame = SseFrame {
264 id: format!("{checkpoint}:{step}:{seq}"),
265 event: event.to_string(),
266 data,
267 seq,
268 };
269 {
270 let mut log = lock_recover(&self.log);
271 if log.len() >= self.capacity {
272 log.pop_front();
273 }
274 log.push_back(frame.clone());
275 }
276 // No live subscribers is normal (background runs); not an error.
277 let _ = self.bcast.send(frame);
278 }
279
280 /// Point subsequent frame ids at a freshly persisted checkpoint.
281 pub(crate) fn note_checkpoint(&self, checkpoint_id: &str) {
282 *lock_recover(&self.last_checkpoint) = checkpoint_id.to_string();
283 }
284
285 /// The super-step of the most recently pushed frame.
286 pub(crate) fn current_step(&self) -> usize {
287 self.last_step.load(Ordering::Relaxed) as usize
288 }
289}
290
291/// Everything the executor task needs, snapshotted from a [`RunHandle`].
292pub(crate) struct RunSnapshot {
293 /// Internal (tenant-scoped) thread id: used for the checkpointer, the
294 /// executor config, and RunManager bookkeeping.
295 pub thread_id: String,
296 /// External thread id as the client knows it — the only form that may
297 /// appear on the wire (SSE frames, terminal JSON).
298 pub wire_thread_id: String,
299 pub graph: String,
300 pub attempt: usize,
301 pub payload: RunPayload,
302 /// The registry binding resolved at admission (R0.11 wave 2).
303 pub admission: Option<crate::registry::RegistryAdmission>,
304 pub sink: FrameSink,
305 pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
306 /// This run's own cancellation token (R0.7 wave 2): a child of the
307 /// server's drain token, so a run-level cancel ([`RunManager::cancel_run`]
308 /// — the cancellation tree's run half) stops this run at its next
309 /// super-step boundary without touching any other run, while a server
310 /// drain still stops them all. Observed by the executor exactly where
311 /// the drain token always was.
312 pub cancel: tokio_util::sync::CancellationToken,
313}
314
315/// Read-only view of a run (used by the rollback and status endpoints).
316pub(crate) struct RunInfo {
317 /// Internal (tenant-scoped) thread id — handlers check tenant ownership
318 /// against it before revealing anything about the run.
319 pub thread_id: String,
320 /// External thread id for wire responses.
321 pub wire_thread_id: String,
322 pub graph: String,
323 pub attempt: usize,
324 pub status: RunStatus,
325 /// The terminal JSON once the run has finished (`None` while active).
326 pub terminal: Option<Value>,
327 pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
328}
329
330/// Handle for one scheduled run, owned by the [`RunManager`]. Crate-private
331/// surface: external users interact with runs over HTTP, not this type.
332pub struct RunHandle {
333 /// Run id (UUID v4).
334 pub(crate) run_id: String,
335 /// Internal (tenant-scoped) thread id this run executes against.
336 pub(crate) thread_id: String,
337 /// External thread id reported on the wire.
338 pub(crate) wire_thread_id: String,
339 /// Registered graph name.
340 pub(crate) graph: String,
341 /// 1-based attempt counter for the thread.
342 pub(crate) attempt: usize,
343 /// Lifecycle status.
344 pub(crate) status: RunStatus,
345 /// Original run payload.
346 pub(crate) payload: RunPayload,
347 /// The registry binding resolved at admission (R0.11 wave 2) — `None`
348 /// for an unbound run, which behaves byte-identically to before.
349 pub(crate) admission: Option<crate::registry::RegistryAdmission>,
350 sink: FrameSink,
351 terminal: watch::Sender<Option<Value>>,
352 checkpoint_ids: Arc<StdMutex<Vec<String>>>,
353 /// This run's own cancellation token (see [`RunSnapshot::cancel`]).
354 /// Firing it is the run-level half of the R0.7 cancellation tree; the
355 /// executor observes it at super-step boundaries, after the boundary
356 /// checkpoint has landed.
357 cancel: tokio_util::sync::CancellationToken,
358}
359
360impl RunHandle {
361 /// Subscribe to the live frame stream.
362 pub(crate) fn subscribe(&self) -> broadcast::Receiver<SseFrame> {
363 self.sink.bcast.subscribe()
364 }
365
366 /// A point-in-time copy of the event log (for replay).
367 pub(crate) fn log_snapshot(&self) -> Vec<SseFrame> {
368 lock_recover(&self.sink.log).iter().cloned().collect()
369 }
370}
371
372/// What [`RunManager::insert`] decided for a freshly scheduled run.
373pub(crate) enum ScheduleDecision {
374 /// The thread slot was free; the run must be spawned now.
375 Started,
376 /// The run was queued behind the active run.
377 Queued,
378}
379
380/// What [`RunManager::cancel_run`] did (R0.7 wave 2).
381#[derive(Debug, Clone, Copy, PartialEq, Eq)]
382pub(crate) enum RunCancel {
383 /// The run was executing; its cancellation token fired. The terminal
384 /// transition lands when the executor observes it at the next
385 /// super-step boundary.
386 Signalled,
387 /// The run was queued behind another; it was dequeued and finished
388 /// terminal-`cancelled` without ever starting.
389 CancelledQueued,
390 /// The run was already terminal; nothing changed.
391 Terminal,
392 /// No such run.
393 Unknown,
394}
395
396/// What [`RunManager::cancel_thread_runs`] did to one thread's runs
397/// (R0.7 wave 2), split by how each run's cancellation lands — mirroring
398/// the task queue's [`crate::tasks::RunCancellation`] shape.
399#[derive(Debug, Default)]
400pub(crate) struct ThreadCancellation {
401 /// Running runs whose cancellation tokens fired (terminal at their
402 /// next boundary).
403 pub signalled: Vec<String>,
404 /// Queued runs dequeued into terminal-`cancelled` immediately.
405 pub cancelled: Vec<String>,
406}
407
408impl ThreadCancellation {
409 /// `true` when no run was touched (the thread had no active or queued
410 /// runs) — the cancel route journals an `AgentExit` only when the
411 /// cancellation actually landed somewhere.
412 pub(crate) fn is_empty(&self) -> bool {
413 self.signalled.is_empty() && self.cancelled.is_empty()
414 }
415}
416
417#[derive(Default)]
418struct RunManagerInner {
419 runs: HashMap<String, RunHandle>,
420 active_by_thread: HashMap<String, String>,
421 queues: HashMap<String, VecDeque<String>>,
422 attempts: HashMap<String, usize>,
423 /// Insertion order of run ids, feeding terminal-run eviction.
424 order: VecDeque<String>,
425}
426
427/// Cap on retained runs (see the module docs' retention note). Without a
428/// cap, `runs` would grow by one record — payload clone, terminal JSON, and
429/// up to `event_log_capacity` SSE frames — per run for the process
430/// lifetime: a steady memory leak on any busy cron schedule.
431const MAX_RETAINED_RUNS: usize = 1024;
432
433/// Registry of all runs, plus per-thread scheduling state. Cheap to clone
434/// (shared inner).
435#[derive(Default, Clone)]
436pub struct RunManager {
437 inner: Arc<Mutex<RunManagerInner>>,
438}
439
440impl RunManager {
441 /// An empty manager.
442 pub fn new() -> Self {
443 Self::default()
444 }
445
446 /// Insert a new run under the given multitask strategy, assigning its
447 /// per-thread attempt number.
448 pub(crate) async fn insert(
449 &self,
450 mut handle: RunHandle,
451 strategy: MultitaskStrategy,
452 queue_cap: usize,
453 ) -> Result<ScheduleDecision, ApiError> {
454 let mut inner = self.inner.lock().await;
455 let busy = inner.active_by_thread.contains_key(&handle.thread_id);
456 let attempt = {
457 let counter = inner.attempts.entry(handle.thread_id.clone()).or_insert(0);
458 *counter += 1;
459 *counter
460 };
461 handle.attempt = attempt;
462 inner.order.push_back(handle.run_id.clone());
463
464 match strategy {
465 MultitaskStrategy::Reject if busy => Err(ApiError::conflict(format!(
466 "thread `{}` already has an active run",
467 handle.thread_id
468 ))),
469 _ if busy => {
470 let queue = inner.queues.entry(handle.thread_id.clone()).or_default();
471 if queue.len() >= queue_cap {
472 return Err(ApiError::conflict(format!(
473 "thread `{}` run queue is full (cap {queue_cap})",
474 handle.thread_id
475 )));
476 }
477 queue.push_back(handle.run_id.clone());
478 inner.runs.insert(handle.run_id.clone(), handle);
479 Ok(ScheduleDecision::Queued)
480 }
481 _ => {
482 inner
483 .active_by_thread
484 .insert(handle.thread_id.clone(), handle.run_id.clone());
485 handle.status = RunStatus::Running;
486 inner.runs.insert(handle.run_id.clone(), handle);
487 Ok(ScheduleDecision::Started)
488 }
489 }
490 }
491
492 /// Snapshot everything the executor task needs for `run_id`.
493 pub(crate) async fn snapshot(&self, run_id: &str) -> Option<RunSnapshot> {
494 let inner = self.inner.lock().await;
495 inner.runs.get(run_id).map(|h| RunSnapshot {
496 thread_id: h.thread_id.clone(),
497 wire_thread_id: h.wire_thread_id.clone(),
498 graph: h.graph.clone(),
499 attempt: h.attempt,
500 payload: h.payload.clone(),
501 admission: h.admission.clone(),
502 sink: h.sink.clone(),
503 checkpoint_ids: Arc::clone(&h.checkpoint_ids),
504 cancel: h.cancel.clone(),
505 })
506 }
507
508 /// Read-only run info for API endpoints.
509 pub(crate) async fn info(&self, run_id: &str) -> Option<RunInfo> {
510 let inner = self.inner.lock().await;
511 inner.runs.get(run_id).map(|h| RunInfo {
512 thread_id: h.thread_id.clone(),
513 wire_thread_id: h.wire_thread_id.clone(),
514 graph: h.graph.clone(),
515 attempt: h.attempt,
516 status: h.status,
517 terminal: h.terminal.borrow().clone(),
518 checkpoint_ids: Arc::clone(&h.checkpoint_ids),
519 })
520 }
521
522 /// Replay log + live subscription + internal thread id for the
523 /// SSE attach endpoint (`GET /runs/{id}/stream`).
524 pub(crate) async fn stream_parts(
525 &self,
526 run_id: &str,
527 ) -> Option<(Vec<SseFrame>, broadcast::Receiver<SseFrame>, String)> {
528 let inner = self.inner.lock().await;
529 inner
530 .runs
531 .get(run_id)
532 .map(|h| (h.log_snapshot(), h.subscribe(), h.thread_id.clone()))
533 }
534
535 /// `true` while the thread has an active run or a non-empty queue —
536 /// rollback refuses to delete checkpoints out from under them.
537 pub(crate) async fn thread_busy(&self, thread_id: &str) -> bool {
538 let inner = self.inner.lock().await;
539 inner.active_by_thread.contains_key(thread_id)
540 || inner
541 .queues
542 .get(thread_id)
543 .is_some_and(|queue| !queue.is_empty())
544 }
545
546 /// Cancel one run (R0.7 wave 2 — the run-level half of the
547 /// cancellation tree). A running run is *signalled*: its own
548 /// cancellation token fires and the executor stops it at the next
549 /// super-step boundary — after the boundary checkpoint has landed —
550 /// ending terminal-`cancelled` and resumable by re-running the thread,
551 /// exactly like the server drain. A queued (pending) run never started,
552 /// so it is dequeued and finished terminal-`cancelled` immediately —
553 /// leaving it queued would let a dead run promote and execute.
554 pub(crate) async fn cancel_run(&self, run_id: &str) -> RunCancel {
555 let mut inner = self.inner.lock().await;
556 let Some(handle) = inner.runs.get_mut(run_id) else {
557 return RunCancel::Unknown;
558 };
559 match handle.status {
560 RunStatus::Running => {
561 handle.cancel.cancel();
562 RunCancel::Signalled
563 }
564 RunStatus::Pending => {
565 let thread_id = handle.thread_id.clone();
566 let wire_thread_id = handle.wire_thread_id.clone();
567 if let Some(queue) = inner.queues.get_mut(&thread_id) {
568 queue.retain(|queued| queued != run_id);
569 }
570 let handle = inner
571 .runs
572 .get_mut(run_id)
573 .expect("the handle was resolved above");
574 handle.status = RunStatus::Cancelled;
575 let terminal = json!({
576 "run_id": run_id,
577 "thread_id": wire_thread_id,
578 "status": "cancelled",
579 "message": "cancelled while queued, before its first step",
580 });
581 handle.terminal.send_replace(Some(terminal));
582 RunCancel::CancelledQueued
583 }
584 // Terminal runs (including an already-cancelled one) are
585 // untouched — cancellation is control flow, idempotent by
586 // no-op, never a second terminal transition.
587 _ => RunCancel::Terminal,
588 }
589 }
590
591 /// Cancel every run of one thread (R0.7 wave 2): the active run is
592 /// signalled, every queued run is dequeued-cancelled — the whole
593 /// per-thread run state, so cancelling an agent's thread leaves no
594 /// pending run that would re-drive it.
595 pub(crate) async fn cancel_thread_runs(&self, thread_id: &str) -> ThreadCancellation {
596 let (active, queued) = {
597 let inner = self.inner.lock().await;
598 (
599 inner.active_by_thread.get(thread_id).cloned(),
600 inner
601 .queues
602 .get(thread_id)
603 .map(|q| q.iter().cloned().collect::<Vec<_>>())
604 .unwrap_or_default(),
605 )
606 };
607 let mut outcome = ThreadCancellation::default();
608 if let Some(run_id) = active {
609 if matches!(self.cancel_run(&run_id).await, RunCancel::Signalled) {
610 outcome.signalled.push(run_id);
611 }
612 }
613 for run_id in queued {
614 if matches!(self.cancel_run(&run_id).await, RunCancel::CancelledQueued) {
615 outcome.cancelled.push(run_id);
616 }
617 }
618 outcome
619 }
620
621 /// Record the terminal status + JSON, wake waiters, release the thread
622 /// slot, and return the next queued run id for the thread (if any), now
623 /// marked active.
624 ///
625 /// `draining` (the server's shutdown drain) suppresses queue promotion:
626 /// the slot frees but nothing is returned — a run promoted into a
627 /// shutting-down process would only be cancelled at its first boundary.
628 /// Queued runs stay `Pending`; their threads' checkpoints are intact
629 /// for the next process to re-drive.
630 pub(crate) async fn finish(
631 &self,
632 run_id: &str,
633 status: RunStatus,
634 terminal: Value,
635 draining: bool,
636 ) -> Option<String> {
637 let mut inner = self.inner.lock().await;
638 let handle = inner.runs.get_mut(run_id)?;
639 handle.status = status;
640 // `send_replace` (not `send`) so the terminal JSON is stored even
641 // when no waiter holds a receiver (background runs); status polling
642 // via `info` reads it back through `watch::Sender::borrow`.
643 handle.terminal.send_replace(Some(terminal));
644 let thread_id = handle.thread_id.clone();
645
646 if inner
647 .active_by_thread
648 .get(&thread_id)
649 .is_some_and(|active| active == run_id)
650 {
651 inner.active_by_thread.remove(&thread_id);
652 }
653
654 let next = if draining {
655 None
656 } else {
657 inner
658 .queues
659 .get_mut(&thread_id)
660 .and_then(VecDeque::pop_front)
661 };
662 if let Some(next_id) = &next {
663 if let Some(h) = inner.runs.get_mut(next_id) {
664 h.status = RunStatus::Running;
665 }
666 inner.active_by_thread.insert(thread_id, next_id.clone());
667 }
668
669 // Evict the oldest terminal runs beyond the retention cap; active
670 // and queued runs keep their slots in `order`.
671 let mut excess = inner.runs.len().saturating_sub(MAX_RETAINED_RUNS);
672 let mut skipped = Vec::new();
673 while excess > 0 {
674 let Some(candidate) = inner.order.pop_front() else {
675 break;
676 };
677 let evictable = inner
678 .runs
679 .get(&candidate)
680 .is_some_and(|h| h.status.is_terminal());
681 if evictable {
682 inner.runs.remove(&candidate);
683 excess -= 1;
684 } else {
685 skipped.push(candidate);
686 }
687 }
688 inner.order.extend(skipped);
689
690 next
691 }
692}
693
694/// Everything the run machinery needs from the application: registry,
695/// checkpointer, manager, and caps. Cheap to clone.
696#[derive(Clone)]
697pub(crate) struct RunDeps {
698 pub registry: GraphRegistry,
699 pub checkpointer: Arc<dyn Checkpointer>,
700 pub manager: RunManager,
701 /// Flight Recorder journal persistence (`GET /runs/{id}/events`).
702 pub server_store: Arc<dyn ServerStore>,
703 pub queue_cap: usize,
704 pub log_capacity: usize,
705 /// The server's drain control (R0.6 wave 2c): threaded into every run's
706 /// executor, which observes it at super-step boundaries.
707 pub shutdown: tokio_util::sync::CancellationToken,
708 /// The deployment's default environment tag (R0.11 wave 2): the
709 /// promotion target a registry-bound run resolves against when its
710 /// binding names no environment (`None`: the untagged surface).
711 pub default_environment_tag: Option<rusty_agent_runtime::learn::EnvironmentTag>,
712}
713
714/// The result of successfully scheduling a run: everything an endpoint
715/// needs to answer (background ack, wait, or stream).
716pub(crate) struct Scheduled {
717 pub run_id: String,
718 pub status: RunStatus,
719 pub terminal: watch::Receiver<Option<Value>>,
720 pub broadcast: broadcast::Receiver<SseFrame>,
721 pub replay: Vec<SseFrame>,
722}
723
724/// Create a run handle, apply the multitask strategy, and spawn execution
725/// immediately when the thread slot is free.
726///
727/// `thread_id` is the internal (tenant-scoped) id used for the checkpointer,
728/// executor, and RunManager bookkeeping; `wire_thread_id` is the external id
729/// reported in SSE frames and terminal JSON.
730pub(crate) async fn schedule(
731 deps: &RunDeps,
732 thread_id: &str,
733 wire_thread_id: &str,
734 graph: &str,
735 payload: RunPayload,
736 strategy: MultitaskStrategy,
737) -> Result<Scheduled, ApiError> {
738 // A draining server must not take new runs: the token would cancel
739 // them at their first boundary anyway, and a 503 lets the caller (or
740 // its load balancer) retry against a pod that is still serving.
741 if deps.shutdown.is_cancelled() {
742 return Err(ApiError::shutting_down(format!(
743 "server is draining; resubmit run on thread `{wire_thread_id}` against a running instance"
744 )));
745 }
746 let run_id = uuid::Uuid::new_v4().to_string();
747 // Registry admission (R0.11 wave 2): the binding resolves now, at
748 // admission — a promotion landing afterwards never reaches this run
749 // (the conservatism checkpoint pinning has kept since R0.7), and a
750 // queued run binds at admission, not at dequeue. The tenant comes
751 // from the internal thread id, so every entry point (HTTP, cron,
752 // trigger, bridge) resolves in the submitter's namespace without
753 // threading the request context through. A resolution failure is an
754 // admission failure: the run never enters the manager.
755 let admission = match &payload.registry {
756 Some(binding) => Some(
757 crate::registry::resolve_admission(
758 &deps.server_store,
759 crate::auth::tenant_of_internal(thread_id),
760 deps.default_environment_tag.as_ref(),
761 &run_id,
762 binding,
763 )
764 .await?,
765 ),
766 None => None,
767 };
768 let (bcast_tx, _bcast_rx) = broadcast::channel(256);
769 let (terminal_tx, terminal_rx) = watch::channel(None);
770 let handle = RunHandle {
771 run_id: run_id.clone(),
772 thread_id: thread_id.to_string(),
773 wire_thread_id: wire_thread_id.to_string(),
774 graph: graph.to_string(),
775 attempt: 0, // assigned by RunManager::insert
776 status: RunStatus::Pending,
777 payload,
778 admission,
779 sink: FrameSink::new(deps.log_capacity, bcast_tx),
780 terminal: terminal_tx,
781 checkpoint_ids: Arc::new(StdMutex::new(Vec::new())),
782 // A child of the server drain token: the drain still stops every
783 // run, and a run-level cancel (R0.7 wave 2) stops only this one.
784 cancel: deps.shutdown.child_token(),
785 };
786 // Subscribe/snapshot before any execution can emit frames.
787 let replay = handle.log_snapshot();
788 let broadcast = handle.subscribe();
789
790 let decision = deps
791 .manager
792 .insert(handle, strategy, deps.queue_cap)
793 .await?;
794 let status = match decision {
795 ScheduleDecision::Started => {
796 spawn_execute(deps.clone(), run_id.clone());
797 RunStatus::Running
798 }
799 ScheduleDecision::Queued => RunStatus::Pending,
800 };
801
802 Ok(Scheduled {
803 run_id,
804 status,
805 terminal: terminal_rx,
806 broadcast,
807 replay,
808 })
809}
810
811/// Drive one run to its terminal state and chain the next queued run.
812async fn execute(deps: RunDeps, run_id: String) {
813 let Some(snap) = deps.manager.snapshot(&run_id).await else {
814 tracing::warn!(%run_id, "scheduled run vanished before execution");
815 return;
816 };
817 let sink = snap.sink.clone();
818 sink.push(
819 "metadata",
820 0,
821 json!({
822 "run_id": run_id,
823 "thread_id": snap.wire_thread_id,
824 "graph": snap.graph,
825 "attempt": snap.attempt,
826 "metadata": snap.payload.metadata,
827 }),
828 );
829
830 let Some((graph, spec)) = deps.registry.get(&snap.graph) else {
831 let message = format!("graph `{}` is no longer registered", snap.graph);
832 tracing::error!(%run_id, %message);
833 sink.push(
834 "error",
835 0,
836 json!({"error": "unknown_graph", "message": message}),
837 );
838 sink.push("end", 0, json!({"status": "error"}));
839 let terminal = json!({
840 "run_id": run_id,
841 "thread_id": snap.wire_thread_id,
842 "status": "error",
843 "error": "unknown_graph",
844 "message": message,
845 });
846 terminate(&deps, &run_id, RunStatus::Error, terminal).await;
847 return;
848 };
849
850 let modes: Vec<String> = snap
851 .payload
852 .stream_mode
853 .clone()
854 .unwrap_or_else(|| vec!["values".to_string(), "updates".to_string()]);
855 // Flight Recorder: one journal per run, keyed by the server-minted run
856 // id. Events carry the external (wire) thread id — the internal
857 // tenant-scoped id must never appear in served evidence. The journal's
858 // clock is the default system clock, so timestamps match pre-R0.5
859 // behavior; attaching it makes the executor read time through it.
860 let journal = Journal::new(run_id.clone(), snap.wire_thread_id.clone(), Clock::System);
861 let (evt_tx, evt_rx) = mpsc::channel::<GraphEvent>(256);
862 let forwarder = tokio::spawn(forward_events(
863 evt_rx,
864 sink.clone(),
865 ForwardDeps {
866 checkpointer: Arc::clone(&deps.checkpointer),
867 server_store: Arc::clone(&deps.server_store),
868 journal: journal.clone(),
869 thread_id: snap.thread_id.clone(),
870 checkpoint_ids: Arc::clone(&snap.checkpoint_ids),
871 modes,
872 },
873 ));
874
875 let mut config = RunConfig::new(snap.thread_id.clone())
876 .with_event_tx(evt_tx)
877 .with_journal(journal.clone())
878 // Cancellation hook: this run's own token (a child of the server
879 // drain token). When either fires, the run stops at its next
880 // super-step boundary — a point where a checkpoint was just
881 // persisted — instead of being torn down mid-step.
882 .with_cancellation(snap.cancel.clone());
883 // Registry admission (R0.11 wave 2): the binding resolved at schedule
884 // time becomes evidence now, ahead of the run's own events — one
885 // `config_resolved` per artifact (chained: each resolution's parent
886 // is the previous, so the admission reads as one causal unit) — and
887 // the resolved manifest stamps every checkpoint header, which is how
888 // the receipt reads it back. Resolution decides nothing about what
889 // will run, so the events are read-only (the `CapsuleResolved`
890 // precedent); a serialization failure here is a bug, not a runtime
891 // condition — the payload type is the server's own.
892 if let Some(admission) = &snap.admission {
893 let mut parent = None;
894 for resolution in &admission.resolutions {
895 let output =
896 serde_json::to_value(resolution).expect("ConfigResolution always serializes");
897 let mut draft =
898 EventDraft::new(RunEventKind::ConfigResolved, Effect::ReadOnly).output(output);
899 if let Some(parent) = parent {
900 draft = draft.parent(parent);
901 }
902 parent = Some(journal.record(draft));
903 }
904 config = config.with_manifest(admission.manifest.clone());
905 }
906 if let Some(command) = &snap.payload.command {
907 if let Some(resume) = &command.resume {
908 config = config.with_resume(resume.clone());
909 }
910 }
911 if let Some(checkpoint) = &snap.payload.checkpoint {
912 config = config.with_checkpoint_id(checkpoint.checkpoint_id.clone());
913 }
914 if let Some(run_cfg) = &snap.payload.config {
915 if let Some(limit) = run_cfg.recursion_limit {
916 config = config.with_max_steps(limit);
917 }
918 }
919 let initial = snap
920 .payload
921 .input
922 .clone()
923 .and_then(|v| State::from_value(v).ok())
924 .unwrap_or_default();
925
926 let mut executor = Executor::with_checkpointer(Arc::clone(&deps.checkpointer));
927 // The resolved middleware chain (R0.11 wave 4) attaches in journaled
928 // order — the same layers the manifest's `middleware` digest pins and
929 // the admission resolution's `layers` field names. Attached after the
930 // admission journal writes above, so the evidence of *what* serves
931 // precedes the run it serves.
932 if let Some(admission) = &snap.admission {
933 if let Some(chain) = &admission.middleware {
934 for layer in chain.layers() {
935 executor = executor.layer_shared(Arc::clone(layer));
936 }
937 }
938 }
939 let result = executor.run(&graph, &spec, initial, config).await;
940 // `config` (holding the only sender) is dropped with the run; the
941 // forwarder drains what remains and exits.
942 let _ = forwarder.await;
943
944 // Final journal write: the complete evidence of the run, including the
945 // events recorded after the last checkpoint boundary. Persisted before
946 // the run goes terminal so `complete: true` on the events endpoint never
947 // races ahead of the snapshot it serves. Evidence of a failed run is
948 // still evidence — this write happens on every outcome.
949 persist_journal(&deps.server_store, &journal).await;
950
951 let step = sink.current_step();
952 let (status, terminal) = match result {
953 Ok(ExecutionOutcome::Done(state)) => {
954 sink.push("end", step, json!({"status": "success"}));
955 let terminal = json!({
956 "run_id": run_id,
957 "thread_id": snap.wire_thread_id,
958 "status": "success",
959 "output": state.to_value(),
960 });
961 (RunStatus::Success, terminal)
962 }
963 Ok(ExecutionOutcome::Interrupted {
964 value,
965 state,
966 checkpoint_id,
967 }) => {
968 sink.push(
969 "end",
970 step,
971 json!({"status": "interrupted", "interrupt": value}),
972 );
973 let terminal = json!({
974 "run_id": run_id,
975 "thread_id": snap.wire_thread_id,
976 "status": "interrupted",
977 "interrupt": value,
978 "checkpoint_id": checkpoint_id,
979 "state": state.to_value(),
980 });
981 (RunStatus::Interrupted, terminal)
982 }
983 Err(error @ RustyError::Cancelled(_)) => {
984 // Drain, not failure: the executor stopped at a super-step
985 // boundary, so the run's last checkpoint is intact and a fresh
986 // run on the thread resumes from it. The wire status is
987 // `cancelled` — matching the task queue's treatment of
988 // cancellation as control flow, never an error.
989 let message = error.to_string();
990 tracing::info!(%run_id, "run drained at a checkpoint boundary; resumable");
991 sink.push("end", step, json!({"status": "cancelled"}));
992 let terminal = json!({
993 "run_id": run_id,
994 "thread_id": snap.wire_thread_id,
995 "status": "cancelled",
996 "message": message,
997 });
998 (RunStatus::Cancelled, terminal)
999 }
1000 Err(error) => {
1001 let kind = error_kind(&error);
1002 let message = error.to_string();
1003 tracing::warn!(%run_id, %error, "run failed");
1004 sink.push("error", step, json!({"error": kind, "message": message}));
1005 sink.push("end", step, json!({"status": "error"}));
1006 let terminal = json!({
1007 "run_id": run_id,
1008 "thread_id": snap.wire_thread_id,
1009 "status": "error",
1010 "error": kind,
1011 "message": message,
1012 });
1013 (RunStatus::Error, terminal)
1014 }
1015 };
1016 terminate(&deps, &run_id, status, terminal).await;
1017}
1018
1019/// Everything [`forward_events`] needs beyond the frame sink, bundled to
1020/// keep the task's argument list readable.
1021struct ForwardDeps {
1022 checkpointer: Arc<dyn Checkpointer>,
1023 server_store: Arc<dyn ServerStore>,
1024 journal: Journal,
1025 /// Internal (tenant-scoped) thread id, for checkpoint read-backs.
1026 thread_id: String,
1027 checkpoint_ids: Arc<StdMutex<Vec<String>>>,
1028 modes: Vec<String>,
1029}
1030
1031/// Map executor events to SSE frames per the design doc's §4 table. Also the
1032/// Flight Recorder's checkpoint-boundary persistence point: every
1033/// `CheckpointSaved` event flushes the journal's current snapshot to the
1034/// server store, so the stored evidence trails the live journal by at most
1035/// one super-step.
1036async fn forward_events(mut rx: mpsc::Receiver<GraphEvent>, sink: FrameSink, deps: ForwardDeps) {
1037 let ForwardDeps {
1038 checkpointer,
1039 server_store,
1040 journal,
1041 thread_id,
1042 checkpoint_ids,
1043 modes,
1044 } = deps;
1045 while let Some(event) = rx.recv().await {
1046 match event {
1047 GraphEvent::StateUpdate { step, updates } => {
1048 if modes.iter().any(|m| m == "updates") {
1049 sink.push("updates", step, json!({"step": step, "updates": updates}));
1050 }
1051 }
1052 GraphEvent::Token { node, delta } => {
1053 if modes.iter().any(|m| m == "messages") {
1054 let step = sink.current_step();
1055 sink.push("messages", step, json!({"node": node, "delta": delta}));
1056 }
1057 }
1058 GraphEvent::CheckpointSaved {
1059 checkpoint_id,
1060 step,
1061 } => {
1062 lock_recover(&checkpoint_ids).push(checkpoint_id.clone());
1063 sink.note_checkpoint(&checkpoint_id);
1064 persist_journal(&server_store, &journal).await;
1065 if modes.iter().any(|m| m == "values") {
1066 match read_back_state(&*checkpointer, &thread_id, &checkpoint_id).await {
1067 Ok(Some(values)) => sink.push("values", step, values),
1068 Ok(None) => {
1069 tracing::debug!(%checkpoint_id, "checkpoint not found for values frame")
1070 }
1071 Err(error) => {
1072 tracing::warn!(%checkpoint_id, %error, "values frame read-back failed")
1073 }
1074 }
1075 }
1076 }
1077 // Reserved for the future `tasks` / `debug` stream modes.
1078 GraphEvent::SuperStep { .. }
1079 | GraphEvent::NodeStart { .. }
1080 | GraphEvent::NodeEnd { .. } => {}
1081 }
1082 }
1083}
1084
1085/// Flush the journal's current snapshot to the server store. A persistence
1086/// failure is logged, not raised: the run's execution must not fail because
1087/// its evidence could not be written, and the next checkpoint boundary (or
1088/// the completion write) retries.
1089async fn persist_journal(server_store: &Arc<dyn ServerStore>, journal: &Journal) {
1090 if let Err(error) = server_store.put_journal(&journal.snapshot()).await {
1091 tracing::warn!(run_id = %journal.run_id(), %error, "journal persistence failed");
1092 }
1093}
1094
1095/// `values` frames carry the full state persisted at a super-step boundary,
1096/// read back from the checkpoint log (design doc §4). A point lookup, not a
1097/// full `list()` scan — that would be O(history) per super-step, O(n²) per
1098/// run.
1099async fn read_back_state(
1100 checkpointer: &dyn Checkpointer,
1101 thread_id: &str,
1102 checkpoint_id: &str,
1103) -> rusty_agent_runtime::error::Result<Option<Value>> {
1104 Ok(checkpointer
1105 .get_by_id(thread_id, checkpoint_id)
1106 .await?
1107 .map(|cp| cp.state.to_value()))
1108}
1109
1110/// Spawn `execute` for a run, guarding the thread's scheduling slot: if the
1111/// task panics (executor bug — the poison-prone lock sites recover via
1112/// [`lock_recover`]), the run is force-finished as `error` so
1113/// `active_by_thread` releases the slot and queued runs drain instead of
1114/// wedging behind a ghost.
1115///
1116/// The future is boxed behind a trait object to break the
1117/// `execute → terminate → spawn(execute)` type cycle, which would otherwise
1118/// make `Send` inference recursive and fail.
1119fn spawn_execute(deps: RunDeps, run_id: String) {
1120 let fut: std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> = Box::pin({
1121 let deps = deps.clone();
1122 let run_id = run_id.clone();
1123 async move { execute(deps, run_id).await }
1124 });
1125 tokio::spawn(async move {
1126 if AssertUnwindSafe(fut).catch_unwind().await.is_ok() {
1127 return;
1128 }
1129 tracing::error!(%run_id, "run task panicked; force-finishing as error");
1130 let Some(snap) = deps.manager.snapshot(&run_id).await else {
1131 return;
1132 };
1133 // If the panic happened after `terminate` completed, the slot is
1134 // already released — finishing again would double-promote the queue.
1135 if matches!(deps.manager.info(&run_id).await, Some(info) if info.status.is_terminal()) {
1136 return;
1137 }
1138 let step = snap.sink.current_step();
1139 snap.sink.push(
1140 "error",
1141 step,
1142 json!({"error": "internal_panic", "message": "run task panicked"}),
1143 );
1144 snap.sink.push("end", step, json!({"status": "error"}));
1145 let terminal = json!({
1146 "run_id": run_id,
1147 "thread_id": snap.wire_thread_id,
1148 "status": "error",
1149 "error": "internal_panic",
1150 "message": "run task panicked",
1151 });
1152 terminate(&deps, &run_id, RunStatus::Error, terminal).await;
1153 });
1154}
1155
1156/// Record the terminal state and spawn the next queued run, if any. While
1157/// the server drains, the queue does not advance (see
1158/// [`RunManager::finish`]'s `draining` flag).
1159async fn terminate(deps: &RunDeps, run_id: &str, status: RunStatus, terminal: Value) {
1160 let draining = deps.shutdown.is_cancelled();
1161 if let Some(next) = deps
1162 .manager
1163 .finish(run_id, status, terminal, draining)
1164 .await
1165 {
1166 spawn_execute(deps.clone(), next);
1167 }
1168}
1169
1170/// Stable error-kind labels for the wire.
1171fn error_kind(error: &RustyError) -> &'static str {
1172 match error {
1173 RustyError::Graph(_) => "graph_error",
1174 RustyError::Node(_) => "node_error",
1175 RustyError::Interrupt { .. } => "interrupted",
1176 RustyError::Checkpoint(_) => "checkpoint_error",
1177 RustyError::Llm(_) => "llm_error",
1178 RustyError::Tool(_) => "tool_error",
1179 RustyError::Serialization(_) => "serialization_error",
1180 RustyError::InvalidUpdate(_) => "invalid_update",
1181 RustyError::Replay(_) => "replay_error",
1182 // Drain cancellation is control flow and takes its own terminal
1183 // path in `execute`; this arm exists for exhaustiveness only.
1184 RustyError::Cancelled(_) => "cancelled",
1185 }
1186}