use std::collections::{HashMap, VecDeque};
use std::panic::AssertUnwindSafe;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex as StdMutex};
use futures::FutureExt;
use rusty_agent_runtime::checkpoint::Checkpointer;
use rusty_agent_runtime::error::RustyError;
use rusty_agent_runtime::executor::{ExecutionOutcome, Executor, GraphEvent, RunConfig};
use rusty_agent_runtime::journal::{Clock, EventDraft, Journal};
use rusty_agent_runtime::record::{Effect, RunEventKind};
use rusty_agent_runtime::state::State;
use serde::Deserialize;
use serde_json::{json, Value};
use tokio::sync::{broadcast, mpsc, watch, Mutex};
use crate::error::ApiError;
use crate::server_store::ServerStore;
use crate::GraphRegistry;
#[derive(Debug, Clone, Default, Deserialize)]
pub struct CommandPayload {
#[serde(default)]
pub resume: Option<Value>,
}
#[derive(Debug, Clone, Default, Deserialize)]
pub struct RunConfigPayload {
#[serde(default)]
pub recursion_limit: Option<usize>,
}
#[derive(Debug, Clone, Default, Deserialize)]
pub struct CheckpointPayload {
pub checkpoint_id: String,
}
#[derive(Debug, Clone, Default, Deserialize)]
pub struct RunPayload {
#[serde(default)]
pub input: Option<Value>,
#[serde(default)]
pub command: Option<CommandPayload>,
#[serde(default)]
pub config: Option<RunConfigPayload>,
#[serde(default)]
pub checkpoint: Option<CheckpointPayload>,
#[serde(default)]
pub metadata: Option<Value>,
#[serde(default)]
pub stream_mode: Option<Vec<String>>,
#[serde(default)]
pub multitask_strategy: Option<String>,
#[serde(default)]
pub assistant_id: Option<String>,
#[serde(default)]
pub registry: Option<crate::registry::RegistryRunBinding>,
#[serde(default)]
pub deployment: Option<crate::deploy::DeploymentRunBinding>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MultitaskStrategy {
Enqueue,
Reject,
}
impl MultitaskStrategy {
pub fn parse(raw: Option<&str>) -> Result<Self, String> {
match raw {
None | Some("enqueue") => Ok(Self::Enqueue),
Some("reject") => Ok(Self::Reject),
Some(other) => Err(format!(
"unknown multitask_strategy `{other}` (expected `enqueue` or `reject`)"
)),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RunStatus {
Pending,
Running,
Success,
Interrupted,
Error,
Cancelled,
}
impl RunStatus {
pub fn as_str(&self) -> &'static str {
match self {
RunStatus::Pending => "pending",
RunStatus::Running => "running",
RunStatus::Success => "success",
RunStatus::Interrupted => "interrupted",
RunStatus::Error => "error",
RunStatus::Cancelled => "cancelled",
}
}
pub(crate) fn is_terminal(self) -> bool {
matches!(
self,
RunStatus::Success | RunStatus::Interrupted | RunStatus::Error | RunStatus::Cancelled
)
}
}
#[derive(Debug, Clone)]
pub struct SseFrame {
pub id: String,
pub event: String,
pub data: Value,
pub seq: u64,
}
#[derive(Clone)]
pub(crate) struct FrameSink {
log: Arc<StdMutex<VecDeque<SseFrame>>>,
bcast: broadcast::Sender<SseFrame>,
seq: Arc<AtomicU64>,
last_checkpoint: Arc<StdMutex<String>>,
last_step: Arc<AtomicU64>,
capacity: usize,
}
pub(crate) fn lock_recover<T>(mutex: &StdMutex<T>) -> std::sync::MutexGuard<'_, T> {
mutex
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
impl FrameSink {
fn new(capacity: usize, bcast: broadcast::Sender<SseFrame>) -> Self {
Self {
log: Arc::new(StdMutex::new(VecDeque::new())),
bcast,
seq: Arc::new(AtomicU64::new(0)),
last_checkpoint: Arc::new(StdMutex::new("-".to_string())),
last_step: Arc::new(AtomicU64::new(0)),
capacity,
}
}
pub(crate) fn push(&self, event: &str, step: usize, data: Value) {
let seq = self.seq.fetch_add(1, Ordering::Relaxed) + 1;
self.last_step.store(step as u64, Ordering::Relaxed);
let checkpoint = lock_recover(&self.last_checkpoint).clone();
let frame = SseFrame {
id: format!("{checkpoint}:{step}:{seq}"),
event: event.to_string(),
data,
seq,
};
{
let mut log = lock_recover(&self.log);
if log.len() >= self.capacity {
log.pop_front();
}
log.push_back(frame.clone());
}
let _ = self.bcast.send(frame);
}
pub(crate) fn note_checkpoint(&self, checkpoint_id: &str) {
*lock_recover(&self.last_checkpoint) = checkpoint_id.to_string();
}
pub(crate) fn current_step(&self) -> usize {
self.last_step.load(Ordering::Relaxed) as usize
}
}
pub(crate) struct RunSnapshot {
pub thread_id: String,
pub wire_thread_id: String,
pub graph: String,
pub attempt: usize,
pub payload: RunPayload,
pub admission: Option<crate::registry::RegistryAdmission>,
pub deployment: Option<crate::deploy::DeploymentAdmission>,
pub sink: FrameSink,
pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
pub cancel: tokio_util::sync::CancellationToken,
}
pub(crate) struct RunInfo {
pub thread_id: String,
pub wire_thread_id: String,
pub graph: String,
pub attempt: usize,
pub status: RunStatus,
pub terminal: Option<Value>,
pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
}
pub struct RunHandle {
pub(crate) run_id: String,
pub(crate) thread_id: String,
pub(crate) wire_thread_id: String,
pub(crate) graph: String,
pub(crate) attempt: usize,
pub(crate) status: RunStatus,
pub(crate) payload: RunPayload,
pub(crate) admission: Option<crate::registry::RegistryAdmission>,
pub(crate) deployment: Option<crate::deploy::DeploymentAdmission>,
sink: FrameSink,
terminal: watch::Sender<Option<Value>>,
checkpoint_ids: Arc<StdMutex<Vec<String>>>,
cancel: tokio_util::sync::CancellationToken,
}
impl RunHandle {
pub(crate) fn subscribe(&self) -> broadcast::Receiver<SseFrame> {
self.sink.bcast.subscribe()
}
pub(crate) fn log_snapshot(&self) -> Vec<SseFrame> {
lock_recover(&self.sink.log).iter().cloned().collect()
}
}
pub(crate) enum ScheduleDecision {
Started,
Queued,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum RunCancel {
Signalled,
CancelledQueued,
Terminal,
Unknown,
}
#[derive(Debug, Default)]
pub(crate) struct ThreadCancellation {
pub signalled: Vec<String>,
pub cancelled: Vec<String>,
}
impl ThreadCancellation {
pub(crate) fn is_empty(&self) -> bool {
self.signalled.is_empty() && self.cancelled.is_empty()
}
}
#[derive(Default)]
struct RunManagerInner {
runs: HashMap<String, RunHandle>,
active_by_thread: HashMap<String, String>,
queues: HashMap<String, VecDeque<String>>,
attempts: HashMap<String, usize>,
order: VecDeque<String>,
}
const MAX_RETAINED_RUNS: usize = 1024;
#[derive(Default, Clone)]
pub struct RunManager {
inner: Arc<Mutex<RunManagerInner>>,
}
impl RunManager {
pub fn new() -> Self {
Self::default()
}
pub(crate) async fn insert(
&self,
mut handle: RunHandle,
strategy: MultitaskStrategy,
queue_cap: usize,
) -> Result<ScheduleDecision, ApiError> {
let mut inner = self.inner.lock().await;
let busy = inner.active_by_thread.contains_key(&handle.thread_id);
let attempt = {
let counter = inner.attempts.entry(handle.thread_id.clone()).or_insert(0);
*counter += 1;
*counter
};
handle.attempt = attempt;
inner.order.push_back(handle.run_id.clone());
match strategy {
MultitaskStrategy::Reject if busy => Err(ApiError::conflict(format!(
"thread `{}` already has an active run",
handle.thread_id
))),
_ if busy => {
let queue = inner.queues.entry(handle.thread_id.clone()).or_default();
if queue.len() >= queue_cap {
return Err(ApiError::conflict(format!(
"thread `{}` run queue is full (cap {queue_cap})",
handle.thread_id
)));
}
queue.push_back(handle.run_id.clone());
inner.runs.insert(handle.run_id.clone(), handle);
Ok(ScheduleDecision::Queued)
}
_ => {
inner
.active_by_thread
.insert(handle.thread_id.clone(), handle.run_id.clone());
handle.status = RunStatus::Running;
inner.runs.insert(handle.run_id.clone(), handle);
Ok(ScheduleDecision::Started)
}
}
}
pub(crate) async fn snapshot(&self, run_id: &str) -> Option<RunSnapshot> {
let inner = self.inner.lock().await;
inner.runs.get(run_id).map(|h| RunSnapshot {
thread_id: h.thread_id.clone(),
wire_thread_id: h.wire_thread_id.clone(),
graph: h.graph.clone(),
attempt: h.attempt,
payload: h.payload.clone(),
admission: h.admission.clone(),
deployment: h.deployment.clone(),
sink: h.sink.clone(),
checkpoint_ids: Arc::clone(&h.checkpoint_ids),
cancel: h.cancel.clone(),
})
}
pub(crate) async fn info(&self, run_id: &str) -> Option<RunInfo> {
let inner = self.inner.lock().await;
inner.runs.get(run_id).map(|h| RunInfo {
thread_id: h.thread_id.clone(),
wire_thread_id: h.wire_thread_id.clone(),
graph: h.graph.clone(),
attempt: h.attempt,
status: h.status,
terminal: h.terminal.borrow().clone(),
checkpoint_ids: Arc::clone(&h.checkpoint_ids),
})
}
pub(crate) async fn stream_parts(
&self,
run_id: &str,
) -> Option<(Vec<SseFrame>, broadcast::Receiver<SseFrame>, String)> {
let inner = self.inner.lock().await;
inner
.runs
.get(run_id)
.map(|h| (h.log_snapshot(), h.subscribe(), h.thread_id.clone()))
}
pub(crate) async fn thread_busy(&self, thread_id: &str) -> bool {
let inner = self.inner.lock().await;
inner.active_by_thread.contains_key(thread_id)
|| inner
.queues
.get(thread_id)
.is_some_and(|queue| !queue.is_empty())
}
pub(crate) async fn cancel_run(&self, run_id: &str) -> RunCancel {
let mut inner = self.inner.lock().await;
let Some(handle) = inner.runs.get_mut(run_id) else {
return RunCancel::Unknown;
};
match handle.status {
RunStatus::Running => {
handle.cancel.cancel();
RunCancel::Signalled
}
RunStatus::Pending => {
let thread_id = handle.thread_id.clone();
let wire_thread_id = handle.wire_thread_id.clone();
if let Some(queue) = inner.queues.get_mut(&thread_id) {
queue.retain(|queued| queued != run_id);
}
let handle = inner
.runs
.get_mut(run_id)
.expect("the handle was resolved above");
handle.status = RunStatus::Cancelled;
let terminal = json!({
"run_id": run_id,
"thread_id": wire_thread_id,
"status": "cancelled",
"message": "cancelled while queued, before its first step",
});
handle.terminal.send_replace(Some(terminal));
RunCancel::CancelledQueued
}
_ => RunCancel::Terminal,
}
}
pub(crate) async fn cancel_thread_runs(&self, thread_id: &str) -> ThreadCancellation {
let (active, queued) = {
let inner = self.inner.lock().await;
(
inner.active_by_thread.get(thread_id).cloned(),
inner
.queues
.get(thread_id)
.map(|q| q.iter().cloned().collect::<Vec<_>>())
.unwrap_or_default(),
)
};
let mut outcome = ThreadCancellation::default();
if let Some(run_id) = active {
if matches!(self.cancel_run(&run_id).await, RunCancel::Signalled) {
outcome.signalled.push(run_id);
}
}
for run_id in queued {
if matches!(self.cancel_run(&run_id).await, RunCancel::CancelledQueued) {
outcome.cancelled.push(run_id);
}
}
outcome
}
pub(crate) async fn finish(
&self,
run_id: &str,
status: RunStatus,
terminal: Value,
draining: bool,
) -> Option<String> {
let mut inner = self.inner.lock().await;
let handle = inner.runs.get_mut(run_id)?;
handle.status = status;
handle.terminal.send_replace(Some(terminal));
let thread_id = handle.thread_id.clone();
if inner
.active_by_thread
.get(&thread_id)
.is_some_and(|active| active == run_id)
{
inner.active_by_thread.remove(&thread_id);
}
let next = if draining {
None
} else {
inner
.queues
.get_mut(&thread_id)
.and_then(VecDeque::pop_front)
};
if let Some(next_id) = &next {
if let Some(h) = inner.runs.get_mut(next_id) {
h.status = RunStatus::Running;
}
inner.active_by_thread.insert(thread_id, next_id.clone());
}
let mut excess = inner.runs.len().saturating_sub(MAX_RETAINED_RUNS);
let mut skipped = Vec::new();
while excess > 0 {
let Some(candidate) = inner.order.pop_front() else {
break;
};
let evictable = inner
.runs
.get(&candidate)
.is_some_and(|h| h.status.is_terminal());
if evictable {
inner.runs.remove(&candidate);
excess -= 1;
} else {
skipped.push(candidate);
}
}
inner.order.extend(skipped);
next
}
}
#[derive(Clone)]
pub(crate) struct RunDeps {
pub registry: GraphRegistry,
pub checkpointer: Arc<dyn Checkpointer>,
pub manager: RunManager,
pub server_store: Arc<dyn ServerStore>,
pub queue_cap: usize,
pub log_capacity: usize,
pub shutdown: tokio_util::sync::CancellationToken,
pub default_environment_tag: Option<rusty_agent_runtime::learn::EnvironmentTag>,
}
pub(crate) struct Scheduled {
pub run_id: String,
pub status: RunStatus,
pub terminal: watch::Receiver<Option<Value>>,
pub broadcast: broadcast::Receiver<SseFrame>,
pub replay: Vec<SseFrame>,
}
pub(crate) async fn schedule(
deps: &RunDeps,
thread_id: &str,
wire_thread_id: &str,
graph: &str,
payload: RunPayload,
strategy: MultitaskStrategy,
) -> Result<Scheduled, ApiError> {
if deps.shutdown.is_cancelled() {
return Err(ApiError::shutting_down(format!(
"server is draining; resubmit run on thread `{wire_thread_id}` against a running instance"
)));
}
let run_id = uuid::Uuid::new_v4().to_string();
let admission = match &payload.registry {
Some(binding) => Some(
crate::registry::resolve_admission(
&deps.server_store,
crate::auth::tenant_of_internal(thread_id),
deps.default_environment_tag.as_ref(),
&run_id,
binding,
)
.await?,
),
None => None,
};
let deployment = match &payload.deployment {
Some(binding) => {
let (graph_obj, _spec) = deps.registry.get(graph).ok_or_else(|| {
ApiError::internal(format!(
"graph `{graph}` left the registry between route validation and admission"
))
})?;
Some(
crate::deploy::resolve_admission(
&deps.server_store,
crate::auth::tenant_of_internal(thread_id),
&run_id,
binding,
graph,
&graph_obj.topology_hash(),
)
.await?,
)
}
None => None,
};
let (bcast_tx, _bcast_rx) = broadcast::channel(256);
let (terminal_tx, terminal_rx) = watch::channel(None);
let handle = RunHandle {
run_id: run_id.clone(),
thread_id: thread_id.to_string(),
wire_thread_id: wire_thread_id.to_string(),
graph: graph.to_string(),
attempt: 0, status: RunStatus::Pending,
payload,
admission,
deployment,
sink: FrameSink::new(deps.log_capacity, bcast_tx),
terminal: terminal_tx,
checkpoint_ids: Arc::new(StdMutex::new(Vec::new())),
cancel: deps.shutdown.child_token(),
};
let replay = handle.log_snapshot();
let broadcast = handle.subscribe();
let decision = deps
.manager
.insert(handle, strategy, deps.queue_cap)
.await?;
let status = match decision {
ScheduleDecision::Started => {
spawn_execute(deps.clone(), run_id.clone());
RunStatus::Running
}
ScheduleDecision::Queued => RunStatus::Pending,
};
Ok(Scheduled {
run_id,
status,
terminal: terminal_rx,
broadcast,
replay,
})
}
async fn execute(deps: RunDeps, run_id: String) {
let Some(snap) = deps.manager.snapshot(&run_id).await else {
tracing::warn!(%run_id, "scheduled run vanished before execution");
return;
};
let sink = snap.sink.clone();
sink.push(
"metadata",
0,
json!({
"run_id": run_id,
"thread_id": snap.wire_thread_id,
"graph": snap.graph,
"attempt": snap.attempt,
"metadata": snap.payload.metadata,
}),
);
let Some((graph, spec)) = deps.registry.get(&snap.graph) else {
let message = format!("graph `{}` is no longer registered", snap.graph);
tracing::error!(%run_id, %message);
sink.push(
"error",
0,
json!({"error": "unknown_graph", "message": message}),
);
sink.push("end", 0, json!({"status": "error"}));
let terminal = json!({
"run_id": run_id,
"thread_id": snap.wire_thread_id,
"status": "error",
"error": "unknown_graph",
"message": message,
});
terminate(&deps, &run_id, RunStatus::Error, terminal).await;
return;
};
let modes: Vec<String> = snap
.payload
.stream_mode
.clone()
.unwrap_or_else(|| vec!["values".to_string(), "updates".to_string()]);
let journal = Journal::new(run_id.clone(), snap.wire_thread_id.clone(), Clock::System);
let (evt_tx, evt_rx) = mpsc::channel::<GraphEvent>(256);
let forwarder = tokio::spawn(forward_events(
evt_rx,
sink.clone(),
ForwardDeps {
checkpointer: Arc::clone(&deps.checkpointer),
server_store: Arc::clone(&deps.server_store),
journal: journal.clone(),
thread_id: snap.thread_id.clone(),
checkpoint_ids: Arc::clone(&snap.checkpoint_ids),
modes,
},
));
let mut config = RunConfig::new(snap.thread_id.clone())
.with_event_tx(evt_tx)
.with_journal(journal.clone())
.with_cancellation(snap.cancel.clone());
let mut parent = None;
if let Some(admission) = &snap.admission {
for resolution in &admission.resolutions {
let output =
serde_json::to_value(resolution).expect("ConfigResolution always serializes");
let mut draft =
EventDraft::new(RunEventKind::ConfigResolved, Effect::ReadOnly).output(output);
if let Some(parent) = parent {
draft = draft.parent(parent);
}
parent = Some(journal.record(draft));
}
config = config.with_manifest(admission.manifest.clone());
}
if let Some(deployment) = &snap.deployment {
let output =
serde_json::to_value(&deployment.resolution).expect("DeploymentResolved serializes");
let mut draft =
EventDraft::new(RunEventKind::DeploymentResolved, Effect::ReadOnly).output(output);
if let Some(parent) = parent {
draft = draft.parent(parent);
}
journal.record(draft);
}
if let Some(command) = &snap.payload.command {
if let Some(resume) = &command.resume {
config = config.with_resume(resume.clone());
}
}
if let Some(checkpoint) = &snap.payload.checkpoint {
config = config.with_checkpoint_id(checkpoint.checkpoint_id.clone());
}
if let Some(run_cfg) = &snap.payload.config {
if let Some(limit) = run_cfg.recursion_limit {
config = config.with_max_steps(limit);
}
}
let initial = snap
.payload
.input
.clone()
.and_then(|v| State::from_value(v).ok())
.unwrap_or_default();
let mut executor = Executor::with_checkpointer(Arc::clone(&deps.checkpointer));
if let Some(admission) = &snap.admission {
if let Some(chain) = &admission.middleware {
for layer in chain.layers() {
executor = executor.layer_shared(Arc::clone(layer));
}
}
}
let result = executor.run(&graph, &spec, initial, config).await;
let _ = forwarder.await;
persist_journal(&deps.server_store, &journal).await;
let step = sink.current_step();
let (status, terminal) = match result {
Ok(ExecutionOutcome::Done(state)) => {
sink.push("end", step, json!({"status": "success"}));
let terminal = json!({
"run_id": run_id,
"thread_id": snap.wire_thread_id,
"status": "success",
"output": state.to_value(),
});
(RunStatus::Success, terminal)
}
Ok(ExecutionOutcome::Interrupted {
value,
state,
checkpoint_id,
}) => {
sink.push(
"end",
step,
json!({"status": "interrupted", "interrupt": value}),
);
let terminal = json!({
"run_id": run_id,
"thread_id": snap.wire_thread_id,
"status": "interrupted",
"interrupt": value,
"checkpoint_id": checkpoint_id,
"state": state.to_value(),
});
(RunStatus::Interrupted, terminal)
}
Err(error @ RustyError::Cancelled(_)) => {
let message = error.to_string();
tracing::info!(%run_id, "run drained at a checkpoint boundary; resumable");
sink.push("end", step, json!({"status": "cancelled"}));
let terminal = json!({
"run_id": run_id,
"thread_id": snap.wire_thread_id,
"status": "cancelled",
"message": message,
});
(RunStatus::Cancelled, terminal)
}
Err(error) => {
let kind = error_kind(&error);
let message = error.to_string();
tracing::warn!(%run_id, %error, "run failed");
sink.push("error", step, json!({"error": kind, "message": message}));
sink.push("end", step, json!({"status": "error"}));
let terminal = json!({
"run_id": run_id,
"thread_id": snap.wire_thread_id,
"status": "error",
"error": kind,
"message": message,
});
(RunStatus::Error, terminal)
}
};
terminate(&deps, &run_id, status, terminal).await;
}
struct ForwardDeps {
checkpointer: Arc<dyn Checkpointer>,
server_store: Arc<dyn ServerStore>,
journal: Journal,
thread_id: String,
checkpoint_ids: Arc<StdMutex<Vec<String>>>,
modes: Vec<String>,
}
async fn forward_events(mut rx: mpsc::Receiver<GraphEvent>, sink: FrameSink, deps: ForwardDeps) {
let ForwardDeps {
checkpointer,
server_store,
journal,
thread_id,
checkpoint_ids,
modes,
} = deps;
while let Some(event) = rx.recv().await {
match event {
GraphEvent::StateUpdate { step, updates } => {
if modes.iter().any(|m| m == "updates") {
sink.push("updates", step, json!({"step": step, "updates": updates}));
}
}
GraphEvent::Token { node, delta } => {
if modes.iter().any(|m| m == "messages") {
let step = sink.current_step();
sink.push("messages", step, json!({"node": node, "delta": delta}));
}
}
GraphEvent::CheckpointSaved {
checkpoint_id,
step,
} => {
lock_recover(&checkpoint_ids).push(checkpoint_id.clone());
sink.note_checkpoint(&checkpoint_id);
persist_journal(&server_store, &journal).await;
if modes.iter().any(|m| m == "values") {
match read_back_state(&*checkpointer, &thread_id, &checkpoint_id).await {
Ok(Some(values)) => sink.push("values", step, values),
Ok(None) => {
tracing::debug!(%checkpoint_id, "checkpoint not found for values frame")
}
Err(error) => {
tracing::warn!(%checkpoint_id, %error, "values frame read-back failed")
}
}
}
}
GraphEvent::SuperStep { .. }
| GraphEvent::NodeStart { .. }
| GraphEvent::NodeEnd { .. } => {}
}
}
}
async fn persist_journal(server_store: &Arc<dyn ServerStore>, journal: &Journal) {
if let Err(error) = server_store.put_journal(&journal.snapshot()).await {
tracing::warn!(run_id = %journal.run_id(), %error, "journal persistence failed");
}
}
async fn read_back_state(
checkpointer: &dyn Checkpointer,
thread_id: &str,
checkpoint_id: &str,
) -> rusty_agent_runtime::error::Result<Option<Value>> {
Ok(checkpointer
.get_by_id(thread_id, checkpoint_id)
.await?
.map(|cp| cp.state.to_value()))
}
fn spawn_execute(deps: RunDeps, run_id: String) {
let fut: std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> = Box::pin({
let deps = deps.clone();
let run_id = run_id.clone();
async move { execute(deps, run_id).await }
});
tokio::spawn(async move {
if AssertUnwindSafe(fut).catch_unwind().await.is_ok() {
return;
}
tracing::error!(%run_id, "run task panicked; force-finishing as error");
let Some(snap) = deps.manager.snapshot(&run_id).await else {
return;
};
if matches!(deps.manager.info(&run_id).await, Some(info) if info.status.is_terminal()) {
return;
}
let step = snap.sink.current_step();
snap.sink.push(
"error",
step,
json!({"error": "internal_panic", "message": "run task panicked"}),
);
snap.sink.push("end", step, json!({"status": "error"}));
let terminal = json!({
"run_id": run_id,
"thread_id": snap.wire_thread_id,
"status": "error",
"error": "internal_panic",
"message": "run task panicked",
});
terminate(&deps, &run_id, RunStatus::Error, terminal).await;
});
}
async fn terminate(deps: &RunDeps, run_id: &str, status: RunStatus, terminal: Value) {
let draining = deps.shutdown.is_cancelled();
if let Some(next) = deps
.manager
.finish(run_id, status, terminal, draining)
.await
{
spawn_execute(deps.clone(), next);
}
}
fn error_kind(error: &RustyError) -> &'static str {
match error {
RustyError::Graph(_) => "graph_error",
RustyError::Node(_) => "node_error",
RustyError::Interrupt { .. } => "interrupted",
RustyError::Checkpoint(_) => "checkpoint_error",
RustyError::Llm(_) => "llm_error",
RustyError::Tool(_) => "tool_error",
RustyError::Serialization(_) => "serialization_error",
RustyError::InvalidUpdate(_) => "invalid_update",
RustyError::Replay(_) => "replay_error",
RustyError::Cancelled(_) => "cancelled",
}
}