1use std::collections::{HashMap, VecDeque};
34use std::panic::AssertUnwindSafe;
35use std::sync::atomic::{AtomicU64, Ordering};
36use std::sync::{Arc, Mutex as StdMutex};
37
38use futures::FutureExt;
39use rusty_agent_runtime::checkpoint::Checkpointer;
40use rusty_agent_runtime::error::RustyError;
41use rusty_agent_runtime::executor::{ExecutionOutcome, Executor, GraphEvent, RunConfig};
42use rusty_agent_runtime::journal::{Clock, Journal};
43use rusty_agent_runtime::state::State;
44use serde::Deserialize;
45use serde_json::{json, Value};
46use tokio::sync::{broadcast, mpsc, watch, Mutex};
47
48use crate::error::ApiError;
49use crate::server_store::ServerStore;
50use crate::GraphRegistry;
51
52#[derive(Debug, Clone, Default, Deserialize)]
59pub struct CommandPayload {
60 #[serde(default)]
62 pub resume: Option<Value>,
63}
64
65#[derive(Debug, Clone, Default, Deserialize)]
67pub struct RunConfigPayload {
68 #[serde(default)]
70 pub recursion_limit: Option<usize>,
71}
72
73#[derive(Debug, Clone, Default, Deserialize)]
77pub struct CheckpointPayload {
78 pub checkpoint_id: String,
80}
81
82#[derive(Debug, Clone, Default, Deserialize)]
84pub struct RunPayload {
85 #[serde(default)]
88 pub input: Option<Value>,
89
90 #[serde(default)]
92 pub command: Option<CommandPayload>,
93
94 #[serde(default)]
96 pub config: Option<RunConfigPayload>,
97
98 #[serde(default)]
103 pub checkpoint: Option<CheckpointPayload>,
104
105 #[serde(default)]
107 pub metadata: Option<Value>,
108
109 #[serde(default)]
113 pub stream_mode: Option<Vec<String>>,
114
115 #[serde(default)]
118 pub multitask_strategy: Option<String>,
119
120 #[serde(default)]
124 pub assistant_id: Option<String>,
125}
126
127#[derive(Debug, Clone, Copy, PartialEq, Eq)]
129pub enum MultitaskStrategy {
130 Enqueue,
132 Reject,
134}
135
136impl MultitaskStrategy {
137 pub fn parse(raw: Option<&str>) -> Result<Self, String> {
139 match raw {
140 None | Some("enqueue") => Ok(Self::Enqueue),
141 Some("reject") => Ok(Self::Reject),
142 Some(other) => Err(format!(
143 "unknown multitask_strategy `{other}` (expected `enqueue` or `reject`)"
144 )),
145 }
146 }
147}
148
149#[derive(Debug, Clone, Copy, PartialEq, Eq)]
155pub enum RunStatus {
156 Pending,
158 Running,
160 Success,
162 Interrupted,
164 Error,
166 Cancelled,
171}
172
173impl RunStatus {
174 pub fn as_str(&self) -> &'static str {
176 match self {
177 RunStatus::Pending => "pending",
178 RunStatus::Running => "running",
179 RunStatus::Success => "success",
180 RunStatus::Interrupted => "interrupted",
181 RunStatus::Error => "error",
182 RunStatus::Cancelled => "cancelled",
183 }
184 }
185
186 pub(crate) fn is_terminal(self) -> bool {
190 matches!(
191 self,
192 RunStatus::Success | RunStatus::Interrupted | RunStatus::Error | RunStatus::Cancelled
193 )
194 }
195}
196
197#[derive(Debug, Clone)]
200pub struct SseFrame {
201 pub id: String,
203 pub event: String,
205 pub data: Value,
207 pub seq: u64,
209}
210
211#[derive(Clone)]
214pub(crate) struct FrameSink {
215 log: Arc<StdMutex<VecDeque<SseFrame>>>,
216 bcast: broadcast::Sender<SseFrame>,
217 seq: Arc<AtomicU64>,
218 last_checkpoint: Arc<StdMutex<String>>,
219 last_step: Arc<AtomicU64>,
220 capacity: usize,
221}
222
223pub(crate) fn lock_recover<T>(mutex: &StdMutex<T>) -> std::sync::MutexGuard<'_, T> {
229 mutex
230 .lock()
231 .unwrap_or_else(|poisoned| poisoned.into_inner())
232}
233
234impl FrameSink {
235 fn new(capacity: usize, bcast: broadcast::Sender<SseFrame>) -> Self {
236 Self {
237 log: Arc::new(StdMutex::new(VecDeque::new())),
238 bcast,
239 seq: Arc::new(AtomicU64::new(0)),
240 last_checkpoint: Arc::new(StdMutex::new("-".to_string())),
241 last_step: Arc::new(AtomicU64::new(0)),
242 capacity,
243 }
244 }
245
246 pub(crate) fn push(&self, event: &str, step: usize, data: Value) {
248 let seq = self.seq.fetch_add(1, Ordering::Relaxed) + 1;
249 self.last_step.store(step as u64, Ordering::Relaxed);
250 let checkpoint = lock_recover(&self.last_checkpoint).clone();
251 let frame = SseFrame {
252 id: format!("{checkpoint}:{step}:{seq}"),
253 event: event.to_string(),
254 data,
255 seq,
256 };
257 {
258 let mut log = lock_recover(&self.log);
259 if log.len() >= self.capacity {
260 log.pop_front();
261 }
262 log.push_back(frame.clone());
263 }
264 let _ = self.bcast.send(frame);
266 }
267
268 pub(crate) fn note_checkpoint(&self, checkpoint_id: &str) {
270 *lock_recover(&self.last_checkpoint) = checkpoint_id.to_string();
271 }
272
273 pub(crate) fn current_step(&self) -> usize {
275 self.last_step.load(Ordering::Relaxed) as usize
276 }
277}
278
279pub(crate) struct RunSnapshot {
281 pub thread_id: String,
284 pub wire_thread_id: String,
287 pub graph: String,
288 pub attempt: usize,
289 pub payload: RunPayload,
290 pub sink: FrameSink,
291 pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
292 pub cancel: tokio_util::sync::CancellationToken,
299}
300
301pub(crate) struct RunInfo {
303 pub thread_id: String,
306 pub wire_thread_id: String,
308 pub graph: String,
309 pub attempt: usize,
310 pub status: RunStatus,
311 pub terminal: Option<Value>,
313 pub checkpoint_ids: Arc<StdMutex<Vec<String>>>,
314}
315
316pub struct RunHandle {
319 pub(crate) run_id: String,
321 pub(crate) thread_id: String,
323 pub(crate) wire_thread_id: String,
325 pub(crate) graph: String,
327 pub(crate) attempt: usize,
329 pub(crate) status: RunStatus,
331 pub(crate) payload: RunPayload,
333 sink: FrameSink,
334 terminal: watch::Sender<Option<Value>>,
335 checkpoint_ids: Arc<StdMutex<Vec<String>>>,
336 cancel: tokio_util::sync::CancellationToken,
341}
342
343impl RunHandle {
344 pub(crate) fn subscribe(&self) -> broadcast::Receiver<SseFrame> {
346 self.sink.bcast.subscribe()
347 }
348
349 pub(crate) fn log_snapshot(&self) -> Vec<SseFrame> {
351 lock_recover(&self.sink.log).iter().cloned().collect()
352 }
353}
354
355pub(crate) enum ScheduleDecision {
357 Started,
359 Queued,
361}
362
363#[derive(Debug, Clone, Copy, PartialEq, Eq)]
365pub(crate) enum RunCancel {
366 Signalled,
370 CancelledQueued,
373 Terminal,
375 Unknown,
377}
378
379#[derive(Debug, Default)]
383pub(crate) struct ThreadCancellation {
384 pub signalled: Vec<String>,
387 pub cancelled: Vec<String>,
389}
390
391impl ThreadCancellation {
392 pub(crate) fn is_empty(&self) -> bool {
396 self.signalled.is_empty() && self.cancelled.is_empty()
397 }
398}
399
400#[derive(Default)]
401struct RunManagerInner {
402 runs: HashMap<String, RunHandle>,
403 active_by_thread: HashMap<String, String>,
404 queues: HashMap<String, VecDeque<String>>,
405 attempts: HashMap<String, usize>,
406 order: VecDeque<String>,
408}
409
410const MAX_RETAINED_RUNS: usize = 1024;
415
416#[derive(Default, Clone)]
419pub struct RunManager {
420 inner: Arc<Mutex<RunManagerInner>>,
421}
422
423impl RunManager {
424 pub fn new() -> Self {
426 Self::default()
427 }
428
429 pub(crate) async fn insert(
432 &self,
433 mut handle: RunHandle,
434 strategy: MultitaskStrategy,
435 queue_cap: usize,
436 ) -> Result<ScheduleDecision, ApiError> {
437 let mut inner = self.inner.lock().await;
438 let busy = inner.active_by_thread.contains_key(&handle.thread_id);
439 let attempt = {
440 let counter = inner.attempts.entry(handle.thread_id.clone()).or_insert(0);
441 *counter += 1;
442 *counter
443 };
444 handle.attempt = attempt;
445 inner.order.push_back(handle.run_id.clone());
446
447 match strategy {
448 MultitaskStrategy::Reject if busy => Err(ApiError::conflict(format!(
449 "thread `{}` already has an active run",
450 handle.thread_id
451 ))),
452 _ if busy => {
453 let queue = inner.queues.entry(handle.thread_id.clone()).or_default();
454 if queue.len() >= queue_cap {
455 return Err(ApiError::conflict(format!(
456 "thread `{}` run queue is full (cap {queue_cap})",
457 handle.thread_id
458 )));
459 }
460 queue.push_back(handle.run_id.clone());
461 inner.runs.insert(handle.run_id.clone(), handle);
462 Ok(ScheduleDecision::Queued)
463 }
464 _ => {
465 inner
466 .active_by_thread
467 .insert(handle.thread_id.clone(), handle.run_id.clone());
468 handle.status = RunStatus::Running;
469 inner.runs.insert(handle.run_id.clone(), handle);
470 Ok(ScheduleDecision::Started)
471 }
472 }
473 }
474
475 pub(crate) async fn snapshot(&self, run_id: &str) -> Option<RunSnapshot> {
477 let inner = self.inner.lock().await;
478 inner.runs.get(run_id).map(|h| RunSnapshot {
479 thread_id: h.thread_id.clone(),
480 wire_thread_id: h.wire_thread_id.clone(),
481 graph: h.graph.clone(),
482 attempt: h.attempt,
483 payload: h.payload.clone(),
484 sink: h.sink.clone(),
485 checkpoint_ids: Arc::clone(&h.checkpoint_ids),
486 cancel: h.cancel.clone(),
487 })
488 }
489
490 pub(crate) async fn info(&self, run_id: &str) -> Option<RunInfo> {
492 let inner = self.inner.lock().await;
493 inner.runs.get(run_id).map(|h| RunInfo {
494 thread_id: h.thread_id.clone(),
495 wire_thread_id: h.wire_thread_id.clone(),
496 graph: h.graph.clone(),
497 attempt: h.attempt,
498 status: h.status,
499 terminal: h.terminal.borrow().clone(),
500 checkpoint_ids: Arc::clone(&h.checkpoint_ids),
501 })
502 }
503
504 pub(crate) async fn stream_parts(
507 &self,
508 run_id: &str,
509 ) -> Option<(Vec<SseFrame>, broadcast::Receiver<SseFrame>, String)> {
510 let inner = self.inner.lock().await;
511 inner
512 .runs
513 .get(run_id)
514 .map(|h| (h.log_snapshot(), h.subscribe(), h.thread_id.clone()))
515 }
516
517 pub(crate) async fn thread_busy(&self, thread_id: &str) -> bool {
520 let inner = self.inner.lock().await;
521 inner.active_by_thread.contains_key(thread_id)
522 || inner
523 .queues
524 .get(thread_id)
525 .is_some_and(|queue| !queue.is_empty())
526 }
527
528 pub(crate) async fn cancel_run(&self, run_id: &str) -> RunCancel {
537 let mut inner = self.inner.lock().await;
538 let Some(handle) = inner.runs.get_mut(run_id) else {
539 return RunCancel::Unknown;
540 };
541 match handle.status {
542 RunStatus::Running => {
543 handle.cancel.cancel();
544 RunCancel::Signalled
545 }
546 RunStatus::Pending => {
547 let thread_id = handle.thread_id.clone();
548 let wire_thread_id = handle.wire_thread_id.clone();
549 if let Some(queue) = inner.queues.get_mut(&thread_id) {
550 queue.retain(|queued| queued != run_id);
551 }
552 let handle = inner
553 .runs
554 .get_mut(run_id)
555 .expect("the handle was resolved above");
556 handle.status = RunStatus::Cancelled;
557 let terminal = json!({
558 "run_id": run_id,
559 "thread_id": wire_thread_id,
560 "status": "cancelled",
561 "message": "cancelled while queued, before its first step",
562 });
563 handle.terminal.send_replace(Some(terminal));
564 RunCancel::CancelledQueued
565 }
566 _ => RunCancel::Terminal,
570 }
571 }
572
573 pub(crate) async fn cancel_thread_runs(&self, thread_id: &str) -> ThreadCancellation {
578 let (active, queued) = {
579 let inner = self.inner.lock().await;
580 (
581 inner.active_by_thread.get(thread_id).cloned(),
582 inner
583 .queues
584 .get(thread_id)
585 .map(|q| q.iter().cloned().collect::<Vec<_>>())
586 .unwrap_or_default(),
587 )
588 };
589 let mut outcome = ThreadCancellation::default();
590 if let Some(run_id) = active {
591 if matches!(self.cancel_run(&run_id).await, RunCancel::Signalled) {
592 outcome.signalled.push(run_id);
593 }
594 }
595 for run_id in queued {
596 if matches!(self.cancel_run(&run_id).await, RunCancel::CancelledQueued) {
597 outcome.cancelled.push(run_id);
598 }
599 }
600 outcome
601 }
602
603 pub(crate) async fn finish(
613 &self,
614 run_id: &str,
615 status: RunStatus,
616 terminal: Value,
617 draining: bool,
618 ) -> Option<String> {
619 let mut inner = self.inner.lock().await;
620 let handle = inner.runs.get_mut(run_id)?;
621 handle.status = status;
622 handle.terminal.send_replace(Some(terminal));
626 let thread_id = handle.thread_id.clone();
627
628 if inner
629 .active_by_thread
630 .get(&thread_id)
631 .is_some_and(|active| active == run_id)
632 {
633 inner.active_by_thread.remove(&thread_id);
634 }
635
636 let next = if draining {
637 None
638 } else {
639 inner
640 .queues
641 .get_mut(&thread_id)
642 .and_then(VecDeque::pop_front)
643 };
644 if let Some(next_id) = &next {
645 if let Some(h) = inner.runs.get_mut(next_id) {
646 h.status = RunStatus::Running;
647 }
648 inner.active_by_thread.insert(thread_id, next_id.clone());
649 }
650
651 let mut excess = inner.runs.len().saturating_sub(MAX_RETAINED_RUNS);
654 let mut skipped = Vec::new();
655 while excess > 0 {
656 let Some(candidate) = inner.order.pop_front() else {
657 break;
658 };
659 let evictable = inner
660 .runs
661 .get(&candidate)
662 .is_some_and(|h| h.status.is_terminal());
663 if evictable {
664 inner.runs.remove(&candidate);
665 excess -= 1;
666 } else {
667 skipped.push(candidate);
668 }
669 }
670 inner.order.extend(skipped);
671
672 next
673 }
674}
675
676#[derive(Clone)]
679pub(crate) struct RunDeps {
680 pub registry: GraphRegistry,
681 pub checkpointer: Arc<dyn Checkpointer>,
682 pub manager: RunManager,
683 pub server_store: Arc<dyn ServerStore>,
685 pub queue_cap: usize,
686 pub log_capacity: usize,
687 pub shutdown: tokio_util::sync::CancellationToken,
690}
691
692pub(crate) struct Scheduled {
695 pub run_id: String,
696 pub status: RunStatus,
697 pub terminal: watch::Receiver<Option<Value>>,
698 pub broadcast: broadcast::Receiver<SseFrame>,
699 pub replay: Vec<SseFrame>,
700}
701
702pub(crate) async fn schedule(
709 deps: &RunDeps,
710 thread_id: &str,
711 wire_thread_id: &str,
712 graph: &str,
713 payload: RunPayload,
714 strategy: MultitaskStrategy,
715) -> Result<Scheduled, ApiError> {
716 if deps.shutdown.is_cancelled() {
720 return Err(ApiError::shutting_down(format!(
721 "server is draining; resubmit run on thread `{wire_thread_id}` against a running instance"
722 )));
723 }
724 let run_id = uuid::Uuid::new_v4().to_string();
725 let (bcast_tx, _bcast_rx) = broadcast::channel(256);
726 let (terminal_tx, terminal_rx) = watch::channel(None);
727 let handle = RunHandle {
728 run_id: run_id.clone(),
729 thread_id: thread_id.to_string(),
730 wire_thread_id: wire_thread_id.to_string(),
731 graph: graph.to_string(),
732 attempt: 0, status: RunStatus::Pending,
734 payload,
735 sink: FrameSink::new(deps.log_capacity, bcast_tx),
736 terminal: terminal_tx,
737 checkpoint_ids: Arc::new(StdMutex::new(Vec::new())),
738 cancel: deps.shutdown.child_token(),
741 };
742 let replay = handle.log_snapshot();
744 let broadcast = handle.subscribe();
745
746 let decision = deps
747 .manager
748 .insert(handle, strategy, deps.queue_cap)
749 .await?;
750 let status = match decision {
751 ScheduleDecision::Started => {
752 spawn_execute(deps.clone(), run_id.clone());
753 RunStatus::Running
754 }
755 ScheduleDecision::Queued => RunStatus::Pending,
756 };
757
758 Ok(Scheduled {
759 run_id,
760 status,
761 terminal: terminal_rx,
762 broadcast,
763 replay,
764 })
765}
766
767async fn execute(deps: RunDeps, run_id: String) {
769 let Some(snap) = deps.manager.snapshot(&run_id).await else {
770 tracing::warn!(%run_id, "scheduled run vanished before execution");
771 return;
772 };
773 let sink = snap.sink.clone();
774 sink.push(
775 "metadata",
776 0,
777 json!({
778 "run_id": run_id,
779 "thread_id": snap.wire_thread_id,
780 "graph": snap.graph,
781 "attempt": snap.attempt,
782 "metadata": snap.payload.metadata,
783 }),
784 );
785
786 let Some((graph, spec)) = deps.registry.get(&snap.graph) else {
787 let message = format!("graph `{}` is no longer registered", snap.graph);
788 tracing::error!(%run_id, %message);
789 sink.push(
790 "error",
791 0,
792 json!({"error": "unknown_graph", "message": message}),
793 );
794 sink.push("end", 0, json!({"status": "error"}));
795 let terminal = json!({
796 "run_id": run_id,
797 "thread_id": snap.wire_thread_id,
798 "status": "error",
799 "error": "unknown_graph",
800 "message": message,
801 });
802 terminate(&deps, &run_id, RunStatus::Error, terminal).await;
803 return;
804 };
805
806 let modes: Vec<String> = snap
807 .payload
808 .stream_mode
809 .clone()
810 .unwrap_or_else(|| vec!["values".to_string(), "updates".to_string()]);
811 let journal = Journal::new(run_id.clone(), snap.wire_thread_id.clone(), Clock::System);
817 let (evt_tx, evt_rx) = mpsc::channel::<GraphEvent>(256);
818 let forwarder = tokio::spawn(forward_events(
819 evt_rx,
820 sink.clone(),
821 ForwardDeps {
822 checkpointer: Arc::clone(&deps.checkpointer),
823 server_store: Arc::clone(&deps.server_store),
824 journal: journal.clone(),
825 thread_id: snap.thread_id.clone(),
826 checkpoint_ids: Arc::clone(&snap.checkpoint_ids),
827 modes,
828 },
829 ));
830
831 let mut config = RunConfig::new(snap.thread_id.clone())
832 .with_event_tx(evt_tx)
833 .with_journal(journal.clone())
834 .with_cancellation(snap.cancel.clone());
839 if let Some(command) = &snap.payload.command {
840 if let Some(resume) = &command.resume {
841 config = config.with_resume(resume.clone());
842 }
843 }
844 if let Some(checkpoint) = &snap.payload.checkpoint {
845 config = config.with_checkpoint_id(checkpoint.checkpoint_id.clone());
846 }
847 if let Some(run_cfg) = &snap.payload.config {
848 if let Some(limit) = run_cfg.recursion_limit {
849 config = config.with_max_steps(limit);
850 }
851 }
852 let initial = snap
853 .payload
854 .input
855 .clone()
856 .and_then(|v| State::from_value(v).ok())
857 .unwrap_or_default();
858
859 let result = Executor::with_checkpointer(Arc::clone(&deps.checkpointer))
860 .run(&graph, &spec, initial, config)
861 .await;
862 let _ = forwarder.await;
865
866 persist_journal(&deps.server_store, &journal).await;
872
873 let step = sink.current_step();
874 let (status, terminal) = match result {
875 Ok(ExecutionOutcome::Done(state)) => {
876 sink.push("end", step, json!({"status": "success"}));
877 let terminal = json!({
878 "run_id": run_id,
879 "thread_id": snap.wire_thread_id,
880 "status": "success",
881 "output": state.to_value(),
882 });
883 (RunStatus::Success, terminal)
884 }
885 Ok(ExecutionOutcome::Interrupted {
886 value,
887 state,
888 checkpoint_id,
889 }) => {
890 sink.push(
891 "end",
892 step,
893 json!({"status": "interrupted", "interrupt": value}),
894 );
895 let terminal = json!({
896 "run_id": run_id,
897 "thread_id": snap.wire_thread_id,
898 "status": "interrupted",
899 "interrupt": value,
900 "checkpoint_id": checkpoint_id,
901 "state": state.to_value(),
902 });
903 (RunStatus::Interrupted, terminal)
904 }
905 Err(error @ RustyError::Cancelled(_)) => {
906 let message = error.to_string();
912 tracing::info!(%run_id, "run drained at a checkpoint boundary; resumable");
913 sink.push("end", step, json!({"status": "cancelled"}));
914 let terminal = json!({
915 "run_id": run_id,
916 "thread_id": snap.wire_thread_id,
917 "status": "cancelled",
918 "message": message,
919 });
920 (RunStatus::Cancelled, terminal)
921 }
922 Err(error) => {
923 let kind = error_kind(&error);
924 let message = error.to_string();
925 tracing::warn!(%run_id, %error, "run failed");
926 sink.push("error", step, json!({"error": kind, "message": message}));
927 sink.push("end", step, json!({"status": "error"}));
928 let terminal = json!({
929 "run_id": run_id,
930 "thread_id": snap.wire_thread_id,
931 "status": "error",
932 "error": kind,
933 "message": message,
934 });
935 (RunStatus::Error, terminal)
936 }
937 };
938 terminate(&deps, &run_id, status, terminal).await;
939}
940
941struct ForwardDeps {
944 checkpointer: Arc<dyn Checkpointer>,
945 server_store: Arc<dyn ServerStore>,
946 journal: Journal,
947 thread_id: String,
949 checkpoint_ids: Arc<StdMutex<Vec<String>>>,
950 modes: Vec<String>,
951}
952
953async fn forward_events(mut rx: mpsc::Receiver<GraphEvent>, sink: FrameSink, deps: ForwardDeps) {
959 let ForwardDeps {
960 checkpointer,
961 server_store,
962 journal,
963 thread_id,
964 checkpoint_ids,
965 modes,
966 } = deps;
967 while let Some(event) = rx.recv().await {
968 match event {
969 GraphEvent::StateUpdate { step, updates } => {
970 if modes.iter().any(|m| m == "updates") {
971 sink.push("updates", step, json!({"step": step, "updates": updates}));
972 }
973 }
974 GraphEvent::Token { node, delta } => {
975 if modes.iter().any(|m| m == "messages") {
976 let step = sink.current_step();
977 sink.push("messages", step, json!({"node": node, "delta": delta}));
978 }
979 }
980 GraphEvent::CheckpointSaved {
981 checkpoint_id,
982 step,
983 } => {
984 lock_recover(&checkpoint_ids).push(checkpoint_id.clone());
985 sink.note_checkpoint(&checkpoint_id);
986 persist_journal(&server_store, &journal).await;
987 if modes.iter().any(|m| m == "values") {
988 match read_back_state(&*checkpointer, &thread_id, &checkpoint_id).await {
989 Ok(Some(values)) => sink.push("values", step, values),
990 Ok(None) => {
991 tracing::debug!(%checkpoint_id, "checkpoint not found for values frame")
992 }
993 Err(error) => {
994 tracing::warn!(%checkpoint_id, %error, "values frame read-back failed")
995 }
996 }
997 }
998 }
999 GraphEvent::SuperStep { .. }
1001 | GraphEvent::NodeStart { .. }
1002 | GraphEvent::NodeEnd { .. } => {}
1003 }
1004 }
1005}
1006
1007async fn persist_journal(server_store: &Arc<dyn ServerStore>, journal: &Journal) {
1012 if let Err(error) = server_store.put_journal(&journal.snapshot()).await {
1013 tracing::warn!(run_id = %journal.run_id(), %error, "journal persistence failed");
1014 }
1015}
1016
1017async fn read_back_state(
1022 checkpointer: &dyn Checkpointer,
1023 thread_id: &str,
1024 checkpoint_id: &str,
1025) -> rusty_agent_runtime::error::Result<Option<Value>> {
1026 Ok(checkpointer
1027 .get_by_id(thread_id, checkpoint_id)
1028 .await?
1029 .map(|cp| cp.state.to_value()))
1030}
1031
1032fn spawn_execute(deps: RunDeps, run_id: String) {
1042 let fut: std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> = Box::pin({
1043 let deps = deps.clone();
1044 let run_id = run_id.clone();
1045 async move { execute(deps, run_id).await }
1046 });
1047 tokio::spawn(async move {
1048 if AssertUnwindSafe(fut).catch_unwind().await.is_ok() {
1049 return;
1050 }
1051 tracing::error!(%run_id, "run task panicked; force-finishing as error");
1052 let Some(snap) = deps.manager.snapshot(&run_id).await else {
1053 return;
1054 };
1055 if matches!(deps.manager.info(&run_id).await, Some(info) if info.status.is_terminal()) {
1058 return;
1059 }
1060 let step = snap.sink.current_step();
1061 snap.sink.push(
1062 "error",
1063 step,
1064 json!({"error": "internal_panic", "message": "run task panicked"}),
1065 );
1066 snap.sink.push("end", step, json!({"status": "error"}));
1067 let terminal = json!({
1068 "run_id": run_id,
1069 "thread_id": snap.wire_thread_id,
1070 "status": "error",
1071 "error": "internal_panic",
1072 "message": "run task panicked",
1073 });
1074 terminate(&deps, &run_id, RunStatus::Error, terminal).await;
1075 });
1076}
1077
1078async fn terminate(deps: &RunDeps, run_id: &str, status: RunStatus, terminal: Value) {
1082 let draining = deps.shutdown.is_cancelled();
1083 if let Some(next) = deps
1084 .manager
1085 .finish(run_id, status, terminal, draining)
1086 .await
1087 {
1088 spawn_execute(deps.clone(), next);
1089 }
1090}
1091
1092fn error_kind(error: &RustyError) -> &'static str {
1094 match error {
1095 RustyError::Graph(_) => "graph_error",
1096 RustyError::Node(_) => "node_error",
1097 RustyError::Interrupt { .. } => "interrupted",
1098 RustyError::Checkpoint(_) => "checkpoint_error",
1099 RustyError::Llm(_) => "llm_error",
1100 RustyError::Tool(_) => "tool_error",
1101 RustyError::Serialization(_) => "serialization_error",
1102 RustyError::InvalidUpdate(_) => "invalid_update",
1103 RustyError::Replay(_) => "replay_error",
1104 RustyError::Cancelled(_) => "cancelled",
1107 }
1108}