Skip to main content

incurs_codemode/
runtime.rs

1use std::collections::BTreeMap;
2use std::sync::Arc;
3use std::sync::atomic::{AtomicU64, Ordering};
4
5use async_trait::async_trait;
6use serde::{Deserialize, Serialize};
7use serde_json::Value;
8use sha2::{Digest, Sha256};
9use thiserror::Error;
10use tokio::sync::Mutex;
11
12use crate::ConnectorDescription;
13
14/// Default number of terminal executions retained by a runtime.
15pub const DEFAULT_MAX_EXECUTIONS: usize = 50;
16/// Maximum serialized bytes for one durable value.
17pub const MAX_DURABLE_VALUE_BYTES: usize = 1_000_000;
18/// Default age after which a paused or abandoned running execution expires.
19pub const DEFAULT_PAUSED_TTL_MS: u64 = 24 * 60 * 60 * 1_000;
20/// Maximum number of ordered lifecycle events retained per execution.
21pub const DEFAULT_MAX_EVENTS: usize = 1_000;
22
23static EXECUTION_SEQUENCE: AtomicU64 = AtomicU64::new(0);
24
25/// Immutable connector metadata captured when an execution begins.
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct CapabilitySnapshot {
28    /// Connector schemas, instructions, annotations, and resolved policies.
29    pub connectors: Vec<ConnectorDescription>,
30    /// Stable SHA-256 fingerprint of the serialized connector set.
31    pub fingerprint: String,
32}
33
34impl CapabilitySnapshot {
35    /// Captures a stable snapshot from resolved connector descriptions.
36    pub fn new(connectors: Vec<ConnectorDescription>) -> Result<Self, String> {
37        let bytes = serde_json::to_vec(&connectors).map_err(|error| error.to_string())?;
38        Ok(Self {
39            connectors,
40            fingerprint: format!("{:x}", Sha256::digest(bytes)),
41        })
42    }
43}
44
45/// Durable reference to a JSON artifact stored outside execution state.
46#[derive(Debug, Clone, Serialize, Deserialize)]
47pub struct ArtifactRef {
48    /// Stable artifact identifier.
49    pub id: String,
50    /// Owning execution.
51    pub execution_id: String,
52    /// Serialized artifact size.
53    pub bytes: usize,
54    /// Bounded human-readable preview.
55    pub preview: String,
56}
57
58/// Storage contract for oversized replay and final values.
59#[async_trait]
60pub trait ArtifactStore: Send + Sync {
61    /// Stores one JSON value and returns its durable reference.
62    async fn put(&self, execution_id: &str, value: &Value) -> Result<ArtifactRef, String>;
63    /// Loads one JSON value when it belongs to the supplied execution.
64    async fn get(&self, execution_id: &str, artifact_id: &str) -> Result<Option<Value>, String>;
65    /// Deletes artifacts owned by one execution.
66    async fn delete_execution(&self, execution_id: &str) -> Result<(), String>;
67}
68
69#[derive(Clone)]
70struct MemoryArtifact {
71    execution_id: String,
72    value: Value,
73}
74
75/// In-memory artifact store for local use and tests.
76#[derive(Default)]
77pub struct MemoryArtifactStore {
78    values: Mutex<BTreeMap<String, MemoryArtifact>>,
79}
80
81#[async_trait]
82impl ArtifactStore for MemoryArtifactStore {
83    async fn put(&self, execution_id: &str, value: &Value) -> Result<ArtifactRef, String> {
84        let bytes = serde_json::to_vec(value).map_err(|error| error.to_string())?;
85        let id = format!(
86            "{:x}",
87            Sha256::digest([execution_id.as_bytes(), &bytes].concat())
88        );
89        self.values.lock().await.insert(
90            id.clone(),
91            MemoryArtifact {
92                execution_id: execution_id.to_string(),
93                value: value.clone(),
94            },
95        );
96        Ok(ArtifactRef {
97            id,
98            execution_id: execution_id.to_string(),
99            bytes: bytes.len(),
100            preview: truncate_bytes(String::from_utf8_lossy(&bytes).into_owned(), 512),
101        })
102    }
103
104    async fn get(&self, execution_id: &str, artifact_id: &str) -> Result<Option<Value>, String> {
105        Ok(self
106            .values
107            .lock()
108            .await
109            .get(artifact_id)
110            .filter(|artifact| artifact.execution_id == execution_id)
111            .map(|artifact| artifact.value.clone()))
112    }
113
114    async fn delete_execution(&self, execution_id: &str) -> Result<(), String> {
115        self.values
116            .lock()
117            .await
118            .retain(|_, artifact| artifact.execution_id != execution_id);
119        Ok(())
120    }
121}
122
123/// Durable execution status.
124#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
125#[serde(rename_all = "snake_case")]
126pub enum ExecutionStatus {
127    /// A sandbox pass may make progress.
128    Running,
129    /// A connector action is awaiting approval.
130    Paused,
131    /// The program completed successfully.
132    Completed,
133    /// The program or replay failed.
134    Error,
135    /// A pending action was rejected or expired.
136    Rejected,
137    /// Applied actions were compensated.
138    RolledBack,
139    /// The execution was cancelled by its caller.
140    Cancelled,
141}
142
143impl ExecutionStatus {
144    fn terminal(self) -> bool {
145        matches!(
146            self,
147            Self::Completed | Self::Error | Self::Rejected | Self::RolledBack | Self::Cancelled
148        )
149    }
150}
151
152/// Ordered event retained with one execution.
153#[derive(Debug, Clone, Serialize, Deserialize)]
154#[serde(tag = "type", rename_all = "snake_case")]
155pub enum ExecutionEvent {
156    /// Execution lifecycle transition.
157    Status {
158        /// New lifecycle status.
159        status: ExecutionStatus,
160        /// Event timestamp in Unix milliseconds.
161        at: u64,
162    },
163    /// Incremental structured value.
164    Chunk {
165        /// Streamed value.
166        data: Value,
167        /// Event timestamp in Unix milliseconds.
168        at: u64,
169    },
170    /// Runtime log entry.
171    Log {
172        /// Log severity.
173        level: String,
174        /// Log message.
175        message: String,
176        /// Event timestamp in Unix milliseconds.
177        at: u64,
178    },
179    /// Execution progress update.
180    Progress {
181        /// Human-readable progress message.
182        message: String,
183        /// Optional completion fraction between 0.0 and 1.0.
184        fraction: Option<f64>,
185        /// Event timestamp in Unix milliseconds.
186        at: u64,
187    },
188}
189
190/// Durable state of one logged call.
191#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
192#[serde(rename_all = "snake_case")]
193pub enum LogEntryState {
194    /// The action is waiting for approval.
195    Pending,
196    /// The host may be executing the call.
197    Executing,
198    /// The result was applied and may be replayed.
199    Applied,
200    /// The action was rejected, compensated, or reset.
201    Reverted,
202}
203
204/// One entry in the deterministic replay spine.
205#[derive(Debug, Clone, Serialize, Deserialize)]
206pub struct LogEntry {
207    /// Host-assigned call sequence.
208    pub seq: u64,
209    /// Connector namespace.
210    pub connector: String,
211    /// Connector method.
212    pub method: String,
213    /// Original arguments.
214    pub arguments: Value,
215    /// Recorded result for non-ephemeral applied calls.
216    pub result: Option<Value>,
217    /// Whether the call needed approval.
218    pub requires_approval: bool,
219    /// Whether replay re-executes the call.
220    pub ephemeral: bool,
221    /// Current durable state.
222    pub state: LogEntryState,
223}
224
225/// Durable state for one sandbox program.
226#[derive(Debug, Clone, Serialize, Deserialize)]
227pub struct ExecutionState {
228    /// Stable execution identifier.
229    pub id: String,
230    /// Original sandbox program.
231    pub code: String,
232    /// Current lifecycle status.
233    pub status: ExecutionStatus,
234    /// Ordered deterministic call log.
235    pub log: Vec<LogEntry>,
236    /// Final successful result.
237    pub result: Option<Value>,
238    /// Terminal error.
239    pub error: Option<String>,
240    /// Captured sandbox console output.
241    pub logs: Vec<String>,
242    /// Connector names available when the run began.
243    pub connectors: Vec<String>,
244    /// Full immutable capability snapshot for deterministic resume.
245    #[serde(default)]
246    pub capabilities: Option<CapabilitySnapshot>,
247    /// Ordered bounded lifecycle and streaming events.
248    #[serde(default)]
249    pub events: Vec<ExecutionEvent>,
250    /// Creation time in Unix milliseconds.
251    pub created_at: u64,
252    /// Last state transition in Unix milliseconds.
253    pub updated_at: u64,
254}
255
256/// A pending connector action shown to an approval UI.
257#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
258pub struct PendingAction {
259    /// Owning execution.
260    pub execution_id: String,
261    /// Sequence within the execution.
262    pub seq: u64,
263    /// Connector namespace.
264    pub connector: String,
265    /// Connector method.
266    pub method: String,
267    /// Arguments awaiting approval.
268    pub arguments: Value,
269}
270
271/// Runtime decision for the next deterministic call.
272#[derive(Debug, Clone, PartialEq)]
273pub enum ToolDecision {
274    /// Return a recorded result without executing.
275    Replay(Value),
276    /// Execute the call and report its result at the supplied sequence.
277    Execute(u64),
278    /// Abort this sandbox pass.
279    Pause(u64),
280}
281
282/// A saved, addressable sandbox program.
283#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
284pub struct Snippet {
285    /// Unique snippet name.
286    pub name: String,
287    /// Human-readable search description.
288    pub description: String,
289    /// Async JavaScript function source.
290    pub code: String,
291    /// Save time in Unix milliseconds.
292    pub saved_at: u64,
293    /// Optional input JSON Schema.
294    pub input_schema: Option<Value>,
295    /// Connectors required by the saved program.
296    pub connectors: Vec<String>,
297}
298
299/// Persistent operations required by the portable runtime.
300#[async_trait]
301pub trait RuntimeStore: Send + Sync {
302    /// Loads one execution.
303    async fn get_execution(&self, id: &str) -> Result<Option<ExecutionState>, String>;
304    /// Replaces one execution.
305    async fn put_execution(&self, execution: &ExecutionState) -> Result<(), String>;
306    /// Lists every execution.
307    async fn list_executions(&self) -> Result<Vec<ExecutionState>, String>;
308    /// Deletes one execution and its log.
309    async fn delete_execution(&self, id: &str) -> Result<(), String>;
310    /// Loads one snippet.
311    async fn get_snippet(&self, name: &str) -> Result<Option<Snippet>, String>;
312    /// Replaces one snippet.
313    async fn put_snippet(&self, snippet: &Snippet) -> Result<(), String>;
314    /// Lists snippets.
315    async fn list_snippets(&self) -> Result<Vec<Snippet>, String>;
316    /// Deletes one snippet.
317    async fn delete_snippet(&self, name: &str) -> Result<bool, String>;
318}
319
320#[derive(Default)]
321struct MemoryState {
322    executions: BTreeMap<String, ExecutionState>,
323    snippets: BTreeMap<String, Snippet>,
324}
325
326/// In-memory runtime store for native use and deterministic tests.
327#[derive(Default)]
328pub struct MemoryStore {
329    state: Mutex<MemoryState>,
330}
331
332#[async_trait]
333impl RuntimeStore for MemoryStore {
334    async fn get_execution(&self, id: &str) -> Result<Option<ExecutionState>, String> {
335        Ok(self.state.lock().await.executions.get(id).cloned())
336    }
337
338    async fn put_execution(&self, execution: &ExecutionState) -> Result<(), String> {
339        self.state
340            .lock()
341            .await
342            .executions
343            .insert(execution.id.clone(), execution.clone());
344        Ok(())
345    }
346
347    async fn list_executions(&self) -> Result<Vec<ExecutionState>, String> {
348        Ok(self
349            .state
350            .lock()
351            .await
352            .executions
353            .values()
354            .cloned()
355            .collect())
356    }
357
358    async fn delete_execution(&self, id: &str) -> Result<(), String> {
359        self.state.lock().await.executions.remove(id);
360        Ok(())
361    }
362
363    async fn get_snippet(&self, name: &str) -> Result<Option<Snippet>, String> {
364        Ok(self.state.lock().await.snippets.get(name).cloned())
365    }
366
367    async fn put_snippet(&self, snippet: &Snippet) -> Result<(), String> {
368        self.state
369            .lock()
370            .await
371            .snippets
372            .insert(snippet.name.clone(), snippet.clone());
373        Ok(())
374    }
375
376    async fn list_snippets(&self) -> Result<Vec<Snippet>, String> {
377        Ok(self.state.lock().await.snippets.values().cloned().collect())
378    }
379
380    async fn delete_snippet(&self, name: &str) -> Result<bool, String> {
381        Ok(self.state.lock().await.snippets.remove(name).is_some())
382    }
383}
384
385/// Runtime state and persistence failures.
386#[derive(Debug, Error)]
387pub enum RuntimeError {
388    /// The requested execution does not exist.
389    #[error("Execution \"{0}\" not found")]
390    MissingExecution(String),
391    /// The execution cannot make the requested transition.
392    #[error("{0}")]
393    InvalidState(String),
394    /// A value cannot be recorded without corrupting replay.
395    #[error("{0}")]
396    DurableValue(String),
397    /// The configured store failed.
398    #[error("Runtime store failed: {0}")]
399    Store(String),
400}
401
402/// Portable deterministic replay and approval state machine.
403pub struct CodeModeRuntime {
404    store: Arc<dyn RuntimeStore>,
405    artifacts: Arc<dyn ArtifactStore>,
406    gate: Mutex<()>,
407}
408
409impl CodeModeRuntime {
410    /// Creates a runtime over a persistence implementation.
411    pub fn new(store: Arc<dyn RuntimeStore>) -> Self {
412        Self::with_artifacts(store, Arc::new(MemoryArtifactStore::default()))
413    }
414
415    /// Creates a runtime with an explicit oversized-value store.
416    pub fn with_artifacts(store: Arc<dyn RuntimeStore>, artifacts: Arc<dyn ArtifactStore>) -> Self {
417        Self {
418            store,
419            artifacts,
420            gate: Mutex::new(()),
421        }
422    }
423
424    /// Starts a fresh execution and prunes old terminal history.
425    pub async fn begin(
426        &self,
427        code: impl Into<String>,
428        connectors: Vec<String>,
429        now: u64,
430    ) -> Result<String, RuntimeError> {
431        self.begin_inner(code.into(), connectors, None, now).await
432    }
433
434    /// Creates a durable execution with an immutable capability snapshot.
435    pub async fn begin_with_capabilities(
436        &self,
437        code: &str,
438        capabilities: CapabilitySnapshot,
439        now: u64,
440    ) -> Result<String, RuntimeError> {
441        let connectors = capabilities
442            .connectors
443            .iter()
444            .map(|connector| connector.name.clone())
445            .collect();
446        self.begin_inner(code.to_string(), connectors, Some(capabilities), now)
447            .await
448    }
449
450    async fn begin_inner(
451        &self,
452        code: String,
453        connectors: Vec<String>,
454        capabilities: Option<CapabilitySnapshot>,
455        now: u64,
456    ) -> Result<String, RuntimeError> {
457        let _guard = self.gate.lock().await;
458        ensure_size("The execution code", &Value::String(code.clone()))?;
459        let sequence = EXECUTION_SEQUENCE.fetch_add(1, Ordering::Relaxed);
460        let id = format!("exec_{now:016}_{sequence:016x}");
461        self.store
462            .put_execution(&ExecutionState {
463                id: id.clone(),
464                code,
465                status: ExecutionStatus::Running,
466                log: Vec::new(),
467                result: None,
468                error: None,
469                logs: Vec::new(),
470                connectors,
471                capabilities,
472                events: vec![ExecutionEvent::Status {
473                    status: ExecutionStatus::Running,
474                    at: now,
475                }],
476                created_at: now,
477                updated_at: now,
478            })
479            .await
480            .map_err(RuntimeError::Store)?;
481        self.prune_locked(DEFAULT_MAX_EXECUTIONS, Some(&id)).await?;
482        Ok(id)
483    }
484
485    /// Moves a paused execution back to running for a replay pass.
486    pub async fn resume(&self, id: &str, now: u64) -> Result<ExecutionState, RuntimeError> {
487        let _guard = self.gate.lock().await;
488        let mut execution = self.require(id).await?;
489        if execution.status != ExecutionStatus::Paused {
490            return Err(RuntimeError::InvalidState(format!(
491                "Execution \"{id}\" is {:?}, not paused",
492                execution.status
493            )));
494        }
495        if execution
496            .log
497            .iter()
498            .any(|entry| entry.state == LogEntryState::Pending)
499        {
500            return Err(RuntimeError::InvalidState(format!(
501                "Execution \"{id}\" is awaiting approval; use approve or reject"
502            )));
503        }
504        execution.status = ExecutionStatus::Running;
505        push_event(
506            &mut execution,
507            ExecutionEvent::Status {
508                status: ExecutionStatus::Running,
509                at: now,
510            },
511        );
512        execution.updated_at = now;
513        self.save(&execution).await?;
514        Ok(execution)
515    }
516
517    /// Cancels a running or paused execution.
518    pub async fn cancel(&self, id: &str, now: u64) -> Result<bool, RuntimeError> {
519        let _guard = self.gate.lock().await;
520        let mut execution = self.require(id).await?;
521        if execution.status.terminal() {
522            return Ok(false);
523        }
524        execution.status = ExecutionStatus::Cancelled;
525        execution.error = Some("Execution cancelled".to_string());
526        execution.updated_at = now;
527        push_event(
528            &mut execution,
529            ExecutionEvent::Status {
530                status: ExecutionStatus::Cancelled,
531                at: now,
532            },
533        );
534        self.save(&execution).await?;
535        Ok(true)
536    }
537
538    /// Appends one bounded execution event.
539    pub async fn event(&self, id: &str, event: ExecutionEvent) -> Result<(), RuntimeError> {
540        let _guard = self.gate.lock().await;
541        let mut execution = self.require(id).await?;
542        push_event(&mut execution, event);
543        self.save(&execution).await
544    }
545
546    /// Loads an artifact value by owning execution and artifact identifiers.
547    pub async fn artifact(
548        &self,
549        execution_id: &str,
550        artifact_id: &str,
551    ) -> Result<Option<Value>, RuntimeError> {
552        self.artifacts
553            .get(execution_id, artifact_id)
554            .await
555            .map_err(RuntimeError::Store)
556    }
557
558    async fn spill(&self, execution_id: &str, value: Value) -> Result<Value, RuntimeError> {
559        if serialized_size(&value) <= MAX_DURABLE_VALUE_BYTES {
560            return Ok(value);
561        }
562        let artifact = self
563            .artifacts
564            .put(execution_id, &value)
565            .await
566            .map_err(RuntimeError::Store)?;
567        Ok(serde_json::json!({ "$artifact": artifact }))
568    }
569
570    async fn rehydrate(&self, execution_id: &str, value: Value) -> Result<Value, RuntimeError> {
571        let Some(artifact) = artifact_ref(&value)? else {
572            return Ok(value);
573        };
574        self.artifacts
575            .get(execution_id, &artifact.id)
576            .await
577            .map_err(RuntimeError::Store)?
578            .ok_or_else(|| {
579                RuntimeError::Store(format!("Artifact \"{}\" is unavailable", artifact.id))
580            })
581    }
582
583    async fn rehydrate_execution(
584        &self,
585        mut execution: ExecutionState,
586    ) -> Result<ExecutionState, RuntimeError> {
587        for entry in &mut execution.log {
588            entry.arguments = self
589                .rehydrate(&execution.id, entry.arguments.clone())
590                .await?;
591            if let Some(result) = entry.result.take() {
592                entry.result = Some(self.rehydrate(&execution.id, result).await?);
593            }
594        }
595        if let Some(result) = execution.result.take() {
596            execution.result = Some(self.rehydrate(&execution.id, result).await?);
597        }
598        Ok(execution)
599    }
600
601    async fn rehydrate_entry(
602        &self,
603        execution_id: &str,
604        mut entry: LogEntry,
605    ) -> Result<LogEntry, RuntimeError> {
606        entry.arguments = self.rehydrate(execution_id, entry.arguments).await?;
607        if let Some(result) = entry.result.take() {
608            entry.result = Some(self.rehydrate(execution_id, result).await?);
609        }
610        Ok(entry)
611    }
612
613    async fn rehydrate_pending(
614        &self,
615        execution_id: &str,
616        mut action: PendingAction,
617    ) -> Result<PendingAction, RuntimeError> {
618        action.arguments = self.rehydrate(execution_id, action.arguments).await?;
619        Ok(action)
620    }
621
622    /// Decides whether the next call replays, executes, or pauses.
623    #[allow(clippy::too_many_arguments)]
624    pub async fn decide(
625        &self,
626        id: &str,
627        seq: u64,
628        connector: &str,
629        method: &str,
630        arguments: Value,
631        requires_approval: bool,
632        ephemeral: bool,
633        now: u64,
634    ) -> Result<ToolDecision, RuntimeError> {
635        let _guard = self.gate.lock().await;
636        let mut execution = self.require(id).await?;
637        if execution.status != ExecutionStatus::Running {
638            return Ok(ToolDecision::Pause(seq));
639        }
640        if requires_approval && ephemeral {
641            return Err(RuntimeError::InvalidState(format!(
642                "{connector}.{method} cannot require approval and re-execute on replay"
643            )));
644        }
645        let arguments = self.spill(id, arguments).await?;
646        if let Some(entry) = execution.log.iter_mut().find(|entry| entry.seq == seq) {
647            if entry.connector != connector
648                || entry.method != method
649                || stable_json(&entry.arguments) != stable_json(&arguments)
650            {
651                execution.status = ExecutionStatus::Error;
652                execution.error = Some(format!(
653                    "Replay diverged at step {seq}: expected {}.{}, got {connector}.{method}. \
654                     Code must be deterministic up to tool calls and steps.",
655                    entry.connector, entry.method
656                ));
657                execution.updated_at = now;
658                self.save(&execution).await?;
659                return Ok(ToolDecision::Pause(seq));
660            }
661            return match entry.state {
662                LogEntryState::Applied if !entry.ephemeral => Ok(ToolDecision::Replay(
663                    self.rehydrate(id, entry.result.clone().unwrap_or(Value::Null))
664                        .await?,
665                )),
666                LogEntryState::Pending | LogEntryState::Executing | LogEntryState::Applied => {
667                    entry.state = LogEntryState::Executing;
668                    execution.updated_at = now;
669                    self.save(&execution).await?;
670                    Ok(ToolDecision::Execute(seq))
671                }
672                LogEntryState::Reverted => {
673                    entry.state = if requires_approval {
674                        LogEntryState::Pending
675                    } else {
676                        LogEntryState::Executing
677                    };
678                    entry.arguments = arguments;
679                    entry.result = None;
680                    entry.requires_approval = requires_approval;
681                    entry.ephemeral = ephemeral;
682                    if requires_approval {
683                        execution.status = ExecutionStatus::Paused;
684                        push_event(
685                            &mut execution,
686                            ExecutionEvent::Status {
687                                status: ExecutionStatus::Paused,
688                                at: now,
689                            },
690                        );
691                    }
692                    execution.updated_at = now;
693                    self.save(&execution).await?;
694                    Ok(if requires_approval {
695                        ToolDecision::Pause(seq)
696                    } else {
697                        ToolDecision::Execute(seq)
698                    })
699                }
700            };
701        }
702        execution.log.push(LogEntry {
703            seq,
704            connector: connector.to_string(),
705            method: method.to_string(),
706            arguments,
707            result: None,
708            requires_approval,
709            ephemeral,
710            state: if requires_approval {
711                LogEntryState::Pending
712            } else {
713                LogEntryState::Executing
714            },
715        });
716        if requires_approval {
717            execution.status = ExecutionStatus::Paused;
718            push_event(
719                &mut execution,
720                ExecutionEvent::Status {
721                    status: ExecutionStatus::Paused,
722                    at: now,
723                },
724            );
725        }
726        execution.updated_at = now;
727        self.save(&execution).await?;
728        Ok(if requires_approval {
729            ToolDecision::Pause(seq)
730        } else {
731            ToolDecision::Execute(seq)
732        })
733    }
734
735    /// Records a completed connector call for deterministic replay.
736    pub async fn record_result(
737        &self,
738        id: &str,
739        seq: u64,
740        result: Value,
741        now: u64,
742    ) -> Result<(), RuntimeError> {
743        let _guard = self.gate.lock().await;
744        let mut execution = self.require(id).await?;
745        if execution.status != ExecutionStatus::Running {
746            return Err(RuntimeError::InvalidState(format!(
747                "Execution \"{id}\" is {:?}, not running",
748                execution.status
749            )));
750        }
751        let result = self.spill(id, result).await?;
752        if execution.status != ExecutionStatus::Running {
753            return Err(RuntimeError::InvalidState(format!(
754                "Execution \"{id}\" is {:?}, not running",
755                execution.status
756            )));
757        }
758        let Some(entry) = execution.log.iter_mut().find(|entry| entry.seq == seq) else {
759            return Err(RuntimeError::InvalidState(format!(
760                "No log entry at step {seq}"
761            )));
762        };
763        if entry.state != LogEntryState::Executing {
764            return Err(RuntimeError::InvalidState(format!(
765                "Log entry at step {seq} is {:?}, not executing",
766                entry.state
767            )));
768        }
769        if !entry.ephemeral {
770            entry.result = Some(result);
771        }
772        entry.state = LogEntryState::Applied;
773        execution.updated_at = now;
774        self.save(&execution).await
775    }
776
777    /// Marks an execution completed and records bounded audit output.
778    pub async fn complete(
779        &self,
780        id: &str,
781        result: Value,
782        logs: Vec<String>,
783        now: u64,
784    ) -> Result<(), RuntimeError> {
785        let _guard = self.gate.lock().await;
786        let mut execution = self.require(id).await?;
787        if execution.status != ExecutionStatus::Running {
788            return Err(RuntimeError::InvalidState(format!(
789                "Execution \"{id}\" is {:?}, not running",
790                execution.status
791            )));
792        }
793        let result = self.spill(id, result).await?;
794        execution.status = ExecutionStatus::Completed;
795        push_event(
796            &mut execution,
797            ExecutionEvent::Status {
798                status: ExecutionStatus::Completed,
799                at: now,
800            },
801        );
802        execution.result = Some(result);
803        execution.logs = bounded_logs(logs);
804        execution.updated_at = now;
805        self.save(&execution).await
806    }
807
808    /// Marks an execution failed.
809    pub async fn fail(
810        &self,
811        id: &str,
812        error: impl Into<String>,
813        logs: Vec<String>,
814        now: u64,
815    ) -> Result<(), RuntimeError> {
816        let _guard = self.gate.lock().await;
817        let mut execution = self.require(id).await?;
818        if execution.status != ExecutionStatus::Running {
819            return Err(RuntimeError::InvalidState(format!(
820                "Execution \"{id}\" is {:?}, not running",
821                execution.status
822            )));
823        }
824        execution.status = ExecutionStatus::Error;
825        push_event(
826            &mut execution,
827            ExecutionEvent::Status {
828                status: ExecutionStatus::Error,
829                at: now,
830            },
831        );
832        execution.error = Some(truncate_bytes(error.into(), MAX_DURABLE_VALUE_BYTES));
833        execution.logs = bounded_logs(logs);
834        execution.updated_at = now;
835        self.save(&execution).await
836    }
837
838    /// Approves a pending action without racing a stale second approval.
839    pub async fn approve(&self, id: &str, seq: u64, now: u64) -> Result<bool, RuntimeError> {
840        let _guard = self.gate.lock().await;
841        let mut execution = self.require(id).await?;
842        if execution.status != ExecutionStatus::Paused {
843            return Ok(false);
844        }
845        let Some(entry) = execution
846            .log
847            .iter()
848            .find(|entry| entry.seq == seq && entry.state == LogEntryState::Pending)
849        else {
850            return Ok(false);
851        };
852        let _ = entry;
853        execution.status = ExecutionStatus::Running;
854        execution.updated_at = now;
855        self.save(&execution).await?;
856        Ok(true)
857    }
858
859    /// Rejects a pending action and terminates its execution.
860    pub async fn reject(&self, id: &str, seq: u64, now: u64) -> Result<bool, RuntimeError> {
861        let _guard = self.gate.lock().await;
862        let mut execution = self.require(id).await?;
863        if execution.status != ExecutionStatus::Paused {
864            return Err(RuntimeError::InvalidState(format!(
865                "Execution \"{id}\" is {:?}, not paused",
866                execution.status
867            )));
868        }
869        let Some(entry) = execution
870            .log
871            .iter_mut()
872            .find(|entry| entry.seq == seq && entry.state == LogEntryState::Pending)
873        else {
874            return Ok(false);
875        };
876        entry.state = LogEntryState::Reverted;
877        let error = format!(
878            "Action {}.{} rejected by user",
879            entry.connector, entry.method
880        );
881        execution.status = ExecutionStatus::Rejected;
882        push_event(
883            &mut execution,
884            ExecutionEvent::Status {
885                status: ExecutionStatus::Rejected,
886                at: now,
887            },
888        );
889        execution.error = Some(error);
890        execution.updated_at = now;
891        self.save(&execution).await?;
892        Ok(true)
893    }
894
895    /// Lists actionable approvals, optionally scoped to one execution.
896    pub async fn pending(&self, id: Option<&str>) -> Result<Vec<PendingAction>, RuntimeError> {
897        let executions = if let Some(id) = id {
898            vec![self.require(id).await?]
899        } else {
900            self.store
901                .list_executions()
902                .await
903                .map_err(RuntimeError::Store)?
904        };
905        let mut result = Vec::new();
906        for execution in executions
907            .into_iter()
908            .filter(|execution| execution.status == ExecutionStatus::Paused)
909        {
910            for entry in execution
911                .log
912                .into_iter()
913                .filter(|entry| entry.state == LogEntryState::Pending)
914            {
915                let action = PendingAction {
916                    execution_id: execution.id.clone(),
917                    seq: entry.seq,
918                    connector: entry.connector,
919                    method: entry.method,
920                    arguments: entry.arguments,
921                };
922                result.push(self.rehydrate_pending(&execution.id, action).await?);
923            }
924        }
925        result.sort_by(|left, right| {
926            right
927                .execution_id
928                .cmp(&left.execution_id)
929                .then_with(|| left.seq.cmp(&right.seq))
930        });
931        Ok(result)
932    }
933
934    /// Returns applied connector actions in reverse order for compensation.
935    pub async fn actions_to_revert(&self, id: &str) -> Result<Vec<LogEntry>, RuntimeError> {
936        let execution = self.require(id).await?;
937        validate_rollback_state(&execution)?;
938        let mut actions = Vec::new();
939        for entry in execution.log.into_iter().filter(|entry| {
940            entry.state == LogEntryState::Applied && !entry.ephemeral && entry.connector != "__step"
941        }) {
942            actions.push(self.rehydrate_entry(&execution.id, entry).await?);
943        }
944        actions.sort_by_key(|entry| std::cmp::Reverse(entry.seq));
945        Ok(actions)
946    }
947
948    /// Marks one compensated action reverted.
949    pub async fn mark_reverted(&self, id: &str, seq: u64, now: u64) -> Result<(), RuntimeError> {
950        let _guard = self.gate.lock().await;
951        let mut execution = self.require(id).await?;
952        if let Some(entry) = execution.log.iter_mut().find(|entry| entry.seq == seq) {
953            entry.state = LogEntryState::Reverted;
954            execution.updated_at = now;
955            self.save(&execution).await?;
956        }
957        Ok(())
958    }
959
960    /// Marks an execution rolled back after compensation completes.
961    pub async fn finish_rollback(&self, id: &str, now: u64) -> Result<(), RuntimeError> {
962        let _guard = self.gate.lock().await;
963        let mut execution = self.require(id).await?;
964        validate_rollback_state(&execution)?;
965        execution.status = ExecutionStatus::RolledBack;
966        push_event(
967            &mut execution,
968            ExecutionEvent::Status {
969                status: ExecutionStatus::RolledBack,
970                at: now,
971            },
972        );
973        execution.updated_at = now;
974        self.save(&execution).await
975    }
976
977    /// Expires abandoned paused and running executions.
978    pub async fn expire(&self, now: u64, max_age_ms: u64) -> Result<Vec<String>, RuntimeError> {
979        let _guard = self.gate.lock().await;
980        let mut expired = Vec::new();
981        for mut execution in self
982            .store
983            .list_executions()
984            .await
985            .map_err(RuntimeError::Store)?
986        {
987            if !matches!(
988                execution.status,
989                ExecutionStatus::Paused | ExecutionStatus::Running
990            ) || now.saturating_sub(execution.updated_at) <= max_age_ms
991            {
992                continue;
993            }
994            execution.status = if execution.status == ExecutionStatus::Paused {
995                ExecutionStatus::Rejected
996            } else {
997                ExecutionStatus::Error
998            };
999            execution.error = Some(if execution.status == ExecutionStatus::Rejected {
1000                "Expired awaiting approval".to_string()
1001            } else {
1002                "Expired while running - the host never completed the pass".to_string()
1003            });
1004            for entry in &mut execution.log {
1005                if entry.state == LogEntryState::Pending {
1006                    entry.state = LogEntryState::Reverted;
1007                }
1008            }
1009            execution.updated_at = now;
1010            self.save(&execution).await?;
1011            expired.push(execution.id);
1012        }
1013        Ok(expired)
1014    }
1015
1016    /// Loads one execution.
1017    pub async fn execution(&self, id: &str) -> Result<Option<ExecutionState>, RuntimeError> {
1018        let Some(execution) = self
1019            .store
1020            .get_execution(id)
1021            .await
1022            .map_err(RuntimeError::Store)?
1023        else {
1024            return Ok(None);
1025        };
1026        self.rehydrate_execution(execution).await.map(Some)
1027    }
1028
1029    /// Loads one bounded execution snapshot with artifact references intact.
1030    pub async fn execution_snapshot(
1031        &self,
1032        id: &str,
1033    ) -> Result<Option<ExecutionState>, RuntimeError> {
1034        self.store
1035            .get_execution(id)
1036            .await
1037            .map_err(RuntimeError::Store)
1038    }
1039
1040    /// Lists executions newest first.
1041    pub async fn executions(&self) -> Result<Vec<ExecutionState>, RuntimeError> {
1042        let mut executions = self
1043            .store
1044            .list_executions()
1045            .await
1046            .map_err(RuntimeError::Store)?;
1047        executions.sort_by_key(|execution| {
1048            (
1049                std::cmp::Reverse(execution.created_at),
1050                std::cmp::Reverse(execution.id.clone()),
1051            )
1052        });
1053        Ok(executions)
1054    }
1055
1056    /// Prunes terminal history while retaining every live execution.
1057    pub async fn prune(&self, keep: usize) -> Result<usize, RuntimeError> {
1058        let _guard = self.gate.lock().await;
1059        self.prune_locked(keep, None).await
1060    }
1061
1062    /// Saves a working execution as a reusable snippet.
1063    pub async fn save_snippet(
1064        &self,
1065        name: impl Into<String>,
1066        description: impl Into<String>,
1067        execution_id: &str,
1068        input_schema: Option<Value>,
1069        now: u64,
1070    ) -> Result<Snippet, RuntimeError> {
1071        let execution = self.require(execution_id).await?;
1072        if execution.status != ExecutionStatus::Completed {
1073            return Err(RuntimeError::InvalidState(
1074                "Only completed executions can be saved as snippets".to_string(),
1075            ));
1076        }
1077        let snippet = Snippet {
1078            name: name.into(),
1079            description: description.into(),
1080            code: execution.code,
1081            saved_at: now,
1082            input_schema,
1083            connectors: execution.connectors,
1084        };
1085        self.store
1086            .put_snippet(&snippet)
1087            .await
1088            .map_err(RuntimeError::Store)?;
1089        Ok(snippet)
1090    }
1091
1092    /// Lists snippets in stable name order.
1093    pub async fn snippets(&self) -> Result<Vec<Snippet>, RuntimeError> {
1094        let mut snippets = self
1095            .store
1096            .list_snippets()
1097            .await
1098            .map_err(RuntimeError::Store)?;
1099        snippets.sort_by(|left, right| left.name.cmp(&right.name));
1100        Ok(snippets)
1101    }
1102
1103    /// Deletes one snippet.
1104    pub async fn delete_snippet(&self, name: &str) -> Result<bool, RuntimeError> {
1105        self.store
1106            .delete_snippet(name)
1107            .await
1108            .map_err(RuntimeError::Store)
1109    }
1110
1111    async fn require(&self, id: &str) -> Result<ExecutionState, RuntimeError> {
1112        self.store
1113            .get_execution(id)
1114            .await
1115            .map_err(RuntimeError::Store)?
1116            .ok_or_else(|| RuntimeError::MissingExecution(id.to_string()))
1117    }
1118
1119    async fn save(&self, execution: &ExecutionState) -> Result<(), RuntimeError> {
1120        self.store
1121            .put_execution(execution)
1122            .await
1123            .map_err(RuntimeError::Store)
1124    }
1125
1126    async fn prune_locked(
1127        &self,
1128        keep: usize,
1129        excluding: Option<&str>,
1130    ) -> Result<usize, RuntimeError> {
1131        let mut terminal = self
1132            .store
1133            .list_executions()
1134            .await
1135            .map_err(RuntimeError::Store)?
1136            .into_iter()
1137            .filter(|execution| {
1138                execution.status.terminal()
1139                    && excluding.is_none_or(|id| execution.id.as_str() != id)
1140            })
1141            .collect::<Vec<_>>();
1142        terminal.sort_by_key(|execution| std::cmp::Reverse(execution.created_at));
1143        let remove = terminal.len().saturating_sub(keep);
1144        for execution in terminal.into_iter().rev().take(remove) {
1145            self.store
1146                .delete_execution(&execution.id)
1147                .await
1148                .map_err(RuntimeError::Store)?;
1149            self.artifacts
1150                .delete_execution(&execution.id)
1151                .await
1152                .map_err(RuntimeError::Store)?;
1153        }
1154        Ok(remove)
1155    }
1156}
1157
1158fn ensure_size(what: &str, value: &Value) -> Result<(), RuntimeError> {
1159    let size = serialized_size(value);
1160    if size > MAX_DURABLE_VALUE_BYTES {
1161        Err(RuntimeError::DurableValue(format!(
1162            "{what} is too large to record durably ({size} bytes > \
1163             {MAX_DURABLE_VALUE_BYTES} byte limit). Write large data to a file or workspace \
1164             instead and pass or return a small reference."
1165        )))
1166    } else {
1167        Ok(())
1168    }
1169}
1170
1171fn serialized_size(value: &Value) -> usize {
1172    serde_json::to_vec(value).map_or(usize::MAX, |value| value.len())
1173}
1174
1175fn stable_json(value: &Value) -> String {
1176    match value {
1177        Value::Object(values) => {
1178            let mut values = values.iter().collect::<Vec<_>>();
1179            values.sort_by_key(|(key, _)| *key);
1180            format!(
1181                "{{{}}}",
1182                values
1183                    .into_iter()
1184                    .map(|(key, value)| format!(
1185                        "{}:{}",
1186                        serde_json::to_string(key).unwrap(),
1187                        stable_json(value)
1188                    ))
1189                    .collect::<Vec<_>>()
1190                    .join(",")
1191            )
1192        }
1193        Value::Array(values) => format!(
1194            "[{}]",
1195            values.iter().map(stable_json).collect::<Vec<_>>().join(",")
1196        ),
1197        value => serde_json::to_string(value).unwrap_or_default(),
1198    }
1199}
1200
1201fn bounded_logs(logs: Vec<String>) -> Vec<String> {
1202    let mut size = 2;
1203    logs.into_iter()
1204        .take_while(|log| {
1205            size += log.len() + 3;
1206            size <= MAX_DURABLE_VALUE_BYTES
1207        })
1208        .collect()
1209}
1210
1211fn push_event(execution: &mut ExecutionState, event: ExecutionEvent) {
1212    execution.events.push(event);
1213    if execution.events.len() > DEFAULT_MAX_EVENTS {
1214        execution
1215            .events
1216            .drain(..execution.events.len() - DEFAULT_MAX_EVENTS);
1217    }
1218}
1219
1220fn artifact_ref(value: &Value) -> Result<Option<ArtifactRef>, RuntimeError> {
1221    let Value::Object(values) = value else {
1222        return Ok(None);
1223    };
1224    if values.len() != 1 {
1225        return Ok(None);
1226    }
1227    let Some(artifact) = values.get("$artifact") else {
1228        return Ok(None);
1229    };
1230    serde_json::from_value::<ArtifactRef>(artifact.clone())
1231        .map(Some)
1232        .map_err(|error| RuntimeError::Store(error.to_string()))
1233}
1234
1235fn truncate_bytes(mut value: String, limit: usize) -> String {
1236    if value.len() <= limit {
1237        return value;
1238    }
1239    let mut end = limit;
1240    while !value.is_char_boundary(end) {
1241        end -= 1;
1242    }
1243    value.truncate(end);
1244    value
1245}
1246
1247fn validate_rollback_state(execution: &ExecutionState) -> Result<(), RuntimeError> {
1248    if !execution.status.terminal() || execution.status == ExecutionStatus::RolledBack {
1249        return Err(RuntimeError::InvalidState(format!(
1250            "Execution \"{}\" is {:?}, not rollback eligible",
1251            execution.id, execution.status
1252        )));
1253    }
1254    if execution.log.iter().any(|entry| {
1255        matches!(
1256            entry.state,
1257            LogEntryState::Pending | LogEntryState::Executing
1258        )
1259    }) {
1260        return Err(RuntimeError::InvalidState(format!(
1261            "Execution \"{}\" has an unfinished action",
1262            execution.id
1263        )));
1264    }
1265    Ok(())
1266}
1267
1268#[cfg(test)]
1269mod tests {
1270    use serde_json::json;
1271
1272    use super::*;
1273
1274    fn runtime() -> CodeModeRuntime {
1275        CodeModeRuntime::new(Arc::new(MemoryStore::default()))
1276    }
1277
1278    fn runtime_parts() -> (CodeModeRuntime, Arc<MemoryStore>, Arc<MemoryArtifactStore>) {
1279        let store = Arc::new(MemoryStore::default());
1280        let artifacts = Arc::new(MemoryArtifactStore::default());
1281        (
1282            CodeModeRuntime::with_artifacts(store.clone(), artifacts.clone()),
1283            store,
1284            artifacts,
1285        )
1286    }
1287
1288    fn artifact_id(value: &Value) -> String {
1289        serde_json::from_value::<ArtifactRef>(value["$artifact"].clone())
1290            .unwrap()
1291            .id
1292    }
1293
1294    #[tokio::test]
1295    async fn pauses_approves_and_replays_without_reapplying() {
1296        let runtime = runtime();
1297        let id = runtime
1298            .begin("async () => {}", vec!["db".into()], 1)
1299            .await
1300            .unwrap();
1301        assert_eq!(
1302            runtime
1303                .decide(&id, 0, "db", "write", json!({"id": 1}), true, false, 2)
1304                .await
1305                .unwrap(),
1306            ToolDecision::Pause(0)
1307        );
1308        assert!(runtime.approve(&id, 0, 3).await.unwrap());
1309        assert_eq!(
1310            runtime
1311                .decide(&id, 0, "db", "write", json!({"id": 1}), true, false, 4)
1312                .await
1313                .unwrap(),
1314            ToolDecision::Execute(0)
1315        );
1316        runtime
1317            .record_result(&id, 0, json!({"ok": true}), 5)
1318            .await
1319            .unwrap();
1320        assert_eq!(
1321            runtime
1322                .decide(&id, 0, "db", "write", json!({"id": 1}), true, false, 6)
1323                .await
1324                .unwrap(),
1325            ToolDecision::Replay(json!({"ok": true}))
1326        );
1327    }
1328
1329    #[tokio::test]
1330    async fn rejects_results_for_pending_approval_entries() {
1331        let runtime = runtime();
1332        let id = runtime
1333            .begin("async () => {}", vec!["db".into()], 1)
1334            .await
1335            .unwrap();
1336        runtime
1337            .decide(&id, 0, "db", "write", json!({"id": 1}), true, false, 2)
1338            .await
1339            .unwrap();
1340
1341        let error = runtime
1342            .record_result(&id, 0, json!({"forged": true}), 3)
1343            .await
1344            .unwrap_err();
1345
1346        assert!(matches!(error, RuntimeError::InvalidState(_)));
1347        let execution = runtime.execution(&id).await.unwrap().unwrap();
1348        assert_eq!(execution.status, ExecutionStatus::Paused);
1349        assert_eq!(execution.log[0].state, LogEntryState::Pending);
1350    }
1351
1352    #[tokio::test]
1353    async fn terminal_state_cannot_be_overwritten_by_late_completion() {
1354        let runtime = runtime();
1355        let id = runtime.begin("async () => 42", vec![], 1).await.unwrap();
1356        runtime.cancel(&id, 2).await.unwrap();
1357
1358        let error = runtime
1359            .complete(&id, json!(42), Vec::new(), 3)
1360            .await
1361            .unwrap_err();
1362
1363        assert!(matches!(error, RuntimeError::InvalidState(_)));
1364        assert_eq!(
1365            runtime.execution(&id).await.unwrap().unwrap().status,
1366            ExecutionStatus::Cancelled
1367        );
1368    }
1369
1370    #[tokio::test]
1371    async fn stale_rejection_cannot_replace_cancellation() {
1372        let runtime = runtime();
1373        let id = runtime
1374            .begin("async () => {}", vec!["db".into()], 1)
1375            .await
1376            .unwrap();
1377        runtime
1378            .decide(&id, 0, "db", "write", json!({}), true, false, 2)
1379            .await
1380            .unwrap();
1381        runtime.cancel(&id, 3).await.unwrap();
1382
1383        let error = runtime.reject(&id, 0, 4).await.unwrap_err();
1384
1385        assert!(matches!(error, RuntimeError::InvalidState(_)));
1386        assert_eq!(
1387            runtime.execution(&id).await.unwrap().unwrap().status,
1388            ExecutionStatus::Cancelled
1389        );
1390    }
1391
1392    #[tokio::test]
1393    async fn rollback_rejects_running_executions() {
1394        let runtime = runtime();
1395        let id = runtime.begin("async () => 42", vec![], 1).await.unwrap();
1396
1397        let error = runtime.actions_to_revert(&id).await.unwrap_err();
1398
1399        assert!(matches!(error, RuntimeError::InvalidState(_)));
1400        assert_eq!(
1401            runtime.execution(&id).await.unwrap().unwrap().status,
1402            ExecutionStatus::Running
1403        );
1404    }
1405
1406    #[tokio::test]
1407    async fn resume_cannot_bypass_pending_approval() {
1408        let runtime = runtime();
1409        let id = runtime
1410            .begin("async () => {}", vec!["db".into()], 1)
1411            .await
1412            .unwrap();
1413        runtime
1414            .decide(&id, 0, "db", "write", json!({"id": 1}), true, false, 2)
1415            .await
1416            .unwrap();
1417
1418        let error = runtime.resume(&id, 3).await.unwrap_err();
1419
1420        assert!(matches!(error, RuntimeError::InvalidState(_)));
1421        let execution = runtime.execution(&id).await.unwrap().unwrap();
1422        assert_eq!(execution.status, ExecutionStatus::Paused);
1423        assert_eq!(execution.log[0].state, LogEntryState::Pending);
1424    }
1425
1426    #[tokio::test]
1427    async fn rollback_targets_only_logged_connector_actions() {
1428        let runtime = runtime();
1429        let id = runtime
1430            .begin("async () => {}", vec!["db".into()], 1)
1431            .await
1432            .unwrap();
1433        runtime
1434            .decide(&id, 0, "db", "read", json!({}), false, true, 2)
1435            .await
1436            .unwrap();
1437        runtime.record_result(&id, 0, json!(1), 3).await.unwrap();
1438        runtime
1439            .decide(&id, 1, "db", "write", json!({}), false, false, 4)
1440            .await
1441            .unwrap();
1442        runtime.record_result(&id, 1, json!(2), 5).await.unwrap();
1443        runtime
1444            .complete(&id, json!({"ok": true}), Vec::new(), 6)
1445            .await
1446            .unwrap();
1447
1448        let actions = runtime.actions_to_revert(&id).await.unwrap();
1449
1450        assert_eq!(actions.len(), 1);
1451        assert_eq!(actions[0].seq, 1);
1452    }
1453
1454    #[tokio::test]
1455    async fn divergence_is_terminal() {
1456        let runtime = runtime();
1457        let id = runtime.begin("async () => {}", vec![], 1).await.unwrap();
1458        runtime
1459            .decide(&id, 0, "db", "read", json!({"id": 1}), false, false, 2)
1460            .await
1461            .unwrap();
1462        runtime.record_result(&id, 0, json!(1), 3).await.unwrap();
1463        assert_eq!(
1464            runtime
1465                .decide(&id, 0, "db", "read", json!({"id": 2}), false, false, 4)
1466                .await
1467                .unwrap(),
1468            ToolDecision::Pause(0)
1469        );
1470        assert_eq!(
1471            runtime.execution(&id).await.unwrap().unwrap().status,
1472            ExecutionStatus::Error
1473        );
1474    }
1475
1476    #[tokio::test]
1477    async fn spills_and_rehydrates_oversized_replay_values() {
1478        let (runtime, store, _) = runtime_parts();
1479        let id = runtime
1480            .begin("async () => {}", vec!["db".into()], 1)
1481            .await
1482            .unwrap();
1483        runtime
1484            .decide(&id, 0, "db", "read", json!({}), false, false, 2)
1485            .await
1486            .unwrap();
1487        let value = json!("x".repeat(MAX_DURABLE_VALUE_BYTES + 1));
1488        runtime
1489            .record_result(&id, 0, value.clone(), 3)
1490            .await
1491            .unwrap();
1492        let raw = store.get_execution(&id).await.unwrap().unwrap();
1493        assert!(raw.log[0].result.as_ref().unwrap()["$artifact"].is_object());
1494        let execution = runtime.execution(&id).await.unwrap().unwrap();
1495        assert_eq!(execution.log[0].result, Some(value.clone()));
1496        assert_eq!(
1497            runtime
1498                .decide(&id, 0, "db", "read", json!({}), false, false, 4)
1499                .await
1500                .unwrap(),
1501            ToolDecision::Replay(value)
1502        );
1503    }
1504
1505    #[tokio::test]
1506    async fn spills_and_rehydrates_oversized_arguments() {
1507        let (runtime, store, _) = runtime_parts();
1508        let id = runtime
1509            .begin("async () => {}", vec!["db".into()], 1)
1510            .await
1511            .unwrap();
1512        let arguments = json!({"payload": "x".repeat(MAX_DURABLE_VALUE_BYTES + 1)});
1513
1514        assert_eq!(
1515            runtime
1516                .decide(&id, 0, "db", "write", arguments.clone(), false, false, 2)
1517                .await
1518                .unwrap(),
1519            ToolDecision::Execute(0)
1520        );
1521
1522        let raw = store.get_execution(&id).await.unwrap().unwrap();
1523        assert!(raw.log[0].arguments["$artifact"].is_object());
1524        let execution = runtime.execution(&id).await.unwrap().unwrap();
1525        assert_eq!(execution.log[0].arguments, arguments);
1526    }
1527
1528    #[tokio::test]
1529    async fn pending_actions_rehydrate_oversized_arguments() {
1530        let runtime = runtime();
1531        let id = runtime
1532            .begin("async () => {}", vec!["db".into()], 1)
1533            .await
1534            .unwrap();
1535        let arguments = json!({"payload": "x".repeat(MAX_DURABLE_VALUE_BYTES + 1)});
1536
1537        assert_eq!(
1538            runtime
1539                .decide(&id, 0, "db", "write", arguments.clone(), true, false, 2)
1540                .await
1541                .unwrap(),
1542            ToolDecision::Pause(0)
1543        );
1544
1545        let pending = runtime.pending(Some(&id)).await.unwrap();
1546        assert_eq!(pending[0].arguments, arguments);
1547    }
1548
1549    #[tokio::test]
1550    async fn retrieves_oversized_final_result_by_execution_and_artifact_ids() {
1551        let (runtime, store, _) = runtime_parts();
1552        let id = runtime.begin("async () => {}", vec![], 1).await.unwrap();
1553        let value = json!("x".repeat(MAX_DURABLE_VALUE_BYTES + 1));
1554
1555        runtime
1556            .complete(&id, value.clone(), Vec::new(), 2)
1557            .await
1558            .unwrap();
1559
1560        let raw = store.get_execution(&id).await.unwrap().unwrap();
1561        let artifact_id = artifact_id(raw.result.as_ref().unwrap());
1562        assert_eq!(
1563            runtime.artifact(&id, &artifact_id).await.unwrap(),
1564            Some(value.clone())
1565        );
1566        assert_eq!(
1567            runtime.execution(&id).await.unwrap().unwrap().result,
1568            Some(value)
1569        );
1570    }
1571
1572    #[tokio::test]
1573    async fn denies_artifact_lookup_for_wrong_execution_owner() {
1574        let (runtime, store, _) = runtime_parts();
1575        let owner = runtime.begin("async () => {}", vec![], 1).await.unwrap();
1576        let other = runtime.begin("async () => {}", vec![], 2).await.unwrap();
1577        let value = json!("x".repeat(MAX_DURABLE_VALUE_BYTES + 1));
1578        runtime
1579            .complete(&owner, value, Vec::new(), 3)
1580            .await
1581            .unwrap();
1582        let raw = store.get_execution(&owner).await.unwrap().unwrap();
1583        let artifact_id = artifact_id(raw.result.as_ref().unwrap());
1584
1585        assert_eq!(runtime.artifact(&other, &artifact_id).await.unwrap(), None);
1586    }
1587
1588    #[tokio::test]
1589    async fn missing_artifact_fails_rehydrated_execution_lookup() {
1590        let (runtime, _, artifacts) = runtime_parts();
1591        let id = runtime.begin("async () => {}", vec![], 1).await.unwrap();
1592        let value = json!("x".repeat(MAX_DURABLE_VALUE_BYTES + 1));
1593        runtime.complete(&id, value, Vec::new(), 2).await.unwrap();
1594
1595        artifacts.delete_execution(&id).await.unwrap();
1596        let error = runtime.execution(&id).await.unwrap_err();
1597
1598        assert!(matches!(error, RuntimeError::Store(message) if message.contains("unavailable")));
1599    }
1600
1601    #[tokio::test]
1602    async fn prune_deletes_artifacts_for_removed_executions() {
1603        let (runtime, store, _) = runtime_parts();
1604        let id = runtime.begin("async () => {}", vec![], 1).await.unwrap();
1605        let value = json!("x".repeat(MAX_DURABLE_VALUE_BYTES + 1));
1606        runtime
1607            .complete(&id, value.clone(), Vec::new(), 2)
1608            .await
1609            .unwrap();
1610        let raw = store.get_execution(&id).await.unwrap().unwrap();
1611        let artifact_id = artifact_id(raw.result.as_ref().unwrap());
1612        assert_eq!(
1613            runtime.artifact(&id, &artifact_id).await.unwrap(),
1614            Some(value)
1615        );
1616
1617        assert_eq!(runtime.prune(0).await.unwrap(), 1);
1618
1619        assert_eq!(runtime.artifact(&id, &artifact_id).await.unwrap(), None);
1620        assert!(runtime.execution(&id).await.unwrap().is_none());
1621    }
1622}