Skip to main content

graphrt_core/
store.rs

1//! Durable run records for hosts that serve many runs (an HTTP server, a job queue).
2//!
3//! A [`RunStore`] keeps, per run: a [`RunRecord`] (status, outputs, pause
4//! reasons...), the run's events as JSON, and the [`Snapshot`] of a paused run, so
5//! a run can be resumed by id after the client went away or the host restarted.
6//!
7//! Nothing secret is persisted: event values serialize with secrets masked, and
8//! snapshots never contain secret plaintext (see [`crate::snapshot`]).
9
10use std::fmt;
11
12use futures::future::BoxFuture;
13use indexmap::IndexMap;
14use serde::{Deserialize, Serialize};
15
16use crate::event::{Event, PauseReason, Usage};
17use crate::snapshot::Snapshot;
18use crate::value::Value;
19
20/// Lifecycle status of a stored run.
21#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
22#[serde(rename_all = "snake_case")]
23#[non_exhaustive]
24pub enum RunStatus {
25    Running,
26    Paused,
27    Succeeded,
28    Failed,
29    Aborted,
30}
31
32impl RunStatus {
33    /// Whether the run reached a final state.
34    pub fn is_terminal(self) -> bool {
35        matches!(self, RunStatus::Succeeded | RunStatus::Failed | RunStatus::Aborted)
36    }
37
38    /// Parse the wire name (`running`, `paused`, ...).
39    pub fn parse(text: &str) -> Option<Self> {
40        match text {
41            "running" => Some(RunStatus::Running),
42            "paused" => Some(RunStatus::Paused),
43            "succeeded" => Some(RunStatus::Succeeded),
44            "failed" => Some(RunStatus::Failed),
45            "aborted" => Some(RunStatus::Aborted),
46            _ => None,
47        }
48    }
49}
50
51/// Persisted summary of one run.
52#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
53pub struct RunRecord {
54    pub run_id: String,
55    /// Name the workflow is served under.
56    pub workflow: String,
57    /// Fingerprint of the workflow graph (resume requires the same graph).
58    pub fingerprint: String,
59    pub status: RunStatus,
60    pub created_ms: u64,
61    pub updated_ms: u64,
62    /// Why the run is paused (empty unless `status == paused`).
63    #[serde(default)]
64    pub pause_reasons: Vec<PauseReason>,
65    #[serde(default)]
66    pub outputs: IndexMap<String, Value>,
67    /// Ordered answer text assembled from response nodes.
68    #[serde(default)]
69    pub answer: String,
70    #[serde(default)]
71    pub error: Option<String>,
72    #[serde(default)]
73    pub error_type: Option<String>,
74    #[serde(default)]
75    pub usage: Usage,
76    /// Events stored so far (may stop growing at the store's cap).
77    #[serde(default)]
78    pub event_count: u64,
79    /// Whether events were dropped because the store's per-run cap was reached.
80    #[serde(default)]
81    pub events_truncated: bool,
82}
83
84impl RunRecord {
85    /// A new `running` record.
86    pub fn running(
87        run_id: impl Into<String>,
88        workflow: impl Into<String>,
89        fingerprint: impl Into<String>,
90        now_ms: u64,
91    ) -> Self {
92        Self {
93            run_id: run_id.into(),
94            workflow: workflow.into(),
95            fingerprint: fingerprint.into(),
96            status: RunStatus::Running,
97            created_ms: now_ms,
98            updated_ms: now_ms,
99            pause_reasons: Vec::new(),
100            outputs: IndexMap::new(),
101            answer: String::new(),
102            error: None,
103            error_type: None,
104            usage: Usage::default(),
105            event_count: 0,
106            events_truncated: false,
107        }
108    }
109
110    /// Apply a terminal event (`RunSucceeded`, `RunFailed`, `RunAborted`, `RunPaused`).
111    /// Returns the snapshot of a paused run so the caller can persist it.
112    pub fn apply_terminal(&mut self, event: &Event, now_ms: u64) -> Option<Snapshot> {
113        self.updated_ms = now_ms;
114        match event {
115            Event::RunSucceeded { outputs, usage, .. } => {
116                self.status = RunStatus::Succeeded;
117                self.outputs.clone_from(outputs);
118                self.usage = *usage;
119                self.pause_reasons.clear();
120                None
121            }
122            Event::RunFailed { error, error_type, usage, .. } => {
123                self.status = RunStatus::Failed;
124                self.error = Some(error.clone());
125                self.error_type = Some(error_type.clone());
126                self.usage = *usage;
127                self.pause_reasons.clear();
128                None
129            }
130            Event::RunAborted { reason, .. } => {
131                self.status = RunStatus::Aborted;
132                self.error = Some(reason.clone());
133                self.pause_reasons.clear();
134                None
135            }
136            Event::RunPaused { reasons, snapshot } => {
137                self.status = RunStatus::Paused;
138                self.pause_reasons.clone_from(reasons);
139                self.usage = snapshot.usage;
140                Some((**snapshot).clone())
141            }
142            _ => None,
143        }
144    }
145}
146
147/// Query for [`RunStore::list`].
148#[derive(Clone, Debug, Default, PartialEq, Eq)]
149pub struct RunFilter {
150    pub workflow: Option<String>,
151    pub status: Option<RunStatus>,
152    /// Maximum records returned (stores cap this; 0 means the store default).
153    pub limit: usize,
154    /// Continue after this run id (the `next_cursor` of a previous page).
155    pub cursor: Option<String>,
156}
157
158/// One page of [`RunStore::list`], newest first.
159#[derive(Clone, Debug, Default, PartialEq, Serialize)]
160pub struct RunPage {
161    pub runs: Vec<RunRecord>,
162    pub next_cursor: Option<String>,
163}
164
165/// Store failure.
166#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
167#[non_exhaustive]
168pub enum StoreError {
169    #[error("run `{0}` not found")]
170    NotFound(String),
171    /// The operation conflicts with the run's current state (already exists,
172    /// not paused, already being resumed).
173    #[error("{0}")]
174    Conflict(String),
175    #[error("invalid run id `{0}`")]
176    InvalidId(String),
177    #[error("run store I/O error: {0}")]
178    Io(String),
179    #[error("corrupt run store entry: {0}")]
180    Corrupt(String),
181}
182
183/// Result type of [`RunStore`] operations.
184pub type StoreResult<T> = Result<T, StoreError>;
185
186/// Durable storage of run records, events and snapshots.
187///
188/// Implementations must make [`RunStore::claim_resume`] atomic: of two concurrent
189/// claims on the same paused run, exactly one succeeds.
190pub trait RunStore: Send + Sync + fmt::Debug {
191    /// Insert a new record. Fails with [`StoreError::Conflict`] if the id exists.
192    fn create(&self, record: RunRecord) -> BoxFuture<'_, StoreResult<()>>;
193    /// Insert or replace a record.
194    fn put(&self, record: RunRecord) -> BoxFuture<'_, StoreResult<()>>;
195    fn get<'a>(&'a self, run_id: &'a str) -> BoxFuture<'a, StoreResult<Option<RunRecord>>>;
196    fn list(&self, filter: RunFilter) -> BoxFuture<'_, StoreResult<RunPage>>;
197    /// Remove a run and everything stored for it. Returns whether it existed.
198    fn delete<'a>(&'a self, run_id: &'a str) -> BoxFuture<'a, StoreResult<bool>>;
199    /// Append events (already serialized by [`event_to_stored`], secrets masked).
200    /// Stores may cap the number of events kept per run; they then set
201    /// `events_truncated`.
202    fn append_events<'a>(&'a self, run_id: &'a str, events: Vec<StoredEvent>) -> BoxFuture<'a, StoreResult<()>>;
203    /// Stored events, starting at index `from`.
204    fn events<'a>(&'a self, run_id: &'a str, from: u64) -> BoxFuture<'a, StoreResult<Vec<StoredEvent>>>;
205    fn save_snapshot<'a>(&'a self, run_id: &'a str, snapshot: &'a Snapshot) -> BoxFuture<'a, StoreResult<()>>;
206    fn load_snapshot<'a>(&'a self, run_id: &'a str) -> BoxFuture<'a, StoreResult<Option<Snapshot>>>;
207    /// Atomically move a `paused` run to `running` and return the updated record.
208    /// Fails with [`StoreError::Conflict`] when the run is not paused (including when
209    /// another request claimed it first).
210    fn claim_resume<'a>(&'a self, run_id: &'a str, now_ms: u64) -> BoxFuture<'a, StoreResult<RunRecord>>;
211    /// After a host restart: mark every `running` run as `failed` ("interrupted by
212    /// restart"). Paused runs stay resumable. Returns how many runs were marked.
213    fn recover_interrupted(&self, now_ms: u64) -> BoxFuture<'_, StoreResult<usize>>;
214    /// Delete runs (any status except `running`) last updated before `cutoff_ms`.
215    fn purge_before(&self, cutoff_ms: u64) -> BoxFuture<'_, StoreResult<usize>>;
216}
217
218/// Error message stored on runs interrupted by a host restart.
219pub const INTERRUPTED_BY_RESTART: &str = "interrupted by restart";
220
221/// An event serialized as one JSON object, the form in which stores keep events.
222///
223/// Holding the JSON text instead of a [`serde_json::Value`] tree saves building
224/// a map per event on the way in and re-serializing it on the way out.
225pub type StoredEvent = Box<serde_json::value::RawValue>;
226
227/// Serialize an event for storage: the same JSON as [`event_to_json`], written
228/// straight to text.
229pub fn event_to_stored(event: &Event) -> StoredEvent {
230    let text = match event {
231        // Rare, and the snapshot has to be dropped: go through the tree.
232        Event::RunPaused { .. } => serde_json::to_string(&event_to_json(event)),
233        _ => serde_json::to_string(event),
234    };
235    text.ok()
236        .and_then(|text| serde_json::value::RawValue::from_string(text).ok())
237        .unwrap_or_else(|| serde_json::value::RawValue::NULL.to_owned())
238}
239
240/// The `type` field of a stored event (e.g. `node_started`), if it has one.
241pub fn stored_event_type(event: &StoredEvent) -> Option<&str> {
242    #[derive(Deserialize)]
243    struct TypeOnly<'a> {
244        #[serde(rename = "type", borrow)]
245        kind: Option<&'a str>,
246    }
247    serde_json::from_str::<TypeOnly<'_>>(event.get()).ok().and_then(|t| t.kind)
248}
249
250/// Serialize an event for storage. Secrets are masked by the value serializer; the
251/// snapshot carried by `run_paused` is dropped (stores keep it separately).
252pub fn event_to_json(event: &Event) -> serde_json::Value {
253    let mut json = serde_json::to_value(event).unwrap_or(serde_json::Value::Null);
254    if let Some(obj) = json.as_object_mut() {
255        if obj.get("type").and_then(serde_json::Value::as_str) == Some("run_paused") {
256            obj.remove("snapshot");
257        }
258    }
259    json
260}
261
262/// Validate a run id used as a storage key: 1–128 chars of `[A-Za-z0-9_.-]`, not
263/// starting with `.`.
264pub fn validate_run_id(run_id: &str) -> StoreResult<()> {
265    let ok = !run_id.is_empty()
266        && run_id.len() <= 128
267        && !run_id.starts_with('.')
268        && run_id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'));
269    if ok { Ok(()) } else { Err(StoreError::InvalidId(run_id.chars().take(64).collect())) }
270}
271
272/// Current Unix time in milliseconds (0 where no clock exists).
273pub fn now_ms() -> u64 {
274    #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
275    {
276        0
277    }
278    #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
279    {
280        std::time::SystemTime::now()
281            .duration_since(std::time::UNIX_EPOCH)
282            .map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX))
283            .unwrap_or(0)
284    }
285}
286
287/// Apply `filter` to records already sorted newest first (shared by implementations).
288pub fn page_of(mut sorted: Vec<RunRecord>, filter: &RunFilter, max_limit: usize) -> RunPage {
289    sorted.retain(|r| {
290        filter.workflow.as_deref().is_none_or(|w| r.workflow == w) && filter.status.is_none_or(|s| r.status == s)
291    });
292    let start = match &filter.cursor {
293        Some(cursor) => sorted.iter().position(|r| &r.run_id == cursor).map_or(sorted.len(), |i| i + 1),
294        None => 0,
295    };
296    let limit = if filter.limit == 0 { max_limit } else { filter.limit.min(max_limit) };
297    let runs: Vec<RunRecord> = sorted.into_iter().skip(start).take(limit.saturating_add(1)).collect();
298    let mut runs = runs;
299    let next_cursor = if runs.len() > limit {
300        runs.truncate(limit);
301        runs.last().map(|r| r.run_id.clone())
302    } else {
303        None
304    };
305    RunPage { runs, next_cursor }
306}
307
308/// Sort records newest first (ties broken by id, descending) for [`page_of`].
309pub fn sort_newest_first(records: &mut [RunRecord]) {
310    records.sort_by(|a, b| b.created_ms.cmp(&a.created_ms).then_with(|| b.run_id.cmp(&a.run_id)));
311}
312
313#[cfg(test)]
314mod tests {
315    use super::*;
316
317    fn rec(id: &str, wf: &str, created: u64, status: RunStatus) -> RunRecord {
318        let mut r = RunRecord::running(id, wf, "fp", created);
319        r.status = status;
320        r
321    }
322
323    #[test]
324    fn run_ids_are_validated() {
325        assert!(validate_run_id("run-1a2b_3.c").is_ok());
326        for bad in ["", "../x", ".hidden", "a/b", "a b", &"x".repeat(129)] {
327            assert!(validate_run_id(bad).is_err(), "{bad}");
328        }
329    }
330
331    #[test]
332    fn paging_and_filtering() {
333        let mut all = vec![
334            rec("a", "w1", 1, RunStatus::Succeeded),
335            rec("b", "w1", 2, RunStatus::Paused),
336            rec("c", "w2", 3, RunStatus::Paused),
337            rec("d", "w1", 4, RunStatus::Failed),
338        ];
339        sort_newest_first(&mut all);
340        let page = page_of(all.clone(), &RunFilter { limit: 2, ..Default::default() }, 100);
341        assert_eq!(page.runs.iter().map(|r| r.run_id.as_str()).collect::<Vec<_>>(), ["d", "c"]);
342        assert_eq!(page.next_cursor.as_deref(), Some("c"));
343        let next = page_of(all.clone(), &RunFilter { limit: 2, cursor: page.next_cursor, ..Default::default() }, 100);
344        assert_eq!(next.runs.iter().map(|r| r.run_id.as_str()).collect::<Vec<_>>(), ["b", "a"]);
345        assert_eq!(next.next_cursor, None);
346        let paused = page_of(
347            all.clone(),
348            &RunFilter { status: Some(RunStatus::Paused), workflow: Some("w1".into()), ..Default::default() },
349            100,
350        );
351        assert_eq!(paused.runs.len(), 1);
352        assert_eq!(paused.runs[0].run_id, "b");
353    }
354
355    #[test]
356    fn paused_event_json_drops_snapshot() {
357        let snapshot = Snapshot {
358            format: crate::snapshot::SNAPSHOT_FORMAT,
359            run_id: "r".into(),
360            graph_fingerprint: "fp".into(),
361            elapsed_ms: 0,
362            steps: 0,
363            usage: Usage::default(),
364            handled_errors: 0,
365            root: crate::snapshot::FrameSnapshot::default(),
366        };
367        let event = Event::RunPaused { reasons: Vec::new(), snapshot: Box::new(snapshot) };
368        let json = event_to_json(&event);
369        assert_eq!(json["type"], "run_paused");
370        assert!(json.get("snapshot").is_none());
371        let stored = event_to_stored(&event);
372        assert_eq!(serde_json::from_str::<serde_json::Value>(stored.get()).unwrap(), json);
373        assert_eq!(stored_event_type(&stored), Some("run_paused"));
374    }
375
376    #[test]
377    fn stored_event_text_matches_the_json_tree() {
378        let mut outputs = IndexMap::new();
379        outputs.insert("z".to_owned(), Value::from("last"));
380        outputs.insert("a".to_owned(), Value::Secret(std::sync::Arc::from("sk-live")));
381        outputs.insert("n".to_owned(), Value::from_json(serde_json::json!(1.5)));
382        let event = Event::RunSucceeded { outputs, usage: Usage::default(), handled_errors: 0, elapsed_ms: 7 };
383        let stored = event_to_stored(&event);
384        assert_eq!(stored.get(), event_to_json(&event).to_string());
385        assert!(!stored.get().contains("sk-live"));
386        assert_eq!(stored_event_type(&stored), Some("run_succeeded"));
387    }
388}