Skip to main content

stateknot_runtime/
failure_close.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! Finite failure-close maintenance, independent of execution ownership.
5
6use crate::{
7    DurableGraphLifecycleOptions, JsonSchemaRegistry, JsonSchemaRegistryBuilder,
8    JsonSchemaRegistryError,
9};
10use serde_json::Value;
11use stateknot_core::{
12    BoundedJson, BoxFuture, CancellationSignal, Digest, EventId, FailureId, JournalAppend,
13    JournalEventIntent, JournalEventKind, JournalExpectation, JournalPayload, SchemaReference,
14    TenantId, Version,
15};
16use stateknot_store_postgres::{
17    PostgresStore, RunFailureCloseCursor, RunFailureCloseOutcome, StoreError,
18};
19use std::time::Duration;
20use thiserror::Error;
21
22/// Offline immutable release identity; never fetched at runtime.
23pub const STANDARD_RUN_FAILURE_CLOSE_EVENT_SCHEMA_ID: &str =
24    "https://stknot.com/schemas/runtime/run-failure-close-event/1.0.0";
25
26/// Returns the exact closed embedded audit schema.
27pub fn standard_run_failure_close_event_schema()
28-> Result<(SchemaReference, Value), RunFailureCloseBuildError> {
29    let document: Value = serde_json::from_str(include_str!(
30        "../schemas/run-failure-close-event-1.0.0.json"
31    ))
32    .map_err(|_| RunFailureCloseBuildError::SchemaDefinition)?;
33    let bytes = serde_json_canonicalizer::to_vec(&document)
34        .map_err(|_| RunFailureCloseBuildError::SchemaDefinition)?;
35    Ok((
36        SchemaReference::new(
37            STANDARD_RUN_FAILURE_CLOSE_EVENT_SCHEMA_ID
38                .parse()
39                .map_err(|_| RunFailureCloseBuildError::SchemaDefinition)?,
40            Version::new(1, 0, 0),
41            Digest::sha256(bytes),
42        ),
43        document,
44    ))
45}
46/// Register before freezing a deployment with child-enabled graph lifecycles.
47pub fn register_standard_run_failure_close_event_schema(
48    builder: &mut JsonSchemaRegistryBuilder,
49) -> Result<SchemaReference, RunFailureCloseBuildError> {
50    let (reference, document) = standard_run_failure_close_event_schema()?;
51    builder.register(reference.clone(), document)?;
52    Ok(reference)
53}
54/// Startup fails closed if the exact offline schema is unavailable.
55#[derive(Clone, Debug, Eq, Error, PartialEq)]
56pub enum RunFailureCloseBuildError {
57    /// Malformed embedded release artifact.
58    #[error("invalid embedded failure close schema")]
59    SchemaDefinition,
60    /// Deployment omitted the exact release schema.
61    #[error("failure close schema is unavailable")]
62    SchemaUnavailable,
63    /// Schema registry refused registration.
64    #[error(transparent)]
65    Registry(#[from] JsonSchemaRegistryError),
66}
67
68/// Tenant-bound continuation. Reset after process loss; not a durable watermark.
69#[derive(Clone, Debug)]
70pub struct RunFailureCloseSweepCursor {
71    tenant: TenantId,
72    after: Option<RunFailureCloseCursor>,
73}
74/// One bounded candidate and its outcome; inspect errors even when later items succeed.
75#[derive(Debug)]
76pub struct RunFailureCloseItem {
77    candidate: RunFailureCloseCursor,
78    result: Result<RunFailureCloseOutcome, StoreError>,
79}
80impl RunFailureCloseItem {
81    /// Tenant/run identity, not a metric label.
82    #[must_use]
83    pub const fn candidate(&self) -> &RunFailureCloseCursor {
84        &self.candidate
85    }
86    /// Pending/unknown child evidence remains an error, never fabricated completion.
87    pub fn result(&self) -> Result<&RunFailureCloseOutcome, &StoreError> {
88        self.result.as_ref()
89    }
90}
91/// At most 16 attempts at closure per tick, each with finite mutation retries.
92#[derive(Debug)]
93pub struct RunFailureCloseTick {
94    cursor: RunFailureCloseSweepCursor,
95    items: Vec<RunFailureCloseItem>,
96    cancelled: bool,
97}
98impl RunFailureCloseTick {
99    /// Preserve across ticks including item failures; exhausted sweeps reset automatically.
100    #[must_use]
101    pub const fn cursor(&self) -> &RunFailureCloseSweepCursor {
102        &self.cursor
103    }
104    /// Every item is available for host supervision and alerting.
105    #[must_use]
106    pub fn items(&self) -> &[RunFailureCloseItem] {
107        &self.items
108    }
109    /// Shutdown interrupted the tick; ambiguous commits remain recoverable.
110    #[must_use]
111    pub const fn is_cancelled(&self) -> bool {
112        self.cancelled
113    }
114}
115
116/// Completes original failures after child cancellation/settlement reconciliation.
117/// The host must authorize tenants, supply fair recurring finite ticks, and inspect
118/// every error. No task is implicitly spawned. Direct evidence was frozen by the
119/// worker before lease release; this coordinator must not call providers again.
120///
121/// ```no_run
122/// use stateknot_core::{CancellationSignal, TenantId};
123/// use stateknot_runtime::{DurableRunFailureCloser, DurableGraphLifecycleOptions,
124///     JsonSchemaRegistryBuilder, RunFailureCloseSweepCursor, RunFailureCloseTick,
125///     register_standard_run_failure_close_event_schema};
126/// use stateknot_store_postgres::{PostgresStore, StoreError};
127/// fn configure(store: PostgresStore, mut schemas: JsonSchemaRegistryBuilder)
128///     -> Result<DurableRunFailureCloser, Box<dyn std::error::Error>> {
129///     register_standard_run_failure_close_event_schema(&mut schemas)?;
130///     Ok(DurableRunFailureCloser::new(store, schemas.build()?,
131///         DurableGraphLifecycleOptions::default())?)
132/// }
133/// async fn step(worker: &DurableRunFailureCloser, tenant: TenantId,
134///     cursor: &mut Option<RunFailureCloseSweepCursor>, shutdown: CancellationSignal)
135///     -> Result<RunFailureCloseTick, StoreError> {
136///     let report = worker.tick(tenant, cursor.clone(), shutdown).await?;
137///     *cursor = Some(report.cursor().clone());
138///     // Inspect every item result; schedule another fair tick even after errors.
139///     Ok(report)
140/// }
141/// ```
142#[derive(Clone, Debug)]
143pub struct DurableRunFailureCloser {
144    store: PostgresStore,
145    schemas: JsonSchemaRegistry,
146    schema: SchemaReference,
147    options: DurableGraphLifecycleOptions,
148}
149impl DurableRunFailureCloser {
150    /// Requires the exact release schema and finite retry policy.
151    pub fn new(
152        store: PostgresStore,
153        schemas: JsonSchemaRegistry,
154        options: DurableGraphLifecycleOptions,
155    ) -> Result<Self, RunFailureCloseBuildError> {
156        let (schema, _) = standard_run_failure_close_event_schema()?;
157        if !schemas.contains(&schema) {
158            return Err(RunFailureCloseBuildError::SchemaUnavailable);
159        }
160        Ok(Self {
161            store,
162            schemas,
163            schema,
164            options,
165        })
166    }
167    /// Independent of runnable discovery. A bad child/run does not starve later candidates.
168    pub fn tick(
169        &self,
170        tenant: TenantId,
171        after: Option<RunFailureCloseSweepCursor>,
172        shutdown: CancellationSignal,
173    ) -> BoxFuture<'_, Result<RunFailureCloseTick, StoreError>> {
174        Box::pin(async move {
175            if after.as_ref().is_some_and(|cursor| cursor.tenant != tenant) {
176                return Err(StoreError::InvalidRunFailureClose);
177            }
178            let mut tick = RunFailureCloseTick {
179                cursor: after.unwrap_or(RunFailureCloseSweepCursor {
180                    tenant: tenant.clone(),
181                    after: None,
182                }),
183                items: Vec::with_capacity(16),
184                cancelled: false,
185            };
186            let candidates = tokio::select! {
187                biased;
188                ()=shutdown.cancelled()=> {tick.cancelled=true;return Ok(tick);}
189                result=self.store.pending_run_failure_closes_after(&tenant,tick.cursor.after.as_ref())=>result?,
190            };
191            let exhausted = candidates.len() < 16;
192            for candidate in candidates {
193                let result = tokio::select! {
194                    biased;
195                    ()=shutdown.cancelled()=> {tick.cancelled=true;break;}
196                    result=self.complete_with_retry(&candidate)=>result,
197                };
198                tick.cursor.after = Some(candidate.clone());
199                tick.items.push(RunFailureCloseItem { candidate, result });
200            }
201            if exhausted && !tick.cancelled {
202                tick.cursor.after = None;
203            }
204            Ok(tick)
205        })
206    }
207    async fn complete_with_retry(
208        &self,
209        candidate: &RunFailureCloseCursor,
210    ) -> Result<RunFailureCloseOutcome, StoreError> {
211        let event = EventId::generate();
212        for attempt in 1..=self.options.maximum_mutation_attempts() {
213            let result = self.complete(candidate, event).await;
214            match result {
215                Err(error)
216                    if attempt < self.options.maximum_mutation_attempts()
217                        && (error.is_retryable()
218                            || matches!(
219                                error,
220                                StoreError::StaleJournalHead | StoreError::StaleLifecycleRevision
221                            )) =>
222                {
223                    tokio::time::sleep(
224                        self.options
225                            .mutation_retry_initial_delay()
226                            .saturating_mul(1_u32 << (attempt - 1))
227                            .min(Duration::from_secs(1)),
228                    )
229                    .await;
230                }
231                other => return other,
232            }
233        }
234        unreachable!("validated positive attempt count")
235    }
236    async fn complete(
237        &self,
238        candidate: &RunFailureCloseCursor,
239        event: EventId,
240    ) -> Result<RunFailureCloseOutcome, StoreError> {
241        let record = self
242            .store
243            .load_run_failure_close(candidate.tenant_id(), candidate.run_id())
244            .await?
245            .ok_or(StoreError::InvalidRunFailureClose)?;
246        if record.completed_at().is_some() {
247            return Ok(RunFailureCloseOutcome::Existing(record));
248        }
249        let run = self
250            .store
251            .load_run(candidate.tenant_id(), candidate.run_id())
252            .await?;
253        let payload = payload(&self.schemas, &self.schema, record.failure().id(), true)?;
254        let append = JournalAppend::new(
255            JournalExpectation::exact(
256                run.journal_head()
257                    .cloned()
258                    .ok_or(StoreError::InvalidRunFailureClose)?,
259            ),
260            JournalEventIntent::control_plane(
261                candidate.tenant_id().clone(),
262                candidate.run_id(),
263                event,
264                payload,
265            )
266            .map_err(|_| StoreError::InvalidRunFailureClose)?,
267        )
268        .map_err(|_| StoreError::InvalidRunFailureClose)?;
269        self.store.complete_run_failure_close(append).await
270    }
271}
272
273pub(crate) fn payload(
274    schemas: &JsonSchemaRegistry,
275    schema: &SchemaReference,
276    failure: FailureId,
277    complete: bool,
278) -> Result<JournalPayload, StoreError> {
279    let (operation, kind) = if complete {
280        ("run_failure_close_completed", "run-failure-close-completed")
281    } else {
282        ("run_failure_close_requested", "run-failure-close-requested")
283    };
284    let data = BoundedJson::try_from_value(
285        serde_json::json!({"operation":operation,"failure_id":failure.to_string()}),
286    )
287    .map_err(|_| StoreError::InvalidRunFailureClose)?;
288    schemas
289        .validate_bounded(schema, &data)
290        .map_err(|_| StoreError::InvalidRunFailureClose)?;
291    JournalPayload::new(
292        schema.clone(),
293        JournalEventKind::new(kind).expect("static event kind"),
294        data,
295    )
296    .map_err(|_| StoreError::InvalidRunFailureClose)
297}
298
299#[cfg(test)]
300mod tests {
301    use super::*;
302    #[test]
303    fn schema_is_offline_closed_and_exact() {
304        let mut builder =
305            JsonSchemaRegistryBuilder::new(crate::JsonSchemaRegistryLimits::default());
306        let reference = register_standard_run_failure_close_event_schema(&mut builder).unwrap();
307        assert_eq!(
308            reference,
309            standard_run_failure_close_event_schema().unwrap().0
310        );
311        let schemas = builder.build().unwrap();
312        for complete in [false, true] {
313            payload(&schemas, &reference, FailureId::generate(), complete).unwrap();
314        }
315        let data=BoundedJson::try_from_value(serde_json::json!({"operation":"run_failure_close_requested","failure_id":FailureId::generate().to_string(),"private_failure":"forbidden"})).unwrap();
316        assert!(schemas.validate_bounded(&reference, &data).is_err());
317    }
318}