Skip to main content

stateknot_store_postgres/
failure_closes.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! Sealed failure decisions: settled direct work, then durable child drain.
5
6#[allow(clippy::wildcard_imports)]
7use super::*;
8use stateknot_core::{FailureCategory, RunFailure};
9
10#[derive(Clone, Debug, Deserialize, Serialize)]
11#[serde(deny_unknown_fields)]
12struct Intent {
13    failure: Failure,
14    direct_usage: BudgetUsage,
15    checkpoint: CheckpointHead,
16    lifecycle: RunLifecycle,
17}
18
19/// Verified immutable failure decision. Active business state is not runnable
20/// while this record is pending; completion retains the same original failure.
21#[derive(Clone, Debug)]
22pub struct RunFailureCloseRecord {
23    intent: Intent,
24    event: JournalEvent,
25    completed_at: Option<Timestamp>,
26}
27impl RunFailureCloseRecord {
28    /// Original failure occurrence; later candidates never replace it.
29    #[must_use]
30    pub const fn failure(&self) -> &Failure {
31        &self.intent.failure
32    }
33    /// Complete, priced DIRECT usage frozen before releasing execution ownership.
34    #[must_use]
35    pub const fn direct_usage(&self) -> &BudgetUsage {
36        &self.intent.direct_usage
37    }
38    /// Exact sealed checkpoint; no successor is allowed.
39    #[must_use]
40    pub const fn checkpoint(&self) -> &CheckpointHead {
41        &self.intent.checkpoint
42    }
43    /// Active lifecycle at registration; not a cancellation request.
44    #[must_use]
45    pub const fn lifecycle(&self) -> &RunLifecycle {
46        &self.intent.lifecycle
47    }
48    /// Worker-fenced registration atomically captured with lease release and child work.
49    #[must_use]
50    pub const fn registration(&self) -> &JournalEvent {
51        &self.event
52    }
53    /// Database observation at final Failed commit, if complete.
54    #[must_use]
55    pub const fn completed_at(&self) -> Option<Timestamp> {
56        self.completed_at
57    }
58}
59
60/// First decision wins; Existing does not assert equivalence of retry candidates.
61#[derive(Clone, Debug)]
62pub enum RunFailureCloseOutcome {
63    /// The requested decision was committed atomically.
64    Committed(RunFailureCloseRecord),
65    /// The original decision was recovered, including after a lost acknowledgement.
66    Existing(RunFailureCloseRecord),
67}
68impl RunFailureCloseOutcome {
69    /// Returns verified original evidence for either outcome.
70    #[must_use]
71    pub const fn record(&self) -> &RunFailureCloseRecord {
72        match self {
73            Self::Committed(record) | Self::Existing(record) => record,
74        }
75    }
76}
77
78/// Tenant-bound stable range cursor; not an authorization grant or permanent watermark.
79#[derive(Clone, Debug)]
80pub struct RunFailureCloseCursor {
81    tenant: TenantId,
82    run_id: RunId,
83    requested_at: Timestamp,
84}
85impl RunFailureCloseCursor {
86    /// Source event time for measuring oldest pending work and sweep lag.
87    #[must_use]
88    pub const fn requested_at(&self) -> Timestamp {
89        self.requested_at
90    }
91    /// Tenant selected by the authenticated maintenance host.
92    #[must_use]
93    pub const fn tenant_id(&self) -> &TenantId {
94        &self.tenant
95    }
96    /// Run identity, still valid after completion removes the candidate.
97    #[must_use]
98    pub const fn run_id(&self) -> RunId {
99        self.run_id
100    }
101}
102
103impl PostgresStore {
104    /// Persist a non-cancellation failure for an Active run whose direct work is
105    /// fully settled. Caller supplies trusted COMPLETE direct usage, never child
106    /// usage. Unknown direct cost or unfinished physical work rejects the whole
107    /// transaction without releasing its recovery lease. All immediate unsettled
108    /// children receive durable cancellation work; no recursive run locks.
109    #[allow(clippy::too_many_lines)]
110    pub async fn request_run_failure_close(
111        &self,
112        checkpoint: &CheckpointHead,
113        failure: Failure,
114        direct_usage: BudgetUsage,
115        append: JournalAppend,
116    ) -> Result<RunFailureCloseOutcome, StoreError> {
117        let tenant = checkpoint.tenant_id();
118        let run_id = checkpoint.run_id();
119        if append.intent().tenant_id() != tenant || append.intent().run_id() != run_id {
120            return Err(StoreError::InvalidRunFailureClose);
121        }
122        let mut tx = self.begin_mutation("failure close request").await?;
123        let run = decode_run(fetch_locked_run_row(&mut tx, tenant, run_id).await?)?;
124        if let Some(record) = load(&mut tx, &run).await? {
125            return Ok(RunFailureCloseOutcome::Existing(record));
126        }
127        if append.intent().payload().kind().as_str() != "run-failure-close-requested"
128            || failure.category() == FailureCategory::Cancelled
129            || direct_usage.unpriced_cost_events().get() != 0
130        {
131            return Err(StoreError::InvalidRunFailureClose);
132        }
133        let at = child_runs::authorize_parent_append(&mut tx, &run, &append, true).await?;
134        let current = load_locked_current_checkpoint(&mut tx, &run, tenant, run_id)
135            .await?
136            .ok_or(StoreError::StaleCheckpointHead)?;
137        if current.head() != *checkpoint {
138            return Err(StoreError::StaleCheckpointHead);
139        }
140        let fence = append
141            .worker_fence()
142            .ok_or(StoreError::WrongAppendAuthority)?
143            .clone();
144        ensure_no_unsettled_tool_invocations(&mut tx, &current).await?;
145        ensure_no_unsettled_model_invocations(&mut tx, &current).await?;
146        let unfinished = query_scalar::<_, bool>("SELECT EXISTS (SELECT 1 FROM stateknot.node_attempts n WHERE n.tenant_id=$1 AND n.run_id=$2 AND n.fence_attempt_id=$3 AND NOT EXISTS (SELECT 1 FROM stateknot.node_attempt_completions c WHERE c.tenant_id=n.tenant_id AND c.run_id=n.run_id AND c.attempt_id=n.attempt_id))")
147            .bind(tenant.as_str()).bind(*run_id.as_uuid()).bind(*fence.attempt_id().as_uuid())
148            .fetch_one(&mut *tx).await.map_err(|source| StoreError::database("failure close direct boundary",source))?;
149        if unfinished {
150            return Err(StoreError::InvalidRunFailureClose);
151        }
152        if let Some(account) =
153            child_runs::load_account_inner(&mut tx, tenant, run_id, false).await?
154        {
155            direct_usage
156                .validate_monotonic_after(account.direct_usage())
157                .map_err(|_| StoreError::IncompleteChildAccounting)?;
158        }
159        let intent = Intent {
160            failure,
161            direct_usage,
162            checkpoint: checkpoint.clone(),
163            lifecycle: run.lifecycle().clone(),
164        };
165        let bytes = encode(&intent)?;
166        if bytes.len() > 4_194_304 {
167            return Err(StoreError::InvalidRunFailureClose);
168        }
169        let event =
170            JournalEvent::commit(append, at).map_err(|error| map_event_commit_error(&error))?;
171        insert_event(&mut tx, &event, projection(&bytes, &event)?).await?;
172        update_run_head(&mut tx, &event, None).await?;
173        query("INSERT INTO stateknot.run_failure_closes (tenant_id,run_id,intent_bytes,intent_digest,journal_sequence,journal_event_id,journal_recorded_at,journal_digest) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)")
174            .bind(tenant.as_str()).bind(*run_id.as_uuid()).bind(&bytes).bind(Digest::sha256(&bytes).as_bytes())
175            .bind(i64::try_from(event.sequence().get()).map_err(|_| StoreError::InvalidRunFailureClose)?)
176            .bind(*event.event_id().as_uuid()).bind(to_database_time(at)?).bind(event.digest().as_bytes())
177            .execute(&mut *tx).await.map_err(|source| StoreError::database("failure close insert",source))?;
178        let released = query("UPDATE stateknot.runs SET lease_attempt_id=NULL,lease_acquired_at=NULL,lease_renewed_at=NULL,lease_expires_at=NULL,scheduler_not_before=NULL,updated_at=$5 WHERE tenant_id=$1 AND run_id=$2 AND lease_attempt_id=$3 AND fencing_epoch=$4 AND lease_expires_at>clock_timestamp()")
179            .bind(tenant.as_str()).bind(*run_id.as_uuid()).bind(*fence.attempt_id().as_uuid())
180            .bind(i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?).bind(to_database_time(at)?)
181            .execute(&mut *tx).await.map_err(|source| StoreError::database("failure close lease release",source))?.rows_affected();
182        if released != 1 {
183            return Err(StoreError::LeaseExpired);
184        }
185        tx.commit()
186            .await
187            .map_err(|source| StoreError::database("failure close commit", source))?;
188        Ok(RunFailureCloseOutcome::Committed(RunFailureCloseRecord {
189            intent,
190            event,
191            completed_at: None,
192        }))
193    }
194
195    /// Verified read under a consistent snapshot, including original evidence after completion.
196    pub async fn load_run_failure_close(
197        &self,
198        tenant: &TenantId,
199        run_id: RunId,
200    ) -> Result<Option<RunFailureCloseRecord>, StoreError> {
201        let mut tx = self.begin_mutation("failure close read").await?;
202        let run = decode_run(fetch_locked_run_row(&mut tx, tenant, run_id).await?)?;
203        load(&mut tx, &run).await
204    }
205
206    /// At most 16 pending decisions. Preserve cursor through item failures, reset
207    /// to None at sweep end and after process loss. Quarantine is not hidden.
208    pub async fn pending_run_failure_closes_after(
209        &self,
210        tenant: &TenantId,
211        after: Option<&RunFailureCloseCursor>,
212    ) -> Result<Vec<RunFailureCloseCursor>, StoreError> {
213        if after.is_some_and(|cursor| cursor.tenant_id() != tenant) {
214            return Err(StoreError::InvalidRunFailureClose);
215        }
216        let sql = if after.is_some() {
217            "SELECT run_id,journal_recorded_at FROM stateknot.run_failure_closes WHERE tenant_id=$1 AND completed_at IS NULL AND (journal_recorded_at,run_id)>($2,$3) ORDER BY journal_recorded_at,run_id LIMIT 16"
218        } else {
219            "SELECT run_id,journal_recorded_at FROM stateknot.run_failure_closes WHERE tenant_id=$1 AND completed_at IS NULL ORDER BY journal_recorded_at,run_id LIMIT 16"
220        };
221        let mut listing = query_as::<_, (Uuid, DateTime<Utc>)>(sql).bind(tenant.as_str());
222        if let Some(cursor) = after {
223            listing = listing
224                .bind(to_database_time(cursor.requested_at)?)
225                .bind(*cursor.run_id.as_uuid());
226        }
227        listing
228            .fetch_all(&self.pool)
229            .await
230            .map_err(|source| StoreError::database("failure close discovery", source))?
231            .into_iter()
232            .map(|(id, at)| {
233                Ok(RunFailureCloseCursor {
234                    tenant: tenant.clone(),
235                    run_id: RunId::from_uuid(id).map_err(|_| StoreError::InvalidRunFailureClose)?,
236                    requested_at: from_database_time(at)?,
237                })
238            })
239            .collect()
240    }
241
242    /// Finalizes only with exact frozen direct plus fully verified settled child
243    /// usage. Call with a control-plane audit append; no execution lease is acquired.
244    pub async fn complete_run_failure_close(
245        &self,
246        append: JournalAppend,
247    ) -> Result<RunFailureCloseOutcome, StoreError> {
248        let mut tx = self.begin_mutation("failure close completion").await?;
249        let run = decode_run(
250            fetch_locked_run_row(
251                &mut tx,
252                append.intent().tenant_id(),
253                append.intent().run_id(),
254            )
255            .await?,
256        )?;
257        let record = load(&mut tx, &run)
258            .await?
259            .ok_or(StoreError::InvalidRunFailureClose)?;
260        if record.completed_at().is_some() {
261            return Ok(RunFailureCloseOutcome::Existing(record));
262        }
263        if append.worker_fence().is_some()
264            || append.intent().payload().kind().as_str() != "run-failure-close-completed"
265        {
266            return Err(StoreError::InvalidRunFailureClose);
267        }
268        let at = child_runs::authorize_parent_append(&mut tx, &run, &append, false).await?;
269        let mut usage = record.direct_usage().clone();
270        if let Some(account) = child_runs::load_account_inner(
271            &mut tx,
272            append.intent().tenant_id(),
273            append.intent().run_id(),
274            false,
275        )
276        .await?
277        {
278            child_runs::ensure_settled(&account)?;
279            usage = usage
280                .checked_accumulate(
281                    &account
282                        .delegated_usage()
283                        .map_err(|_| StoreError::IncompleteChildAccounting)?,
284                )
285                .map_err(|_| StoreError::IncompleteChildAccounting)?;
286        }
287        let failure = RunFailure::new(record.failure().clone(), at, usage)
288            .map_err(|_| StoreError::InvalidRunFailureClose)?;
289        let transition = RunTransition::Fail { failure };
290        let projection = RunProjection::transition(run.lifecycle().revision(), transition.clone());
291        let prepared = prepare_durable_wait_projection(
292            &run,
293            append.intent().tenant_id(),
294            append.intent().run_id(),
295            run.lifecycle().revision(),
296            transition,
297            at,
298        )?;
299        let event =
300            JournalEvent::commit(append, at).map_err(|error| map_event_commit_error(&error))?;
301        insert_event(&mut tx, &event, projection_digest(&projection)?).await?;
302        update_run_head(&mut tx, &event, Some(&prepared)).await?;
303        tx.commit()
304            .await
305            .map_err(|source| StoreError::database("failure close final commit", source))?;
306        Ok(RunFailureCloseOutcome::Committed(RunFailureCloseRecord {
307            completed_at: Some(at),
308            ..record
309        }))
310    }
311}
312
313pub(super) async fn exists(
314    tx: &mut Transaction<'_, Postgres>,
315    tenant: &TenantId,
316    run: RunId,
317) -> Result<bool, StoreError> {
318    query_scalar("SELECT EXISTS (SELECT 1 FROM stateknot.run_failure_closes WHERE tenant_id=$1 AND run_id=$2)")
319        .bind(tenant.as_str()).bind(*run.as_uuid()).fetch_one(&mut **tx).await.map_err(|source| StoreError::database("failure close gate",source))
320}
321
322fn encode(intent: &Intent) -> Result<Vec<u8>, StoreError> {
323    serde_json_canonicalizer::to_vec(intent).map_err(|_| StoreError::InvalidRunFailureClose)
324}
325fn projection(bytes: &[u8], event: &JournalEvent) -> Result<Digest, StoreError> {
326    Ok(Digest::sha256(
327        serde_json_canonicalizer::to_vec(&(
328            "stateknot.run-failure-close.v1",
329            Digest::sha256(bytes),
330            event.intent_digest(),
331        ))
332        .map_err(|_| StoreError::InvalidRunFailureClose)?,
333    ))
334}
335
336pub(super) async fn load(
337    tx: &mut Transaction<'_, Postgres>,
338    run: &StoredRun,
339) -> Result<Option<RunFailureCloseRecord>, StoreError> {
340    let scope = run.lifecycle().provenance();
341    let row = query("SELECT * FROM stateknot.run_failure_closes WHERE tenant_id=$1 AND run_id=$2")
342        .bind(scope.tenant_id().as_str())
343        .bind(*scope.run_id().as_uuid())
344        .fetch_optional(&mut **tx)
345        .await
346        .map_err(|source| StoreError::database("failure close load", source))?;
347    let Some(row) = row else {
348        return Ok(None);
349    };
350    let err = |source| StoreError::database("failure close decode", source);
351    let bytes: Vec<u8> = row.try_get("intent_bytes").map_err(err)?;
352    if bytes.is_empty() || bytes.len() > 4_194_304 {
353        return Err(StoreError::corrupt("failure close size"));
354    }
355    let intent: Intent =
356        serde_json::from_slice(&bytes).map_err(|_| StoreError::corrupt("failure close intent"))?;
357    let head = child_runs::row_head(&row, scope.tenant_id(), scope.run_id())?;
358    let (event, digest) = child_runs::anchored_event(
359        tx,
360        scope.tenant_id(),
361        scope.run_id(),
362        i64::try_from(head.sequence().get()).map_err(|_| StoreError::InvalidRunFailureClose)?,
363    )
364    .await?;
365    let completed_at = row
366        .try_get::<Option<DateTime<Utc>>, _>("completed_at")
367        .map_err(err)?
368        .map(from_database_time)
369        .transpose()?;
370    let checkpoint = load_locked_current_checkpoint(tx, run, scope.tenant_id(), scope.run_id())
371        .await?
372        .ok_or(StoreError::StaleCheckpointHead)?;
373    if encode(&intent)? != bytes
374        || Digest::sha256(&bytes).as_bytes()
375            != row.try_get::<Vec<u8>, _>("intent_digest").map_err(err)?
376        || digest != Some(projection(&bytes, &event)?)
377        || event.head() != head
378        || event.source().worker_fence().is_none()
379        || event.payload().kind().as_str() != "run-failure-close-requested"
380        || intent.lifecycle.provenance() != scope
381        || intent.lifecycle.status() != RunStatus::Active
382        || intent.lifecycle.changed_at() > head.recorded_at()
383        || intent.checkpoint != checkpoint.head()
384        || intent.failure.category() == FailureCategory::Cancelled
385        || intent.direct_usage.unpriced_cost_events().get() != 0
386        || run
387            .journal_head()
388            .is_none_or(|current| current.sequence() < head.sequence())
389    {
390        return Err(StoreError::corrupt("failure close binding"));
391    }
392    let valid = if let Some(at) = completed_at {
393        run.lifecycle().status() == RunStatus::Failed
394            && at == run.lifecycle().changed_at()
395            && at >= head.recorded_at()
396            && serde_json_canonicalizer::to_vec(&run.lifecycle().terminal_failure()).ok()
397                == serde_json_canonicalizer::to_vec(&Some(&intent.failure)).ok()
398            && run.lifecycle().revision().get()
399                == intent
400                    .lifecycle
401                    .revision()
402                    .get()
403                    .checked_add(1)
404                    .ok_or(StoreError::InvalidRunFailureClose)?
405    } else {
406        encode_lifecycle(run.lifecycle())? == encode_lifecycle(&intent.lifecycle)?
407    };
408    if !valid || run.lease().is_some() {
409        return Err(StoreError::corrupt("failure close lifecycle"));
410    }
411    Ok(Some(RunFailureCloseRecord {
412        intent,
413        event,
414        completed_at,
415    }))
416}
417
418pub(super) async fn validate_direct_usage(
419    tx: &mut Transaction<'_, Postgres>,
420    event: &JournalEvent,
421    direct: &BudgetUsage,
422) -> Result<(), StoreError> {
423    let run = decode_run(fetch_locked_run_row(tx, event.tenant_id(), event.run_id()).await?)?;
424    if let Some(record) = load(tx, &run).await? {
425        if direct != record.direct_usage() {
426            return Err(StoreError::IncompleteChildAccounting);
427        }
428    }
429    Ok(())
430}
431
432pub(super) async fn verify_schema(pool: &PgPool) -> Result<(), StoreError> {
433    let installed = query_scalar::<_, String>(CATALOG_QUERY)
434        .fetch_one(pool)
435        .await
436        .map_err(|source| StoreError::database("failure close catalog", source))?;
437    let expected: serde_json::Value =
438        serde_json::from_str(include_str!("failure_close_catalog.json"))
439            .map_err(|_| StoreError::IncompleteSchema)?;
440    if serde_json::from_str::<serde_json::Value>(&installed)
441        .map_err(|_| StoreError::IncompleteSchema)?
442        != expected
443    {
444        return Err(StoreError::IncompleteSchema);
445    }
446    for function in include_str!("../migrations/0024_run_failure_closes.sql")
447        .split("CREATE FUNCTION stateknot.")
448        .skip(1)
449    {
450        let (name, _) = function
451            .split_once("() RETURNS trigger")
452            .ok_or(StoreError::IncompleteSchema)?;
453        let body = function
454            .split("$$")
455            .nth(1)
456            .ok_or(StoreError::IncompleteSchema)?;
457        let installed = query_scalar::<_,String>("SELECT prosrc FROM pg_proc JOIN pg_language ON pg_language.oid=prolang WHERE pg_proc.oid=to_regprocedure($1) AND NOT prosecdef AND lanname='plpgsql'")
458            .bind(format!("stateknot.{name}()")).fetch_optional(pool).await.map_err(|source| StoreError::database("failure close guard",source))?;
459        if installed.as_deref() != Some(body) {
460            return Err(StoreError::IncompleteSchema);
461        }
462    }
463    Ok(())
464}
465
466const CATALOG_QUERY: &str = r"
467WITH relations AS (SELECT oid,relname FROM pg_class WHERE relnamespace='stateknot'::regnamespace AND relname IN ('run_failure_closes'))
468SELECT jsonb_build_object(
469 'columns',(SELECT jsonb_agg(jsonb_build_array(r.relname,a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull) ORDER BY r.relname,a.attnum) FROM relations r JOIN pg_attribute a ON a.attrelid=r.oid WHERE a.attnum>0 AND NOT a.attisdropped),
470 'constraints',(SELECT jsonb_agg(jsonb_build_array(r.relname,c.conname,pg_get_constraintdef(c.oid),c.convalidated,c.condeferrable) ORDER BY r.relname,c.conname) FROM relations r JOIN pg_constraint c ON c.conrelid=r.oid),
471 'indexes',(SELECT jsonb_agg(jsonb_build_array(r.relname,pg_get_indexdef(i.indexrelid),i.indisvalid,i.indisready,i.indislive) ORDER BY r.relname,pg_get_indexdef(i.indexrelid)) FROM relations r JOIN pg_index i ON i.indrelid=r.oid),
472 'triggers',(SELECT jsonb_agg(jsonb_build_array(t.tgname,pg_get_triggerdef(t.oid),t.tgenabled) ORDER BY t.tgname) FROM pg_trigger t WHERE NOT t.tgisinternal AND (t.tgrelid IN (SELECT oid FROM relations) OR (t.tgrelid='stateknot.runs'::regclass AND t.tgname IN ('runs_failure_close_guard','runs_failure_close_complete')) OR (t.tgrelid='stateknot.child_run_ownership'::regclass AND t.tgname='child_ownership_failure_close_guard')))
473)::text
474";