1#[allow(clippy::wildcard_imports)]
7use super::*;
8use stateknot_core::{FailureCategory, RunFailure};
9
10#[derive(Clone, Debug, Deserialize, Serialize)]
11#[serde(deny_unknown_fields)]
12struct Intent {
13 failure: Failure,
14 direct_usage: BudgetUsage,
15 checkpoint: CheckpointHead,
16 lifecycle: RunLifecycle,
17}
18
19#[derive(Clone, Debug)]
22pub struct RunFailureCloseRecord {
23 intent: Intent,
24 event: JournalEvent,
25 completed_at: Option<Timestamp>,
26}
27impl RunFailureCloseRecord {
28 #[must_use]
30 pub const fn failure(&self) -> &Failure {
31 &self.intent.failure
32 }
33 #[must_use]
35 pub const fn direct_usage(&self) -> &BudgetUsage {
36 &self.intent.direct_usage
37 }
38 #[must_use]
40 pub const fn checkpoint(&self) -> &CheckpointHead {
41 &self.intent.checkpoint
42 }
43 #[must_use]
45 pub const fn lifecycle(&self) -> &RunLifecycle {
46 &self.intent.lifecycle
47 }
48 #[must_use]
50 pub const fn registration(&self) -> &JournalEvent {
51 &self.event
52 }
53 #[must_use]
55 pub const fn completed_at(&self) -> Option<Timestamp> {
56 self.completed_at
57 }
58}
59
60#[derive(Clone, Debug)]
62pub enum RunFailureCloseOutcome {
63 Committed(RunFailureCloseRecord),
65 Existing(RunFailureCloseRecord),
67}
68impl RunFailureCloseOutcome {
69 #[must_use]
71 pub const fn record(&self) -> &RunFailureCloseRecord {
72 match self {
73 Self::Committed(record) | Self::Existing(record) => record,
74 }
75 }
76}
77
78#[derive(Clone, Debug)]
80pub struct RunFailureCloseCursor {
81 tenant: TenantId,
82 run_id: RunId,
83 requested_at: Timestamp,
84}
85impl RunFailureCloseCursor {
86 #[must_use]
88 pub const fn requested_at(&self) -> Timestamp {
89 self.requested_at
90 }
91 #[must_use]
93 pub const fn tenant_id(&self) -> &TenantId {
94 &self.tenant
95 }
96 #[must_use]
98 pub const fn run_id(&self) -> RunId {
99 self.run_id
100 }
101}
102
103impl PostgresStore {
104 #[allow(clippy::too_many_lines)]
110 pub async fn request_run_failure_close(
111 &self,
112 checkpoint: &CheckpointHead,
113 failure: Failure,
114 direct_usage: BudgetUsage,
115 append: JournalAppend,
116 ) -> Result<RunFailureCloseOutcome, StoreError> {
117 let tenant = checkpoint.tenant_id();
118 let run_id = checkpoint.run_id();
119 if append.intent().tenant_id() != tenant || append.intent().run_id() != run_id {
120 return Err(StoreError::InvalidRunFailureClose);
121 }
122 let mut tx = self.begin_mutation("failure close request").await?;
123 let run = decode_run(fetch_locked_run_row(&mut tx, tenant, run_id).await?)?;
124 if let Some(record) = load(&mut tx, &run).await? {
125 return Ok(RunFailureCloseOutcome::Existing(record));
126 }
127 if append.intent().payload().kind().as_str() != "run-failure-close-requested"
128 || failure.category() == FailureCategory::Cancelled
129 || direct_usage.unpriced_cost_events().get() != 0
130 {
131 return Err(StoreError::InvalidRunFailureClose);
132 }
133 let at = child_runs::authorize_parent_append(&mut tx, &run, &append, true).await?;
134 let current = load_locked_current_checkpoint(&mut tx, &run, tenant, run_id)
135 .await?
136 .ok_or(StoreError::StaleCheckpointHead)?;
137 if current.head() != *checkpoint {
138 return Err(StoreError::StaleCheckpointHead);
139 }
140 let fence = append
141 .worker_fence()
142 .ok_or(StoreError::WrongAppendAuthority)?
143 .clone();
144 ensure_no_unsettled_tool_invocations(&mut tx, ¤t).await?;
145 ensure_no_unsettled_model_invocations(&mut tx, ¤t).await?;
146 let unfinished = query_scalar::<_, bool>("SELECT EXISTS (SELECT 1 FROM stateknot.node_attempts n WHERE n.tenant_id=$1 AND n.run_id=$2 AND n.fence_attempt_id=$3 AND NOT EXISTS (SELECT 1 FROM stateknot.node_attempt_completions c WHERE c.tenant_id=n.tenant_id AND c.run_id=n.run_id AND c.attempt_id=n.attempt_id))")
147 .bind(tenant.as_str()).bind(*run_id.as_uuid()).bind(*fence.attempt_id().as_uuid())
148 .fetch_one(&mut *tx).await.map_err(|source| StoreError::database("failure close direct boundary",source))?;
149 if unfinished {
150 return Err(StoreError::InvalidRunFailureClose);
151 }
152 if let Some(account) =
153 child_runs::load_account_inner(&mut tx, tenant, run_id, false).await?
154 {
155 direct_usage
156 .validate_monotonic_after(account.direct_usage())
157 .map_err(|_| StoreError::IncompleteChildAccounting)?;
158 }
159 let intent = Intent {
160 failure,
161 direct_usage,
162 checkpoint: checkpoint.clone(),
163 lifecycle: run.lifecycle().clone(),
164 };
165 let bytes = encode(&intent)?;
166 if bytes.len() > 4_194_304 {
167 return Err(StoreError::InvalidRunFailureClose);
168 }
169 let event =
170 JournalEvent::commit(append, at).map_err(|error| map_event_commit_error(&error))?;
171 insert_event(&mut tx, &event, projection(&bytes, &event)?).await?;
172 update_run_head(&mut tx, &event, None).await?;
173 query("INSERT INTO stateknot.run_failure_closes (tenant_id,run_id,intent_bytes,intent_digest,journal_sequence,journal_event_id,journal_recorded_at,journal_digest) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)")
174 .bind(tenant.as_str()).bind(*run_id.as_uuid()).bind(&bytes).bind(Digest::sha256(&bytes).as_bytes())
175 .bind(i64::try_from(event.sequence().get()).map_err(|_| StoreError::InvalidRunFailureClose)?)
176 .bind(*event.event_id().as_uuid()).bind(to_database_time(at)?).bind(event.digest().as_bytes())
177 .execute(&mut *tx).await.map_err(|source| StoreError::database("failure close insert",source))?;
178 let released = query("UPDATE stateknot.runs SET lease_attempt_id=NULL,lease_acquired_at=NULL,lease_renewed_at=NULL,lease_expires_at=NULL,scheduler_not_before=NULL,updated_at=$5 WHERE tenant_id=$1 AND run_id=$2 AND lease_attempt_id=$3 AND fencing_epoch=$4 AND lease_expires_at>clock_timestamp()")
179 .bind(tenant.as_str()).bind(*run_id.as_uuid()).bind(*fence.attempt_id().as_uuid())
180 .bind(i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?).bind(to_database_time(at)?)
181 .execute(&mut *tx).await.map_err(|source| StoreError::database("failure close lease release",source))?.rows_affected();
182 if released != 1 {
183 return Err(StoreError::LeaseExpired);
184 }
185 tx.commit()
186 .await
187 .map_err(|source| StoreError::database("failure close commit", source))?;
188 Ok(RunFailureCloseOutcome::Committed(RunFailureCloseRecord {
189 intent,
190 event,
191 completed_at: None,
192 }))
193 }
194
195 pub async fn load_run_failure_close(
197 &self,
198 tenant: &TenantId,
199 run_id: RunId,
200 ) -> Result<Option<RunFailureCloseRecord>, StoreError> {
201 let mut tx = self.begin_mutation("failure close read").await?;
202 let run = decode_run(fetch_locked_run_row(&mut tx, tenant, run_id).await?)?;
203 load(&mut tx, &run).await
204 }
205
206 pub async fn pending_run_failure_closes_after(
209 &self,
210 tenant: &TenantId,
211 after: Option<&RunFailureCloseCursor>,
212 ) -> Result<Vec<RunFailureCloseCursor>, StoreError> {
213 if after.is_some_and(|cursor| cursor.tenant_id() != tenant) {
214 return Err(StoreError::InvalidRunFailureClose);
215 }
216 let sql = if after.is_some() {
217 "SELECT run_id,journal_recorded_at FROM stateknot.run_failure_closes WHERE tenant_id=$1 AND completed_at IS NULL AND (journal_recorded_at,run_id)>($2,$3) ORDER BY journal_recorded_at,run_id LIMIT 16"
218 } else {
219 "SELECT run_id,journal_recorded_at FROM stateknot.run_failure_closes WHERE tenant_id=$1 AND completed_at IS NULL ORDER BY journal_recorded_at,run_id LIMIT 16"
220 };
221 let mut listing = query_as::<_, (Uuid, DateTime<Utc>)>(sql).bind(tenant.as_str());
222 if let Some(cursor) = after {
223 listing = listing
224 .bind(to_database_time(cursor.requested_at)?)
225 .bind(*cursor.run_id.as_uuid());
226 }
227 listing
228 .fetch_all(&self.pool)
229 .await
230 .map_err(|source| StoreError::database("failure close discovery", source))?
231 .into_iter()
232 .map(|(id, at)| {
233 Ok(RunFailureCloseCursor {
234 tenant: tenant.clone(),
235 run_id: RunId::from_uuid(id).map_err(|_| StoreError::InvalidRunFailureClose)?,
236 requested_at: from_database_time(at)?,
237 })
238 })
239 .collect()
240 }
241
242 pub async fn complete_run_failure_close(
245 &self,
246 append: JournalAppend,
247 ) -> Result<RunFailureCloseOutcome, StoreError> {
248 let mut tx = self.begin_mutation("failure close completion").await?;
249 let run = decode_run(
250 fetch_locked_run_row(
251 &mut tx,
252 append.intent().tenant_id(),
253 append.intent().run_id(),
254 )
255 .await?,
256 )?;
257 let record = load(&mut tx, &run)
258 .await?
259 .ok_or(StoreError::InvalidRunFailureClose)?;
260 if record.completed_at().is_some() {
261 return Ok(RunFailureCloseOutcome::Existing(record));
262 }
263 if append.worker_fence().is_some()
264 || append.intent().payload().kind().as_str() != "run-failure-close-completed"
265 {
266 return Err(StoreError::InvalidRunFailureClose);
267 }
268 let at = child_runs::authorize_parent_append(&mut tx, &run, &append, false).await?;
269 let mut usage = record.direct_usage().clone();
270 if let Some(account) = child_runs::load_account_inner(
271 &mut tx,
272 append.intent().tenant_id(),
273 append.intent().run_id(),
274 false,
275 )
276 .await?
277 {
278 child_runs::ensure_settled(&account)?;
279 usage = usage
280 .checked_accumulate(
281 &account
282 .delegated_usage()
283 .map_err(|_| StoreError::IncompleteChildAccounting)?,
284 )
285 .map_err(|_| StoreError::IncompleteChildAccounting)?;
286 }
287 let failure = RunFailure::new(record.failure().clone(), at, usage)
288 .map_err(|_| StoreError::InvalidRunFailureClose)?;
289 let transition = RunTransition::Fail { failure };
290 let projection = RunProjection::transition(run.lifecycle().revision(), transition.clone());
291 let prepared = prepare_durable_wait_projection(
292 &run,
293 append.intent().tenant_id(),
294 append.intent().run_id(),
295 run.lifecycle().revision(),
296 transition,
297 at,
298 )?;
299 let event =
300 JournalEvent::commit(append, at).map_err(|error| map_event_commit_error(&error))?;
301 insert_event(&mut tx, &event, projection_digest(&projection)?).await?;
302 update_run_head(&mut tx, &event, Some(&prepared)).await?;
303 tx.commit()
304 .await
305 .map_err(|source| StoreError::database("failure close final commit", source))?;
306 Ok(RunFailureCloseOutcome::Committed(RunFailureCloseRecord {
307 completed_at: Some(at),
308 ..record
309 }))
310 }
311}
312
313pub(super) async fn exists(
314 tx: &mut Transaction<'_, Postgres>,
315 tenant: &TenantId,
316 run: RunId,
317) -> Result<bool, StoreError> {
318 query_scalar("SELECT EXISTS (SELECT 1 FROM stateknot.run_failure_closes WHERE tenant_id=$1 AND run_id=$2)")
319 .bind(tenant.as_str()).bind(*run.as_uuid()).fetch_one(&mut **tx).await.map_err(|source| StoreError::database("failure close gate",source))
320}
321
322fn encode(intent: &Intent) -> Result<Vec<u8>, StoreError> {
323 serde_json_canonicalizer::to_vec(intent).map_err(|_| StoreError::InvalidRunFailureClose)
324}
325fn projection(bytes: &[u8], event: &JournalEvent) -> Result<Digest, StoreError> {
326 Ok(Digest::sha256(
327 serde_json_canonicalizer::to_vec(&(
328 "stateknot.run-failure-close.v1",
329 Digest::sha256(bytes),
330 event.intent_digest(),
331 ))
332 .map_err(|_| StoreError::InvalidRunFailureClose)?,
333 ))
334}
335
336pub(super) async fn load(
337 tx: &mut Transaction<'_, Postgres>,
338 run: &StoredRun,
339) -> Result<Option<RunFailureCloseRecord>, StoreError> {
340 let scope = run.lifecycle().provenance();
341 let row = query("SELECT * FROM stateknot.run_failure_closes WHERE tenant_id=$1 AND run_id=$2")
342 .bind(scope.tenant_id().as_str())
343 .bind(*scope.run_id().as_uuid())
344 .fetch_optional(&mut **tx)
345 .await
346 .map_err(|source| StoreError::database("failure close load", source))?;
347 let Some(row) = row else {
348 return Ok(None);
349 };
350 let err = |source| StoreError::database("failure close decode", source);
351 let bytes: Vec<u8> = row.try_get("intent_bytes").map_err(err)?;
352 if bytes.is_empty() || bytes.len() > 4_194_304 {
353 return Err(StoreError::corrupt("failure close size"));
354 }
355 let intent: Intent =
356 serde_json::from_slice(&bytes).map_err(|_| StoreError::corrupt("failure close intent"))?;
357 let head = child_runs::row_head(&row, scope.tenant_id(), scope.run_id())?;
358 let (event, digest) = child_runs::anchored_event(
359 tx,
360 scope.tenant_id(),
361 scope.run_id(),
362 i64::try_from(head.sequence().get()).map_err(|_| StoreError::InvalidRunFailureClose)?,
363 )
364 .await?;
365 let completed_at = row
366 .try_get::<Option<DateTime<Utc>>, _>("completed_at")
367 .map_err(err)?
368 .map(from_database_time)
369 .transpose()?;
370 let checkpoint = load_locked_current_checkpoint(tx, run, scope.tenant_id(), scope.run_id())
371 .await?
372 .ok_or(StoreError::StaleCheckpointHead)?;
373 if encode(&intent)? != bytes
374 || Digest::sha256(&bytes).as_bytes()
375 != row.try_get::<Vec<u8>, _>("intent_digest").map_err(err)?
376 || digest != Some(projection(&bytes, &event)?)
377 || event.head() != head
378 || event.source().worker_fence().is_none()
379 || event.payload().kind().as_str() != "run-failure-close-requested"
380 || intent.lifecycle.provenance() != scope
381 || intent.lifecycle.status() != RunStatus::Active
382 || intent.lifecycle.changed_at() > head.recorded_at()
383 || intent.checkpoint != checkpoint.head()
384 || intent.failure.category() == FailureCategory::Cancelled
385 || intent.direct_usage.unpriced_cost_events().get() != 0
386 || run
387 .journal_head()
388 .is_none_or(|current| current.sequence() < head.sequence())
389 {
390 return Err(StoreError::corrupt("failure close binding"));
391 }
392 let valid = if let Some(at) = completed_at {
393 run.lifecycle().status() == RunStatus::Failed
394 && at == run.lifecycle().changed_at()
395 && at >= head.recorded_at()
396 && serde_json_canonicalizer::to_vec(&run.lifecycle().terminal_failure()).ok()
397 == serde_json_canonicalizer::to_vec(&Some(&intent.failure)).ok()
398 && run.lifecycle().revision().get()
399 == intent
400 .lifecycle
401 .revision()
402 .get()
403 .checked_add(1)
404 .ok_or(StoreError::InvalidRunFailureClose)?
405 } else {
406 encode_lifecycle(run.lifecycle())? == encode_lifecycle(&intent.lifecycle)?
407 };
408 if !valid || run.lease().is_some() {
409 return Err(StoreError::corrupt("failure close lifecycle"));
410 }
411 Ok(Some(RunFailureCloseRecord {
412 intent,
413 event,
414 completed_at,
415 }))
416}
417
418pub(super) async fn validate_direct_usage(
419 tx: &mut Transaction<'_, Postgres>,
420 event: &JournalEvent,
421 direct: &BudgetUsage,
422) -> Result<(), StoreError> {
423 let run = decode_run(fetch_locked_run_row(tx, event.tenant_id(), event.run_id()).await?)?;
424 if let Some(record) = load(tx, &run).await? {
425 if direct != record.direct_usage() {
426 return Err(StoreError::IncompleteChildAccounting);
427 }
428 }
429 Ok(())
430}
431
432pub(super) async fn verify_schema(pool: &PgPool) -> Result<(), StoreError> {
433 let installed = query_scalar::<_, String>(CATALOG_QUERY)
434 .fetch_one(pool)
435 .await
436 .map_err(|source| StoreError::database("failure close catalog", source))?;
437 let expected: serde_json::Value =
438 serde_json::from_str(include_str!("failure_close_catalog.json"))
439 .map_err(|_| StoreError::IncompleteSchema)?;
440 if serde_json::from_str::<serde_json::Value>(&installed)
441 .map_err(|_| StoreError::IncompleteSchema)?
442 != expected
443 {
444 return Err(StoreError::IncompleteSchema);
445 }
446 for function in include_str!("../migrations/0024_run_failure_closes.sql")
447 .split("CREATE FUNCTION stateknot.")
448 .skip(1)
449 {
450 let (name, _) = function
451 .split_once("() RETURNS trigger")
452 .ok_or(StoreError::IncompleteSchema)?;
453 let body = function
454 .split("$$")
455 .nth(1)
456 .ok_or(StoreError::IncompleteSchema)?;
457 let installed = query_scalar::<_,String>("SELECT prosrc FROM pg_proc JOIN pg_language ON pg_language.oid=prolang WHERE pg_proc.oid=to_regprocedure($1) AND NOT prosecdef AND lanname='plpgsql'")
458 .bind(format!("stateknot.{name}()")).fetch_optional(pool).await.map_err(|source| StoreError::database("failure close guard",source))?;
459 if installed.as_deref() != Some(body) {
460 return Err(StoreError::IncompleteSchema);
461 }
462 }
463 Ok(())
464}
465
466const CATALOG_QUERY: &str = r"
467WITH relations AS (SELECT oid,relname FROM pg_class WHERE relnamespace='stateknot'::regnamespace AND relname IN ('run_failure_closes'))
468SELECT jsonb_build_object(
469 'columns',(SELECT jsonb_agg(jsonb_build_array(r.relname,a.attname,format_type(a.atttypid,a.atttypmod),a.attnotnull) ORDER BY r.relname,a.attnum) FROM relations r JOIN pg_attribute a ON a.attrelid=r.oid WHERE a.attnum>0 AND NOT a.attisdropped),
470 'constraints',(SELECT jsonb_agg(jsonb_build_array(r.relname,c.conname,pg_get_constraintdef(c.oid),c.convalidated,c.condeferrable) ORDER BY r.relname,c.conname) FROM relations r JOIN pg_constraint c ON c.conrelid=r.oid),
471 'indexes',(SELECT jsonb_agg(jsonb_build_array(r.relname,pg_get_indexdef(i.indexrelid),i.indisvalid,i.indisready,i.indislive) ORDER BY r.relname,pg_get_indexdef(i.indexrelid)) FROM relations r JOIN pg_index i ON i.indrelid=r.oid),
472 'triggers',(SELECT jsonb_agg(jsonb_build_array(t.tgname,pg_get_triggerdef(t.oid),t.tgenabled) ORDER BY t.tgname) FROM pg_trigger t WHERE NOT t.tgisinternal AND (t.tgrelid IN (SELECT oid FROM relations) OR (t.tgrelid='stateknot.runs'::regclass AND t.tgname IN ('runs_failure_close_guard','runs_failure_close_complete')) OR (t.tgrelid='stateknot.child_run_ownership'::regclass AND t.tgname='child_ownership_failure_close_guard')))
473)::text
474";