1use crate::{
7 DurableGraphLifecycleOptions, JsonSchemaRegistry, JsonSchemaRegistryBuilder,
8 JsonSchemaRegistryError,
9};
10use serde_json::Value;
11use stateknot_core::{
12 BoundedJson, BoxFuture, CancellationSignal, Digest, EventId, FailureId, JournalAppend,
13 JournalEventIntent, JournalEventKind, JournalExpectation, JournalPayload, SchemaReference,
14 TenantId, Version,
15};
16use stateknot_store_postgres::{
17 PostgresStore, RunFailureCloseCursor, RunFailureCloseOutcome, StoreError,
18};
19use std::time::Duration;
20use thiserror::Error;
21
22pub const STANDARD_RUN_FAILURE_CLOSE_EVENT_SCHEMA_ID: &str =
24 "https://stknot.com/schemas/runtime/run-failure-close-event/1.0.0";
25
26pub fn standard_run_failure_close_event_schema()
28-> Result<(SchemaReference, Value), RunFailureCloseBuildError> {
29 let document: Value = serde_json::from_str(include_str!(
30 "../schemas/run-failure-close-event-1.0.0.json"
31 ))
32 .map_err(|_| RunFailureCloseBuildError::SchemaDefinition)?;
33 let bytes = serde_json_canonicalizer::to_vec(&document)
34 .map_err(|_| RunFailureCloseBuildError::SchemaDefinition)?;
35 Ok((
36 SchemaReference::new(
37 STANDARD_RUN_FAILURE_CLOSE_EVENT_SCHEMA_ID
38 .parse()
39 .map_err(|_| RunFailureCloseBuildError::SchemaDefinition)?,
40 Version::new(1, 0, 0),
41 Digest::sha256(bytes),
42 ),
43 document,
44 ))
45}
46pub fn register_standard_run_failure_close_event_schema(
48 builder: &mut JsonSchemaRegistryBuilder,
49) -> Result<SchemaReference, RunFailureCloseBuildError> {
50 let (reference, document) = standard_run_failure_close_event_schema()?;
51 builder.register(reference.clone(), document)?;
52 Ok(reference)
53}
54#[derive(Clone, Debug, Eq, Error, PartialEq)]
56pub enum RunFailureCloseBuildError {
57 #[error("invalid embedded failure close schema")]
59 SchemaDefinition,
60 #[error("failure close schema is unavailable")]
62 SchemaUnavailable,
63 #[error(transparent)]
65 Registry(#[from] JsonSchemaRegistryError),
66}
67
68#[derive(Clone, Debug)]
70pub struct RunFailureCloseSweepCursor {
71 tenant: TenantId,
72 after: Option<RunFailureCloseCursor>,
73}
74#[derive(Debug)]
76pub struct RunFailureCloseItem {
77 candidate: RunFailureCloseCursor,
78 result: Result<RunFailureCloseOutcome, StoreError>,
79}
80impl RunFailureCloseItem {
81 #[must_use]
83 pub const fn candidate(&self) -> &RunFailureCloseCursor {
84 &self.candidate
85 }
86 pub fn result(&self) -> Result<&RunFailureCloseOutcome, &StoreError> {
88 self.result.as_ref()
89 }
90}
91#[derive(Debug)]
93pub struct RunFailureCloseTick {
94 cursor: RunFailureCloseSweepCursor,
95 items: Vec<RunFailureCloseItem>,
96 cancelled: bool,
97}
98impl RunFailureCloseTick {
99 #[must_use]
101 pub const fn cursor(&self) -> &RunFailureCloseSweepCursor {
102 &self.cursor
103 }
104 #[must_use]
106 pub fn items(&self) -> &[RunFailureCloseItem] {
107 &self.items
108 }
109 #[must_use]
111 pub const fn is_cancelled(&self) -> bool {
112 self.cancelled
113 }
114}
115
116#[derive(Clone, Debug)]
143pub struct DurableRunFailureCloser {
144 store: PostgresStore,
145 schemas: JsonSchemaRegistry,
146 schema: SchemaReference,
147 options: DurableGraphLifecycleOptions,
148}
149impl DurableRunFailureCloser {
150 pub fn new(
152 store: PostgresStore,
153 schemas: JsonSchemaRegistry,
154 options: DurableGraphLifecycleOptions,
155 ) -> Result<Self, RunFailureCloseBuildError> {
156 let (schema, _) = standard_run_failure_close_event_schema()?;
157 if !schemas.contains(&schema) {
158 return Err(RunFailureCloseBuildError::SchemaUnavailable);
159 }
160 Ok(Self {
161 store,
162 schemas,
163 schema,
164 options,
165 })
166 }
167 pub fn tick(
169 &self,
170 tenant: TenantId,
171 after: Option<RunFailureCloseSweepCursor>,
172 shutdown: CancellationSignal,
173 ) -> BoxFuture<'_, Result<RunFailureCloseTick, StoreError>> {
174 Box::pin(async move {
175 if after.as_ref().is_some_and(|cursor| cursor.tenant != tenant) {
176 return Err(StoreError::InvalidRunFailureClose);
177 }
178 let mut tick = RunFailureCloseTick {
179 cursor: after.unwrap_or(RunFailureCloseSweepCursor {
180 tenant: tenant.clone(),
181 after: None,
182 }),
183 items: Vec::with_capacity(16),
184 cancelled: false,
185 };
186 let candidates = tokio::select! {
187 biased;
188 ()=shutdown.cancelled()=> {tick.cancelled=true;return Ok(tick);}
189 result=self.store.pending_run_failure_closes_after(&tenant,tick.cursor.after.as_ref())=>result?,
190 };
191 let exhausted = candidates.len() < 16;
192 for candidate in candidates {
193 let result = tokio::select! {
194 biased;
195 ()=shutdown.cancelled()=> {tick.cancelled=true;break;}
196 result=self.complete_with_retry(&candidate)=>result,
197 };
198 tick.cursor.after = Some(candidate.clone());
199 tick.items.push(RunFailureCloseItem { candidate, result });
200 }
201 if exhausted && !tick.cancelled {
202 tick.cursor.after = None;
203 }
204 Ok(tick)
205 })
206 }
207 async fn complete_with_retry(
208 &self,
209 candidate: &RunFailureCloseCursor,
210 ) -> Result<RunFailureCloseOutcome, StoreError> {
211 let event = EventId::generate();
212 for attempt in 1..=self.options.maximum_mutation_attempts() {
213 let result = self.complete(candidate, event).await;
214 match result {
215 Err(error)
216 if attempt < self.options.maximum_mutation_attempts()
217 && (error.is_retryable()
218 || matches!(
219 error,
220 StoreError::StaleJournalHead | StoreError::StaleLifecycleRevision
221 )) =>
222 {
223 tokio::time::sleep(
224 self.options
225 .mutation_retry_initial_delay()
226 .saturating_mul(1_u32 << (attempt - 1))
227 .min(Duration::from_secs(1)),
228 )
229 .await;
230 }
231 other => return other,
232 }
233 }
234 unreachable!("validated positive attempt count")
235 }
236 async fn complete(
237 &self,
238 candidate: &RunFailureCloseCursor,
239 event: EventId,
240 ) -> Result<RunFailureCloseOutcome, StoreError> {
241 let record = self
242 .store
243 .load_run_failure_close(candidate.tenant_id(), candidate.run_id())
244 .await?
245 .ok_or(StoreError::InvalidRunFailureClose)?;
246 if record.completed_at().is_some() {
247 return Ok(RunFailureCloseOutcome::Existing(record));
248 }
249 let run = self
250 .store
251 .load_run(candidate.tenant_id(), candidate.run_id())
252 .await?;
253 let payload = payload(&self.schemas, &self.schema, record.failure().id(), true)?;
254 let append = JournalAppend::new(
255 JournalExpectation::exact(
256 run.journal_head()
257 .cloned()
258 .ok_or(StoreError::InvalidRunFailureClose)?,
259 ),
260 JournalEventIntent::control_plane(
261 candidate.tenant_id().clone(),
262 candidate.run_id(),
263 event,
264 payload,
265 )
266 .map_err(|_| StoreError::InvalidRunFailureClose)?,
267 )
268 .map_err(|_| StoreError::InvalidRunFailureClose)?;
269 self.store.complete_run_failure_close(append).await
270 }
271}
272
273pub(crate) fn payload(
274 schemas: &JsonSchemaRegistry,
275 schema: &SchemaReference,
276 failure: FailureId,
277 complete: bool,
278) -> Result<JournalPayload, StoreError> {
279 let (operation, kind) = if complete {
280 ("run_failure_close_completed", "run-failure-close-completed")
281 } else {
282 ("run_failure_close_requested", "run-failure-close-requested")
283 };
284 let data = BoundedJson::try_from_value(
285 serde_json::json!({"operation":operation,"failure_id":failure.to_string()}),
286 )
287 .map_err(|_| StoreError::InvalidRunFailureClose)?;
288 schemas
289 .validate_bounded(schema, &data)
290 .map_err(|_| StoreError::InvalidRunFailureClose)?;
291 JournalPayload::new(
292 schema.clone(),
293 JournalEventKind::new(kind).expect("static event kind"),
294 data,
295 )
296 .map_err(|_| StoreError::InvalidRunFailureClose)
297}
298
299#[cfg(test)]
300mod tests {
301 use super::*;
302 #[test]
303 fn schema_is_offline_closed_and_exact() {
304 let mut builder =
305 JsonSchemaRegistryBuilder::new(crate::JsonSchemaRegistryLimits::default());
306 let reference = register_standard_run_failure_close_event_schema(&mut builder).unwrap();
307 assert_eq!(
308 reference,
309 standard_run_failure_close_event_schema().unwrap().0
310 );
311 let schemas = builder.build().unwrap();
312 for complete in [false, true] {
313 payload(&schemas, &reference, FailureId::generate(), complete).unwrap();
314 }
315 let data=BoundedJson::try_from_value(serde_json::json!({"operation":"run_failure_close_requested","failure_id":FailureId::generate().to_string(),"private_failure":"forbidden"})).unwrap();
316 assert!(schemas.validate_bounded(&reference, &data).is_err());
317 }
318}