1use crate::{JsonSchemaRegistry, JsonSchemaRegistryBuilder, JsonSchemaRegistryError};
7use serde_json::{Value, json};
8use stateknot_core::{
9 BoundedJson, BoxFuture, CancellationSignal, ChildRunKey, Digest, EventId, Failure,
10 FailureCategory, FailureCode, FailureId, FailureMessage, FailureOrigin, JournalAppend,
11 JournalEventIntent, JournalEventKind, JournalExpectation, RetryAdvice, RunCancellationRequest,
12 SchemaReference, TenantId, Version,
13};
14use stateknot_store_postgres::{
15 ChildCancellationDelivery, ChildCancellationOutcome, ChildRunSettlementOutcome, PostgresStore,
16 StoreError,
17};
18use std::{fmt, fmt::Write, time::Duration};
19use thiserror::Error;
20
21pub const STANDARD_CHILD_RECONCILIATION_EVENT_SCHEMA_ID: &str =
23 "https://stknot.com/schemas/runtime/child-reconciliation-event/1.0.0";
24
25pub fn standard_child_reconciliation_event_schema()
27-> Result<(SchemaReference, Value), ChildReconcilerBuildError> {
28 let document: Value = serde_json::from_str(include_str!(
29 "../schemas/child-reconciliation-event-1.0.0.json"
30 ))
31 .map_err(|_| ChildReconcilerBuildError::SchemaDefinition)?;
32 let canonical = serde_json_canonicalizer::to_vec(&document)
33 .map_err(|_| ChildReconcilerBuildError::SchemaDefinition)?;
34 let id = STANDARD_CHILD_RECONCILIATION_EVENT_SCHEMA_ID
35 .parse()
36 .map_err(|_| ChildReconcilerBuildError::SchemaDefinition)?;
37 Ok((
38 SchemaReference::new(id, Version::new(1, 0, 0), Digest::sha256(canonical)),
39 document,
40 ))
41}
42
43pub fn register_standard_child_reconciliation_event_schema(
45 builder: &mut JsonSchemaRegistryBuilder,
46) -> Result<SchemaReference, ChildReconcilerBuildError> {
47 let (reference, document) = standard_child_reconciliation_event_schema()?;
48 builder.register(reference.clone(), document)?;
49 Ok(reference)
50}
51
52#[derive(Clone, Copy, Debug, Eq, PartialEq)]
54pub struct DurableChildReconcilerOptions {
55 maximum_mutation_attempts: u8,
56 retry_initial_delay: Duration,
57}
58
59impl DurableChildReconcilerOptions {
60 pub fn new(
62 maximum_mutation_attempts: u8,
63 retry_initial_delay: Duration,
64 ) -> Result<Self, ChildReconcilerBuildError> {
65 if !(1..=10).contains(&maximum_mutation_attempts)
66 || retry_initial_delay.is_zero()
67 || retry_initial_delay > Duration::from_secs(1)
68 {
69 return Err(ChildReconcilerBuildError::InvalidOptions);
70 }
71 Ok(Self {
72 maximum_mutation_attempts,
73 retry_initial_delay,
74 })
75 }
76 #[must_use]
78 pub const fn maximum_mutation_attempts(self) -> u8 {
79 self.maximum_mutation_attempts
80 }
81
82 pub(crate) fn retry_delay(self, attempt: u8) -> Duration {
83 self.retry_initial_delay
84 .saturating_mul(1_u32 << (attempt - 1))
85 .min(Duration::from_secs(1))
86 }
87}
88impl Default for DurableChildReconcilerOptions {
89 fn default() -> Self {
90 Self {
91 maximum_mutation_attempts: 3,
92 retry_initial_delay: Duration::from_millis(25),
93 }
94 }
95}
96
97#[derive(Clone, Debug)]
100pub struct ChildReconciliationCursor {
101 tenant: TenantId,
102 cancellation_after: Option<ChildRunKey>,
103 settlement_after: Option<ChildRunKey>,
104}
105
106#[derive(Clone, Copy, Debug, Eq, PartialEq)]
108pub enum ChildReconciliationKind {
109 Cancellation,
111 Settlement,
113}
114
115#[derive(Clone, Copy, Debug, Eq, PartialEq)]
117pub enum ChildReconciliationCommit {
118 Cancellation(ChildCancellationOutcome),
120 CancellationRecovered,
122 Settlement,
124 SettlementRecovered,
126}
127
128pub struct ChildReconciliationItem {
130 key: ChildRunKey,
131 kind: ChildReconciliationKind,
132 result: Result<ChildReconciliationCommit, StoreError>,
133}
134impl ChildReconciliationItem {
135 #[must_use]
137 pub const fn key(&self) -> &ChildRunKey {
138 &self.key
139 }
140 #[must_use]
142 pub const fn kind(&self) -> ChildReconciliationKind {
143 self.kind
144 }
145 pub fn result(&self) -> Result<ChildReconciliationCommit, &StoreError> {
147 self.result.as_ref().copied()
148 }
149}
150impl fmt::Debug for ChildReconciliationItem {
151 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
152 f.debug_struct("ChildReconciliationItem")
153 .field("key", &self.key)
154 .field("kind", &self.kind)
155 .field("result", &self.result.as_ref().map_err(ToString::to_string))
156 .finish()
157 }
158}
159
160#[derive(Debug)]
162pub struct ChildReconciliationTick {
163 cursor: ChildReconciliationCursor,
164 items: Vec<ChildReconciliationItem>,
165 cancelled: bool,
166}
167impl ChildReconciliationTick {
168 #[must_use]
170 pub const fn cursor(&self) -> &ChildReconciliationCursor {
171 &self.cursor
172 }
173 #[must_use]
175 pub fn items(&self) -> &[ChildReconciliationItem] {
176 &self.items
177 }
178 #[must_use]
180 pub const fn is_cancelled(&self) -> bool {
181 self.cancelled
182 }
183}
184
185#[derive(Debug, Error)]
187pub enum ChildReconcilerBuildError {
188 #[error("invalid child reconciler retry policy")]
190 InvalidOptions,
191 #[error("invalid embedded child reconciliation schema")]
193 SchemaDefinition,
194 #[error("child reconciliation schema is absent from the deployment registry")]
196 SchemaUnavailable,
197 #[error(transparent)]
199 Registry(#[from] JsonSchemaRegistryError),
200}
201
202#[derive(Clone, Debug)]
243pub struct DurableChildReconciler {
244 store: PostgresStore,
245 schemas: JsonSchemaRegistry,
246 schema: SchemaReference,
247 options: DurableChildReconcilerOptions,
248}
249impl DurableChildReconciler {
250 pub fn new(
252 store: PostgresStore,
253 schemas: JsonSchemaRegistry,
254 options: DurableChildReconcilerOptions,
255 ) -> Result<Self, ChildReconcilerBuildError> {
256 let (schema, _) = standard_child_reconciliation_event_schema()?;
257 if !schemas.contains(&schema) {
258 return Err(ChildReconcilerBuildError::SchemaUnavailable);
259 }
260 Ok(Self {
261 store,
262 schemas,
263 schema,
264 options,
265 })
266 }
267
268 pub fn tick(
272 &self,
273 tenant: TenantId,
274 after: Option<ChildReconciliationCursor>,
275 shutdown: CancellationSignal,
276 ) -> BoxFuture<'_, Result<ChildReconciliationTick, StoreError>> {
277 Box::pin(self.tick_inner(tenant, after, shutdown))
278 }
279
280 async fn tick_inner(
281 &self,
282 tenant: TenantId,
283 after: Option<ChildReconciliationCursor>,
284 shutdown: CancellationSignal,
285 ) -> Result<ChildReconciliationTick, StoreError> {
286 if after.as_ref().is_some_and(|cursor| cursor.tenant != tenant) {
287 return Err(StoreError::ChildRunRejected);
288 }
289 let mut tick = ChildReconciliationTick {
290 cursor: after.unwrap_or(ChildReconciliationCursor {
291 tenant: tenant.clone(),
292 cancellation_after: None,
293 settlement_after: None,
294 }),
295 items: Vec::with_capacity(32),
296 cancelled: false,
297 };
298 for kind in [
299 ChildReconciliationKind::Cancellation,
300 ChildReconciliationKind::Settlement,
301 ] {
302 if shutdown.is_cancelled() {
303 tick.cancelled = true;
304 break;
305 }
306 let keys = tokio::select! {
307 biased;
308 () = shutdown.cancelled() => { tick.cancelled = true; break; }
309 keys = async {
310 match kind {
311 ChildReconciliationKind::Cancellation => self.store.pending_child_cancellations_after(&tenant, tick.cursor.cancellation_after.as_ref()).await,
312 ChildReconciliationKind::Settlement => self.store.pending_child_settlements_after(&tenant, tick.cursor.settlement_after.as_ref()).await,
313 }
314 } => keys?,
315 };
316 let exhausted = keys.len() < 16;
317 for key in keys {
318 let result = tokio::select! {
319 biased;
320 () = shutdown.cancelled() => { tick.cancelled = true; break; }
321 result = self.reconcile_with_retry(&key, kind) => result,
322 };
323 match kind {
324 ChildReconciliationKind::Cancellation => {
325 tick.cursor.cancellation_after = Some(key.clone());
326 }
327 ChildReconciliationKind::Settlement => {
328 tick.cursor.settlement_after = Some(key.clone());
329 }
330 }
331 tick.items
332 .push(ChildReconciliationItem { key, kind, result });
333 }
334 if tick.cancelled {
335 break;
336 }
337 if exhausted {
338 match kind {
339 ChildReconciliationKind::Cancellation => tick.cursor.cancellation_after = None,
340 ChildReconciliationKind::Settlement => tick.cursor.settlement_after = None,
341 }
342 }
343 }
344 Ok(tick)
345 }
346
347 async fn reconcile_with_retry(
348 &self,
349 key: &ChildRunKey,
350 kind: ChildReconciliationKind,
351 ) -> Result<ChildReconciliationCommit, StoreError> {
352 let event_id = EventId::generate();
353 let failure_id = FailureId::generate();
354 for attempt in 1..=self.options.maximum_mutation_attempts {
355 let result = match kind {
356 ChildReconciliationKind::Cancellation => {
357 Box::pin(self.cancel(key, event_id, failure_id)).await
358 }
359 ChildReconciliationKind::Settlement => Box::pin(self.settle(key, event_id)).await,
360 };
361 match result {
362 Err(error)
363 if attempt < self.options.maximum_mutation_attempts
364 && (error.is_retryable()
365 || matches!(
366 error,
367 StoreError::StaleJournalHead | StoreError::StaleLifecycleRevision
368 )) =>
369 {
370 let delay = self
371 .options
372 .retry_initial_delay
373 .saturating_mul(1_u32 << (attempt - 1))
374 .min(Duration::from_secs(1));
375 tokio::time::sleep(delay).await;
376 }
377 other => return other,
378 }
379 }
380 unreachable!("validated positive attempt count always returns")
381 }
382
383 async fn cancel(
384 &self,
385 key: &ChildRunKey,
386 event_id: EventId,
387 failure_id: FailureId,
388 ) -> Result<ChildReconciliationCommit, StoreError> {
389 let record = self.store.load_child_cancellation(key).await?;
390 if record.receipt().is_some() {
391 return Ok(ChildReconciliationCommit::CancellationRecovered);
392 }
393 let child = self
394 .store
395 .load_run(key.tenant_id(), record.child_run_id())
396 .await?;
397 let at = self.store.observe_database_clock().await?;
398 let failure = Failure::new(
399 failure_id,
400 FailureCategory::Cancelled,
401 FailureCode::new(if record.is_parent_failure_close() {
402 "child.parent_failed"
403 } else {
404 "child.parent_cancelled"
405 })
406 .expect("static code"),
407 FailureOrigin::new("stateknot.runtime.child_reconciler").expect("static origin"),
408 FailureMessage::new(if record.is_parent_failure_close() {
409 "The owning parent is closing after failure."
410 } else {
411 "The owning parent requested cancellation."
412 })
413 .expect("static public message"),
414 RetryAdvice::Never,
415 )
416 .expect("static cancellation semantics")
417 .with_caused_by_event(event_id);
418 let request =
419 RunCancellationRequest::new(failure, at).map_err(|_| StoreError::ChildRunRejected)?;
420 let append = self.append(
421 key,
422 record.child_run_id(),
423 child
424 .journal_head()
425 .cloned()
426 .ok_or(StoreError::ChildRunRejected)?,
427 event_id,
428 "child-run-cancellation-requested",
429 "child_run_cancellation_requested",
430 )?;
431 match Box::pin(self.store.deliver_child_cancellation(key, append, request)).await? {
432 ChildCancellationDelivery::Committed(record) => {
433 Ok(ChildReconciliationCommit::Cancellation(
434 record
435 .receipt()
436 .ok_or(StoreError::ChildRunRejected)?
437 .outcome(),
438 ))
439 }
440 ChildCancellationDelivery::Idempotent(_) => {
441 Ok(ChildReconciliationCommit::CancellationRecovered)
442 }
443 }
444 }
445
446 async fn settle(
447 &self,
448 key: &ChildRunKey,
449 event_id: EventId,
450 ) -> Result<ChildReconciliationCommit, StoreError> {
451 let parent = self
452 .store
453 .load_run(key.tenant_id(), key.parent_run_id())
454 .await?;
455 let append = self.append(
456 key,
457 key.parent_run_id(),
458 parent
459 .journal_head()
460 .cloned()
461 .ok_or(StoreError::ChildRunRejected)?,
462 event_id,
463 "child-run-settled",
464 "child_run_settled",
465 )?;
466 match Box::pin(self.store.settle_child_run(key, append)).await? {
467 ChildRunSettlementOutcome::Committed { .. } => {
468 Ok(ChildReconciliationCommit::Settlement)
469 }
470 ChildRunSettlementOutcome::Idempotent { .. } => {
471 Ok(ChildReconciliationCommit::SettlementRecovered)
472 }
473 _ => Err(StoreError::ChildRunRejected),
474 }
475 }
476
477 fn append(
478 &self,
479 key: &ChildRunKey,
480 run: stateknot_core::RunId,
481 head: stateknot_core::JournalHead,
482 id: EventId,
483 kind: &str,
484 operation: &str,
485 ) -> Result<JournalAppend, StoreError> {
486 let mut digest = String::with_capacity(64);
487 for value in key.digest().as_bytes() {
488 write!(digest, "{value:02x}").expect("String writes are infallible");
489 }
490 let data = BoundedJson::try_from_value(
491 json!({ "operation": operation, "ownership_digest": digest }),
492 )
493 .map_err(|_| StoreError::ChildRunRejected)?;
494 self.schemas
495 .validate_bounded(&self.schema, &data)
496 .map_err(|_| StoreError::ChildRunRejected)?;
497 let payload = stateknot_core::JournalPayload::new(
498 self.schema.clone(),
499 JournalEventKind::new(kind).map_err(|_| StoreError::ChildRunRejected)?,
500 data,
501 )
502 .map_err(|_| StoreError::ChildRunRejected)?;
503 let intent = JournalEventIntent::control_plane(key.tenant_id().clone(), run, id, payload)
504 .map_err(|_| StoreError::ChildRunRejected)?;
505 JournalAppend::new(JournalExpectation::exact(head), intent)
506 .map_err(|_| StoreError::ChildRunRejected)
507 }
508}
509
510#[cfg(test)]
511mod tests {
512 use super::*;
513 use crate::JsonSchemaRegistryLimits;
514 #[test]
515 fn schema_is_closed_pinned_and_retry_bounds_are_finite() {
516 let mut builder = JsonSchemaRegistryBuilder::new(JsonSchemaRegistryLimits::default());
517 let schema = register_standard_child_reconciliation_event_schema(&mut builder).unwrap();
518 assert_eq!(
519 schema.digest().to_string(),
520 "sha256:54346f985269669cbdd22b6c70c77aebbadb5953fc96c45518af3086f4cdfd08"
521 );
522 let registry = builder.build().unwrap();
523 for operation in ["child_run_settled", "child_run_cancellation_requested"] {
524 let mut value = json!({ "operation": operation, "ownership_digest": "a".repeat(64) });
525 registry
526 .validate_bounded(
527 &schema,
528 &BoundedJson::try_from_value(value.clone()).unwrap(),
529 )
530 .unwrap();
531 value["private_parent_reason"] = json!("must not leak");
532 assert!(
533 registry
534 .validate_bounded(&schema, &BoundedJson::try_from_value(value).unwrap())
535 .is_err()
536 );
537 }
538 assert!(DurableChildReconcilerOptions::new(0, Duration::from_millis(1)).is_err());
539 assert!(DurableChildReconcilerOptions::new(11, Duration::from_millis(1)).is_err());
540 assert!(DurableChildReconcilerOptions::new(3, Duration::ZERO).is_err());
541 assert!(DurableChildReconcilerOptions::new(3, Duration::from_secs(2)).is_err());
542 }
543}