Skip to main content

stateknot_runtime/
child_reconciler.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! Trusted tenant-scoped child cancellation and accounting maintenance.
5
6use crate::{JsonSchemaRegistry, JsonSchemaRegistryBuilder, JsonSchemaRegistryError};
7use serde_json::{Value, json};
8use stateknot_core::{
9    BoundedJson, BoxFuture, CancellationSignal, ChildRunKey, Digest, EventId, Failure,
10    FailureCategory, FailureCode, FailureId, FailureMessage, FailureOrigin, JournalAppend,
11    JournalEventIntent, JournalEventKind, JournalExpectation, RetryAdvice, RunCancellationRequest,
12    SchemaReference, TenantId, Version,
13};
14use stateknot_store_postgres::{
15    ChildCancellationDelivery, ChildCancellationOutcome, ChildRunSettlementOutcome, PostgresStore,
16    StoreError,
17};
18use std::{fmt, fmt::Write, time::Duration};
19use thiserror::Error;
20
21/// Offline identity of the embedded reconciliation audit schema.
22pub const STANDARD_CHILD_RECONCILIATION_EVENT_SCHEMA_ID: &str =
23    "https://stknot.com/schemas/runtime/child-reconciliation-event/1.0.0";
24
25/// Returns the closed schema shipped in the binary; no HTTP lookup is performed.
26pub fn standard_child_reconciliation_event_schema()
27-> Result<(SchemaReference, Value), ChildReconcilerBuildError> {
28    let document: Value = serde_json::from_str(include_str!(
29        "../schemas/child-reconciliation-event-1.0.0.json"
30    ))
31    .map_err(|_| ChildReconcilerBuildError::SchemaDefinition)?;
32    let canonical = serde_json_canonicalizer::to_vec(&document)
33        .map_err(|_| ChildReconcilerBuildError::SchemaDefinition)?;
34    let id = STANDARD_CHILD_RECONCILIATION_EVENT_SCHEMA_ID
35        .parse()
36        .map_err(|_| ChildReconcilerBuildError::SchemaDefinition)?;
37    Ok((
38        SchemaReference::new(id, Version::new(1, 0, 0), Digest::sha256(canonical)),
39        document,
40    ))
41}
42
43/// Registers the exact audit schema before freezing the deployment registry.
44pub fn register_standard_child_reconciliation_event_schema(
45    builder: &mut JsonSchemaRegistryBuilder,
46) -> Result<SchemaReference, ChildReconcilerBuildError> {
47    let (reference, document) = standard_child_reconciliation_event_schema()?;
48    builder.register(reference.clone(), document)?;
49    Ok(reference)
50}
51
52/// Finite retry policy. Each tick scans at most 16 cancellation and 16 settlement keys.
53#[derive(Clone, Copy, Debug, Eq, PartialEq)]
54pub struct DurableChildReconcilerOptions {
55    maximum_mutation_attempts: u8,
56    retry_initial_delay: Duration,
57}
58
59impl DurableChildReconcilerOptions {
60    /// Rejects attempts outside 1–10 and initial delay outside (0, 1 second].
61    pub fn new(
62        maximum_mutation_attempts: u8,
63        retry_initial_delay: Duration,
64    ) -> Result<Self, ChildReconcilerBuildError> {
65        if !(1..=10).contains(&maximum_mutation_attempts)
66            || retry_initial_delay.is_zero()
67            || retry_initial_delay > Duration::from_secs(1)
68        {
69            return Err(ChildReconcilerBuildError::InvalidOptions);
70        }
71        Ok(Self {
72            maximum_mutation_attempts,
73            retry_initial_delay,
74        })
75    }
76    /// Returns the hard bound on fresh/read-recovery attempts for one key.
77    #[must_use]
78    pub const fn maximum_mutation_attempts(self) -> u8 {
79        self.maximum_mutation_attempts
80    }
81
82    pub(crate) fn retry_delay(self, attempt: u8) -> Duration {
83        self.retry_initial_delay
84            .saturating_mul(1_u32 << (attempt - 1))
85            .min(Duration::from_secs(1))
86    }
87}
88impl Default for DurableChildReconcilerOptions {
89    fn default() -> Self {
90        Self {
91            maximum_mutation_attempts: 3,
92            retry_initial_delay: Duration::from_millis(25),
93        }
94    }
95}
96
97/// In-process scan continuation. Retain it between ticks; restart at `None` after a crash.
98/// Each lane automatically starts a new sweep once it reaches the end.
99#[derive(Clone, Debug)]
100pub struct ChildReconciliationCursor {
101    tenant: TenantId,
102    cancellation_after: Option<ChildRunKey>,
103    settlement_after: Option<ChildRunKey>,
104}
105
106/// Maintenance lane, for bounded-cardinality counters.
107#[derive(Clone, Copy, Debug, Eq, PartialEq)]
108pub enum ChildReconciliationKind {
109    /// Deliver durable parent cancellation without confirming child termination.
110    Cancellation,
111    /// Replace one reservation with verified terminal usage once.
112    Settlement,
113}
114
115/// Successful per-key maintenance observation.
116#[derive(Clone, Copy, Debug, Eq, PartialEq)]
117pub enum ChildReconciliationCommit {
118    /// Newly consumed cancellation work; includes whether the child was already terminal.
119    Cancellation(ChildCancellationOutcome),
120    /// Original cancellation receipt was recovered without another mutation.
121    CancellationRecovered,
122    /// Newly recorded terminal settlement.
123    Settlement,
124    /// Original settlement was recovered without another charge.
125    SettlementRecovered,
126}
127
128/// One bounded, inspectable result. Errors do not prevent other keys from progressing.
129pub struct ChildReconciliationItem {
130    key: ChildRunKey,
131    kind: ChildReconciliationKind,
132    result: Result<ChildReconciliationCommit, StoreError>,
133}
134impl ChildReconciliationItem {
135    /// Returns ownership for secure operator correlation, not a metric label.
136    #[must_use]
137    pub const fn key(&self) -> &ChildRunKey {
138        &self.key
139    }
140    /// Returns the lane that processed this key.
141    #[must_use]
142    pub const fn kind(&self) -> ChildReconciliationKind {
143        self.kind
144    }
145    /// Returns complete typed success or failure. Unknown cost remains an explicit error.
146    pub fn result(&self) -> Result<ChildReconciliationCommit, &StoreError> {
147        self.result.as_ref().copied()
148    }
149}
150impl fmt::Debug for ChildReconciliationItem {
151    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
152        f.debug_struct("ChildReconciliationItem")
153            .field("key", &self.key)
154            .field("kind", &self.kind)
155            .field("result", &self.result.as_ref().map_err(ToString::to_string))
156            .finish()
157    }
158}
159
160/// Result of one finite tick; this is neither a child Join nor an Agent execution result.
161#[derive(Debug)]
162pub struct ChildReconciliationTick {
163    cursor: ChildReconciliationCursor,
164    items: Vec<ChildReconciliationItem>,
165    cancelled: bool,
166}
167impl ChildReconciliationTick {
168    /// Pass this continuation to the next tick, including after per-item errors.
169    #[must_use]
170    pub const fn cursor(&self) -> &ChildReconciliationCursor {
171        &self.cursor
172    }
173    /// Returns up to 32 results. Inspect errors and alert on unresolved/corrupt evidence.
174    #[must_use]
175    pub fn items(&self) -> &[ChildReconciliationItem] {
176        &self.items
177    }
178    /// Indicates cooperative shutdown; unfinished work remains in durable queues.
179    #[must_use]
180    pub const fn is_cancelled(&self) -> bool {
181        self.cancelled
182    }
183}
184
185/// Startup validation error; no runtime network schema loading is allowed.
186#[derive(Debug, Error)]
187pub enum ChildReconcilerBuildError {
188    /// Retry policy would be unbounded or outside supported limits.
189    #[error("invalid child reconciler retry policy")]
190    InvalidOptions,
191    /// Embedded release schema is invalid.
192    #[error("invalid embedded child reconciliation schema")]
193    SchemaDefinition,
194    /// Deployment omitted the exact embedded schema.
195    #[error("child reconciliation schema is absent from the deployment registry")]
196    SchemaUnavailable,
197    /// Registry rejected schema registration.
198    #[error(transparent)]
199    Registry(#[from] JsonSchemaRegistryError),
200}
201
202/// Trusted control-plane worker for owned children; never dispatches user/provider code.
203/// The caller authenticates the tenant and controls scheduling, fairness and tick cadence.
204/// Schedule alongside execution workers. No lease or transaction is held between ticks.
205///
206/// Register the schema in the same builder as the deployment's other schemas.
207/// The host retains the continuation and inspects each result; this helper is
208/// one finite maintenance step, not an execution loop or a child Join.
209///
210/// ```no_run
211/// use stateknot_core::{CancellationSignal, TenantId};
212/// use stateknot_runtime::{
213///     ChildReconciliationCursor, ChildReconciliationTick, DurableChildReconciler,
214///     DurableChildReconcilerOptions, JsonSchemaRegistryBuilder,
215///     register_standard_child_reconciliation_event_schema,
216/// };
217/// use stateknot_store_postgres::{PostgresStore, StoreError};
218///
219/// fn configure(
220///     store: PostgresStore,
221///     mut deployment_schemas: JsonSchemaRegistryBuilder,
222/// ) -> Result<DurableChildReconciler, Box<dyn std::error::Error>> {
223///     register_standard_child_reconciliation_event_schema(&mut deployment_schemas)?;
224///     Ok(DurableChildReconciler::new(
225///         store, deployment_schemas.build()?, DurableChildReconcilerOptions::default(),
226///     )?)
227/// }
228///
229/// async fn maintenance_step(
230///     worker: &DurableChildReconciler,
231///     authorized_tenant: TenantId,
232///     continuation: &mut Option<ChildReconciliationCursor>,
233///     shutdown: CancellationSignal,
234/// ) -> Result<ChildReconciliationTick, StoreError> {
235///     let report = worker.tick(authorized_tenant, continuation.clone(), shutdown).await?;
236///     *continuation = Some(report.cursor().clone());
237///     // Return every per-item error to the host's alerting/reconciliation policy.
238///     // After process loss start a new sweep with None; no durable work is lost.
239///     Ok(report)
240/// }
241/// ```
242#[derive(Clone, Debug)]
243pub struct DurableChildReconciler {
244    store: PostgresStore,
245    schemas: JsonSchemaRegistry,
246    schema: SchemaReference,
247    options: DurableChildReconcilerOptions,
248}
249impl DurableChildReconciler {
250    /// Freezes the pool, retry policy and exact offline audit schema at startup.
251    pub fn new(
252        store: PostgresStore,
253        schemas: JsonSchemaRegistry,
254        options: DurableChildReconcilerOptions,
255    ) -> Result<Self, ChildReconcilerBuildError> {
256        let (schema, _) = standard_child_reconciliation_event_schema()?;
257        if !schemas.contains(&schema) {
258            return Err(ChildReconcilerBuildError::SchemaUnavailable);
259        }
260        Ok(Self {
261            store,
262            schemas,
263            schema,
264            options,
265        })
266    }
267
268    /// Reconciles one page per lane. Inspect per-key errors and retain the cursor so
269    /// an unknown-price/quarantined item cannot permanently block later work.
270    /// Shutdown may win during an ambiguous commit; next ticks recover durable facts.
271    pub fn tick(
272        &self,
273        tenant: TenantId,
274        after: Option<ChildReconciliationCursor>,
275        shutdown: CancellationSignal,
276    ) -> BoxFuture<'_, Result<ChildReconciliationTick, StoreError>> {
277        Box::pin(self.tick_inner(tenant, after, shutdown))
278    }
279
280    async fn tick_inner(
281        &self,
282        tenant: TenantId,
283        after: Option<ChildReconciliationCursor>,
284        shutdown: CancellationSignal,
285    ) -> Result<ChildReconciliationTick, StoreError> {
286        if after.as_ref().is_some_and(|cursor| cursor.tenant != tenant) {
287            return Err(StoreError::ChildRunRejected);
288        }
289        let mut tick = ChildReconciliationTick {
290            cursor: after.unwrap_or(ChildReconciliationCursor {
291                tenant: tenant.clone(),
292                cancellation_after: None,
293                settlement_after: None,
294            }),
295            items: Vec::with_capacity(32),
296            cancelled: false,
297        };
298        for kind in [
299            ChildReconciliationKind::Cancellation,
300            ChildReconciliationKind::Settlement,
301        ] {
302            if shutdown.is_cancelled() {
303                tick.cancelled = true;
304                break;
305            }
306            let keys = tokio::select! {
307                biased;
308                () = shutdown.cancelled() => { tick.cancelled = true; break; }
309                keys = async {
310                    match kind {
311                        ChildReconciliationKind::Cancellation => self.store.pending_child_cancellations_after(&tenant, tick.cursor.cancellation_after.as_ref()).await,
312                        ChildReconciliationKind::Settlement => self.store.pending_child_settlements_after(&tenant, tick.cursor.settlement_after.as_ref()).await,
313                    }
314                } => keys?,
315            };
316            let exhausted = keys.len() < 16;
317            for key in keys {
318                let result = tokio::select! {
319                    biased;
320                    () = shutdown.cancelled() => { tick.cancelled = true; break; }
321                    result = self.reconcile_with_retry(&key, kind) => result,
322                };
323                match kind {
324                    ChildReconciliationKind::Cancellation => {
325                        tick.cursor.cancellation_after = Some(key.clone());
326                    }
327                    ChildReconciliationKind::Settlement => {
328                        tick.cursor.settlement_after = Some(key.clone());
329                    }
330                }
331                tick.items
332                    .push(ChildReconciliationItem { key, kind, result });
333            }
334            if tick.cancelled {
335                break;
336            }
337            if exhausted {
338                match kind {
339                    ChildReconciliationKind::Cancellation => tick.cursor.cancellation_after = None,
340                    ChildReconciliationKind::Settlement => tick.cursor.settlement_after = None,
341                }
342            }
343        }
344        Ok(tick)
345    }
346
347    async fn reconcile_with_retry(
348        &self,
349        key: &ChildRunKey,
350        kind: ChildReconciliationKind,
351    ) -> Result<ChildReconciliationCommit, StoreError> {
352        let event_id = EventId::generate();
353        let failure_id = FailureId::generate();
354        for attempt in 1..=self.options.maximum_mutation_attempts {
355            let result = match kind {
356                ChildReconciliationKind::Cancellation => {
357                    Box::pin(self.cancel(key, event_id, failure_id)).await
358                }
359                ChildReconciliationKind::Settlement => Box::pin(self.settle(key, event_id)).await,
360            };
361            match result {
362                Err(error)
363                    if attempt < self.options.maximum_mutation_attempts
364                        && (error.is_retryable()
365                            || matches!(
366                                error,
367                                StoreError::StaleJournalHead | StoreError::StaleLifecycleRevision
368                            )) =>
369                {
370                    let delay = self
371                        .options
372                        .retry_initial_delay
373                        .saturating_mul(1_u32 << (attempt - 1))
374                        .min(Duration::from_secs(1));
375                    tokio::time::sleep(delay).await;
376                }
377                other => return other,
378            }
379        }
380        unreachable!("validated positive attempt count always returns")
381    }
382
383    async fn cancel(
384        &self,
385        key: &ChildRunKey,
386        event_id: EventId,
387        failure_id: FailureId,
388    ) -> Result<ChildReconciliationCommit, StoreError> {
389        let record = self.store.load_child_cancellation(key).await?;
390        if record.receipt().is_some() {
391            return Ok(ChildReconciliationCommit::CancellationRecovered);
392        }
393        let child = self
394            .store
395            .load_run(key.tenant_id(), record.child_run_id())
396            .await?;
397        let at = self.store.observe_database_clock().await?;
398        let failure = Failure::new(
399            failure_id,
400            FailureCategory::Cancelled,
401            FailureCode::new(if record.is_parent_failure_close() {
402                "child.parent_failed"
403            } else {
404                "child.parent_cancelled"
405            })
406            .expect("static code"),
407            FailureOrigin::new("stateknot.runtime.child_reconciler").expect("static origin"),
408            FailureMessage::new(if record.is_parent_failure_close() {
409                "The owning parent is closing after failure."
410            } else {
411                "The owning parent requested cancellation."
412            })
413            .expect("static public message"),
414            RetryAdvice::Never,
415        )
416        .expect("static cancellation semantics")
417        .with_caused_by_event(event_id);
418        let request =
419            RunCancellationRequest::new(failure, at).map_err(|_| StoreError::ChildRunRejected)?;
420        let append = self.append(
421            key,
422            record.child_run_id(),
423            child
424                .journal_head()
425                .cloned()
426                .ok_or(StoreError::ChildRunRejected)?,
427            event_id,
428            "child-run-cancellation-requested",
429            "child_run_cancellation_requested",
430        )?;
431        match Box::pin(self.store.deliver_child_cancellation(key, append, request)).await? {
432            ChildCancellationDelivery::Committed(record) => {
433                Ok(ChildReconciliationCommit::Cancellation(
434                    record
435                        .receipt()
436                        .ok_or(StoreError::ChildRunRejected)?
437                        .outcome(),
438                ))
439            }
440            ChildCancellationDelivery::Idempotent(_) => {
441                Ok(ChildReconciliationCommit::CancellationRecovered)
442            }
443        }
444    }
445
446    async fn settle(
447        &self,
448        key: &ChildRunKey,
449        event_id: EventId,
450    ) -> Result<ChildReconciliationCommit, StoreError> {
451        let parent = self
452            .store
453            .load_run(key.tenant_id(), key.parent_run_id())
454            .await?;
455        let append = self.append(
456            key,
457            key.parent_run_id(),
458            parent
459                .journal_head()
460                .cloned()
461                .ok_or(StoreError::ChildRunRejected)?,
462            event_id,
463            "child-run-settled",
464            "child_run_settled",
465        )?;
466        match Box::pin(self.store.settle_child_run(key, append)).await? {
467            ChildRunSettlementOutcome::Committed { .. } => {
468                Ok(ChildReconciliationCommit::Settlement)
469            }
470            ChildRunSettlementOutcome::Idempotent { .. } => {
471                Ok(ChildReconciliationCommit::SettlementRecovered)
472            }
473            _ => Err(StoreError::ChildRunRejected),
474        }
475    }
476
477    fn append(
478        &self,
479        key: &ChildRunKey,
480        run: stateknot_core::RunId,
481        head: stateknot_core::JournalHead,
482        id: EventId,
483        kind: &str,
484        operation: &str,
485    ) -> Result<JournalAppend, StoreError> {
486        let mut digest = String::with_capacity(64);
487        for value in key.digest().as_bytes() {
488            write!(digest, "{value:02x}").expect("String writes are infallible");
489        }
490        let data = BoundedJson::try_from_value(
491            json!({ "operation": operation, "ownership_digest": digest }),
492        )
493        .map_err(|_| StoreError::ChildRunRejected)?;
494        self.schemas
495            .validate_bounded(&self.schema, &data)
496            .map_err(|_| StoreError::ChildRunRejected)?;
497        let payload = stateknot_core::JournalPayload::new(
498            self.schema.clone(),
499            JournalEventKind::new(kind).map_err(|_| StoreError::ChildRunRejected)?,
500            data,
501        )
502        .map_err(|_| StoreError::ChildRunRejected)?;
503        let intent = JournalEventIntent::control_plane(key.tenant_id().clone(), run, id, payload)
504            .map_err(|_| StoreError::ChildRunRejected)?;
505        JournalAppend::new(JournalExpectation::exact(head), intent)
506            .map_err(|_| StoreError::ChildRunRejected)
507    }
508}
509
510#[cfg(test)]
511mod tests {
512    use super::*;
513    use crate::JsonSchemaRegistryLimits;
514    #[test]
515    fn schema_is_closed_pinned_and_retry_bounds_are_finite() {
516        let mut builder = JsonSchemaRegistryBuilder::new(JsonSchemaRegistryLimits::default());
517        let schema = register_standard_child_reconciliation_event_schema(&mut builder).unwrap();
518        assert_eq!(
519            schema.digest().to_string(),
520            "sha256:54346f985269669cbdd22b6c70c77aebbadb5953fc96c45518af3086f4cdfd08"
521        );
522        let registry = builder.build().unwrap();
523        for operation in ["child_run_settled", "child_run_cancellation_requested"] {
524            let mut value = json!({ "operation": operation, "ownership_digest": "a".repeat(64) });
525            registry
526                .validate_bounded(
527                    &schema,
528                    &BoundedJson::try_from_value(value.clone()).unwrap(),
529                )
530                .unwrap();
531            value["private_parent_reason"] = json!("must not leak");
532            assert!(
533                registry
534                    .validate_bounded(&schema, &BoundedJson::try_from_value(value).unwrap())
535                    .is_err()
536            );
537        }
538        assert!(DurableChildReconcilerOptions::new(0, Duration::from_millis(1)).is_err());
539        assert!(DurableChildReconcilerOptions::new(11, Duration::from_millis(1)).is_err());
540        assert!(DurableChildReconcilerOptions::new(3, Duration::ZERO).is_err());
541        assert!(DurableChildReconcilerOptions::new(3, Duration::from_secs(2)).is_err());
542    }
543}