Skip to main content

stateknot_runtime/
agent_loop.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! One bounded, lease-safe durable graph agent loop.
5
6use std::{sync::Arc, time::Duration};
7
8use stateknot_core::{BoxFuture, CancellationSignal, RunFence, Timestamp};
9use stateknot_store_postgres::{
10    AppendOutcome, BarrierCommitOutcome, DelayedRetryScheduleOutcome, LeaseReleaseOutcome,
11    PostgresStore, StoreError, WaitCheckpointCommitOutcome,
12};
13use thiserror::Error;
14
15use crate::{
16    DurableGraphDriver, DurableGraphDriverBuildError, DurableGraphDriverOptions,
17    DurableGraphLifecycle, DurableGraphLifecycleBuildError, DurableGraphLifecycleOptions,
18    ExecutableGraphRegistry, GraphBarrierLifecycleOutcome, GraphDriveOutcome, GraphDriveReport,
19    GraphDriverError, GraphLifecycleError, GraphLifecycleEvidenceProvider,
20};
21
22const MAX_MUTATION_RETRY_DELAY: Duration = Duration::from_secs(1);
23
24/// Closed result of one bounded claimed-run execution quantum.
25#[derive(Clone, Debug)]
26#[non_exhaustive]
27pub enum AgentLoopOutcome {
28    /// Original failure sealed and execution lease released; maintenance drains children.
29    FailureClosing(Box<stateknot_store_postgres::RunFailureCloseOutcome>),
30    /// Dedicated child Join registration released the parent lease. This is
31    /// neither a terminal failure nor a checkpointed timer/user wait.
32    ChildJoin(Box<stateknot_core::ChildRunJoinRequest>),
33    /// Durable cancellation intent and exact cumulative usage were acknowledged.
34    CancellationConfirmed(AppendOutcome),
35    /// The graph suspended with a complete durable wait batch.
36    Waiting(WaitCheckpointCommitOutcome),
37    /// The graph and admitted agent invocation succeeded.
38    Succeeded(BarrierCommitOutcome),
39    /// Durable supervision committed terminal failure.
40    Failed(AppendOutcome),
41    /// Same-fence in-flight work released ownership for crash takeover.
42    RecoveryReleased(LeaseReleaseOutcome),
43    /// All unsettled work is durably gated until this database instant.
44    Deferred {
45        /// Inclusive database retry boundary.
46        not_before: Timestamp,
47        /// Exact durable scheduling convergence result.
48        schedule: DelayedRetryScheduleOutcome,
49    },
50    /// The configured durable work quantum was exhausted safely.
51    Yielded {
52        /// Exact-fence lease release convergence.
53        release: LeaseReleaseOutcome,
54    },
55    /// Cooperative shutdown won and ownership was released.
56    Cancelled {
57        /// Exact-fence lease release convergence.
58        release: LeaseReleaseOutcome,
59    },
60}
61
62/// Outcome and bounded execution counters for one loop call.
63#[derive(Debug)]
64pub struct AgentLoopResult {
65    outcome: AgentLoopOutcome,
66    report: GraphDriveReport,
67}
68
69impl AgentLoopResult {
70    const fn new(outcome: AgentLoopOutcome, report: GraphDriveReport) -> Self {
71        Self { outcome, report }
72    }
73
74    /// Returns why this bounded loop call stopped.
75    #[must_use]
76    pub const fn outcome(&self) -> &AgentLoopOutcome {
77        &self.outcome
78    }
79
80    /// Returns replay, execution, renewal, and retry counters.
81    #[must_use]
82    pub const fn report(&self) -> GraphDriveReport {
83        self.report
84    }
85
86    /// Consumes the result into its outcome and report.
87    #[must_use]
88    pub fn into_parts(self) -> (AgentLoopOutcome, GraphDriveReport) {
89        (self.outcome, self.report)
90    }
91}
92
93/// Production binding of driver execution and lifecycle coordination.
94#[derive(Clone)]
95pub struct DurableAgentLoop {
96    store: PostgresStore,
97    driver: DurableGraphDriver,
98    lifecycle: DurableGraphLifecycle,
99    cleanup_options: DurableGraphLifecycleOptions,
100}
101
102impl DurableAgentLoop {
103    /// Returns shared node-future completion accounting for this loop and clones.
104    pub fn execution_activity(&self) -> crate::GraphExecutionActivity {
105        self.driver.execution_activity()
106    }
107
108    /// Builds a loop over one shared store and immutable deployment registry.
109    ///
110    /// Constructing both layers here prevents a caller from accidentally
111    /// pairing a driver and lifecycle coordinator backed by different pools or
112    /// executable snapshots.
113    ///
114    /// # Errors
115    ///
116    /// Returns the exact driver or lifecycle startup binding failure.
117    pub fn new(
118        store: PostgresStore,
119        registry: ExecutableGraphRegistry,
120        evidence: Arc<dyn GraphLifecycleEvidenceProvider>,
121        driver_options: DurableGraphDriverOptions,
122        lifecycle_options: DurableGraphLifecycleOptions,
123    ) -> Result<Self, DurableAgentLoopBuildError> {
124        let driver = DurableGraphDriver::new(store.clone(), registry.clone(), driver_options)?;
125        let lifecycle =
126            DurableGraphLifecycle::new(store.clone(), registry, evidence, lifecycle_options)?;
127        Ok(Self {
128            store,
129            driver,
130            lifecycle,
131            cleanup_options: lifecycle_options,
132        })
133    }
134
135    /// Drives one already-claimed exact fence to the next durable scheduling or
136    /// lifecycle boundary.
137    ///
138    /// Driver and lifecycle failures trigger a bounded best-effort exact-fence
139    /// release. A cleanup database failure is returned alongside the primary
140    /// failure; stale, expired, absent, non-runnable, or quarantined ownership
141    /// already permits safe takeover and does not replace the primary error.
142    pub fn run(
143        &self,
144        fence: RunFence,
145        shutdown: CancellationSignal,
146    ) -> BoxFuture<'_, Result<AgentLoopResult, AgentLoopError>> {
147        Box::pin(self.run_inner(fence, shutdown))
148    }
149
150    async fn run_inner(
151        &self,
152        fence: RunFence,
153        shutdown: CancellationSignal,
154    ) -> Result<AgentLoopResult, AgentLoopError> {
155        let driven = match self.driver.drive(fence.clone(), shutdown).await {
156            Ok(driven) => driven,
157            Err(source) => return Err(self.cleanup_driver_error(&fence, source).await),
158        };
159        let (outcome, report) = driven.into_parts();
160        let outcome = match outcome {
161            GraphDriveOutcome::CancellationRequested(handoff) => {
162                let lifecycle = match self.lifecycle.confirm_cancellation(*handoff).await {
163                    Ok(outcome) => outcome,
164                    Err(source) => {
165                        return Err(self.cleanup_lifecycle_error(&fence, source).await);
166                    }
167                };
168                lifecycle_outcome(lifecycle)
169            }
170            GraphDriveOutcome::LifecycleBarrierReady(handoff) => {
171                let lifecycle = match self.lifecycle.commit_barrier(*handoff).await {
172                    Ok(outcome) => outcome,
173                    Err(source) => {
174                        return Err(self.cleanup_lifecycle_error(&fence, source).await);
175                    }
176                };
177                lifecycle_outcome(lifecycle)
178            }
179            GraphDriveOutcome::Blocked(handoff) => {
180                let lifecycle = match self.lifecycle.resolve_blocked(*handoff).await {
181                    Ok(outcome) => outcome,
182                    Err(source) => {
183                        return Err(self.cleanup_lifecycle_error(&fence, source).await);
184                    }
185                };
186                lifecycle_outcome(lifecycle)
187            }
188            GraphDriveOutcome::Deferred {
189                not_before,
190                schedule,
191            } => AgentLoopOutcome::Deferred {
192                not_before,
193                schedule,
194            },
195            GraphDriveOutcome::ChildJoin(request) => AgentLoopOutcome::ChildJoin(request),
196            GraphDriveOutcome::Yielded { release } => AgentLoopOutcome::Yielded { release },
197            GraphDriveOutcome::Cancelled { release } => AgentLoopOutcome::Cancelled { release },
198        };
199        Ok(AgentLoopResult::new(outcome, report))
200    }
201
202    async fn cleanup_driver_error(
203        &self,
204        fence: &RunFence,
205        source: GraphDriverError,
206    ) -> AgentLoopError {
207        match self.release_with_retry(fence).await {
208            Ok(_) => AgentLoopError::Driver {
209                source: Box::new(source),
210            },
211            Err(cleanup) if cleanup_error_is_benign(&cleanup) => AgentLoopError::Driver {
212                source: Box::new(source),
213            },
214            Err(cleanup) => AgentLoopError::DriverCleanup {
215                source: Box::new(source),
216                cleanup: Box::new(cleanup),
217            },
218        }
219    }
220
221    async fn cleanup_lifecycle_error(
222        &self,
223        fence: &RunFence,
224        source: GraphLifecycleError,
225    ) -> AgentLoopError {
226        match self.release_with_retry(fence).await {
227            Ok(_) => AgentLoopError::Lifecycle {
228                source: Box::new(source),
229            },
230            Err(cleanup) if cleanup_error_is_benign(&cleanup) => AgentLoopError::Lifecycle {
231                source: Box::new(source),
232            },
233            Err(cleanup) => AgentLoopError::LifecycleCleanup {
234                source: Box::new(source),
235                cleanup: Box::new(cleanup),
236            },
237        }
238    }
239
240    async fn release_with_retry(
241        &self,
242        fence: &RunFence,
243    ) -> Result<LeaseReleaseOutcome, StoreError> {
244        let mut attempt = 1_u8;
245        loop {
246            match self.store.release_lease(fence).await {
247                Ok(outcome) => return Ok(outcome),
248                Err(error)
249                    if attempt < self.cleanup_options.maximum_mutation_attempts()
250                        && error.is_retryable() =>
251                {
252                    self.cleanup_backoff(attempt).await;
253                    attempt = attempt.saturating_add(1);
254                }
255                Err(error) => return Err(error),
256            }
257        }
258    }
259
260    async fn cleanup_backoff(&self, attempt: u8) {
261        let multiplier = 1_u32
262            .checked_shl(u32::from(attempt.saturating_sub(1)))
263            .unwrap_or(u32::MAX);
264        let delay = self
265            .cleanup_options
266            .mutation_retry_initial_delay()
267            .checked_mul(multiplier)
268            .unwrap_or(MAX_MUTATION_RETRY_DELAY)
269            .min(MAX_MUTATION_RETRY_DELAY);
270        tokio::time::sleep(delay).await;
271    }
272}
273
274fn lifecycle_outcome(outcome: GraphBarrierLifecycleOutcome) -> AgentLoopOutcome {
275    match outcome {
276        GraphBarrierLifecycleOutcome::FailureClosing(outcome) => {
277            AgentLoopOutcome::FailureClosing(outcome)
278        }
279        GraphBarrierLifecycleOutcome::Cancelled(outcome) => {
280            AgentLoopOutcome::CancellationConfirmed(outcome)
281        }
282        GraphBarrierLifecycleOutcome::Waiting(outcome) => AgentLoopOutcome::Waiting(outcome),
283        GraphBarrierLifecycleOutcome::Succeeded(outcome) => AgentLoopOutcome::Succeeded(outcome),
284        GraphBarrierLifecycleOutcome::Failed(outcome) => AgentLoopOutcome::Failed(outcome),
285        GraphBarrierLifecycleOutcome::Released(outcome) => {
286            AgentLoopOutcome::RecoveryReleased(outcome)
287        }
288    }
289}
290
291fn cleanup_error_is_benign(error: &StoreError) -> bool {
292    matches!(
293        error,
294        StoreError::NoActiveLease
295            | StoreError::StaleFence
296            | StoreError::LeaseExpired
297            | StoreError::RunNotRunnable
298            | StoreError::RunQuarantined
299    )
300}
301
302/// Startup failure while building a consistent durable loop.
303#[derive(Clone, Debug, Eq, Error, PartialEq)]
304#[non_exhaustive]
305pub enum DurableAgentLoopBuildError {
306    /// The graph driver could not bind to the deployment snapshot.
307    #[error(transparent)]
308    Driver(#[from] DurableGraphDriverBuildError),
309    /// The lifecycle coordinator could not bind to the deployment snapshot.
310    #[error(transparent)]
311    Lifecycle(#[from] DurableGraphLifecycleBuildError),
312}
313
314/// Payload-redacted durable loop failure.
315#[derive(Debug, Error)]
316#[non_exhaustive]
317pub enum AgentLoopError {
318    /// Graph driving failed; ownership was released or already reclaimable.
319    #[error("durable agent graph driving failed: {source}")]
320    Driver {
321        /// Exact payload-redacted driver failure.
322        #[source]
323        source: Box<GraphDriverError>,
324    },
325    /// Lifecycle coordination failed; ownership was released or reclaimable.
326    #[error("durable agent lifecycle coordination failed: {source}")]
327    Lifecycle {
328        /// Exact payload-redacted lifecycle failure.
329        #[source]
330        source: Box<GraphLifecycleError>,
331    },
332    /// Driver failure was followed by a database cleanup failure.
333    #[error("durable agent graph driving and lease cleanup both failed")]
334    DriverCleanup {
335        /// Exact primary driver failure.
336        source: Box<GraphDriverError>,
337        /// Exact payload-redacted lease cleanup failure.
338        cleanup: Box<StoreError>,
339    },
340    /// Lifecycle failure was followed by a database cleanup failure.
341    #[error("durable agent lifecycle coordination and lease cleanup both failed")]
342    LifecycleCleanup {
343        /// Exact primary lifecycle failure.
344        source: Box<GraphLifecycleError>,
345        /// Exact payload-redacted lease cleanup failure.
346        cleanup: Box<StoreError>,
347    },
348}
349
350impl AgentLoopError {
351    /// Returns a cleanup failure when both the primary operation and release
352    /// failed.
353    #[must_use]
354    pub const fn cleanup_error(&self) -> Option<&StoreError> {
355        match self {
356            Self::DriverCleanup { cleanup, .. } | Self::LifecycleCleanup { cleanup, .. } => {
357                Some(cleanup)
358            }
359            Self::Driver { .. } | Self::Lifecycle { .. } => None,
360        }
361    }
362}