1use std::{sync::Arc, time::Duration};
7
8use stateknot_core::{BoxFuture, CancellationSignal, RunFence, Timestamp};
9use stateknot_store_postgres::{
10 AppendOutcome, BarrierCommitOutcome, DelayedRetryScheduleOutcome, LeaseReleaseOutcome,
11 PostgresStore, StoreError, WaitCheckpointCommitOutcome,
12};
13use thiserror::Error;
14
15use crate::{
16 DurableGraphDriver, DurableGraphDriverBuildError, DurableGraphDriverOptions,
17 DurableGraphLifecycle, DurableGraphLifecycleBuildError, DurableGraphLifecycleOptions,
18 ExecutableGraphRegistry, GraphBarrierLifecycleOutcome, GraphDriveOutcome, GraphDriveReport,
19 GraphDriverError, GraphLifecycleError, GraphLifecycleEvidenceProvider,
20};
21
22const MAX_MUTATION_RETRY_DELAY: Duration = Duration::from_secs(1);
23
24#[derive(Clone, Debug)]
26#[non_exhaustive]
27pub enum AgentLoopOutcome {
28 FailureClosing(Box<stateknot_store_postgres::RunFailureCloseOutcome>),
30 ChildJoin(Box<stateknot_core::ChildRunJoinRequest>),
33 CancellationConfirmed(AppendOutcome),
35 Waiting(WaitCheckpointCommitOutcome),
37 Succeeded(BarrierCommitOutcome),
39 Failed(AppendOutcome),
41 RecoveryReleased(LeaseReleaseOutcome),
43 Deferred {
45 not_before: Timestamp,
47 schedule: DelayedRetryScheduleOutcome,
49 },
50 Yielded {
52 release: LeaseReleaseOutcome,
54 },
55 Cancelled {
57 release: LeaseReleaseOutcome,
59 },
60}
61
62#[derive(Debug)]
64pub struct AgentLoopResult {
65 outcome: AgentLoopOutcome,
66 report: GraphDriveReport,
67}
68
69impl AgentLoopResult {
70 const fn new(outcome: AgentLoopOutcome, report: GraphDriveReport) -> Self {
71 Self { outcome, report }
72 }
73
74 #[must_use]
76 pub const fn outcome(&self) -> &AgentLoopOutcome {
77 &self.outcome
78 }
79
80 #[must_use]
82 pub const fn report(&self) -> GraphDriveReport {
83 self.report
84 }
85
86 #[must_use]
88 pub fn into_parts(self) -> (AgentLoopOutcome, GraphDriveReport) {
89 (self.outcome, self.report)
90 }
91}
92
93#[derive(Clone)]
95pub struct DurableAgentLoop {
96 store: PostgresStore,
97 driver: DurableGraphDriver,
98 lifecycle: DurableGraphLifecycle,
99 cleanup_options: DurableGraphLifecycleOptions,
100}
101
102impl DurableAgentLoop {
103 pub fn execution_activity(&self) -> crate::GraphExecutionActivity {
105 self.driver.execution_activity()
106 }
107
108 pub fn new(
118 store: PostgresStore,
119 registry: ExecutableGraphRegistry,
120 evidence: Arc<dyn GraphLifecycleEvidenceProvider>,
121 driver_options: DurableGraphDriverOptions,
122 lifecycle_options: DurableGraphLifecycleOptions,
123 ) -> Result<Self, DurableAgentLoopBuildError> {
124 let driver = DurableGraphDriver::new(store.clone(), registry.clone(), driver_options)?;
125 let lifecycle =
126 DurableGraphLifecycle::new(store.clone(), registry, evidence, lifecycle_options)?;
127 Ok(Self {
128 store,
129 driver,
130 lifecycle,
131 cleanup_options: lifecycle_options,
132 })
133 }
134
135 pub fn run(
143 &self,
144 fence: RunFence,
145 shutdown: CancellationSignal,
146 ) -> BoxFuture<'_, Result<AgentLoopResult, AgentLoopError>> {
147 Box::pin(self.run_inner(fence, shutdown))
148 }
149
150 async fn run_inner(
151 &self,
152 fence: RunFence,
153 shutdown: CancellationSignal,
154 ) -> Result<AgentLoopResult, AgentLoopError> {
155 let driven = match self.driver.drive(fence.clone(), shutdown).await {
156 Ok(driven) => driven,
157 Err(source) => return Err(self.cleanup_driver_error(&fence, source).await),
158 };
159 let (outcome, report) = driven.into_parts();
160 let outcome = match outcome {
161 GraphDriveOutcome::CancellationRequested(handoff) => {
162 let lifecycle = match self.lifecycle.confirm_cancellation(*handoff).await {
163 Ok(outcome) => outcome,
164 Err(source) => {
165 return Err(self.cleanup_lifecycle_error(&fence, source).await);
166 }
167 };
168 lifecycle_outcome(lifecycle)
169 }
170 GraphDriveOutcome::LifecycleBarrierReady(handoff) => {
171 let lifecycle = match self.lifecycle.commit_barrier(*handoff).await {
172 Ok(outcome) => outcome,
173 Err(source) => {
174 return Err(self.cleanup_lifecycle_error(&fence, source).await);
175 }
176 };
177 lifecycle_outcome(lifecycle)
178 }
179 GraphDriveOutcome::Blocked(handoff) => {
180 let lifecycle = match self.lifecycle.resolve_blocked(*handoff).await {
181 Ok(outcome) => outcome,
182 Err(source) => {
183 return Err(self.cleanup_lifecycle_error(&fence, source).await);
184 }
185 };
186 lifecycle_outcome(lifecycle)
187 }
188 GraphDriveOutcome::Deferred {
189 not_before,
190 schedule,
191 } => AgentLoopOutcome::Deferred {
192 not_before,
193 schedule,
194 },
195 GraphDriveOutcome::ChildJoin(request) => AgentLoopOutcome::ChildJoin(request),
196 GraphDriveOutcome::Yielded { release } => AgentLoopOutcome::Yielded { release },
197 GraphDriveOutcome::Cancelled { release } => AgentLoopOutcome::Cancelled { release },
198 };
199 Ok(AgentLoopResult::new(outcome, report))
200 }
201
202 async fn cleanup_driver_error(
203 &self,
204 fence: &RunFence,
205 source: GraphDriverError,
206 ) -> AgentLoopError {
207 match self.release_with_retry(fence).await {
208 Ok(_) => AgentLoopError::Driver {
209 source: Box::new(source),
210 },
211 Err(cleanup) if cleanup_error_is_benign(&cleanup) => AgentLoopError::Driver {
212 source: Box::new(source),
213 },
214 Err(cleanup) => AgentLoopError::DriverCleanup {
215 source: Box::new(source),
216 cleanup: Box::new(cleanup),
217 },
218 }
219 }
220
221 async fn cleanup_lifecycle_error(
222 &self,
223 fence: &RunFence,
224 source: GraphLifecycleError,
225 ) -> AgentLoopError {
226 match self.release_with_retry(fence).await {
227 Ok(_) => AgentLoopError::Lifecycle {
228 source: Box::new(source),
229 },
230 Err(cleanup) if cleanup_error_is_benign(&cleanup) => AgentLoopError::Lifecycle {
231 source: Box::new(source),
232 },
233 Err(cleanup) => AgentLoopError::LifecycleCleanup {
234 source: Box::new(source),
235 cleanup: Box::new(cleanup),
236 },
237 }
238 }
239
240 async fn release_with_retry(
241 &self,
242 fence: &RunFence,
243 ) -> Result<LeaseReleaseOutcome, StoreError> {
244 let mut attempt = 1_u8;
245 loop {
246 match self.store.release_lease(fence).await {
247 Ok(outcome) => return Ok(outcome),
248 Err(error)
249 if attempt < self.cleanup_options.maximum_mutation_attempts()
250 && error.is_retryable() =>
251 {
252 self.cleanup_backoff(attempt).await;
253 attempt = attempt.saturating_add(1);
254 }
255 Err(error) => return Err(error),
256 }
257 }
258 }
259
260 async fn cleanup_backoff(&self, attempt: u8) {
261 let multiplier = 1_u32
262 .checked_shl(u32::from(attempt.saturating_sub(1)))
263 .unwrap_or(u32::MAX);
264 let delay = self
265 .cleanup_options
266 .mutation_retry_initial_delay()
267 .checked_mul(multiplier)
268 .unwrap_or(MAX_MUTATION_RETRY_DELAY)
269 .min(MAX_MUTATION_RETRY_DELAY);
270 tokio::time::sleep(delay).await;
271 }
272}
273
274fn lifecycle_outcome(outcome: GraphBarrierLifecycleOutcome) -> AgentLoopOutcome {
275 match outcome {
276 GraphBarrierLifecycleOutcome::FailureClosing(outcome) => {
277 AgentLoopOutcome::FailureClosing(outcome)
278 }
279 GraphBarrierLifecycleOutcome::Cancelled(outcome) => {
280 AgentLoopOutcome::CancellationConfirmed(outcome)
281 }
282 GraphBarrierLifecycleOutcome::Waiting(outcome) => AgentLoopOutcome::Waiting(outcome),
283 GraphBarrierLifecycleOutcome::Succeeded(outcome) => AgentLoopOutcome::Succeeded(outcome),
284 GraphBarrierLifecycleOutcome::Failed(outcome) => AgentLoopOutcome::Failed(outcome),
285 GraphBarrierLifecycleOutcome::Released(outcome) => {
286 AgentLoopOutcome::RecoveryReleased(outcome)
287 }
288 }
289}
290
291fn cleanup_error_is_benign(error: &StoreError) -> bool {
292 matches!(
293 error,
294 StoreError::NoActiveLease
295 | StoreError::StaleFence
296 | StoreError::LeaseExpired
297 | StoreError::RunNotRunnable
298 | StoreError::RunQuarantined
299 )
300}
301
302#[derive(Clone, Debug, Eq, Error, PartialEq)]
304#[non_exhaustive]
305pub enum DurableAgentLoopBuildError {
306 #[error(transparent)]
308 Driver(#[from] DurableGraphDriverBuildError),
309 #[error(transparent)]
311 Lifecycle(#[from] DurableGraphLifecycleBuildError),
312}
313
314#[derive(Debug, Error)]
316#[non_exhaustive]
317pub enum AgentLoopError {
318 #[error("durable agent graph driving failed: {source}")]
320 Driver {
321 #[source]
323 source: Box<GraphDriverError>,
324 },
325 #[error("durable agent lifecycle coordination failed: {source}")]
327 Lifecycle {
328 #[source]
330 source: Box<GraphLifecycleError>,
331 },
332 #[error("durable agent graph driving and lease cleanup both failed")]
334 DriverCleanup {
335 source: Box<GraphDriverError>,
337 cleanup: Box<StoreError>,
339 },
340 #[error("durable agent lifecycle coordination and lease cleanup both failed")]
342 LifecycleCleanup {
343 source: Box<GraphLifecycleError>,
345 cleanup: Box<StoreError>,
347 },
348}
349
350impl AgentLoopError {
351 #[must_use]
354 pub const fn cleanup_error(&self) -> Option<&StoreError> {
355 match self {
356 Self::DriverCleanup { cleanup, .. } | Self::LifecycleCleanup { cleanup, .. } => {
357 Some(cleanup)
358 }
359 Self::Driver { .. } | Self::Lifecycle { .. } => None,
360 }
361 }
362}