1use futures_util::FutureExt;
8use stateknot_core::{BoxFuture, CancellationObserver, CancellationSignal};
9use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration};
10use thiserror::Error;
11use tokio::{
12 task::{JoinHandle, JoinSet},
13 time::{sleep, timeout},
14};
15use tokio_util::sync::CancellationToken;
16
17mod binding;
18mod health;
19mod options;
20pub use binding::{AgentMaintenanceBinding, AgentMaintenanceMutationOptions};
21pub use health::{AgentMaintenanceHealth, AgentMaintenanceStatus};
22pub use options::AgentMaintenanceOptions;
23
24pub trait AgentMaintenanceReadiness: Send + Sync {
27 fn check(&self) -> BoxFuture<'_, Result<(), AgentMaintenanceReadinessError>>;
29}
30
31#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)]
33#[error("maintenance host dependencies are unavailable")]
34pub struct AgentMaintenanceReadinessError;
35
36#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)]
38pub enum AgentMaintenanceError {
39 #[error("maintenance options are invalid")]
41 InvalidOptions,
42 #[error("maintenance tenant allowlist is invalid")]
44 InvalidTenants,
45 #[error("maintenance schemas are unavailable")]
47 SchemaUnavailable,
48 #[error("maintenance dependencies are unavailable")]
50 Unavailable,
51 #[error("maintenance coordinator is stopped")]
53 Stopped,
54}
55
56#[derive(Clone, Copy, Debug, Eq, PartialEq)]
58pub enum AgentMaintenanceJob {
59 Deadline,
61 Child,
63 Join,
65 FailureClose,
67}
68impl AgentMaintenanceJob {
69 pub const ALL: [Self; 4] = [Self::Deadline, Self::Child, Self::Join, Self::FailureClose];
71 const fn index(self) -> usize {
72 self as usize
73 }
74}
75
76#[derive(Clone, Copy, Debug, Eq, PartialEq)]
78pub enum AgentMaintenanceFailure {
79 Store,
81 TickDeadline,
83 Task,
85}
86
87#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
89pub struct AgentMaintenanceJobReport {
90 pub completed_ticks: u64,
92 pub items: u64,
94 pub item_failures: u64,
96}
97
98#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
100pub struct AgentMaintenanceReport {
101 pub started_ticks: u64,
103 pub readiness_failures: u64,
105 pub forced_ticks: usize,
107 pub failure: Option<AgentMaintenanceFailure>,
109 jobs: [AgentMaintenanceJobReport; 4],
110}
111impl AgentMaintenanceReport {
112 #[must_use]
114 pub const fn job(&self, job: AgentMaintenanceJob) -> AgentMaintenanceJobReport {
115 self.jobs[job.index()]
116 }
117}
118
119pub struct AgentMaintenance {
136 health: AgentMaintenanceHealth,
137 stop: CancellationToken,
138 task: Option<JoinHandle<AgentMaintenanceReport>>,
139}
140impl AgentMaintenance {
141 pub async fn start(
144 binding: AgentMaintenanceBinding,
145 host: Arc<dyn AgentMaintenanceReadiness>,
146 options: AgentMaintenanceOptions,
147 ) -> Result<Self, AgentMaintenanceError> {
148 if !probe(&binding, host.as_ref(), &options)
149 .await
150 .unwrap_or(false)
151 {
152 return Err(AgentMaintenanceError::Unavailable);
153 }
154 let health = AgentMaintenanceHealth::new(options.freshness);
155 health.probe(true);
156 let stop = CancellationToken::new();
157 let guard = RuntimeGuard {
159 tasks: JoinSet::new(),
160 health: health.clone(),
161 stop: stop.clone(),
162 };
163 let task = tokio::spawn(run(Arc::new(binding), host, options, guard));
164 Ok(Self {
165 health,
166 stop,
167 task: Some(task),
168 })
169 }
170 pub fn health(&self) -> AgentMaintenanceHealth {
172 self.health.clone()
173 }
174 pub fn begin_shutdown(&self) {
176 self.health.draining();
177 self.stop.cancel();
178 }
179 pub async fn wait(&mut self) -> Result<AgentMaintenanceReport, AgentMaintenanceError> {
181 let result = self
182 .task
183 .as_mut()
184 .ok_or(AgentMaintenanceError::Stopped)?
185 .await;
186 self.task = None;
187 result.map_err(|_| AgentMaintenanceError::Stopped)
188 }
189 pub async fn shutdown(&mut self) -> Result<AgentMaintenanceReport, AgentMaintenanceError> {
191 self.begin_shutdown();
192 self.wait().await
193 }
194}
195impl Drop for AgentMaintenance {
196 fn drop(&mut self) {
197 self.begin_shutdown();
198 if let Some(task) = &self.task {
199 task.abort();
200 }
201 }
202}
203struct Signal(CancellationToken);
204impl CancellationObserver for Signal {
205 fn is_cancelled(&self) -> bool {
206 self.0.is_cancelled()
207 }
208 fn cancelled(&self) -> BoxFuture<'_, ()> {
209 Box::pin(self.0.cancelled())
210 }
211}
212struct RuntimeGuard {
213 tasks: JoinSet<()>,
214 health: AgentMaintenanceHealth,
215 stop: CancellationToken,
216}
217impl Drop for RuntimeGuard {
218 fn drop(&mut self) {
219 self.stop.cancel();
220 self.tasks.abort_all();
221 self.health.stopped();
222 }
223}
224async fn probe(
225 binding: &AgentMaintenanceBinding,
226 host: &dyn AgentMaintenanceReadiness,
227 options: &AgentMaintenanceOptions,
228) -> Result<bool, ()> {
229 AssertUnwindSafe(async {
230 timeout(options.probe_timeout, async {
231 binding.check().await?;
232 host.check()
233 .await
234 .map_err(|_| AgentMaintenanceError::Unavailable)
235 })
236 .await
237 .is_ok_and(|r| r.is_ok())
238 })
239 .catch_unwind()
240 .await
241 .map_err(|_| ())
242}
243async fn pause(duration: Duration, stop: &CancellationToken) {
244 tokio::select! { biased; () = stop.cancelled() => {}, () = sleep(duration) => {} }
245}
246async fn monitor(
247 binding: Arc<AgentMaintenanceBinding>,
248 host: Arc<dyn AgentMaintenanceReadiness>,
249 options: AgentMaintenanceOptions,
250 health: AgentMaintenanceHealth,
251 stop: CancellationToken,
252) {
253 loop {
254 pause(options.probe_interval, &stop).await;
255 let result = tokio::select! { biased; () = stop.cancelled() => return, result = probe(&binding, host.as_ref(), &options) => result };
256 if let Ok(ready) = result {
257 health.probe(ready);
258 } else {
259 health.fail(AgentMaintenanceFailure::Task);
260 stop.cancel();
261 return;
262 }
263 }
264}
265async fn sweep(
266 binding: Arc<AgentMaintenanceBinding>,
267 options: AgentMaintenanceOptions,
268 health: AgentMaintenanceHealth,
269 stop: CancellationToken,
270) {
271 let mut rotation = binding.rotation();
272 let rotation_len = rotation.len();
273 let mut next = 0;
274 let signal = CancellationSignal::new(Signal(stop.clone()));
275 while !stop.is_cancelled() {
276 let Some(active) = health.admit() else {
277 pause(options.delay, &stop).await;
278 continue;
279 };
280 let progress = &mut rotation[next];
281 let job = progress.job();
282 let tick = binding.tick(progress, signal.clone());
283 tokio::pin!(tick);
284 let result = tokio::select! {
285 result = &mut tick => result,
286 () = sleep(options.tick_timeout) => {
287 health.fail(AgentMaintenanceFailure::TickDeadline); stop.cancel();
288 tick.await
289 }
290 };
291 let delay = if let Ok(counts) = result {
292 health.complete(job, counts);
293 if counts.item_failures > 0 {
294 options.failure_delay
295 } else {
296 options.delay
297 }
298 } else {
299 health.fail(AgentMaintenanceFailure::Store);
300 stop.cancel();
301 options.failure_delay
302 };
303 next = (next + 1) % rotation_len;
304 drop(active);
305 pause(delay, &stop).await;
306 }
307}
308async fn run(
309 binding: Arc<AgentMaintenanceBinding>,
310 host: Arc<dyn AgentMaintenanceReadiness>,
311 options: AgentMaintenanceOptions,
312 mut guard: RuntimeGuard,
313) -> AgentMaintenanceReport {
314 guard.tasks.spawn(monitor(
315 binding.clone(),
316 host,
317 options.clone(),
318 guard.health.clone(),
319 guard.stop.clone(),
320 ));
321 guard.tasks.spawn(sweep(
322 binding,
323 options.clone(),
324 guard.health.clone(),
325 guard.stop.clone(),
326 ));
327 tokio::select! {
328 biased;
329 () = guard.stop.cancelled() => {},
330 _ = guard.tasks.join_next() => {
331 if !guard.stop.is_cancelled() { guard.health.fail(AgentMaintenanceFailure::Task); }
332 guard.stop.cancel();
333 }
334 }
335 guard.health.draining();
336 drain(&mut guard, options.drain_timeout).await;
337 guard.health.stopped();
338 guard.health.report()
339}
340
341async fn drain(guard: &mut RuntimeGuard, deadline: Duration) {
342 let drained = timeout(deadline, async {
343 while let Some(result) = guard.tasks.join_next().await {
344 if result.is_err() {
345 guard.health.fail(AgentMaintenanceFailure::Task);
346 }
347 }
348 })
349 .await
350 .is_ok();
351 if !drained {
352 guard.health.forced();
353 guard.tasks.abort_all();
354 while guard.tasks.join_next().await.is_some() {}
355 }
356}
357
358#[cfg(test)]
359mod tests {
360 use super::*;
361 #[tokio::test]
362 async fn forced_drain_joins_future_destruction() {
363 let health = AgentMaintenanceHealth::new(Duration::from_secs(1));
364 health.probe(true);
365 let active = health.admit().unwrap();
366 let mut guard = RuntimeGuard {
367 tasks: JoinSet::new(),
368 health: health.clone(),
369 stop: CancellationToken::new(),
370 };
371 guard.tasks.spawn(async move {
372 let _active = active;
373 std::future::pending::<()>().await;
374 });
375 health.draining();
376 guard.stop.cancel();
377 drain(&mut guard, Duration::from_millis(10)).await;
378 assert!(guard.tasks.is_empty());
379 assert_eq!(health.active_ticks(), 0);
380 assert_eq!(health.report().forced_ticks, 1);
381 }
382 #[tokio::test]
383 async fn abort_before_first_poll_closes_health() {
384 let health = AgentMaintenanceHealth::new(Duration::from_secs(1));
385 let guard = RuntimeGuard {
386 tasks: JoinSet::new(),
387 health: health.clone(),
388 stop: CancellationToken::new(),
389 };
390 let task = tokio::spawn(async move {
391 let _guard = guard;
392 std::future::pending::<()>().await;
393 });
394 task.abort();
395 assert!(task.await.unwrap_err().is_cancelled());
396 assert_eq!(health.status(), AgentMaintenanceStatus::Stopped);
397 }
398}