use futures_util::FutureExt;
use stateknot_core::{BoxFuture, CancellationObserver, CancellationSignal};
use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration};
use thiserror::Error;
use tokio::{
task::{JoinHandle, JoinSet},
time::{sleep, timeout},
};
use tokio_util::sync::CancellationToken;
mod binding;
mod health;
mod options;
pub use binding::{AgentMaintenanceBinding, AgentMaintenanceMutationOptions};
pub use health::{AgentMaintenanceHealth, AgentMaintenanceStatus};
pub use options::AgentMaintenanceOptions;
pub trait AgentMaintenanceReadiness: Send + Sync {
fn check(&self) -> BoxFuture<'_, Result<(), AgentMaintenanceReadinessError>>;
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)]
#[error("maintenance host dependencies are unavailable")]
pub struct AgentMaintenanceReadinessError;
#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)]
pub enum AgentMaintenanceError {
#[error("maintenance options are invalid")]
InvalidOptions,
#[error("maintenance tenant allowlist is invalid")]
InvalidTenants,
#[error("maintenance schemas are unavailable")]
SchemaUnavailable,
#[error("maintenance dependencies are unavailable")]
Unavailable,
#[error("maintenance coordinator is stopped")]
Stopped,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum AgentMaintenanceJob {
Deadline,
Child,
Join,
FailureClose,
}
impl AgentMaintenanceJob {
pub const ALL: [Self; 4] = [Self::Deadline, Self::Child, Self::Join, Self::FailureClose];
const fn index(self) -> usize {
self as usize
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum AgentMaintenanceFailure {
Store,
TickDeadline,
Task,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct AgentMaintenanceJobReport {
pub completed_ticks: u64,
pub items: u64,
pub item_failures: u64,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct AgentMaintenanceReport {
pub started_ticks: u64,
pub readiness_failures: u64,
pub forced_ticks: usize,
pub failure: Option<AgentMaintenanceFailure>,
jobs: [AgentMaintenanceJobReport; 4],
}
impl AgentMaintenanceReport {
#[must_use]
pub const fn job(&self, job: AgentMaintenanceJob) -> AgentMaintenanceJobReport {
self.jobs[job.index()]
}
}
pub struct AgentMaintenance {
health: AgentMaintenanceHealth,
stop: CancellationToken,
task: Option<JoinHandle<AgentMaintenanceReport>>,
}
impl AgentMaintenance {
pub async fn start(
binding: AgentMaintenanceBinding,
host: Arc<dyn AgentMaintenanceReadiness>,
options: AgentMaintenanceOptions,
) -> Result<Self, AgentMaintenanceError> {
if !probe(&binding, host.as_ref(), &options)
.await
.unwrap_or(false)
{
return Err(AgentMaintenanceError::Unavailable);
}
let health = AgentMaintenanceHealth::new(options.freshness);
health.probe(true);
let stop = CancellationToken::new();
let guard = RuntimeGuard {
tasks: JoinSet::new(),
health: health.clone(),
stop: stop.clone(),
};
let task = tokio::spawn(run(Arc::new(binding), host, options, guard));
Ok(Self {
health,
stop,
task: Some(task),
})
}
pub fn health(&self) -> AgentMaintenanceHealth {
self.health.clone()
}
pub fn begin_shutdown(&self) {
self.health.draining();
self.stop.cancel();
}
pub async fn wait(&mut self) -> Result<AgentMaintenanceReport, AgentMaintenanceError> {
let result = self
.task
.as_mut()
.ok_or(AgentMaintenanceError::Stopped)?
.await;
self.task = None;
result.map_err(|_| AgentMaintenanceError::Stopped)
}
pub async fn shutdown(&mut self) -> Result<AgentMaintenanceReport, AgentMaintenanceError> {
self.begin_shutdown();
self.wait().await
}
}
impl Drop for AgentMaintenance {
fn drop(&mut self) {
self.begin_shutdown();
if let Some(task) = &self.task {
task.abort();
}
}
}
struct Signal(CancellationToken);
impl CancellationObserver for Signal {
fn is_cancelled(&self) -> bool {
self.0.is_cancelled()
}
fn cancelled(&self) -> BoxFuture<'_, ()> {
Box::pin(self.0.cancelled())
}
}
struct RuntimeGuard {
tasks: JoinSet<()>,
health: AgentMaintenanceHealth,
stop: CancellationToken,
}
impl Drop for RuntimeGuard {
fn drop(&mut self) {
self.stop.cancel();
self.tasks.abort_all();
self.health.stopped();
}
}
async fn probe(
binding: &AgentMaintenanceBinding,
host: &dyn AgentMaintenanceReadiness,
options: &AgentMaintenanceOptions,
) -> Result<bool, ()> {
AssertUnwindSafe(async {
timeout(options.probe_timeout, async {
binding.check().await?;
host.check()
.await
.map_err(|_| AgentMaintenanceError::Unavailable)
})
.await
.is_ok_and(|r| r.is_ok())
})
.catch_unwind()
.await
.map_err(|_| ())
}
async fn pause(duration: Duration, stop: &CancellationToken) {
tokio::select! { biased; () = stop.cancelled() => {}, () = sleep(duration) => {} }
}
async fn monitor(
binding: Arc<AgentMaintenanceBinding>,
host: Arc<dyn AgentMaintenanceReadiness>,
options: AgentMaintenanceOptions,
health: AgentMaintenanceHealth,
stop: CancellationToken,
) {
loop {
pause(options.probe_interval, &stop).await;
let result = tokio::select! { biased; () = stop.cancelled() => return, result = probe(&binding, host.as_ref(), &options) => result };
if let Ok(ready) = result {
health.probe(ready);
} else {
health.fail(AgentMaintenanceFailure::Task);
stop.cancel();
return;
}
}
}
async fn sweep(
binding: Arc<AgentMaintenanceBinding>,
options: AgentMaintenanceOptions,
health: AgentMaintenanceHealth,
stop: CancellationToken,
) {
let mut rotation = binding.rotation();
let rotation_len = rotation.len();
let mut next = 0;
let signal = CancellationSignal::new(Signal(stop.clone()));
while !stop.is_cancelled() {
let Some(active) = health.admit() else {
pause(options.delay, &stop).await;
continue;
};
let progress = &mut rotation[next];
let job = progress.job();
let tick = binding.tick(progress, signal.clone());
tokio::pin!(tick);
let result = tokio::select! {
result = &mut tick => result,
() = sleep(options.tick_timeout) => {
health.fail(AgentMaintenanceFailure::TickDeadline); stop.cancel();
tick.await
}
};
let delay = if let Ok(counts) = result {
health.complete(job, counts);
if counts.item_failures > 0 {
options.failure_delay
} else {
options.delay
}
} else {
health.fail(AgentMaintenanceFailure::Store);
stop.cancel();
options.failure_delay
};
next = (next + 1) % rotation_len;
drop(active);
pause(delay, &stop).await;
}
}
async fn run(
binding: Arc<AgentMaintenanceBinding>,
host: Arc<dyn AgentMaintenanceReadiness>,
options: AgentMaintenanceOptions,
mut guard: RuntimeGuard,
) -> AgentMaintenanceReport {
guard.tasks.spawn(monitor(
binding.clone(),
host,
options.clone(),
guard.health.clone(),
guard.stop.clone(),
));
guard.tasks.spawn(sweep(
binding,
options.clone(),
guard.health.clone(),
guard.stop.clone(),
));
tokio::select! {
biased;
() = guard.stop.cancelled() => {},
_ = guard.tasks.join_next() => {
if !guard.stop.is_cancelled() { guard.health.fail(AgentMaintenanceFailure::Task); }
guard.stop.cancel();
}
}
guard.health.draining();
drain(&mut guard, options.drain_timeout).await;
guard.health.stopped();
guard.health.report()
}
async fn drain(guard: &mut RuntimeGuard, deadline: Duration) {
let drained = timeout(deadline, async {
while let Some(result) = guard.tasks.join_next().await {
if result.is_err() {
guard.health.fail(AgentMaintenanceFailure::Task);
}
}
})
.await
.is_ok();
if !drained {
guard.health.forced();
guard.tasks.abort_all();
while guard.tasks.join_next().await.is_some() {}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn forced_drain_joins_future_destruction() {
let health = AgentMaintenanceHealth::new(Duration::from_secs(1));
health.probe(true);
let active = health.admit().unwrap();
let mut guard = RuntimeGuard {
tasks: JoinSet::new(),
health: health.clone(),
stop: CancellationToken::new(),
};
guard.tasks.spawn(async move {
let _active = active;
std::future::pending::<()>().await;
});
health.draining();
guard.stop.cancel();
drain(&mut guard, Duration::from_millis(10)).await;
assert!(guard.tasks.is_empty());
assert_eq!(health.active_ticks(), 0);
assert_eq!(health.report().forced_ticks, 1);
}
#[tokio::test]
async fn abort_before_first_poll_closes_health() {
let health = AgentMaintenanceHealth::new(Duration::from_secs(1));
let guard = RuntimeGuard {
tasks: JoinSet::new(),
health: health.clone(),
stop: CancellationToken::new(),
};
let task = tokio::spawn(async move {
let _guard = guard;
std::future::pending::<()>().await;
});
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
assert_eq!(health.status(), AgentMaintenanceStatus::Stopped);
}
}