Skip to main content

stateknot_runtime/
agent_maintenance.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! Owned, bounded maintenance over the existing durable Agent state machines.
5//! No node execution, implicit migrations, retention, pool closure or public routes.
6
7use futures_util::FutureExt;
8use stateknot_core::{BoxFuture, CancellationObserver, CancellationSignal};
9use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration};
10use thiserror::Error;
11use tokio::{
12    task::{JoinHandle, JoinSet},
13    time::{sleep, timeout},
14};
15use tokio_util::sync::CancellationToken;
16
17mod binding;
18mod health;
19mod options;
20pub use binding::{AgentMaintenanceBinding, AgentMaintenanceMutationOptions};
21pub use health::{AgentMaintenanceHealth, AgentMaintenanceStatus};
22pub use options::AgentMaintenanceOptions;
23
24/// Mandatory read-only qualification of the actual host dependencies and policy.
25/// Callbacks must yield and be cancellation-cooperative; no default allow-all check.
26pub trait AgentMaintenanceReadiness: Send + Sync {
27    /// Checks host dependencies in addition to the bound store's schema check.
28    fn check(&self) -> BoxFuture<'_, Result<(), AgentMaintenanceReadinessError>>;
29}
30
31/// Sanitized readiness failure, without database, identity or payload diagnostics.
32#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)]
33#[error("maintenance host dependencies are unavailable")]
34pub struct AgentMaintenanceReadinessError;
35
36/// Closed startup and ownership errors.
37#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)]
38pub enum AgentMaintenanceError {
39    /// A finite timing bound is invalid.
40    #[error("maintenance options are invalid")]
41    InvalidOptions,
42    /// Require 1–128 distinct explicitly authorized tenants.
43    #[error("maintenance tenant allowlist is invalid")]
44    InvalidTenants,
45    /// At least one exact offline maintenance schema is absent.
46    #[error("maintenance schemas are unavailable")]
47    SchemaUnavailable,
48    /// Actual database or host readiness failed, panicked or timed out.
49    #[error("maintenance dependencies are unavailable")]
50    Unavailable,
51    /// Coordinator failed or its report has already been taken.
52    #[error("maintenance coordinator is stopped")]
53    Stopped,
54}
55
56/// Four concrete jobs; child reconciliation includes cancellation and settlement.
57#[derive(Clone, Copy, Debug, Eq, PartialEq)]
58pub enum AgentMaintenanceJob {
59    /// Request cancellation for admitted Agent deadlines using database time.
60    Deadline,
61    /// Propagate child cancellation and settle terminal children.
62    Child,
63    /// Publish eligible durable child Join results.
64    Join,
65    /// Close failed parents after durable child accounting completes.
66    FailureClose,
67}
68impl AgentMaintenanceJob {
69    /// Deterministic job order within each tenant's rotation.
70    pub const ALL: [Self; 4] = [Self::Deadline, Self::Child, Self::Join, Self::FailureClose];
71    const fn index(self) -> usize {
72        self as usize
73    }
74}
75
76/// First fail-stop cause; item errors instead count and continue the sweep.
77#[derive(Clone, Copy, Debug, Eq, PartialEq)]
78pub enum AgentMaintenanceFailure {
79    /// Candidate discovery failed; restart from durable records.
80    Store,
81    /// One admitted tick exceeded its absolute deadline.
82    TickDeadline,
83    /// An owned task panicked or exited unexpectedly.
84    Task,
85}
86
87/// Saturating per-job observations, not durable usage or terminal Run outcomes.
88#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
89pub struct AgentMaintenanceJobReport {
90    /// Returned finite ticks, including interrupted partial pages.
91    pub completed_ticks: u64,
92    /// Candidate observations (including failures and repeated observations).
93    pub items: u64,
94    /// Failed candidate observations; alert even when dependency status is Ready.
95    pub item_failures: u64,
96}
97
98/// Process-local counters, reset on restart and safe for protected health views.
99#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
100pub struct AgentMaintenanceReport {
101    /// Admitted ticks, including interrupted or failed discovery.
102    pub started_ticks: u64,
103    /// Failed or timed-out periodic dependency checks.
104    pub readiness_failures: u64,
105    /// Active tick futures when forced drain began (at most one).
106    pub forced_ticks: usize,
107    /// First fail-stop cause; absence means requested shutdown, not Run success.
108    pub failure: Option<AgentMaintenanceFailure>,
109    jobs: [AgentMaintenanceJobReport; 4],
110}
111impl AgentMaintenanceReport {
112    /// Counters for a concrete job, without tenant or Run labels.
113    #[must_use]
114    pub const fn job(&self, job: AgentMaintenanceJob) -> AgentMaintenanceJobReport {
115        self.jobs[job.index()]
116    }
117}
118
119/// Owned maintenance role. Await `shutdown` for joined completion; Drop only
120/// initiates cleanup. An OS supervisor must bound non-yielding host code.
121///
122/// ```no_run
123/// use std::sync::Arc;
124/// use stateknot_runtime::{agent_maintenance::*, JsonSchemaRegistry};
125/// use stateknot_core::TenantId;
126/// use stateknot_store_postgres::PostgresStore;
127/// async fn start(store: PostgresStore, schemas: JsonSchemaRegistry,
128///     tenants: Vec<TenantId>, host: Arc<dyn AgentMaintenanceReadiness>)
129///     -> Result<AgentMaintenance, AgentMaintenanceError> {
130///     let binding = AgentMaintenanceBinding::new(store, schemas, tenants,
131///         AgentMaintenanceMutationOptions::default())?;
132///     AgentMaintenance::start(binding, host, AgentMaintenanceOptions::default()).await
133/// }
134/// ```
135pub struct AgentMaintenance {
136    health: AgentMaintenanceHealth,
137    stop: CancellationToken,
138    task: Option<JoinHandle<AgentMaintenanceReport>>,
139}
140impl AgentMaintenance {
141    /// Checks actual dependencies before spawning any maintenance task. Failed
142    /// or cancelled startup performs no durable mutation and claims no work.
143    pub async fn start(
144        binding: AgentMaintenanceBinding,
145        host: Arc<dyn AgentMaintenanceReadiness>,
146        options: AgentMaintenanceOptions,
147    ) -> Result<Self, AgentMaintenanceError> {
148        if !probe(&binding, host.as_ref(), &options)
149            .await
150            .unwrap_or(false)
151        {
152            return Err(AgentMaintenanceError::Unavailable);
153        }
154        let health = AgentMaintenanceHealth::new(options.freshness);
155        health.probe(true);
156        let stop = CancellationToken::new();
157        // Construct before spawn so abort-before-first-poll still closes health.
158        let guard = RuntimeGuard {
159            tasks: JoinSet::new(),
160            health: health.clone(),
161            stop: stop.clone(),
162        };
163        let task = tokio::spawn(run(Arc::new(binding), host, options, guard));
164        Ok(Self {
165            health,
166            stop,
167            task: Some(task),
168        })
169    }
170    /// Payload-free cached health without retaining store or schema handles.
171    pub fn health(&self) -> AgentMaintenanceHealth {
172        self.health.clone()
173    }
174    /// Synchronously closes admission; does not request durable Run cancellation.
175    pub fn begin_shutdown(&self) {
176        self.health.draining();
177        self.stop.cancel();
178    }
179    /// Cancellation-safe joined wait; a cancelled wait retains coordinator ownership.
180    pub async fn wait(&mut self) -> Result<AgentMaintenanceReport, AgentMaintenanceError> {
181        let result = self
182            .task
183            .as_mut()
184            .ok_or(AgentMaintenanceError::Stopped)?
185            .await;
186        self.task = None;
187        result.map_err(|_| AgentMaintenanceError::Stopped)
188    }
189    /// Requests shutdown, then joins all owned tasks including forced cancellation.
190    pub async fn shutdown(&mut self) -> Result<AgentMaintenanceReport, AgentMaintenanceError> {
191        self.begin_shutdown();
192        self.wait().await
193    }
194}
195impl Drop for AgentMaintenance {
196    fn drop(&mut self) {
197        self.begin_shutdown();
198        if let Some(task) = &self.task {
199            task.abort();
200        }
201    }
202}
203struct Signal(CancellationToken);
204impl CancellationObserver for Signal {
205    fn is_cancelled(&self) -> bool {
206        self.0.is_cancelled()
207    }
208    fn cancelled(&self) -> BoxFuture<'_, ()> {
209        Box::pin(self.0.cancelled())
210    }
211}
212struct RuntimeGuard {
213    tasks: JoinSet<()>,
214    health: AgentMaintenanceHealth,
215    stop: CancellationToken,
216}
217impl Drop for RuntimeGuard {
218    fn drop(&mut self) {
219        self.stop.cancel();
220        self.tasks.abort_all();
221        self.health.stopped();
222    }
223}
224async fn probe(
225    binding: &AgentMaintenanceBinding,
226    host: &dyn AgentMaintenanceReadiness,
227    options: &AgentMaintenanceOptions,
228) -> Result<bool, ()> {
229    AssertUnwindSafe(async {
230        timeout(options.probe_timeout, async {
231            binding.check().await?;
232            host.check()
233                .await
234                .map_err(|_| AgentMaintenanceError::Unavailable)
235        })
236        .await
237        .is_ok_and(|r| r.is_ok())
238    })
239    .catch_unwind()
240    .await
241    .map_err(|_| ())
242}
243async fn pause(duration: Duration, stop: &CancellationToken) {
244    tokio::select! { biased; () = stop.cancelled() => {}, () = sleep(duration) => {} }
245}
246async fn monitor(
247    binding: Arc<AgentMaintenanceBinding>,
248    host: Arc<dyn AgentMaintenanceReadiness>,
249    options: AgentMaintenanceOptions,
250    health: AgentMaintenanceHealth,
251    stop: CancellationToken,
252) {
253    loop {
254        pause(options.probe_interval, &stop).await;
255        let result = tokio::select! { biased; () = stop.cancelled() => return, result = probe(&binding, host.as_ref(), &options) => result };
256        if let Ok(ready) = result {
257            health.probe(ready);
258        } else {
259            health.fail(AgentMaintenanceFailure::Task);
260            stop.cancel();
261            return;
262        }
263    }
264}
265async fn sweep(
266    binding: Arc<AgentMaintenanceBinding>,
267    options: AgentMaintenanceOptions,
268    health: AgentMaintenanceHealth,
269    stop: CancellationToken,
270) {
271    let mut rotation = binding.rotation();
272    let rotation_len = rotation.len();
273    let mut next = 0;
274    let signal = CancellationSignal::new(Signal(stop.clone()));
275    while !stop.is_cancelled() {
276        let Some(active) = health.admit() else {
277            pause(options.delay, &stop).await;
278            continue;
279        };
280        let progress = &mut rotation[next];
281        let job = progress.job();
282        let tick = binding.tick(progress, signal.clone());
283        tokio::pin!(tick);
284        let result = tokio::select! {
285            result = &mut tick => result,
286            () = sleep(options.tick_timeout) => {
287                health.fail(AgentMaintenanceFailure::TickDeadline); stop.cancel();
288                tick.await
289            }
290        };
291        let delay = if let Ok(counts) = result {
292            health.complete(job, counts);
293            if counts.item_failures > 0 {
294                options.failure_delay
295            } else {
296                options.delay
297            }
298        } else {
299            health.fail(AgentMaintenanceFailure::Store);
300            stop.cancel();
301            options.failure_delay
302        };
303        next = (next + 1) % rotation_len;
304        drop(active);
305        pause(delay, &stop).await;
306    }
307}
308async fn run(
309    binding: Arc<AgentMaintenanceBinding>,
310    host: Arc<dyn AgentMaintenanceReadiness>,
311    options: AgentMaintenanceOptions,
312    mut guard: RuntimeGuard,
313) -> AgentMaintenanceReport {
314    guard.tasks.spawn(monitor(
315        binding.clone(),
316        host,
317        options.clone(),
318        guard.health.clone(),
319        guard.stop.clone(),
320    ));
321    guard.tasks.spawn(sweep(
322        binding,
323        options.clone(),
324        guard.health.clone(),
325        guard.stop.clone(),
326    ));
327    tokio::select! {
328        biased;
329        () = guard.stop.cancelled() => {},
330        _ = guard.tasks.join_next() => {
331            if !guard.stop.is_cancelled() { guard.health.fail(AgentMaintenanceFailure::Task); }
332            guard.stop.cancel();
333        }
334    }
335    guard.health.draining();
336    drain(&mut guard, options.drain_timeout).await;
337    guard.health.stopped();
338    guard.health.report()
339}
340
341async fn drain(guard: &mut RuntimeGuard, deadline: Duration) {
342    let drained = timeout(deadline, async {
343        while let Some(result) = guard.tasks.join_next().await {
344            if result.is_err() {
345                guard.health.fail(AgentMaintenanceFailure::Task);
346            }
347        }
348    })
349    .await
350    .is_ok();
351    if !drained {
352        guard.health.forced();
353        guard.tasks.abort_all();
354        while guard.tasks.join_next().await.is_some() {}
355    }
356}
357
358#[cfg(test)]
359mod tests {
360    use super::*;
361    #[tokio::test]
362    async fn forced_drain_joins_future_destruction() {
363        let health = AgentMaintenanceHealth::new(Duration::from_secs(1));
364        health.probe(true);
365        let active = health.admit().unwrap();
366        let mut guard = RuntimeGuard {
367            tasks: JoinSet::new(),
368            health: health.clone(),
369            stop: CancellationToken::new(),
370        };
371        guard.tasks.spawn(async move {
372            let _active = active;
373            std::future::pending::<()>().await;
374        });
375        health.draining();
376        guard.stop.cancel();
377        drain(&mut guard, Duration::from_millis(10)).await;
378        assert!(guard.tasks.is_empty());
379        assert_eq!(health.active_ticks(), 0);
380        assert_eq!(health.report().forced_ticks, 1);
381    }
382    #[tokio::test]
383    async fn abort_before_first_poll_closes_health() {
384        let health = AgentMaintenanceHealth::new(Duration::from_secs(1));
385        let guard = RuntimeGuard {
386            tasks: JoinSet::new(),
387            health: health.clone(),
388            stop: CancellationToken::new(),
389        };
390        let task = tokio::spawn(async move {
391            let _guard = guard;
392            std::future::pending::<()>().await;
393        });
394        task.abort();
395        assert!(task.await.unwrap_err().is_cancelled());
396        assert_eq!(health.status(), AgentMaintenanceStatus::Stopped);
397    }
398}