stateknot_runtime/agent_maintenance/
health.rs1use super::{
5 AgentMaintenanceFailure, AgentMaintenanceJob, AgentMaintenanceJobReport, AgentMaintenanceReport,
6};
7use std::{
8 sync::{Arc, Mutex, MutexGuard},
9 time::Duration,
10};
11use tokio::time::Instant;
12
13#[derive(Clone, Copy, Debug, Eq, PartialEq)]
15pub enum AgentMaintenanceStatus {
16 Ready,
18 Unavailable,
20 Draining,
22 Stopped,
24}
25struct State {
26 status: AgentMaintenanceStatus,
27 checked: Option<Instant>,
28 active: usize,
29 report: AgentMaintenanceReport,
30}
31#[derive(Clone)]
33pub struct AgentMaintenanceHealth {
34 state: Arc<Mutex<State>>,
35 freshness: Duration,
36}
37impl AgentMaintenanceHealth {
38 pub(super) fn new(freshness: Duration) -> Self {
39 Self {
40 state: Arc::new(Mutex::new(State {
41 status: AgentMaintenanceStatus::Unavailable,
42 checked: None,
43 active: 0,
44 report: AgentMaintenanceReport::default(),
45 })),
46 freshness,
47 }
48 }
49 fn lock(&self) -> MutexGuard<'_, State> {
50 self.state
51 .lock()
52 .unwrap_or_else(std::sync::PoisonError::into_inner)
53 }
54 fn effective(&self, state: &State) -> AgentMaintenanceStatus {
55 if state.status == AgentMaintenanceStatus::Ready
56 && state
57 .checked
58 .is_none_or(|checked| checked.elapsed() >= self.freshness)
59 {
60 AgentMaintenanceStatus::Unavailable
61 } else {
62 state.status
63 }
64 }
65 pub fn status(&self) -> AgentMaintenanceStatus {
67 self.effective(&self.lock())
68 }
69 pub fn active_ticks(&self) -> usize {
71 self.lock().active
72 }
73 pub fn report(&self) -> AgentMaintenanceReport {
75 self.lock().report
76 }
77 pub(super) fn probe(&self, ready: bool) {
78 let mut state = self.lock();
79 if matches!(
80 state.status,
81 AgentMaintenanceStatus::Ready | AgentMaintenanceStatus::Unavailable
82 ) {
83 state.checked = ready.then(Instant::now);
84 state.status = if ready {
85 AgentMaintenanceStatus::Ready
86 } else {
87 AgentMaintenanceStatus::Unavailable
88 };
89 if !ready {
90 state.report.readiness_failures = state.report.readiness_failures.saturating_add(1);
91 }
92 }
93 }
94 pub(super) fn draining(&self) {
95 let mut state = self.lock();
96 if state.status != AgentMaintenanceStatus::Stopped {
97 state.status = AgentMaintenanceStatus::Draining;
98 }
99 }
100 pub(super) fn stopped(&self) {
101 self.lock().status = AgentMaintenanceStatus::Stopped;
102 }
103 pub(super) fn fail(&self, failure: AgentMaintenanceFailure) {
104 let mut state = self.lock();
105 state.report.failure.get_or_insert(failure);
106 state.status = AgentMaintenanceStatus::Draining;
107 }
108 pub(super) fn forced(&self) {
109 let mut state = self.lock();
110 state.report.forced_ticks = state.active;
111 }
112 pub(super) fn admit(&self) -> Option<TickGuard> {
113 let mut state = self.lock();
114 if self.effective(&state) != AgentMaintenanceStatus::Ready {
115 return None;
116 }
117 state.active += 1;
118 state.report.started_ticks = state.report.started_ticks.saturating_add(1);
119 Some(TickGuard(self.clone()))
120 }
121 pub(super) fn complete(&self, job: AgentMaintenanceJob, counts: AgentMaintenanceJobReport) {
122 let mut state = self.lock();
123 let report = &mut state.report.jobs[job.index()];
124 report.completed_ticks = report
125 .completed_ticks
126 .saturating_add(counts.completed_ticks);
127 report.items = report.items.saturating_add(counts.items);
128 report.item_failures = report.item_failures.saturating_add(counts.item_failures);
129 }
130}
131pub(super) struct TickGuard(AgentMaintenanceHealth);
132impl Drop for TickGuard {
133 fn drop(&mut self) {
134 self.0.lock().active -= 1;
135 }
136}
137
138#[cfg(test)]
139mod tests {
140 use super::*;
141 #[tokio::test]
142 async fn stale_health_and_late_probes_cannot_reopen_shutdown() {
143 let health = AgentMaintenanceHealth::new(Duration::from_millis(10));
144 assert!(health.admit().is_none());
145 health.probe(true);
146 let tick = health.admit().unwrap();
147 assert_eq!(health.active_ticks(), 1);
148 tokio::time::sleep(Duration::from_millis(15)).await;
149 assert_eq!(health.status(), AgentMaintenanceStatus::Unavailable);
150 assert!(health.admit().is_none());
151 health.draining();
152 health.probe(true);
153 assert!(health.admit().is_none());
154 drop(tick);
155 assert_eq!(health.active_ticks(), 0);
156 health.stopped();
157 health.probe(true);
158 assert_eq!(health.status(), AgentMaintenanceStatus::Stopped);
159 }
160}