1use std::collections::{HashMap, HashSet};
48use std::sync::{Arc, Mutex};
49use std::time::Duration;
50
51use deadpool_postgres::Pool;
52use interloper::HealthStore;
53use interloper::health::{COOLDOWN_MAX, ProviderHealth, Status, Tier, TierOverlay};
54use jiff::{SignedDuration, Timestamp};
55
56use crate::store::StoreUnavailable;
57
58pub const EVIDENCE_HORIZON: Duration = COOLDOWN_MAX;
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum Vote {
65 Up,
66 Down {
68 until: Timestamp,
69 },
70}
71
72pub fn vote(e: &ProviderHealth, now: Timestamp) -> Option<Vote> {
74 let checked = e.last_checked?;
75 let horizon = SignedDuration::try_from(EVIDENCE_HORIZON).unwrap_or(SignedDuration::MAX);
76 if now.duration_since(checked) > horizon {
77 return None;
78 }
79 match e.status {
80 Status::Ok | Status::Slow => Some(Vote::Up),
81 Status::Down if e.needs_residential_ip => None,
82 Status::Down => match e.retry_at {
83 Some(until) if until > now => Some(Vote::Down { until }),
84 _ => None,
85 },
86 Status::Quarantined => None,
87 }
88}
89
90pub fn peer_votes(
92 peers: &[HashMap<String, ProviderHealth>],
93 now: Timestamp,
94) -> HashMap<String, Vec<Vote>> {
95 let mut out: HashMap<String, Vec<Vote>> = HashMap::new();
96 for snapshot in peers {
97 for (provider, e) in snapshot {
98 if let Some(v) = vote(e, now) {
99 out.entry(provider.clone()).or_default().push(v);
100 }
101 }
102 }
103 out
104}
105
106pub fn decide(
109 local: Option<&ProviderHealth>,
110 tier: Option<Tier>,
111 peers: &[Vote],
112 now: Timestamp,
113) -> Option<Tier> {
114 let tier = tier?;
115 let mut up = 0usize;
116 let mut down = 0usize;
117 let own = local.and_then(|e| vote(e, now));
118 for v in own.iter().chain(peers) {
119 match v {
120 Vote::Up => up += 1,
121 Vote::Down { until } if *until > now => down += 1,
122 Vote::Down { .. } => {}
123 }
124 }
125 if down > up { None } else { Some(tier) }
126}
127
128#[derive(Default)]
130pub struct PeerView {
131 votes: Mutex<HashMap<String, Vec<Vote>>>,
132}
133
134impl PeerView {
135 pub fn set(&self, votes: HashMap<String, Vec<Vote>>) {
136 *self.votes.lock().unwrap() = votes;
137 }
138
139 pub fn vetoes(&self, provider: &str, local: Option<&ProviderHealth>, now: Timestamp) -> bool {
141 let votes = self.votes.lock().unwrap();
142 let peers = votes.get(provider).map_or(&[][..], Vec::as_slice);
143 decide(local, Some(Tier::Healthy), &[], now).is_some()
146 && decide(local, Some(Tier::Healthy), peers, now).is_none()
147 }
148}
149
150impl TierOverlay for PeerView {
151 fn adjust(
152 &self,
153 provider: &str,
154 local: Option<&ProviderHealth>,
155 tier: Option<Tier>,
156 now: Timestamp,
157 ) -> Option<Tier> {
158 let votes = self.votes.lock().unwrap();
159 match votes.get(provider) {
160 Some(peers) => decide(local, tier, peers, now),
161 None => tier,
162 }
163 }
164}
165
166use crate::store::unavailable as db;
167
168pub struct HealthSync {
171 pool: Pool,
172 health: Arc<HealthStore>,
173 instance: String,
174 stale_after: Duration,
175 view: Arc<PeerView>,
176 published: Mutex<HashSet<String>>,
180 lock: tokio::sync::Mutex<()>,
183}
184
185impl HealthSync {
186 pub fn new(
189 pool: Pool,
190 health: Arc<HealthStore>,
191 instance: impl Into<String>,
192 stale_after: Duration,
193 ) -> Arc<Self> {
194 let view = Arc::new(PeerView::default());
195 health.set_overlay(Some(view.clone()));
196 Arc::new(HealthSync {
197 pool,
198 health,
199 instance: instance.into(),
200 stale_after,
201 view,
202 published: Mutex::new(HashSet::new()),
203 lock: tokio::sync::Mutex::new(()),
204 })
205 }
206
207 pub fn instance(&self) -> &str {
208 &self.instance
209 }
210
211 pub fn view(&self) -> &PeerView {
212 &self.view
213 }
214
215 pub async fn load(&self) -> Result<(), StoreUnavailable> {
219 let _g = self.lock.lock().await;
220 let client = self.pool.get().await.map_err(db)?;
221 let row = client
222 .query_opt(
223 "select snapshot from instance_health where instance_id = $1",
224 &[&self.instance],
225 )
226 .await
227 .map_err(db)?;
228 if let Some(row) = row {
229 let v: serde_json::Value = row.get(0);
230 if let Ok(snap) = serde_json::from_value::<HashMap<String, ProviderHealth>>(v) {
231 self.health.seed(snap);
232 }
233 }
234 let accounted: HashSet<String> = self
235 .health
236 .snapshot()
237 .into_iter()
238 .filter(|(_, e)| e.status == Status::Quarantined)
239 .map(|(k, _)| k)
240 .collect();
241 self.reconcile(&client, accounted).await
242 }
243
244 pub async fn sync(&self) -> Result<(), StoreUnavailable> {
247 let _g = self.lock.lock().await;
248 let client = self.pool.get().await.map_err(db)?;
249 let snapshot = self.health.snapshot();
250 let body = serde_json::to_value(&snapshot).expect("health serializes");
251 client
252 .execute(
253 "insert into instance_health (instance_id, snapshot, updated_at)
254 values ($1, $2, now())
255 on conflict (instance_id) do update
256 set snapshot = excluded.snapshot, updated_at = now()",
257 &[&self.instance, &body],
258 )
259 .await
260 .map_err(db)?;
261 let mut accounted = self.published.lock().unwrap().clone();
262 for (provider, e) in &snapshot {
263 if e.status == Status::Quarantined && !accounted.contains(provider) {
264 client
267 .execute(
268 "insert into provider_quarantine (provider, reason, instance_id)
269 values ($1, $2, $3) on conflict (provider) do nothing",
270 &[
271 provider,
272 &e.last_failure.as_deref().unwrap_or("quarantined"),
273 &self.instance,
274 ],
275 )
276 .await
277 .map_err(db)?;
278 accounted.insert(provider.clone());
279 }
280 }
281 let rows = client
282 .query(
283 "select snapshot from instance_health
284 where instance_id <> $1
285 and updated_at > now() - make_interval(secs => $2)",
286 &[&self.instance, &self.stale_after.as_secs_f64()],
287 )
288 .await
289 .map_err(db)?;
290 let peers: Vec<HashMap<String, ProviderHealth>> = rows
291 .into_iter()
292 .filter_map(|r| serde_json::from_value(r.get::<_, serde_json::Value>(0)).ok())
293 .collect();
294 self.view.set(peer_votes(&peers, Timestamp::now()));
295 self.reconcile(&client, accounted).await
296 }
297
298 async fn reconcile(
303 &self,
304 client: &deadpool_postgres::Client,
305 accounted: HashSet<String>,
306 ) -> Result<(), StoreUnavailable> {
307 let rows = client
308 .query("select provider, reason from provider_quarantine", &[])
309 .await
310 .map_err(db)?;
311 let mut table = HashSet::new();
312 for r in rows {
313 let provider: String = r.get(0);
314 let reason: String = r.get(1);
315 if self.health.status(&provider) != Status::Quarantined {
316 self.health.quarantine(&provider, &reason);
317 }
318 table.insert(provider);
319 }
320 for (provider, e) in self.health.snapshot() {
321 if e.status == Status::Quarantined
322 && !table.contains(&provider)
323 && accounted.contains(&provider)
324 {
325 self.health.unquarantine(&provider);
326 }
327 }
328 *self.published.lock().unwrap() = table;
329 Ok(())
330 }
331
332 pub async fn clear_quarantine(&self, provider: &str) -> Result<bool, StoreUnavailable> {
335 let _g = self.lock.lock().await;
336 let client = self.pool.get().await.map_err(db)?;
337 let n = client
338 .execute(
339 "delete from provider_quarantine where provider = $1",
340 &[&provider],
341 )
342 .await
343 .map_err(db)?;
344 self.published.lock().unwrap().remove(provider);
345 let local = self.health.unquarantine(provider);
346 Ok(n > 0 || local)
347 }
348}
349
350#[cfg(test)]
351pub(crate) fn test_database_url() -> Option<String> {
352 std::env::var("INTERLOPER_TEST_DATABASE_URL")
353 .ok()
354 .filter(|s| !s.trim().is_empty())
355}
356
357#[cfg(test)]
358mod tests {
359 use super::*;
360
361 fn t(s: i64) -> Timestamp {
362 Timestamp::from_second(1_800_000_000 + s).unwrap()
363 }
364
365 fn ok_at(s: i64) -> ProviderHealth {
366 ProviderHealth {
367 status: Status::Ok,
368 last_checked: Some(t(s)),
369 ..Default::default()
370 }
371 }
372
373 fn down_at(s: i64, until: i64) -> ProviderHealth {
374 ProviderHealth {
375 status: Status::Down,
376 last_checked: Some(t(s)),
377 failures: 1,
378 retry_at: Some(t(until)),
379 ..Default::default()
380 }
381 }
382
383 #[test]
384 fn votes() {
385 let now = t(0);
386 assert_eq!(vote(&ok_at(-5), now), Some(Vote::Up));
387 assert_eq!(
388 vote(&down_at(-5, 25), now),
389 Some(Vote::Down { until: t(25) })
390 );
391 assert_eq!(vote(&down_at(-60, -30), now), None);
393 let q = ProviderHealth {
394 status: Status::Quarantined,
395 ..ok_at(-1)
396 };
397 assert_eq!(vote(&q, now), None);
398 assert_eq!(vote(&ProviderHealth::default(), now), None);
399 assert_eq!(vote(&ok_at(-(15 * 60) - 1), now), None);
400 let challenged = ProviderHealth {
402 needs_residential_ip: true,
403 ..down_at(-5, 25)
404 };
405 assert_eq!(vote(&challenged, now), None);
406 }
407
408 #[test]
409 fn uninformed_instance_follows_a_peer_that_saw_failure() {
410 let now = t(0);
411 let peers = [Vote::Down { until: t(25) }];
412 assert_eq!(decide(None, Some(Tier::Healthy), &peers, now), None);
413 assert_eq!(
415 decide(None, Some(Tier::Healthy), &peers, t(26)),
416 Some(Tier::Healthy)
417 );
418 }
419
420 #[test]
421 fn ties_route_and_majorities_demote() {
422 let now = t(0);
423 let local = ok_at(-10);
424 let one = [Vote::Down { until: t(25) }];
425 assert_eq!(
426 decide(Some(&local), Some(Tier::Healthy), &one, now),
427 Some(Tier::Healthy),
428 "one peer cannot overrule our own success"
429 );
430 let two = [Vote::Down { until: t(25) }, Vote::Down { until: t(40) }];
431 assert_eq!(decide(Some(&local), Some(Tier::Slow), &two, now), None);
432 let mixed = [
433 Vote::Down { until: t(25) },
434 Vote::Down { until: t(40) },
435 Vote::Up,
436 ];
437 assert_eq!(
438 decide(Some(&local), Some(Tier::Healthy), &mixed, now),
439 Some(Tier::Healthy)
440 );
441 }
442
443 #[test]
444 fn peers_never_promote() {
445 let local = down_at(-1, 30);
446 let peers = [Vote::Up, Vote::Up, Vote::Up];
447 assert_eq!(decide(Some(&local), None, &peers, t(0)), None);
448 }
449
450 #[test]
451 fn probation_keeps_its_tier_unless_outvoted() {
452 let local = down_at(-60, -30);
453 assert_eq!(
454 decide(Some(&local), Some(Tier::Probation), &[Vote::Up], t(0)),
455 Some(Tier::Probation)
456 );
457 assert_eq!(
458 decide(
459 Some(&local),
460 Some(Tier::Probation),
461 &[Vote::Down { until: t(9) }],
462 t(0)
463 ),
464 None
465 );
466 }
467
468 #[test]
469 fn peer_votes_group_by_provider() {
470 let now = t(0);
471 let a = HashMap::from([
472 ("p".to_string(), down_at(-1, 30)),
473 ("q".to_string(), ok_at(-1)),
474 ]);
475 let b = HashMap::from([
476 ("p".to_string(), down_at(-2, 20)),
477 ("r".to_string(), ProviderHealth::default()),
478 ]);
479 let v = peer_votes(&[a, b], now);
480 assert_eq!(v["p"].len(), 2);
481 assert_eq!(v["q"], vec![Vote::Up]);
482 assert!(!v.contains_key("r"));
483 }
484
485 #[test]
486 fn overlay_reaches_routing() {
487 let h = Arc::new(HealthStore::new());
488 let view = Arc::new(PeerView::default());
489 h.set_overlay(Some(view.clone()));
490 assert!(h.tier("p").is_some());
491 let until = Timestamp::now()
492 .checked_add(SignedDuration::from_secs(60))
493 .unwrap();
494 view.set(HashMap::from([("p".into(), vec![Vote::Down { until }])]));
495 assert_eq!(h.tier("p"), None);
496 assert!(view.vetoes("p", None, Timestamp::now()));
497 h.record_success("p", Duration::from_millis(10));
499 assert_eq!(h.tier("p"), Some(Tier::Healthy));
500 assert_eq!(h.snapshot()["p"].status, Status::Ok);
502 }
503}