use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use deadpool_postgres::Pool;
use interloper::HealthStore;
use interloper::health::{COOLDOWN_MAX, ProviderHealth, Status, Tier, TierOverlay};
use jiff::{SignedDuration, Timestamp};
use crate::store::StoreUnavailable;
pub const EVIDENCE_HORIZON: Duration = COOLDOWN_MAX;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Vote {
Up,
Down {
until: Timestamp,
},
}
pub fn vote(e: &ProviderHealth, now: Timestamp) -> Option<Vote> {
let checked = e.last_checked?;
let horizon = SignedDuration::try_from(EVIDENCE_HORIZON).unwrap_or(SignedDuration::MAX);
if now.duration_since(checked) > horizon {
return None;
}
match e.status {
Status::Ok | Status::Slow => Some(Vote::Up),
Status::Down if e.needs_residential_ip => None,
Status::Down => match e.retry_at {
Some(until) if until > now => Some(Vote::Down { until }),
_ => None,
},
Status::Quarantined => None,
}
}
pub fn peer_votes(
peers: &[HashMap<String, ProviderHealth>],
now: Timestamp,
) -> HashMap<String, Vec<Vote>> {
let mut out: HashMap<String, Vec<Vote>> = HashMap::new();
for snapshot in peers {
for (provider, e) in snapshot {
if let Some(v) = vote(e, now) {
out.entry(provider.clone()).or_default().push(v);
}
}
}
out
}
pub fn decide(
local: Option<&ProviderHealth>,
tier: Option<Tier>,
peers: &[Vote],
now: Timestamp,
) -> Option<Tier> {
let tier = tier?;
let mut up = 0usize;
let mut down = 0usize;
let own = local.and_then(|e| vote(e, now));
for v in own.iter().chain(peers) {
match v {
Vote::Up => up += 1,
Vote::Down { until } if *until > now => down += 1,
Vote::Down { .. } => {}
}
}
if down > up { None } else { Some(tier) }
}
#[derive(Default)]
pub struct PeerView {
votes: Mutex<HashMap<String, Vec<Vote>>>,
}
impl PeerView {
pub fn set(&self, votes: HashMap<String, Vec<Vote>>) {
*self.votes.lock().unwrap() = votes;
}
pub fn vetoes(&self, provider: &str, local: Option<&ProviderHealth>, now: Timestamp) -> bool {
let votes = self.votes.lock().unwrap();
let peers = votes.get(provider).map_or(&[][..], Vec::as_slice);
decide(local, Some(Tier::Healthy), &[], now).is_some()
&& decide(local, Some(Tier::Healthy), peers, now).is_none()
}
}
impl TierOverlay for PeerView {
fn adjust(
&self,
provider: &str,
local: Option<&ProviderHealth>,
tier: Option<Tier>,
now: Timestamp,
) -> Option<Tier> {
let votes = self.votes.lock().unwrap();
match votes.get(provider) {
Some(peers) => decide(local, tier, peers, now),
None => tier,
}
}
}
use crate::store::unavailable as db;
pub struct HealthSync {
pool: Pool,
health: Arc<HealthStore>,
instance: String,
stale_after: Duration,
view: Arc<PeerView>,
published: Mutex<HashSet<String>>,
lock: tokio::sync::Mutex<()>,
}
impl HealthSync {
pub fn new(
pool: Pool,
health: Arc<HealthStore>,
instance: impl Into<String>,
stale_after: Duration,
) -> Arc<Self> {
let view = Arc::new(PeerView::default());
health.set_overlay(Some(view.clone()));
Arc::new(HealthSync {
pool,
health,
instance: instance.into(),
stale_after,
view,
published: Mutex::new(HashSet::new()),
lock: tokio::sync::Mutex::new(()),
})
}
pub fn instance(&self) -> &str {
&self.instance
}
pub fn view(&self) -> &PeerView {
&self.view
}
pub async fn load(&self) -> Result<(), StoreUnavailable> {
let _g = self.lock.lock().await;
let client = self.pool.get().await.map_err(db)?;
let row = client
.query_opt(
"select snapshot from instance_health where instance_id = $1",
&[&self.instance],
)
.await
.map_err(db)?;
if let Some(row) = row {
let v: serde_json::Value = row.get(0);
if let Ok(snap) = serde_json::from_value::<HashMap<String, ProviderHealth>>(v) {
self.health.seed(snap);
}
}
let accounted: HashSet<String> = self
.health
.snapshot()
.into_iter()
.filter(|(_, e)| e.status == Status::Quarantined)
.map(|(k, _)| k)
.collect();
self.reconcile(&client, accounted).await
}
pub async fn sync(&self) -> Result<(), StoreUnavailable> {
let _g = self.lock.lock().await;
let client = self.pool.get().await.map_err(db)?;
let snapshot = self.health.snapshot();
let body = serde_json::to_value(&snapshot).expect("health serializes");
client
.execute(
"insert into instance_health (instance_id, snapshot, updated_at)
values ($1, $2, now())
on conflict (instance_id) do update
set snapshot = excluded.snapshot, updated_at = now()",
&[&self.instance, &body],
)
.await
.map_err(db)?;
let mut accounted = self.published.lock().unwrap().clone();
for (provider, e) in &snapshot {
if e.status == Status::Quarantined && !accounted.contains(provider) {
client
.execute(
"insert into provider_quarantine (provider, reason, instance_id)
values ($1, $2, $3) on conflict (provider) do nothing",
&[
provider,
&e.last_failure.as_deref().unwrap_or("quarantined"),
&self.instance,
],
)
.await
.map_err(db)?;
accounted.insert(provider.clone());
}
}
let rows = client
.query(
"select snapshot from instance_health
where instance_id <> $1
and updated_at > now() - make_interval(secs => $2)",
&[&self.instance, &self.stale_after.as_secs_f64()],
)
.await
.map_err(db)?;
let peers: Vec<HashMap<String, ProviderHealth>> = rows
.into_iter()
.filter_map(|r| serde_json::from_value(r.get::<_, serde_json::Value>(0)).ok())
.collect();
self.view.set(peer_votes(&peers, Timestamp::now()));
self.reconcile(&client, accounted).await
}
async fn reconcile(
&self,
client: &deadpool_postgres::Client,
accounted: HashSet<String>,
) -> Result<(), StoreUnavailable> {
let rows = client
.query("select provider, reason from provider_quarantine", &[])
.await
.map_err(db)?;
let mut table = HashSet::new();
for r in rows {
let provider: String = r.get(0);
let reason: String = r.get(1);
if self.health.status(&provider) != Status::Quarantined {
self.health.quarantine(&provider, &reason);
}
table.insert(provider);
}
for (provider, e) in self.health.snapshot() {
if e.status == Status::Quarantined
&& !table.contains(&provider)
&& accounted.contains(&provider)
{
self.health.unquarantine(&provider);
}
}
*self.published.lock().unwrap() = table;
Ok(())
}
pub async fn clear_quarantine(&self, provider: &str) -> Result<bool, StoreUnavailable> {
let _g = self.lock.lock().await;
let client = self.pool.get().await.map_err(db)?;
let n = client
.execute(
"delete from provider_quarantine where provider = $1",
&[&provider],
)
.await
.map_err(db)?;
self.published.lock().unwrap().remove(provider);
let local = self.health.unquarantine(provider);
Ok(n > 0 || local)
}
}
#[cfg(test)]
pub(crate) fn test_database_url() -> Option<String> {
std::env::var("INTERLOPER_TEST_DATABASE_URL")
.ok()
.filter(|s| !s.trim().is_empty())
}
#[cfg(test)]
mod tests {
use super::*;
fn t(s: i64) -> Timestamp {
Timestamp::from_second(1_800_000_000 + s).unwrap()
}
fn ok_at(s: i64) -> ProviderHealth {
ProviderHealth {
status: Status::Ok,
last_checked: Some(t(s)),
..Default::default()
}
}
fn down_at(s: i64, until: i64) -> ProviderHealth {
ProviderHealth {
status: Status::Down,
last_checked: Some(t(s)),
failures: 1,
retry_at: Some(t(until)),
..Default::default()
}
}
#[test]
fn votes() {
let now = t(0);
assert_eq!(vote(&ok_at(-5), now), Some(Vote::Up));
assert_eq!(
vote(&down_at(-5, 25), now),
Some(Vote::Down { until: t(25) })
);
assert_eq!(vote(&down_at(-60, -30), now), None);
let q = ProviderHealth {
status: Status::Quarantined,
..ok_at(-1)
};
assert_eq!(vote(&q, now), None);
assert_eq!(vote(&ProviderHealth::default(), now), None);
assert_eq!(vote(&ok_at(-(15 * 60) - 1), now), None);
let challenged = ProviderHealth {
needs_residential_ip: true,
..down_at(-5, 25)
};
assert_eq!(vote(&challenged, now), None);
}
#[test]
fn uninformed_instance_follows_a_peer_that_saw_failure() {
let now = t(0);
let peers = [Vote::Down { until: t(25) }];
assert_eq!(decide(None, Some(Tier::Healthy), &peers, now), None);
assert_eq!(
decide(None, Some(Tier::Healthy), &peers, t(26)),
Some(Tier::Healthy)
);
}
#[test]
fn ties_route_and_majorities_demote() {
let now = t(0);
let local = ok_at(-10);
let one = [Vote::Down { until: t(25) }];
assert_eq!(
decide(Some(&local), Some(Tier::Healthy), &one, now),
Some(Tier::Healthy),
"one peer cannot overrule our own success"
);
let two = [Vote::Down { until: t(25) }, Vote::Down { until: t(40) }];
assert_eq!(decide(Some(&local), Some(Tier::Slow), &two, now), None);
let mixed = [
Vote::Down { until: t(25) },
Vote::Down { until: t(40) },
Vote::Up,
];
assert_eq!(
decide(Some(&local), Some(Tier::Healthy), &mixed, now),
Some(Tier::Healthy)
);
}
#[test]
fn peers_never_promote() {
let local = down_at(-1, 30);
let peers = [Vote::Up, Vote::Up, Vote::Up];
assert_eq!(decide(Some(&local), None, &peers, t(0)), None);
}
#[test]
fn probation_keeps_its_tier_unless_outvoted() {
let local = down_at(-60, -30);
assert_eq!(
decide(Some(&local), Some(Tier::Probation), &[Vote::Up], t(0)),
Some(Tier::Probation)
);
assert_eq!(
decide(
Some(&local),
Some(Tier::Probation),
&[Vote::Down { until: t(9) }],
t(0)
),
None
);
}
#[test]
fn peer_votes_group_by_provider() {
let now = t(0);
let a = HashMap::from([
("p".to_string(), down_at(-1, 30)),
("q".to_string(), ok_at(-1)),
]);
let b = HashMap::from([
("p".to_string(), down_at(-2, 20)),
("r".to_string(), ProviderHealth::default()),
]);
let v = peer_votes(&[a, b], now);
assert_eq!(v["p"].len(), 2);
assert_eq!(v["q"], vec![Vote::Up]);
assert!(!v.contains_key("r"));
}
#[test]
fn overlay_reaches_routing() {
let h = Arc::new(HealthStore::new());
let view = Arc::new(PeerView::default());
h.set_overlay(Some(view.clone()));
assert!(h.tier("p").is_some());
let until = Timestamp::now()
.checked_add(SignedDuration::from_secs(60))
.unwrap();
view.set(HashMap::from([("p".into(), vec![Vote::Down { until }])]));
assert_eq!(h.tier("p"), None);
assert!(view.vetoes("p", None, Timestamp::now()));
h.record_success("p", Duration::from_millis(10));
assert_eq!(h.tier("p"), Some(Tier::Healthy));
assert_eq!(h.snapshot()["p"].status, Status::Ok);
}
}