Skip to main content

interloper_server/
fleet.rs

1//! Fleet mode: several servers sharing one Postgres.
2//!
3//! The address store is the easy half (see [`crate::store`]). This is the
4//! other half: health. Two tables, because health holds two kinds of fact.
5//!
6//! * `instance_health`: one row per instance, its whole [`HealthStore`]
7//!   snapshot plus `updated_at` (stamped by the database clock, so instance
8//!   clock skew cannot make a dead peer look alive). Each row has exactly one
9//!   writer, so no locking is needed. Rows older than `stale_after` are
10//!   ignored: a crashed or scaled-down instance stops voting by itself.
11//! * `provider_quarantine`: global. Schema drift means the adapter is wrong
12//!   everywhere it runs, so the first instance to see it inserts a row (later
13//!   inserts are no-ops), every instance adopts it, and deleting the row
14//!   (`DELETE /v1/providers/{name}/quarantine` on any instance) clears it
15//!   fleet-wide.
16//!
17//! # The routing rule
18//!
19//! cowbird wrote per-instance rows but never read peers' back, so every
20//! instance paid its own failed requests (and timeouts) to learn that a
21//! provider had died. Here peers' fresh rows feed routing through a
22//! [`TierOverlay`], under a rule chosen so a peer can save us a failed call
23//! but can never cost us much:
24//!
25//! 1. **Peers only demote.** If this instance excludes a provider, it stays
26//!    excluded; no peer can route us to something we saw fail.
27//! 2. **Each informed instance votes, this one included.** An entry votes
28//!    *down* while it is DOWN inside its cooldown (`retry_at` in the future),
29//!    and *up* when it is OK or SLOW. Everything else abstains: no entry,
30//!    probation (cooldown over, outcome unknown), quarantine (the global
31//!    table handles that), and evidence older than [`EVIDENCE_HORIZON`].
32//! 3. **Egress-specific failures abstain.** A DOWN caused by a challenge
33//!    (`needs_residential_ip`) is about that instance's IP; fan-out exists to
34//!    have several, so it must not take the provider away from the others.
35//! 4. **Strict majority demotes; ties route.** The provider leaves routing
36//!    while down votes outnumber up votes. A peer that just saw a failure
37//!    keeps an uninformed instance from repeating it; a single peer cannot
38//!    overrule an instance that saw a success. Being wrong toward "up" costs
39//!    one failed call (the local breaker then trips); being wrong toward
40//!    "down" is bounded by the voters' own cooldowns (at most 15 minutes).
41//!    Down votes are re-checked against their `retry_at` at routing time,
42//!    so a peer's verdict lapses on schedule even between syncs.
43//!
44//! Peers' votes never enter this instance's snapshot, so a verdict cannot
45//! echo around the fleet and outlive the measurement behind it.
46
47use std::collections::{HashMap, HashSet};
48use std::sync::{Arc, Mutex};
49use std::time::Duration;
50
51use deadpool_postgres::Pool;
52use interloper::HealthStore;
53use interloper::health::{COOLDOWN_MAX, ProviderHealth, Status, Tier, TierOverlay};
54use jiff::{SignedDuration, Timestamp};
55
56use crate::store::StoreUnavailable;
57
58/// Evidence older than this abstains. The longest a DOWN can last, so an old
59/// success cannot outvote a failure that is still inside its cooldown for
60/// longer than that failure could have lasted anyway.
61pub const EVIDENCE_HORIZON: Duration = COOLDOWN_MAX;
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum Vote {
65    Up,
66    /// Down until the voter's own cooldown ends.
67    Down {
68        until: Timestamp,
69    },
70}
71
72/// What one instance's entry says about a provider; `None` abstains.
73pub fn vote(e: &ProviderHealth, now: Timestamp) -> Option<Vote> {
74    let checked = e.last_checked?;
75    let horizon = SignedDuration::try_from(EVIDENCE_HORIZON).unwrap_or(SignedDuration::MAX);
76    if now.duration_since(checked) > horizon {
77        return None;
78    }
79    match e.status {
80        Status::Ok | Status::Slow => Some(Vote::Up),
81        Status::Down if e.needs_residential_ip => None,
82        Status::Down => match e.retry_at {
83            Some(until) if until > now => Some(Vote::Down { until }),
84            _ => None,
85        },
86        Status::Quarantined => None,
87    }
88}
89
90/// Every fresh peer's votes, by provider.
91pub fn peer_votes(
92    peers: &[HashMap<String, ProviderHealth>],
93    now: Timestamp,
94) -> HashMap<String, Vec<Vote>> {
95    let mut out: HashMap<String, Vec<Vote>> = HashMap::new();
96    for snapshot in peers {
97        for (provider, e) in snapshot {
98            if let Some(v) = vote(e, now) {
99                out.entry(provider.clone()).or_default().push(v);
100            }
101        }
102    }
103    out
104}
105
106/// The rule in the module docs: the tier to route at, given the local entry,
107/// the local verdict and the peers' votes.
108pub fn decide(
109    local: Option<&ProviderHealth>,
110    tier: Option<Tier>,
111    peers: &[Vote],
112    now: Timestamp,
113) -> Option<Tier> {
114    let tier = tier?;
115    let mut up = 0usize;
116    let mut down = 0usize;
117    let own = local.and_then(|e| vote(e, now));
118    for v in own.iter().chain(peers) {
119        match v {
120            Vote::Up => up += 1,
121            Vote::Down { until } if *until > now => down += 1,
122            Vote::Down { .. } => {}
123        }
124    }
125    if down > up { None } else { Some(tier) }
126}
127
128/// Peers' votes as of the last sync, consulted by [`HealthStore::tier`].
129#[derive(Default)]
130pub struct PeerView {
131    votes: Mutex<HashMap<String, Vec<Vote>>>,
132}
133
134impl PeerView {
135    pub fn set(&self, votes: HashMap<String, Vec<Vote>>) {
136        *self.votes.lock().unwrap() = votes;
137    }
138
139    /// Whether the peers alone take a locally routable provider out.
140    pub fn vetoes(&self, provider: &str, local: Option<&ProviderHealth>, now: Timestamp) -> bool {
141        let votes = self.votes.lock().unwrap();
142        let peers = votes.get(provider).map_or(&[][..], Vec::as_slice);
143        // Only what the peers change: a provider our own vote already holds
144        // down is not theirs to report.
145        decide(local, Some(Tier::Healthy), &[], now).is_some()
146            && decide(local, Some(Tier::Healthy), peers, now).is_none()
147    }
148}
149
150impl TierOverlay for PeerView {
151    fn adjust(
152        &self,
153        provider: &str,
154        local: Option<&ProviderHealth>,
155        tier: Option<Tier>,
156        now: Timestamp,
157    ) -> Option<Tier> {
158        let votes = self.votes.lock().unwrap();
159        match votes.get(provider) {
160            Some(peers) => decide(local, tier, peers, now),
161            None => tier,
162        }
163    }
164}
165
166use crate::store::unavailable as db;
167
168/// Publishes this instance's health, reads its peers', and keeps quarantine
169/// in step with the global table.
170pub struct HealthSync {
171    pool: Pool,
172    health: Arc<HealthStore>,
173    instance: String,
174    stale_after: Duration,
175    view: Arc<PeerView>,
176    /// Quarantines already accounted for against the table: inserted by us,
177    /// or adopted from it. These are never re-inserted, or an instance that
178    /// adopted a quarantine would resurrect the row an operator just deleted.
179    published: Mutex<HashSet<String>>,
180    /// Serializes `sync` and `clear_quarantine`, so a clear cannot interleave
181    /// with a sync that snapshotted the quarantine before it.
182    lock: tokio::sync::Mutex<()>,
183}
184
185impl HealthSync {
186    /// Installs the peer overlay on `health`. `stale_after` is how old a
187    /// peer's row may be and still vote.
188    pub fn new(
189        pool: Pool,
190        health: Arc<HealthStore>,
191        instance: impl Into<String>,
192        stale_after: Duration,
193    ) -> Arc<Self> {
194        let view = Arc::new(PeerView::default());
195        health.set_overlay(Some(view.clone()));
196        Arc::new(HealthSync {
197            pool,
198            health,
199            instance: instance.into(),
200            stale_after,
201            view,
202            published: Mutex::new(HashSet::new()),
203            lock: tokio::sync::Mutex::new(()),
204        })
205    }
206
207    pub fn instance(&self) -> &str {
208        &self.instance
209    }
210
211    pub fn view(&self) -> &PeerView {
212        &self.view
213    }
214
215    /// At startup: seed from this instance's own row, then adopt the table's
216    /// quarantines. The table is the authority here: a local quarantine it
217    /// lacks was cleared while this instance was away.
218    pub async fn load(&self) -> Result<(), StoreUnavailable> {
219        let _g = self.lock.lock().await;
220        let client = self.pool.get().await.map_err(db)?;
221        let row = client
222            .query_opt(
223                "select snapshot from instance_health where instance_id = $1",
224                &[&self.instance],
225            )
226            .await
227            .map_err(db)?;
228        if let Some(row) = row {
229            let v: serde_json::Value = row.get(0);
230            if let Ok(snap) = serde_json::from_value::<HashMap<String, ProviderHealth>>(v) {
231                self.health.seed(snap);
232            }
233        }
234        let accounted: HashSet<String> = self
235            .health
236            .snapshot()
237            .into_iter()
238            .filter(|(_, e)| e.status == Status::Quarantined)
239            .map(|(k, _)| k)
240            .collect();
241        self.reconcile(&client, accounted).await
242    }
243
244    /// One round: publish our row and new quarantines, refresh the peer view,
245    /// reconcile quarantines.
246    pub async fn sync(&self) -> Result<(), StoreUnavailable> {
247        let _g = self.lock.lock().await;
248        let client = self.pool.get().await.map_err(db)?;
249        let snapshot = self.health.snapshot();
250        let body = serde_json::to_value(&snapshot).expect("health serializes");
251        client
252            .execute(
253                "insert into instance_health (instance_id, snapshot, updated_at)
254                 values ($1, $2, now())
255                 on conflict (instance_id) do update
256                 set snapshot = excluded.snapshot, updated_at = now()",
257                &[&self.instance, &body],
258            )
259            .await
260            .map_err(db)?;
261        let mut accounted = self.published.lock().unwrap().clone();
262        for (provider, e) in &snapshot {
263            if e.status == Status::Quarantined && !accounted.contains(provider) {
264                // First writer wins: a later instance noticing the same drift
265                // must not overwrite who saw it first, or when.
266                client
267                    .execute(
268                        "insert into provider_quarantine (provider, reason, instance_id)
269                         values ($1, $2, $3) on conflict (provider) do nothing",
270                        &[
271                            provider,
272                            &e.last_failure.as_deref().unwrap_or("quarantined"),
273                            &self.instance,
274                        ],
275                    )
276                    .await
277                    .map_err(db)?;
278                accounted.insert(provider.clone());
279            }
280        }
281        let rows = client
282            .query(
283                "select snapshot from instance_health
284                 where instance_id <> $1
285                   and updated_at > now() - make_interval(secs => $2)",
286                &[&self.instance, &self.stale_after.as_secs_f64()],
287            )
288            .await
289            .map_err(db)?;
290        let peers: Vec<HashMap<String, ProviderHealth>> = rows
291            .into_iter()
292            .filter_map(|r| serde_json::from_value(r.get::<_, serde_json::Value>(0)).ok())
293            .collect();
294        self.view.set(peer_votes(&peers, Timestamp::now()));
295        self.reconcile(&client, accounted).await
296    }
297
298    /// Match local quarantines to the table in both directions. Only a
299    /// quarantine in `accounted` is lifted when the table lacks it; one that
300    /// appeared locally after this round's snapshot has not been published
301    /// yet, and lifting it would lose a fresh drift (cowbird lifted it).
302    async fn reconcile(
303        &self,
304        client: &deadpool_postgres::Client,
305        accounted: HashSet<String>,
306    ) -> Result<(), StoreUnavailable> {
307        let rows = client
308            .query("select provider, reason from provider_quarantine", &[])
309            .await
310            .map_err(db)?;
311        let mut table = HashSet::new();
312        for r in rows {
313            let provider: String = r.get(0);
314            let reason: String = r.get(1);
315            if self.health.status(&provider) != Status::Quarantined {
316                self.health.quarantine(&provider, &reason);
317            }
318            table.insert(provider);
319        }
320        for (provider, e) in self.health.snapshot() {
321            if e.status == Status::Quarantined
322                && !table.contains(&provider)
323                && accounted.contains(&provider)
324            {
325                self.health.unquarantine(&provider);
326            }
327        }
328        *self.published.lock().unwrap() = table;
329        Ok(())
330    }
331
332    /// Clear a quarantine fleet-wide. Returns whether anything was cleared,
333    /// here or in the table. Peers lift theirs on their next sync.
334    pub async fn clear_quarantine(&self, provider: &str) -> Result<bool, StoreUnavailable> {
335        let _g = self.lock.lock().await;
336        let client = self.pool.get().await.map_err(db)?;
337        let n = client
338            .execute(
339                "delete from provider_quarantine where provider = $1",
340                &[&provider],
341            )
342            .await
343            .map_err(db)?;
344        self.published.lock().unwrap().remove(provider);
345        let local = self.health.unquarantine(provider);
346        Ok(n > 0 || local)
347    }
348}
349
350#[cfg(test)]
351pub(crate) fn test_database_url() -> Option<String> {
352    std::env::var("INTERLOPER_TEST_DATABASE_URL")
353        .ok()
354        .filter(|s| !s.trim().is_empty())
355}
356
357#[cfg(test)]
358mod tests {
359    use super::*;
360
361    fn t(s: i64) -> Timestamp {
362        Timestamp::from_second(1_800_000_000 + s).unwrap()
363    }
364
365    fn ok_at(s: i64) -> ProviderHealth {
366        ProviderHealth {
367            status: Status::Ok,
368            last_checked: Some(t(s)),
369            ..Default::default()
370        }
371    }
372
373    fn down_at(s: i64, until: i64) -> ProviderHealth {
374        ProviderHealth {
375            status: Status::Down,
376            last_checked: Some(t(s)),
377            failures: 1,
378            retry_at: Some(t(until)),
379            ..Default::default()
380        }
381    }
382
383    #[test]
384    fn votes() {
385        let now = t(0);
386        assert_eq!(vote(&ok_at(-5), now), Some(Vote::Up));
387        assert_eq!(
388            vote(&down_at(-5, 25), now),
389            Some(Vote::Down { until: t(25) })
390        );
391        // Probation, quarantine, never checked, and stale evidence abstain.
392        assert_eq!(vote(&down_at(-60, -30), now), None);
393        let q = ProviderHealth {
394            status: Status::Quarantined,
395            ..ok_at(-1)
396        };
397        assert_eq!(vote(&q, now), None);
398        assert_eq!(vote(&ProviderHealth::default(), now), None);
399        assert_eq!(vote(&ok_at(-(15 * 60) - 1), now), None);
400        // A challenge is about the voter's egress IP, not the provider.
401        let challenged = ProviderHealth {
402            needs_residential_ip: true,
403            ..down_at(-5, 25)
404        };
405        assert_eq!(vote(&challenged, now), None);
406    }
407
408    #[test]
409    fn uninformed_instance_follows_a_peer_that_saw_failure() {
410        let now = t(0);
411        let peers = [Vote::Down { until: t(25) }];
412        assert_eq!(decide(None, Some(Tier::Healthy), &peers, now), None);
413        // ...until the peer's own cooldown lapses, without another sync.
414        assert_eq!(
415            decide(None, Some(Tier::Healthy), &peers, t(26)),
416            Some(Tier::Healthy)
417        );
418    }
419
420    #[test]
421    fn ties_route_and_majorities_demote() {
422        let now = t(0);
423        let local = ok_at(-10);
424        let one = [Vote::Down { until: t(25) }];
425        assert_eq!(
426            decide(Some(&local), Some(Tier::Healthy), &one, now),
427            Some(Tier::Healthy),
428            "one peer cannot overrule our own success"
429        );
430        let two = [Vote::Down { until: t(25) }, Vote::Down { until: t(40) }];
431        assert_eq!(decide(Some(&local), Some(Tier::Slow), &two, now), None);
432        let mixed = [
433            Vote::Down { until: t(25) },
434            Vote::Down { until: t(40) },
435            Vote::Up,
436        ];
437        assert_eq!(
438            decide(Some(&local), Some(Tier::Healthy), &mixed, now),
439            Some(Tier::Healthy)
440        );
441    }
442
443    #[test]
444    fn peers_never_promote() {
445        let local = down_at(-1, 30);
446        let peers = [Vote::Up, Vote::Up, Vote::Up];
447        assert_eq!(decide(Some(&local), None, &peers, t(0)), None);
448    }
449
450    #[test]
451    fn probation_keeps_its_tier_unless_outvoted() {
452        let local = down_at(-60, -30);
453        assert_eq!(
454            decide(Some(&local), Some(Tier::Probation), &[Vote::Up], t(0)),
455            Some(Tier::Probation)
456        );
457        assert_eq!(
458            decide(
459                Some(&local),
460                Some(Tier::Probation),
461                &[Vote::Down { until: t(9) }],
462                t(0)
463            ),
464            None
465        );
466    }
467
468    #[test]
469    fn peer_votes_group_by_provider() {
470        let now = t(0);
471        let a = HashMap::from([
472            ("p".to_string(), down_at(-1, 30)),
473            ("q".to_string(), ok_at(-1)),
474        ]);
475        let b = HashMap::from([
476            ("p".to_string(), down_at(-2, 20)),
477            ("r".to_string(), ProviderHealth::default()),
478        ]);
479        let v = peer_votes(&[a, b], now);
480        assert_eq!(v["p"].len(), 2);
481        assert_eq!(v["q"], vec![Vote::Up]);
482        assert!(!v.contains_key("r"));
483    }
484
485    #[test]
486    fn overlay_reaches_routing() {
487        let h = Arc::new(HealthStore::new());
488        let view = Arc::new(PeerView::default());
489        h.set_overlay(Some(view.clone()));
490        assert!(h.tier("p").is_some());
491        let until = Timestamp::now()
492            .checked_add(SignedDuration::from_secs(60))
493            .unwrap();
494        view.set(HashMap::from([("p".into(), vec![Vote::Down { until }])]));
495        assert_eq!(h.tier("p"), None);
496        assert!(view.vetoes("p", None, Timestamp::now()));
497        // Our own fresh success ties the vote, and ties route.
498        h.record_success("p", Duration::from_millis(10));
499        assert_eq!(h.tier("p"), Some(Tier::Healthy));
500        // The overlay is never part of the snapshot we publish.
501        assert_eq!(h.snapshot()["p"].status, Status::Ok);
502    }
503}