Skip to main content

interloper_server/
store.rs

1//! Where the server keeps the addresses it issued.
2//!
3//! An address carries provider state (mail.tm's JWT, for one) that later calls
4//! need. Handing that blob to clients would make `GET /v1/inboxes/{addr}/messages`
5//! impossible to call from plain curl, so the server holds it. cowbird kept it
6//! in a dict by default, so a restart lost every live inbox; here a SQLite file
7//! keeps them when `INTERLOPER_DATABASE` is set, and Postgres
8//! (`INTERLOPER_DATABASE_URL`) lets a fleet of servers share them, so any
9//! instance serves an address another one issued.
10//!
11//! All three backends key by the lowercased address, store the address as
12//! JSON, evict an expired row when it is read, and reap the rest in `sweep`.
13
14use std::collections::HashMap;
15use std::path::Path;
16use std::str::FromStr;
17use std::sync::{Arc, Mutex};
18use std::time::Duration;
19
20use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod, Runtime};
21use interloper::Address;
22use jiff::Timestamp;
23use rusqlite::{Connection, OptionalExtension, params};
24
25#[derive(Debug)]
26pub struct StoreUnavailable(pub String);
27
28impl std::fmt::Display for StoreUnavailable {
29    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
30        f.write_str(&self.0)
31    }
32}
33
34/// Any database failure, pool timeouts included, is an outage to the caller:
35/// a 503 to retry, not a 500.
36pub(crate) fn unavailable(e: impl std::error::Error) -> StoreUnavailable {
37    // tokio-postgres prints a server error as just "db error"; the server's
38    // message is in the source chain.
39    let mut msg = e.to_string();
40    let mut src = e.source();
41    while let Some(s) = src {
42        msg = format!("{msg}: {s}");
43        src = s.source();
44    }
45    StoreUnavailable(msg)
46}
47
48/// How long a request waits for a pooled connection before answering 503.
49const ACQUIRE_TIMEOUT: Duration = Duration::from_secs(5);
50
51/// Tables for fleet mode. `addresses` mirrors the SQLite schema. The two
52/// health tables are described in [`crate::fleet`].
53const PG_SCHEMA: &str = "
54    create table if not exists addresses (
55        value      text primary key,
56        body       text not null,
57        expires_at bigint
58    );
59    create index if not exists addresses_expiry on addresses (expires_at);
60    create table if not exists instance_health (
61        instance_id text primary key,
62        snapshot    jsonb not null,
63        updated_at  timestamptz not null default now()
64    );
65    create table if not exists provider_quarantine (
66        provider    text primary key,
67        reason      text not null,
68        instance_id text not null,
69        at          timestamptz not null default now()
70    );
71";
72
73pub enum Store {
74    Memory(Mutex<HashMap<String, Address>>),
75    Sqlite(Mutex<Connection>),
76    Postgres(Pool),
77}
78
79fn tls_connector() -> tokio_postgres_rustls::MakeRustlsConnect {
80    // An explicit provider: the process-wide default is never installed, and
81    // installing one here could race with anything else that does.
82    let roots = rustls::RootCertStore {
83        roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(),
84    };
85    let config = rustls::ClientConfig::builder_with_provider(Arc::new(
86        rustls::crypto::ring::default_provider(),
87    ))
88    .with_safe_default_protocol_versions()
89    .expect("ring supports the default protocol versions")
90    .with_root_certificates(roots)
91    .with_no_client_auth();
92    tokio_postgres_rustls::MakeRustlsConnect::new(config)
93}
94
95impl Store {
96    pub fn memory() -> Self {
97        Store::Memory(Mutex::new(HashMap::new()))
98    }
99
100    pub fn sqlite(path: &Path) -> Result<Self, String> {
101        if let Some(dir) = path.parent() {
102            std::fs::create_dir_all(dir).map_err(|e| e.to_string())?;
103        }
104        let conn = Connection::open(path).map_err(|e| e.to_string())?;
105        conn.execute_batch(
106            "pragma journal_mode = wal;
107             create table if not exists addresses (
108                 value text primary key,
109                 body text not null,
110                 expires_at integer
111             );
112             create index if not exists addresses_expiry on addresses(expires_at);",
113        )
114        .map_err(|e| e.to_string())?;
115        Ok(Store::Sqlite(Mutex::new(conn)))
116    }
117
118    /// Connect, and create the schema if it is missing. Fails rather than
119    /// starting a server whose every request would 503.
120    ///
121    /// TLS follows the URL's `sslmode`, as libpq does: `prefer` (the default)
122    /// uses TLS when the server offers it, `require` refuses plaintext, and
123    /// `disable` never tries. TLS always verifies the certificate and host
124    /// name against Mozilla's roots, so a managed database (Neon, Supabase,
125    /// RDS, ...) works with its URL as given.
126    pub async fn postgres(url: &str) -> Result<Self, String> {
127        let mut cfg = tokio_postgres::Config::from_str(url).map_err(|e| e.to_string())?;
128        if cfg.get_connect_timeout().is_none() {
129            cfg.connect_timeout(Duration::from_secs(5));
130        }
131        let mgr = Manager::from_config(
132            cfg,
133            tls_connector(),
134            ManagerConfig {
135                recycling_method: RecyclingMethod::Fast,
136            },
137        );
138        let pool = Pool::builder(mgr)
139            .max_size(16)
140            .runtime(Runtime::Tokio1)
141            .wait_timeout(Some(ACQUIRE_TIMEOUT))
142            .create_timeout(Some(ACQUIRE_TIMEOUT))
143            .build()
144            .map_err(|e| e.to_string())?;
145        let client = pool.get().await.map_err(|e| {
146            // deadpool's Display hides the cause, which is the useful part.
147            let mut msg = e.to_string();
148            let mut src = std::error::Error::source(&e);
149            while let Some(c) = src {
150                msg.push_str(": ");
151                msg.push_str(&c.to_string());
152                src = c.source();
153            }
154            if msg.contains("TLS") || msg.contains("certificate") {
155                msg.push_str(
156                    " (the server's certificate must chain to a public CA; on a private \
157                     network use ?sslmode=disable)",
158                );
159            }
160            msg
161        })?;
162        // Two instances booting at once may both run this, and `if not
163        // exists` still races on the catalog. Separate statements, so the
164        // schema commits before the lock is released.
165        client
166            .execute("select pg_advisory_lock(7280431)", &[])
167            .await
168            .map_err(|e| unavailable(e).0)?;
169        let migrated = client
170            .batch_execute(PG_SCHEMA)
171            .await
172            .map_err(|e| unavailable(e).0);
173        let _ = client
174            .execute("select pg_advisory_unlock(7280431)", &[])
175            .await;
176        migrated?;
177        drop(client);
178        Ok(Store::Postgres(pool))
179    }
180
181    /// The connection pool, when this store is Postgres; fleet health sync
182    /// shares it.
183    pub fn pg_pool(&self) -> Option<&Pool> {
184        match self {
185            Store::Postgres(p) => Some(p),
186            _ => None,
187        }
188    }
189
190    pub async fn put(&self, a: &Address) -> Result<(), StoreUnavailable> {
191        let key = a.value.to_lowercase();
192        let body = serde_json::to_string(a).expect("address serializes");
193        let expires = a.expires_at.map(|t| t.as_second());
194        match self {
195            Store::Memory(m) => {
196                m.lock().unwrap().insert(key, a.clone());
197                Ok(())
198            }
199            Store::Sqlite(c) => c
200                .lock()
201                .unwrap()
202                .execute(
203                    "insert or replace into addresses (value, body, expires_at) values (?1, ?2, ?3)",
204                    params![key, body, expires],
205                )
206                .map(|_| ())
207                .map_err(unavailable),
208            // Upsert: one instance issuing and another refreshing the same
209            // address is ordinary, not a conflict.
210            Store::Postgres(p) => p
211                .get()
212                .await
213                .map_err(unavailable)?
214                .execute(
215                    "insert into addresses (value, body, expires_at) values ($1, $2, $3)
216                     on conflict (value) do update
217                     set body = excluded.body, expires_at = excluded.expires_at",
218                    &[&key, &body, &expires],
219                )
220                .await
221                .map(|_| ())
222                .map_err(unavailable),
223        }
224    }
225
226    /// The address, unless unknown or expired (expired rows are evicted on read).
227    pub async fn get(&self, value: &str) -> Result<Option<Address>, StoreUnavailable> {
228        let key = value.to_lowercase();
229        let now = Timestamp::now();
230        let body: Option<String> = match self {
231            Store::Memory(m) => {
232                let found = m.lock().unwrap().get(&key).cloned();
233                return match found {
234                    Some(a) if a.is_expired(now) => {
235                        self.remove(&key).await?;
236                        Ok(None)
237                    }
238                    other => Ok(other),
239                };
240            }
241            Store::Sqlite(c) => c
242                .lock()
243                .unwrap()
244                .query_row(
245                    "select body from addresses where value = ?1",
246                    params![key],
247                    |r| r.get(0),
248                )
249                .optional()
250                .map_err(unavailable)?,
251            Store::Postgres(p) => p
252                .get()
253                .await
254                .map_err(unavailable)?
255                .query_opt("select body from addresses where value = $1", &[&key])
256                .await
257                .map_err(unavailable)?
258                .map(|r| r.get(0)),
259        };
260        match body.and_then(|b| serde_json::from_str::<Address>(&b).ok()) {
261            Some(a) if a.is_expired(now) => {
262                self.remove(&key).await?;
263                Ok(None)
264            }
265            other => Ok(other),
266        }
267    }
268
269    async fn remove(&self, key: &str) -> Result<(), StoreUnavailable> {
270        match self {
271            Store::Memory(m) => {
272                m.lock().unwrap().remove(key);
273                Ok(())
274            }
275            Store::Sqlite(c) => c
276                .lock()
277                .unwrap()
278                .execute("delete from addresses where value = ?1", params![key])
279                .map(|_| ())
280                .map_err(unavailable),
281            Store::Postgres(p) => p
282                .get()
283                .await
284                .map_err(unavailable)?
285                .execute("delete from addresses where value = $1", &[&key])
286                .await
287                .map(|_| ())
288                .map_err(unavailable),
289        }
290    }
291
292    /// Reap expired rows. `get` only evicts rows somebody asks for, and nobody
293    /// asks for an address after it expired; without this they pile up.
294    pub async fn sweep(&self) -> Result<usize, StoreUnavailable> {
295        let now = Timestamp::now();
296        match self {
297            Store::Memory(m) => {
298                let mut m = m.lock().unwrap();
299                let before = m.len();
300                m.retain(|_, a| !a.is_expired(now));
301                Ok(before - m.len())
302            }
303            Store::Sqlite(c) => c
304                .lock()
305                .unwrap()
306                .execute(
307                    "delete from addresses where expires_at is not null and expires_at <= ?1",
308                    params![now.as_second()],
309                )
310                .map_err(unavailable),
311            Store::Postgres(p) => p
312                .get()
313                .await
314                .map_err(unavailable)?
315                .execute(
316                    "delete from addresses where expires_at is not null and expires_at <= $1",
317                    &[&now.as_second()],
318                )
319                .await
320                .map(|n| n as usize)
321                .map_err(unavailable),
322        }
323    }
324}
325
326#[cfg(test)]
327mod tests {
328    use super::*;
329
330    fn past(value: &str) -> Address {
331        let mut a = Address::new(value, "p");
332        a.expires_at = Some(Timestamp::from_second(1).unwrap());
333        a
334    }
335
336    /// The shared contract. `tag` keeps rows from concurrent runs against
337    /// one Postgres apart.
338    async fn check(store: Store, tag: &str) {
339        let me = format!("Me-{tag}@X.test");
340        let old = format!("old-{tag}@x.test");
341        let a = Address::new(&me, "p").with_state("secret");
342        store.put(&a).await.unwrap();
343        assert_eq!(store.get(&me.to_lowercase()).await.unwrap(), Some(a));
344        // Upsert, not a duplicate-key error.
345        let b = Address::new(&me, "p").with_state("refreshed");
346        store.put(&b).await.unwrap();
347        assert_eq!(store.get(&me).await.unwrap(), Some(b));
348        store.put(&past(&old)).await.unwrap();
349        assert_eq!(store.get(&old).await.unwrap(), None);
350        store.put(&past(&old)).await.unwrap();
351        assert!(store.sweep().await.unwrap() >= 1);
352        assert_eq!(store.get(&old).await.unwrap(), None);
353        assert_eq!(
354            store.get(&format!("nobody-{tag}@x.test")).await.unwrap(),
355            None
356        );
357    }
358
359    #[tokio::test]
360    async fn memory_store() {
361        check(Store::memory(), "m").await;
362    }
363
364    #[tokio::test]
365    async fn sqlite_store_survives_reopen() {
366        let dir = std::env::temp_dir().join(format!("ilp-store-{}", std::process::id()));
367        let path = dir.join("a.db");
368        check(Store::sqlite(&path).unwrap(), "s").await;
369        let a = Address::new("keep@x.test", "p").with_state("s");
370        Store::sqlite(&path).unwrap().put(&a).await.unwrap();
371        assert_eq!(
372            Store::sqlite(&path)
373                .unwrap()
374                .get("keep@x.test")
375                .await
376                .unwrap(),
377            Some(a)
378        );
379        std::fs::remove_dir_all(dir).ok();
380    }
381
382    /// Runs only with `INTERLOPER_TEST_DATABASE_URL` set.
383    #[tokio::test]
384    async fn postgres_store() {
385        let Some(url) = crate::fleet::test_database_url() else {
386            eprintln!("skipped: INTERLOPER_TEST_DATABASE_URL is unset");
387            return;
388        };
389        let tag = format!("{:x}", rand::random::<u64>());
390        check(Store::postgres(&url).await.unwrap(), &tag).await;
391        // A second connection (another instance) sees what the first wrote.
392        let a = Address::new(format!("shared-{tag}@x.test"), "p").with_state("s");
393        Store::postgres(&url).await.unwrap().put(&a).await.unwrap();
394        assert_eq!(
395            Store::postgres(&url)
396                .await
397                .unwrap()
398                .get(&a.value)
399                .await
400                .unwrap(),
401            Some(a)
402        );
403    }
404
405    #[tokio::test]
406    async fn unreachable_postgres_fails_fast() {
407        let r = Store::postgres("postgres://u:p@127.0.0.1:1/db").await;
408        assert!(r.is_err());
409    }
410}