interloper-server 0.1.1

HTTP API over interloper: issue inboxes, read mail, long-poll, SSE and webhooks.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
//! Where the server keeps the addresses it issued.
//!
//! An address carries provider state (mail.tm's JWT, for one) that later calls
//! need. Handing that blob to clients would make `GET /v1/inboxes/{addr}/messages`
//! impossible to call from plain curl, so the server holds it. cowbird kept it
//! in a dict by default, so a restart lost every live inbox; here a SQLite file
//! keeps them when `INTERLOPER_DATABASE` is set, and Postgres
//! (`INTERLOPER_DATABASE_URL`) lets a fleet of servers share them, so any
//! instance serves an address another one issued.
//!
//! All three backends key by the lowercased address, store the address as
//! JSON, evict an expired row when it is read, and reap the rest in `sweep`.

use std::collections::HashMap;
use std::path::Path;
use std::str::FromStr;
use std::sync::{Arc, Mutex};
use std::time::Duration;

use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod, Runtime};
use interloper::Address;
use jiff::Timestamp;
use rusqlite::{Connection, OptionalExtension, params};

#[derive(Debug)]
pub struct StoreUnavailable(pub String);

impl std::fmt::Display for StoreUnavailable {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.write_str(&self.0)
    }
}

/// Any database failure, pool timeouts included, is an outage to the caller:
/// a 503 to retry, not a 500.
pub(crate) fn unavailable(e: impl std::error::Error) -> StoreUnavailable {
    // tokio-postgres prints a server error as just "db error"; the server's
    // message is in the source chain.
    let mut msg = e.to_string();
    let mut src = e.source();
    while let Some(s) = src {
        msg = format!("{msg}: {s}");
        src = s.source();
    }
    StoreUnavailable(msg)
}

/// How long a request waits for a pooled connection before answering 503.
const ACQUIRE_TIMEOUT: Duration = Duration::from_secs(5);

/// Tables for fleet mode. `addresses` mirrors the SQLite schema. The two
/// health tables are described in [`crate::fleet`].
const PG_SCHEMA: &str = "
    create table if not exists addresses (
        value      text primary key,
        body       text not null,
        expires_at bigint
    );
    create index if not exists addresses_expiry on addresses (expires_at);
    create table if not exists instance_health (
        instance_id text primary key,
        snapshot    jsonb not null,
        updated_at  timestamptz not null default now()
    );
    create table if not exists provider_quarantine (
        provider    text primary key,
        reason      text not null,
        instance_id text not null,
        at          timestamptz not null default now()
    );
";

pub enum Store {
    Memory(Mutex<HashMap<String, Address>>),
    Sqlite(Mutex<Connection>),
    Postgres(Pool),
}

fn tls_connector() -> tokio_postgres_rustls::MakeRustlsConnect {
    // An explicit provider: the process-wide default is never installed, and
    // installing one here could race with anything else that does.
    let roots = rustls::RootCertStore {
        roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(),
    };
    let config = rustls::ClientConfig::builder_with_provider(Arc::new(
        rustls::crypto::ring::default_provider(),
    ))
    .with_safe_default_protocol_versions()
    .expect("ring supports the default protocol versions")
    .with_root_certificates(roots)
    .with_no_client_auth();
    tokio_postgres_rustls::MakeRustlsConnect::new(config)
}

impl Store {
    pub fn memory() -> Self {
        Store::Memory(Mutex::new(HashMap::new()))
    }

    pub fn sqlite(path: &Path) -> Result<Self, String> {
        if let Some(dir) = path.parent() {
            std::fs::create_dir_all(dir).map_err(|e| e.to_string())?;
        }
        let conn = Connection::open(path).map_err(|e| e.to_string())?;
        conn.execute_batch(
            "pragma journal_mode = wal;
             create table if not exists addresses (
                 value text primary key,
                 body text not null,
                 expires_at integer
             );
             create index if not exists addresses_expiry on addresses(expires_at);",
        )
        .map_err(|e| e.to_string())?;
        Ok(Store::Sqlite(Mutex::new(conn)))
    }

    /// Connect, and create the schema if it is missing. Fails rather than
    /// starting a server whose every request would 503.
    ///
    /// TLS follows the URL's `sslmode`, as libpq does: `prefer` (the default)
    /// uses TLS when the server offers it, `require` refuses plaintext, and
    /// `disable` never tries. TLS always verifies the certificate and host
    /// name against Mozilla's roots, so a managed database (Neon, Supabase,
    /// RDS, ...) works with its URL as given.
    pub async fn postgres(url: &str) -> Result<Self, String> {
        let mut cfg = tokio_postgres::Config::from_str(url).map_err(|e| e.to_string())?;
        if cfg.get_connect_timeout().is_none() {
            cfg.connect_timeout(Duration::from_secs(5));
        }
        let mgr = Manager::from_config(
            cfg,
            tls_connector(),
            ManagerConfig {
                recycling_method: RecyclingMethod::Fast,
            },
        );
        let pool = Pool::builder(mgr)
            .max_size(16)
            .runtime(Runtime::Tokio1)
            .wait_timeout(Some(ACQUIRE_TIMEOUT))
            .create_timeout(Some(ACQUIRE_TIMEOUT))
            .build()
            .map_err(|e| e.to_string())?;
        let client = pool.get().await.map_err(|e| {
            // deadpool's Display hides the cause, which is the useful part.
            let mut msg = e.to_string();
            let mut src = std::error::Error::source(&e);
            while let Some(c) = src {
                msg.push_str(": ");
                msg.push_str(&c.to_string());
                src = c.source();
            }
            if msg.contains("TLS") || msg.contains("certificate") {
                msg.push_str(
                    " (the server's certificate must chain to a public CA; on a private \
                     network use ?sslmode=disable)",
                );
            }
            msg
        })?;
        // Two instances booting at once may both run this, and `if not
        // exists` still races on the catalog. Separate statements, so the
        // schema commits before the lock is released.
        client
            .execute("select pg_advisory_lock(7280431)", &[])
            .await
            .map_err(|e| unavailable(e).0)?;
        let migrated = client
            .batch_execute(PG_SCHEMA)
            .await
            .map_err(|e| unavailable(e).0);
        let _ = client
            .execute("select pg_advisory_unlock(7280431)", &[])
            .await;
        migrated?;
        drop(client);
        Ok(Store::Postgres(pool))
    }

    /// The connection pool, when this store is Postgres; fleet health sync
    /// shares it.
    pub fn pg_pool(&self) -> Option<&Pool> {
        match self {
            Store::Postgres(p) => Some(p),
            _ => None,
        }
    }

    pub async fn put(&self, a: &Address) -> Result<(), StoreUnavailable> {
        let key = a.value.to_lowercase();
        let body = serde_json::to_string(a).expect("address serializes");
        let expires = a.expires_at.map(|t| t.as_second());
        match self {
            Store::Memory(m) => {
                m.lock().unwrap().insert(key, a.clone());
                Ok(())
            }
            Store::Sqlite(c) => c
                .lock()
                .unwrap()
                .execute(
                    "insert or replace into addresses (value, body, expires_at) values (?1, ?2, ?3)",
                    params![key, body, expires],
                )
                .map(|_| ())
                .map_err(unavailable),
            // Upsert: one instance issuing and another refreshing the same
            // address is ordinary, not a conflict.
            Store::Postgres(p) => p
                .get()
                .await
                .map_err(unavailable)?
                .execute(
                    "insert into addresses (value, body, expires_at) values ($1, $2, $3)
                     on conflict (value) do update
                     set body = excluded.body, expires_at = excluded.expires_at",
                    &[&key, &body, &expires],
                )
                .await
                .map(|_| ())
                .map_err(unavailable),
        }
    }

    /// The address, unless unknown or expired (expired rows are evicted on read).
    pub async fn get(&self, value: &str) -> Result<Option<Address>, StoreUnavailable> {
        let key = value.to_lowercase();
        let now = Timestamp::now();
        let body: Option<String> = match self {
            Store::Memory(m) => {
                let found = m.lock().unwrap().get(&key).cloned();
                return match found {
                    Some(a) if a.is_expired(now) => {
                        self.remove(&key).await?;
                        Ok(None)
                    }
                    other => Ok(other),
                };
            }
            Store::Sqlite(c) => c
                .lock()
                .unwrap()
                .query_row(
                    "select body from addresses where value = ?1",
                    params![key],
                    |r| r.get(0),
                )
                .optional()
                .map_err(unavailable)?,
            Store::Postgres(p) => p
                .get()
                .await
                .map_err(unavailable)?
                .query_opt("select body from addresses where value = $1", &[&key])
                .await
                .map_err(unavailable)?
                .map(|r| r.get(0)),
        };
        match body.and_then(|b| serde_json::from_str::<Address>(&b).ok()) {
            Some(a) if a.is_expired(now) => {
                self.remove(&key).await?;
                Ok(None)
            }
            other => Ok(other),
        }
    }

    async fn remove(&self, key: &str) -> Result<(), StoreUnavailable> {
        match self {
            Store::Memory(m) => {
                m.lock().unwrap().remove(key);
                Ok(())
            }
            Store::Sqlite(c) => c
                .lock()
                .unwrap()
                .execute("delete from addresses where value = ?1", params![key])
                .map(|_| ())
                .map_err(unavailable),
            Store::Postgres(p) => p
                .get()
                .await
                .map_err(unavailable)?
                .execute("delete from addresses where value = $1", &[&key])
                .await
                .map(|_| ())
                .map_err(unavailable),
        }
    }

    /// Reap expired rows. `get` only evicts rows somebody asks for, and nobody
    /// asks for an address after it expired; without this they pile up.
    pub async fn sweep(&self) -> Result<usize, StoreUnavailable> {
        let now = Timestamp::now();
        match self {
            Store::Memory(m) => {
                let mut m = m.lock().unwrap();
                let before = m.len();
                m.retain(|_, a| !a.is_expired(now));
                Ok(before - m.len())
            }
            Store::Sqlite(c) => c
                .lock()
                .unwrap()
                .execute(
                    "delete from addresses where expires_at is not null and expires_at <= ?1",
                    params![now.as_second()],
                )
                .map_err(unavailable),
            Store::Postgres(p) => p
                .get()
                .await
                .map_err(unavailable)?
                .execute(
                    "delete from addresses where expires_at is not null and expires_at <= $1",
                    &[&now.as_second()],
                )
                .await
                .map(|n| n as usize)
                .map_err(unavailable),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn past(value: &str) -> Address {
        let mut a = Address::new(value, "p");
        a.expires_at = Some(Timestamp::from_second(1).unwrap());
        a
    }

    /// The shared contract. `tag` keeps rows from concurrent runs against
    /// one Postgres apart.
    async fn check(store: Store, tag: &str) {
        let me = format!("Me-{tag}@X.test");
        let old = format!("old-{tag}@x.test");
        let a = Address::new(&me, "p").with_state("secret");
        store.put(&a).await.unwrap();
        assert_eq!(store.get(&me.to_lowercase()).await.unwrap(), Some(a));
        // Upsert, not a duplicate-key error.
        let b = Address::new(&me, "p").with_state("refreshed");
        store.put(&b).await.unwrap();
        assert_eq!(store.get(&me).await.unwrap(), Some(b));
        store.put(&past(&old)).await.unwrap();
        assert_eq!(store.get(&old).await.unwrap(), None);
        store.put(&past(&old)).await.unwrap();
        assert!(store.sweep().await.unwrap() >= 1);
        assert_eq!(store.get(&old).await.unwrap(), None);
        assert_eq!(
            store.get(&format!("nobody-{tag}@x.test")).await.unwrap(),
            None
        );
    }

    #[tokio::test]
    async fn memory_store() {
        check(Store::memory(), "m").await;
    }

    #[tokio::test]
    async fn sqlite_store_survives_reopen() {
        let dir = std::env::temp_dir().join(format!("ilp-store-{}", std::process::id()));
        let path = dir.join("a.db");
        check(Store::sqlite(&path).unwrap(), "s").await;
        let a = Address::new("keep@x.test", "p").with_state("s");
        Store::sqlite(&path).unwrap().put(&a).await.unwrap();
        assert_eq!(
            Store::sqlite(&path)
                .unwrap()
                .get("keep@x.test")
                .await
                .unwrap(),
            Some(a)
        );
        std::fs::remove_dir_all(dir).ok();
    }

    /// Runs only with `INTERLOPER_TEST_DATABASE_URL` set.
    #[tokio::test]
    async fn postgres_store() {
        let Some(url) = crate::fleet::test_database_url() else {
            eprintln!("skipped: INTERLOPER_TEST_DATABASE_URL is unset");
            return;
        };
        let tag = format!("{:x}", rand::random::<u64>());
        check(Store::postgres(&url).await.unwrap(), &tag).await;
        // A second connection (another instance) sees what the first wrote.
        let a = Address::new(format!("shared-{tag}@x.test"), "p").with_state("s");
        Store::postgres(&url).await.unwrap().put(&a).await.unwrap();
        assert_eq!(
            Store::postgres(&url)
                .await
                .unwrap()
                .get(&a.value)
                .await
                .unwrap(),
            Some(a)
        );
    }

    #[tokio::test]
    async fn unreachable_postgres_fails_fast() {
        let r = Store::postgres("postgres://u:p@127.0.0.1:1/db").await;
        assert!(r.is_err());
    }
}