halephant-core 0.1.0

A proxy and pooler for Postgres
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
//! Connection checkout — the hot path through the pool. Walks
//! candidate nodes, falls back to opening new connections, and
//! enqueues on the shared `(database, user, role)` wait queue when
//! every candidate is at its per-node ceiling.

use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Instant;

use tokio::sync::oneshot;
use tracing::{debug, trace};

use crate::clients;
use crate::connections::server;
use crate::errors::ResolveError;
use crate::o11y;
use crate::pool::PoolManager;
use crate::pool::types::{ConnGuard, PoolKey, Routing, TryCheckout, WaitKey};

impl PoolManager {
    /// Check out a server connection.
    ///
    /// When `read_only` is true, routes to a replica. When false, routes
    /// to the primary. Returns an RAII guard that auto-discards on drop;
    /// call `checkin` to return the connection for reuse.
    ///
    /// If every candidate node for the resolved role is at its per-node
    /// `max_connections`, the checkout enqueues on a shared
    /// `(database, user, role)` wait queue and blocks until either:
    ///
    /// 1. a connection is released on any candidate node and this
    ///    waiter is the next in the FIFO queue, or
    /// 2. `server.checkout_timeout` elapses since the first enqueue,
    ///    in which case the checkout fails with a classified
    ///    `checkout_timeout` error.
    ///
    /// The `ClientGuard` is flipped to `ClientState::Waiting` for the
    /// duration of any blocked wait and back to `ClientState::Idle` on
    /// success, timeout, or spurious wakeup.
    #[tracing::instrument(name = "pool.checkout", skip_all, err(Display), fields(
        db.namespace = %database,
        user = %user,
        halephant.client.id = client.id().as_u64(),
        read_only,
        pool.reused,
        pool.waited,
        otel.status_code,
        otel.status_description,
    ))]
    pub async fn checkout(
        self: &Arc<Self>,
        client: &clients::ClientGuard,
        database: &str,
        user: &str,
        read_only: bool,
    ) -> anyhow::Result<ConnGuard> {
        let routing = if read_only {
            Routing::Replica
        } else {
            Routing::Primary
        };
        let wait_key = WaitKey {
            database: database.to_owned(),
            user: user.to_owned(),
            role: routing,
        };

        // Snapshot config once for the whole checkout. An atomic swap
        // during this operation does not affect in-flight checkouts —
        // they finish under the old view and the next checkout picks
        // up the new one. `cfg` owns an `Arc<Config>`, so borrows
        // derived from it are valid for the function lifetime and
        // survive across `.await`.
        let cfg = self.config.load_full();
        let max_prepared_statements = cfg.server.max_prepared_statements;

        // Resolve config-derived parameters once. The per-pool
        // `checkout_timeout` override takes precedence over the
        // server-wide default.
        let (
            cluster_name,
            connect_timeout,
            idle_timeout,
            max_lifetime,
            checkout_timeout,
            upstream_database,
        ) = {
            let Some((name, cluster, pool_config)) = cfg.find_pool(database) else {
                return Err(ResolveError::UnknownDatabase {
                    database: database.to_owned(),
                }
                .into());
            };
            (
                name.to_owned(),
                cluster.connect_timeout,
                pool_config.idle_timeout,
                pool_config.max_lifetime,
                pool_config
                    .checkout_timeout
                    .unwrap_or(cfg.server.checkout_timeout),
                pool_config.database_name(database).to_owned(),
            )
        };

        // Look up user-specific connection parameters once for the batch.
        // These are only used if we actually open a new connection.
        let (upstream_user, user_params) = {
            let user_config = cfg.find_user(database, user);
            let upstream_user = user_config
                .map_or(user, |u| u.upstream_name(user))
                .to_owned();
            let params = user_config
                .map(|u| u.parameters.clone())
                .unwrap_or_default();
            (upstream_user, params)
        };

        let checkout_start = Instant::now();
        let mut queue_start: Option<Instant> = None;

        async {
            loop {
                // Recompute candidates each loop iteration so a
                // topology change (e.g., primary failover) during a
                // wait is picked up on retry.
                let candidates = self.candidate_nodes(&cfg, database, user, routing);
                if candidates.is_empty() {
                    return Err(match routing {
                        Routing::Primary => ResolveError::NoPrimary {
                            cluster: cluster_name.clone(),
                        },
                        Routing::Replica => ResolveError::NoReplica {
                            cluster: cluster_name.clone(),
                        },
                    }
                    .into());
                }

                // 1. Try every candidate atomically before giving up.
                //    `any_full` tracks whether any candidate was at its
                //    ceiling (meaning a slot will eventually free up and
                //    waiting is productive); `last_connect_error` keeps
                //    the most recent connect/auth failure so we can
                //    surface it if no candidate is worth waiting on.
                let mut any_full = false;
                let mut last_connect_error: Option<anyhow::Error> = None;
                for key in &candidates {
                    let max = self.max_for_key(&cfg, key);
                    match self.try_checkout(key, max, idle_timeout, max_lifetime) {
                        TryCheckout::TookIdle(idle) => {
                            if let Some(queued_at) = queue_start {
                                o11y::metrics::record_wait_duration(
                                    queued_at.elapsed(),
                                    database,
                                    user,
                                    routing,
                                );
                            }
                            o11y::metrics::record_checkout(
                                checkout_start,
                                database,
                                user,
                                &key.node,
                            );
                            let span = tracing::Span::current();
                            span.record("pool.reused", true);
                            span.record("pool.waited", queue_start.is_some());
                            debug!(
                                database,
                                user,
                                node = %key.node,
                                "reusing idle connection",
                            );
                            return Ok(ConnGuard {
                                pools: Arc::clone(self),
                                key: key.clone(),
                                conn: Some(idle.conn),
                            });
                        }
                        TryCheckout::MayOpen => {
                            let password =
                                self.lookup_password(&key.node, &upstream_database, &upstream_user);
                            match server::connect_server(
                                &key.node,
                                &upstream_database,
                                &upstream_user,
                                password.as_deref(),
                                &user_params,
                                max_prepared_statements,
                                connect_timeout,
                            )
                            .await
                            {
                                Ok(conn) => {
                                    if let Some(queued_at) = queue_start {
                                        o11y::metrics::record_wait_duration(
                                            queued_at.elapsed(),
                                            database,
                                            user,
                                            routing,
                                        );
                                    }
                                    o11y::metrics::record_checkout(
                                        checkout_start,
                                        database,
                                        user,
                                        &key.node,
                                    );
                                    let span = tracing::Span::current();
                                    span.record("pool.reused", false);
                                    span.record("pool.waited", queue_start.is_some());
                                    debug!(
                                        database,
                                        user,
                                        node = %key.node,
                                        "opened new server connection",
                                    );
                                    return Ok(ConnGuard {
                                        pools: Arc::clone(self),
                                        key: key.clone(),
                                        conn: Some(conn),
                                    });
                                }
                                Err(e) => {
                                    // Connect/auth failed on this
                                    // candidate — release the slot we
                                    // speculatively claimed and fall
                                    // through to the next candidate so
                                    // a transient failure on one replica
                                    // doesn't forfeit a healthy one
                                    // later in the round-robin order.
                                    self.discard_internal(key);
                                    last_connect_error = Some(e);
                                }
                            }
                        }
                        TryCheckout::Full => {
                            any_full = true;
                        }
                    }
                }

                // 2. Every candidate either refused connections or hit
                //    its ceiling. If nothing was Full, there's no slot
                //    to wait for — surface the connect error directly.
                //    Otherwise fall through to the enqueue-and-wait
                //    path so the Full candidate can wake us when a
                //    slot frees up.
                if !any_full && let Some(e) = last_connect_error {
                    return Err(e);
                }

                // 3. Every candidate is at capacity. Enqueue and wait.
                //    Record Waiting state and the target role only while
                //    actually blocked so fast checkouts don't flicker
                //    through Waiting in admin output.
                let rx = {
                    let (tx, rx) = oneshot::channel();
                    let enqueued_at = Instant::now();
                    let mut inner = self.inner.lock();
                    inner
                        .waits
                        .entry(wait_key.clone())
                        .or_default()
                        .waiters
                        .push_back((enqueued_at, tx));
                    rx
                };
                client.set_waiting_for(Some(clients::WaitTarget {
                    database: database.to_owned(),
                    user: user.to_owned(),
                    role: match routing {
                        Routing::Primary => clients::WaitRole::Primary,
                        Routing::Replica => clients::WaitRole::Replica,
                    },
                }));
                client.set_state(clients::ClientState::Waiting);
                // First enqueue stamps the queue clock; subsequent
                // re-enqueues after a spurious wakeup keep the
                // original value so the metric reflects total queue
                // time, not per-wait segments.
                queue_start.get_or_insert_with(Instant::now);

                // Single timeout budget across the full checkout —
                // spurious wakeups don't get a fresh clock. Measured
                // from `checkout_start` so pre-enqueue candidate
                // attempts count toward the overall deadline.
                let elapsed = checkout_start.elapsed();
                let remaining = checkout_timeout.saturating_sub(elapsed);
                if remaining.is_zero() {
                    client.set_waiting_for(None);
                    client.set_state(clients::ClientState::Idle);
                    anyhow::bail!(
                        "checkout timed out after {checkout_timeout:?} for {database}/{user}",
                    );
                }
                let wake = tokio::time::timeout(remaining, rx).await;
                client.set_waiting_for(None);
                client.set_state(clients::ClientState::Idle);

                if wake.is_err() {
                    // Outer `tokio::time::timeout` elapsed.
                    anyhow::bail!(
                        "checkout timed out after {checkout_timeout:?} for {database}/{user}",
                    );
                }
                // Shutdown wakes everyone via `wake_all_waiters`. A
                // waiter that was spuriously woken by an unrelated
                // checkin right as shutdown started would also loop
                // here, so the check happens after the wake.
                if self.shutting_down.load(Ordering::Acquire) {
                    anyhow::bail!("halephant is shutting down");
                }
                // Woken — loop top re-runs the candidate check.
            }
        }
        .await
        .inspect(|_| {
            tracing::Span::current().record("otel.status_code", "OK");
        })
        .inspect_err(|e| {
            // Classify the failure so `db.client.connection.errors` time
            // series break down cleanly in dashboards.
            let msg = e.to_string();
            let error_type = if let Some(resolve_err) = e.downcast_ref::<ResolveError>() {
                match resolve_err {
                    ResolveError::UnknownDatabase { .. } => "unknown_database",
                    ResolveError::NoPrimary { .. } => "no_primary",
                    ResolveError::NoReplica { .. } => "no_replica",
                }
            } else if msg.contains("shutting down") {
                "shutting_down"
            } else if msg.contains("checkout timed out") {
                "checkout_timeout"
            } else {
                "connect_failed"
            };
            o11y::metrics::record_checkout_error(database, user, error_type);
            let span = tracing::Span::current();
            span.record("otel.status_code", "ERROR");
            span.record("otel.status_description", "checkout failed");
        })
    }

    /// Atomic check-and-claim for a single pool key.
    ///
    /// Discards expired idle connections inline (idle_timeout /
    /// max_lifetime). Increments `pool.active` on success so a concurrent
    /// checkout can't double-claim the same slot. Holds the inner lock
    /// for the duration of one HashMap lookup + VecDeque pop — no
    /// `.await` while locked.
    fn try_checkout(
        &self,
        key: &PoolKey,
        max: u32,
        idle_timeout: std::time::Duration,
        max_lifetime: std::time::Duration,
    ) -> TryCheckout {
        let mut inner = self.inner.lock();
        let pool = inner.pools.entry(key.clone()).or_default();

        // Drain expired idle entries until we find a usable one or the
        // queue is empty. Each iteration drops the front entry.
        while let Some(idle) = pool.idle.pop_front() {
            if idle.idle_since.elapsed() > idle_timeout {
                trace!(
                    database = %key.database,
                    user = %key.user,
                    "discarding idle connection (idle timeout)",
                );
                continue;
            }
            if idle.conn.created_at.elapsed() > max_lifetime {
                trace!(
                    database = %key.database,
                    user = %key.user,
                    "discarding idle connection (max lifetime)",
                );
                continue;
            }
            pool.active += 1;
            return TryCheckout::TookIdle(Box::new(idle));
        }

        if max == 0 || pool.total() < max {
            pool.active += 1;
            TryCheckout::MayOpen
        } else {
            TryCheckout::Full
        }
    }

    /// Release a slot that was speculatively claimed by a failed
    /// connect attempt or a dropped [`ConnGuard`]. Wakes a waiter if
    /// there is one — the just-freed slot is a retry opportunity.
    pub(in crate::pool) fn discard_internal(&self, key: &PoolKey) {
        {
            let mut inner = self.inner.lock();
            if let Some(pool) = inner.pools.get_mut(key) {
                pool.active = pool.active.saturating_sub(1);
                trace!(
                    node = key.node,
                    database = key.database,
                    user = key.user,
                    "connection discarded"
                );
            }
        }
        // Freeing an active slot may allow a waiter to grow the pool.
        self.wake_one_for_node(key);
    }
}