Skip to main content

halephant_core/pool/
refill.rs

1//! Background connection management: warm-up, shortfall refills,
2//! scavenger ticks, and the reset-and-return path taken by
3//! [`crate::pool::types::ConnGuard::checkin`].
4
5use std::sync::Arc;
6use std::time::Instant;
7
8use tracing::{Instrument, debug, trace, warn};
9
10use crate::connections::reset::reset_connection;
11use crate::connections::server::{self, ServerConn};
12use crate::pool::PoolManager;
13use crate::pool::types::{IdleConn, Pool, PoolKey};
14
15/// Maximum in-flight connection opens per call to
16/// [`PoolManager::spawn_connections`]. Because PostgreSQL's postmaster
17/// accepts and forks backends serially, slamming a single node with many
18/// simultaneous connects queues every SCRAM handshake behind the fork
19/// loop and inflates per-connection latency. A small cap keeps each
20/// connect close to its minimum wall time while still parallelizing
21/// across different nodes (each batch gets its own semaphore).
22const PER_NODE_REFILL_CONCURRENCY: usize = 8;
23
24impl PoolManager {
25    /// Open idle connections at startup based on `min_connections` config.
26    pub async fn warm_up(self: &Arc<Self>) {
27        let shortfalls = self.shortfalls();
28        if shortfalls.is_empty() {
29            debug!("warm-up: nothing to do");
30            return;
31        }
32
33        let mut tasks = tokio::task::JoinSet::new();
34        for (key, n) in shortfalls {
35            let this = Arc::clone(self);
36            tasks.spawn(async move { this.spawn_connections(&key, n).await });
37        }
38
39        let mut opened = 0u32;
40        while let Some(result) = tasks.join_next().await {
41            match result {
42                Ok(n) => opened += n,
43                Err(e) => warn!("warm-up task panicked: {e}"),
44            }
45        }
46        debug!(opened, "warm-up complete");
47    }
48
49    /// Remove idle connections that exceed `max_lifetime` or whose socket is
50    /// dead, then apply `idle_timeout` only to connections in excess of
51    /// `min_connections` for that pool entry.
52    ///
53    /// `max_lifetime` and dead-socket detection always apply because those
54    /// represent unhealthy connections. `idle_timeout` represents excess
55    /// capacity that can be released — so it must respect the floor set by
56    /// `min_connections`, otherwise warm-up connections decay away even when
57    /// the user explicitly asked for a minimum.
58    ///
59    /// Returns the number of connections removed across all pool entries.
60    /// Normally invoked from [`PoolManager::run_scavenger`]; exposed so tests
61    /// can drive it directly without spinning up the full background task.
62    pub fn scavenge_idle(&self) -> u32 {
63        let cfg = self.config.load_full();
64        let mut inner = self.inner.lock();
65        let mut removed = 0u32;
66
67        for (key, pool) in &mut inner.pools {
68            let pool_config = cfg.find_pool(&key.database).map(|(_, _, p)| p);
69            let idle_timeout =
70                pool_config.map_or(std::time::Duration::from_secs(300), |p| p.idle_timeout);
71            let max_lifetime =
72                pool_config.map_or(std::time::Duration::from_secs(3600), |p| p.max_lifetime);
73
74            let before = pool.idle.len();
75
76            // Step 1: always drop dead and over-lifetime connections.
77            pool.idle.retain(|idle| {
78                if idle.conn.created_at.elapsed() > max_lifetime {
79                    return false;
80                }
81                crate::connections::sock::is_alive(idle.conn.framed.get_ref())
82            });
83
84            // Step 2: apply idle_timeout, but stop once the pool is at the
85            // floor. The idle queue is FIFO, so the oldest expired connections
86            // are dropped first and the newest survive.
87            let min_idle = self.min_for_key(&cfg, key) as usize;
88            let removable = pool.idle.len().saturating_sub(min_idle);
89            let mut dropped = 0usize;
90            pool.idle.retain(|idle| {
91                if dropped >= removable {
92                    return true;
93                }
94                if idle.idle_since.elapsed() > idle_timeout {
95                    dropped += 1;
96                    return false;
97                }
98                true
99            });
100
101            removed += (before - pool.idle.len()) as u32;
102        }
103
104        // Prune empty wait-queue entries so short-lived
105        // `(database, user, role)` combinations that contended once
106        // and never came back don't accumulate in the map. This runs
107        // while the inner lock is already held for the scavenge pass.
108        inner.waits.retain(|_, q| !q.waiters.is_empty());
109
110        removed
111    }
112
113    /// Enumerate every configured pool key whose connection count is below
114    /// its `min_for_key` floor, paired with the deficit. Walks the
115    /// configured `(cluster, database, user)` tuples and resolves the
116    /// expected primary and replica nodes from current topology — so the
117    /// result includes brand-new replicas that have no pool entry yet.
118    fn shortfalls(&self) -> Vec<(PoolKey, u32)> {
119        let cfg = self.config.load_full();
120        let inner = self.inner.lock();
121        let mut result = Vec::new();
122
123        for (cluster_name, cluster) in &cfg.cluster {
124            let primary = self.topology.primary(cluster_name);
125            let replicas = self.topology.replicas(cluster_name);
126
127            for (db_name, pool_config) in &cluster.pool {
128                for (user_name, user_config) in &pool_config.user {
129                    // Primary shortfall.
130                    if user_config.min_connections.primary > 0
131                        && let Some(ref primary_addr) = primary
132                    {
133                        let key = PoolKey {
134                            node: primary_addr.clone(),
135                            database: db_name.clone(),
136                            user: user_name.clone(),
137                        };
138                        let total = inner.pools.get(&key).map_or(0, Pool::total);
139                        let want = user_config.min_connections.primary;
140                        if total < want {
141                            result.push((key, want - total));
142                        }
143                    }
144
145                    // Replica shortfall — per replica node, not distributed.
146                    if user_config.min_connections.replica > 0 {
147                        for replica_addr in &replicas {
148                            let key = PoolKey {
149                                node: replica_addr.clone(),
150                                database: db_name.clone(),
151                                user: user_name.clone(),
152                            };
153                            let total = inner.pools.get(&key).map_or(0, Pool::total);
154                            let want = user_config.min_connections.replica;
155                            if total < want {
156                                result.push((key, want - total));
157                            }
158                        }
159                    }
160                }
161            }
162        }
163
164        result
165    }
166
167    /// Open up to `n` new connections for the given pool key in parallel
168    /// and insert each successful one into the idle queue. Returns the
169    /// number opened. Per-connection failures are logged and counted as
170    /// zero so a single bad node does not abort the rest of the batch.
171    ///
172    /// Wrapped in a `pool.refill` span so the per-connection `pool.connect`
173    /// spans nest under it in OTel traces.
174    #[tracing::instrument(name = "pool.refill", skip_all, fields(
175        server.address = %key.node,
176        db.namespace = %key.database,
177        user = %key.user,
178        count = n,
179        opened = tracing::field::Empty,
180        otel.status_code,
181    ))]
182    async fn spawn_connections(self: &Arc<Self>, key: &PoolKey, n: u32) -> u32 {
183        if n == 0 {
184            return 0;
185        }
186
187        // Resolve the upstream identity once for the batch. Connection
188        // failures from a missing config or user are surfaced via warn,
189        // not by silently returning 0 — that would mask config drift.
190        let cfg = self.config.load_full();
191        let (upstream_database, upstream_user, params, max_prepared, connect_timeout) = {
192            let Some((_, cluster_config, pool_config)) = cfg.find_pool(&key.database) else {
193                warn!(
194                    database = %key.database,
195                    "refill: database not configured",
196                );
197                return 0;
198            };
199            let Some(user_config) = pool_config.user.get(&key.user) else {
200                warn!(
201                    database = %key.database,
202                    user = %key.user,
203                    "refill: user not configured",
204                );
205                return 0;
206            };
207            (
208                pool_config.database_name(&key.database).to_owned(),
209                user_config.upstream_name(&key.user).to_owned(),
210                user_config.parameters.clone(),
211                cfg.server.max_prepared_statements,
212                cluster_config.connect_timeout,
213            )
214        };
215
216        let password = self.lookup_password(&key.node, &upstream_database, &upstream_user);
217
218        // Cap in-flight opens to the same upstream node. PostgreSQL's
219        // postmaster accepts and forks backends serially — slamming it
220        // with tons of simultaneous connects queues every SCRAM handshake
221        // behind the fork loop, so each individual `pool.connect` span
222        // stretches to hundreds of milliseconds. Capping the batch keeps
223        // the per-connect wall time close to the minimum and gets the
224        // earliest connections into the idle queue sooner. This semaphore
225        // is local to one `spawn_connections` call, so different nodes
226        // still run fully in parallel.
227        let permits = Arc::new(tokio::sync::Semaphore::new(PER_NODE_REFILL_CONCURRENCY));
228
229        let parent_span = tracing::Span::current();
230        let mut tasks = tokio::task::JoinSet::new();
231        for _ in 0..n {
232            let this = Arc::clone(self);
233            let key = key.clone();
234            let upstream_database = upstream_database.clone();
235            let upstream_user = upstream_user.clone();
236            let password = password.clone();
237            let params = params.clone();
238            let permits = Arc::clone(&permits);
239
240            tasks.spawn(
241                async move {
242                    // Acquire before connecting so permits throttle the
243                    // actual handshake work, not just scheduling.
244                    let _permit = permits
245                        .acquire_owned()
246                        .await
247                        .expect("refill semaphore is never closed");
248                    match server::connect_server(
249                        &key.node,
250                        &upstream_database,
251                        &upstream_user,
252                        password.as_deref(),
253                        &params,
254                        max_prepared,
255                        connect_timeout,
256                    )
257                    .await
258                    {
259                        Ok(conn) => {
260                            {
261                                let mut inner = this.inner.lock();
262                                let pool = inner.pools.entry(key.clone()).or_default();
263                                pool.idle.push_back(IdleConn {
264                                    conn,
265                                    idle_since: Instant::now(),
266                                });
267                            }
268                            // New idle connection — wake a waiter if
269                            // anyone is queued for this role.
270                            this.wake_one_for_node(&key);
271                            true
272                        }
273                        Err(e) => {
274                            warn!(
275                                database = key.database,
276                                user = key.user,
277                                node = key.node,
278                                %e,
279                                "refill connection failed",
280                            );
281                            false
282                        }
283                    }
284                }
285                .instrument(parent_span.clone()),
286            );
287        }
288
289        let mut opened = 0u32;
290        while let Some(result) = tasks.join_next().await {
291            match result {
292                Ok(true) => opened += 1,
293                Ok(false) => {}
294                Err(e) => warn!(%e, "refill task panicked"),
295            }
296        }
297        let span = tracing::Span::current();
298        span.record("opened", opened);
299        // Status is OK as long as at least one connection made it into the
300        // pool. A batch where every task failed is recorded as error.
301        if opened > 0 || n == 0 {
302            span.record("otel.status_code", "OK");
303        } else {
304            span.record("otel.status_code", "ERROR");
305        }
306        opened
307    }
308
309    /// Run the idle scavenger and refill loop at a fixed interval.
310    ///
311    /// Each tick prunes expired idle connections (preserving the
312    /// `min_connections` floor), then refills any pool entry below its
313    /// floor by fanning every `spawn_connections` call into a
314    /// `JoinSet` and awaiting them concurrently — matching
315    /// [`PoolManager::warm_up`]. Running them in series here would
316    /// let a single slow node blow most of the 30-second budget and
317    /// leave unrelated pools under their `min_connections` floor.
318    /// The per-tick state is logged at debug level so the loop is
319    /// visible without needing trace-level filters.
320    pub async fn run_scavenger(self: &Arc<Self>) {
321        let interval = std::time::Duration::from_secs(30);
322        loop {
323            tokio::time::sleep(interval).await;
324            let removed = self.scavenge_idle();
325            let shortfalls = self.shortfalls();
326            debug!(removed, refills = shortfalls.len(), "scavenger tick");
327            let mut tasks = tokio::task::JoinSet::new();
328            for (key, n) in shortfalls {
329                let this = Arc::clone(self);
330                tasks.spawn(async move { this.spawn_connections(&key, n).await });
331            }
332            while let Some(result) = tasks.join_next().await {
333                if let Err(e) = result {
334                    warn!(%e, "scavenger refill task panicked");
335                }
336            }
337        }
338    }
339
340    /// Reset a connection in the background and return it to the idle pool.
341    pub(in crate::pool) async fn reset_and_return(
342        self: &Arc<Self>,
343        key: PoolKey,
344        mut conn: ServerConn,
345    ) {
346        match reset_connection(&mut conn).await {
347            Ok(()) => {
348                let returned = {
349                    let mut inner = self.inner.lock();
350                    if let Some(pool) = inner.pools.get_mut(&key) {
351                        pool.resetting = pool.resetting.saturating_sub(1);
352                        pool.idle.push_back(IdleConn {
353                            conn,
354                            idle_since: Instant::now(),
355                        });
356                        trace!(
357                            node = key.node,
358                            database = key.database,
359                            user = key.user,
360                            idle = pool.idle.len(),
361                            "connection reset and returned to pool"
362                        );
363                        true
364                    } else {
365                        // Pool entry gone — connection dropped.
366                        false
367                    }
368                };
369                if returned {
370                    self.wake_one_for_node(&key);
371                }
372            }
373            Err(e) => {
374                debug!(
375                    node = key.node,
376                    database = key.database,
377                    user = key.user,
378                    %e,
379                    "reset failed, discarding connection"
380                );
381                {
382                    let mut inner = self.inner.lock();
383                    if let Some(pool) = inner.pools.get_mut(&key) {
384                        pool.resetting = pool.resetting.saturating_sub(1);
385                    }
386                }
387                // The reset failed so the connection is gone — that
388                // frees a slot on the node, wake a waiter so it can
389                // try to grow the pool.
390                self.wake_one_for_node(&key);
391            }
392        }
393    }
394}