Skip to main content

halephant_core/pool/
refill.rs

1//! Background connection management: warm-up, shortfall refills,
2//! scavenger ticks, and the reset-and-return path taken by
3//! [`crate::pool::types::ConnGuard::checkin`].
4
5use std::sync::Arc;
6use std::time::Instant;
7
8use tracing::{Instrument, debug, warn};
9
10use crate::connections::reset::reset_connection;
11use crate::connections::server::{self, ServerConn};
12use crate::pool::PoolManager;
13use crate::pool::types::{ConnId, IdleConn, PoolKey};
14
15/// Maximum in-flight connection opens per call to
16/// [`PoolManager::spawn_connections`]. Because PostgreSQL's postmaster
17/// accepts and forks backends serially, slamming a single node with many
18/// simultaneous connects queues every SCRAM handshake behind the fork
19/// loop and inflates per-connection latency. A small cap keeps each
20/// connect close to its minimum wall time while still parallelizing
21/// across different nodes (each batch gets its own semaphore).
22const PER_NODE_REFILL_CONCURRENCY: usize = 8;
23
24impl PoolManager {
25    /// Open idle connections at startup based on `min_connections` config.
26    pub async fn warm_up(self: &Arc<Self>) {
27        let shortfalls = self.shortfalls();
28        if shortfalls.is_empty() {
29            debug!("warm-up: nothing to do");
30            return;
31        }
32
33        let mut tasks = tokio::task::JoinSet::new();
34        for (key, n) in shortfalls {
35            let this = Arc::clone(self);
36            tasks.spawn(async move { this.spawn_connections(&key, n).await });
37        }
38
39        let mut opened = 0u32;
40        while let Some(result) = tasks.join_next().await {
41            match result {
42                Ok(n) => opened += n,
43                Err(e) => warn!("warm-up task panicked: {e}"),
44            }
45        }
46        debug!(opened, "warm-up complete");
47    }
48
49    /// Remove idle connections that exceed `max_lifetime` or whose socket is
50    /// dead, then apply `idle_timeout` only to connections in excess of
51    /// `min_connections` for that pool entry.
52    ///
53    /// `max_lifetime` and dead-socket detection always apply because those
54    /// represent unhealthy connections. `idle_timeout` represents excess
55    /// capacity that can be released — so it must respect the floor set by
56    /// `min_connections`, otherwise warm-up connections decay away even when
57    /// the user explicitly asked for a minimum.
58    ///
59    /// Returns the number of connections removed across all pool entries.
60    /// Normally invoked from [`PoolManager::run_scavenger`]; exposed so tests
61    /// can drive it directly without spinning up the full background task.
62    pub fn scavenge_idle(&self) -> u32 {
63        let cfg = self.config.load_full();
64        let mut freed: Vec<(PoolKey, u32)> = Vec::new();
65        let mut removed = 0u32;
66
67        {
68            let mut inner = self.inner.lock();
69
70            for (key, pool) in &mut inner.pools {
71                let pool_config = cfg.find_pool(&key.database).map(|(_, _, p)| p);
72                let idle_timeout =
73                    pool_config.map_or(std::time::Duration::from_secs(300), |p| p.idle_timeout);
74                let max_lifetime =
75                    pool_config.map_or(std::time::Duration::from_secs(3600), |p| p.max_lifetime);
76
77                let before = pool.idle.len();
78
79                // Remove dead sockets and connections past max_lifetime
80                // unconditionally (regardless of min_connections floor).
81                pool.idle.retain(|idle| {
82                    idle.conn.created_at.elapsed() <= max_lifetime
83                        && crate::connections::sock::is_alive(idle.conn.framed.get_ref())
84                });
85
86                // Apply idle_timeout only to excess capacity above
87                // the min_connections floor.
88                let min_idle = self.min_for_key(&cfg, key) as usize;
89                let removable = pool.idle.len().saturating_sub(min_idle);
90                let mut dropped = 0usize;
91                pool.idle.retain(|idle| {
92                    if dropped >= removable {
93                        return true;
94                    }
95                    if idle.idle_since.elapsed() > idle_timeout {
96                        dropped += 1;
97                        return false;
98                    }
99                    true
100                });
101
102                let n = (before - pool.idle.len()) as u32;
103                if n > 0 {
104                    freed.push((key.clone(), n));
105                }
106                removed += n;
107            }
108
109            inner.waits.retain(|_, q| !q.waiters.is_empty());
110        }
111
112        // Removing idle connections frees physical capacity — wake
113        // any waiter that was blocked on node-level limits.
114        for (key, n) in &freed {
115            for _ in 0..*n {
116                self.wake_one_for_node(key);
117            }
118        }
119
120        removed
121    }
122
123    /// Enumerate every configured pool key whose connection count is below
124    /// its `min_for_key` floor, paired with the deficit. Walks the
125    /// configured `(cluster, database, user)` tuples and resolves the
126    /// expected primary and replica nodes from current topology — so the
127    /// result includes brand-new replicas that have no pool entry yet.
128    fn shortfalls(&self) -> Vec<(PoolKey, u32)> {
129        let cfg = self.config.load_full();
130        let inner = self.inner.lock();
131        let mut result = Vec::new();
132
133        for (cluster_name, cluster) in &cfg.cluster {
134            let primary = self.topology.primary(cluster_name);
135            let replicas = self.topology.replicas(cluster_name);
136
137            for (db_name, pool_config) in &cluster.pool {
138                for (user_name, user_config) in &pool_config.user {
139                    // Primary shortfall.
140                    if user_config.min_connections.primary > 0
141                        && let Some(ref primary_addr) = primary
142                    {
143                        let key = PoolKey {
144                            node: primary_addr.clone(),
145                            database: db_name.clone(),
146                            user: user_name.clone(),
147                        };
148                        let total = inner.pools.get(&key).map_or(0, |p| p.total() as u32);
149                        let want = user_config.min_connections.primary;
150                        if total < want {
151                            result.push((key, want - total));
152                        }
153                    }
154
155                    // Replica shortfall — per replica node, not distributed.
156                    if user_config.min_connections.replica > 0 {
157                        for replica_addr in &replicas {
158                            let key = PoolKey {
159                                node: replica_addr.clone(),
160                                database: db_name.clone(),
161                                user: user_name.clone(),
162                            };
163                            let total = inner.pools.get(&key).map_or(0, |p| p.total() as u32);
164                            let want = user_config.min_connections.replica;
165                            if total < want {
166                                result.push((key, want - total));
167                            }
168                        }
169                    }
170                }
171            }
172        }
173
174        result
175    }
176
177    /// Open up to `n` new connections for the given pool key in parallel
178    /// and insert each successful one into the idle queue. Returns the
179    /// number opened. Per-connection failures are logged and counted as
180    /// zero so a single bad node does not abort the rest of the batch.
181    ///
182    /// Wrapped in a `pool.refill` span so the per-connection `pool.connect`
183    /// spans nest under it in OTel traces.
184    #[tracing::instrument(name = "pool.refill", skip_all, fields(
185        server.address = %key.node,
186        db.namespace = %key.database,
187        user = %key.user,
188        count = n,
189        opened = tracing::field::Empty,
190        otel.status_code,
191    ))]
192    async fn spawn_connections(self: &Arc<Self>, key: &PoolKey, n: u32) -> u32 {
193        if n == 0 {
194            return 0;
195        }
196
197        // Resolve the upstream identity once for the batch. Connection
198        // failures from a missing config or user are surfaced via warn,
199        // not by silently returning 0 — that would mask config drift.
200        let cfg = self.config.load_full();
201        let (upstream_database, upstream_user, params, max_prepared, connect_timeout) = {
202            let Some((_, cluster_config, pool_config)) = cfg.find_pool(&key.database) else {
203                warn!(
204                    database = %key.database,
205                    "refill: database not configured",
206                );
207                return 0;
208            };
209            let Some(user_config) = pool_config.user.get(&key.user) else {
210                warn!(
211                    database = %key.database,
212                    user = %key.user,
213                    "refill: user not configured",
214                );
215                return 0;
216            };
217            (
218                pool_config.database_name(&key.database).to_owned(),
219                user_config.upstream_name(&key.user).to_owned(),
220                user_config.parameters.clone(),
221                cfg.server.max_prepared_statements,
222                cluster_config.connect_timeout,
223            )
224        };
225
226        let password = self.lookup_password(&key.node, &upstream_database, &upstream_user);
227
228        // Cap in-flight opens to the same upstream node. PostgreSQL's
229        // postmaster accepts and forks backends serially — slamming it
230        // with tons of simultaneous connects queues every SCRAM handshake
231        // behind the fork loop, so each individual `pool.connect` span
232        // stretches to hundreds of milliseconds. Capping the batch keeps
233        // the per-connect wall time close to the minimum and gets the
234        // earliest connections into the idle queue sooner. This semaphore
235        // is local to one `spawn_connections` call, so different nodes
236        // still run fully in parallel.
237        let permits = Arc::new(tokio::sync::Semaphore::new(PER_NODE_REFILL_CONCURRENCY));
238
239        let parent_span = tracing::Span::current();
240        let mut tasks = tokio::task::JoinSet::new();
241        for _ in 0..n {
242            let this = Arc::clone(self);
243            let key = key.clone();
244            let upstream_database = upstream_database.clone();
245            let upstream_user = upstream_user.clone();
246            let password = password.clone();
247            let params = params.clone();
248            let permits = Arc::clone(&permits);
249
250            tasks.spawn(
251                async move {
252                    // Acquire before connecting so permits throttle the
253                    // actual handshake work, not just scheduling.
254                    let _permit = permits
255                        .acquire_owned()
256                        .await
257                        .expect("refill semaphore is never closed");
258                    match server::connect_server(
259                        &key.node,
260                        &upstream_database,
261                        &upstream_user,
262                        password.as_deref(),
263                        &params,
264                        max_prepared,
265                        connect_timeout,
266                    )
267                    .await
268                    {
269                        Ok(conn) => {
270                            {
271                                let mut inner = this.inner.lock();
272                                let pool = inner.pools.entry(key.clone()).or_default();
273                                pool.idle.push_back(IdleConn {
274                                    id: ConnId::next(),
275                                    conn,
276                                    idle_since: Instant::now(),
277                                });
278                            }
279                            // New idle connection — wake a waiter if
280                            // anyone is queued for this role.
281                            this.wake_one_for_node(&key);
282                            true
283                        }
284                        Err(e) => {
285                            warn!(
286                                database = key.database,
287                                user = key.user,
288                                node = key.node,
289                                %e,
290                                "refill connection failed",
291                            );
292                            false
293                        }
294                    }
295                }
296                .instrument(parent_span.clone()),
297            );
298        }
299
300        let mut opened = 0u32;
301        while let Some(result) = tasks.join_next().await {
302            match result {
303                Ok(true) => opened += 1,
304                Ok(false) => {}
305                Err(e) => warn!(%e, "refill task panicked"),
306            }
307        }
308        let span = tracing::Span::current();
309        span.record("opened", opened);
310        // Status is OK as long as at least one connection made it into the
311        // pool. A batch where every task failed is recorded as error.
312        if opened > 0 || n == 0 {
313            span.record("otel.status_code", "OK");
314        } else {
315            span.record("otel.status_code", "ERROR");
316        }
317        opened
318    }
319
320    /// Run the idle scavenger and refill loop at a fixed interval.
321    ///
322    /// Each tick prunes expired idle connections (preserving the
323    /// `min_connections` floor), then refills any pool entry below its
324    /// floor by fanning every `spawn_connections` call into a
325    /// `JoinSet` and awaiting them concurrently — matching
326    /// [`PoolManager::warm_up`]. Running them in series here would
327    /// let a single slow node blow most of the 30-second budget and
328    /// leave unrelated pools under their `min_connections` floor.
329    /// The per-tick state is logged at debug level so the loop is
330    /// visible without needing trace-level filters.
331    pub async fn run_scavenger(self: &Arc<Self>) {
332        let interval = std::time::Duration::from_secs(30);
333        loop {
334            tokio::time::sleep(interval).await;
335            let removed = self.scavenge_idle();
336            let shortfalls = self.shortfalls();
337            debug!(removed, refills = shortfalls.len(), "scavenger tick");
338            let mut tasks = tokio::task::JoinSet::new();
339            for (key, n) in shortfalls {
340                let this = Arc::clone(self);
341                tasks.spawn(async move { this.spawn_connections(&key, n).await });
342            }
343            while let Some(result) = tasks.join_next().await {
344                if let Err(e) = result {
345                    warn!(%e, "scavenger refill task panicked");
346                }
347            }
348        }
349    }
350
351    pub(in crate::pool) async fn reset_and_return(
352        self: &Arc<Self>,
353        key: PoolKey,
354        id: ConnId,
355        mut conn: ServerConn,
356    ) {
357        match reset_connection(&mut conn).await {
358            Ok(()) => {
359                let returned = {
360                    let mut inner = self.inner.lock();
361                    if let Some(pool) = inner.pools.get_mut(&key) {
362                        pool.resetting.remove(&id);
363                        pool.idle.push_back(IdleConn {
364                            id,
365                            conn,
366                            idle_since: Instant::now(),
367                        });
368                        true
369                    } else {
370                        false
371                    }
372                };
373                if returned {
374                    self.wake_one_for_node(&key);
375                }
376            }
377            Err(e) => {
378                debug!(
379                    node = key.node,
380                    database = key.database,
381                    user = key.user,
382                    %e,
383                    "reset failed, discarding connection"
384                );
385                {
386                    let mut inner = self.inner.lock();
387                    if let Some(pool) = inner.pools.get_mut(&key) {
388                        pool.resetting.remove(&id);
389                    }
390                }
391                self.wake_one_for_node(&key);
392            }
393        }
394    }
395}