Skip to main content

halephant_core/pool/
refill.rs

1//! Background connection management: warm-up, shortfall refills,
2//! scavenger ticks, and the reset-and-return path taken by
3//! [`crate::pool::types::ConnGuard::checkin`].
4
5use std::sync::Arc;
6use std::time::Instant;
7
8use tracing::{Instrument, debug, warn};
9
10use crate::connections::reset::reset_connection;
11use crate::connections::server::ServerConn;
12use crate::pool::PoolManager;
13use crate::pool::candidates::ConnectSpec;
14use crate::pool::types::{ConnId, IdleConn, PoolKey};
15
16/// Maximum in-flight connection opens per call to
17/// [`PoolManager::spawn_connections`]. Because PostgreSQL's postmaster
18/// accepts and forks backends serially, slamming a single node with many
19/// simultaneous connects queues every SCRAM handshake behind the fork
20/// loop and inflates per-connection latency. A small cap keeps each
21/// connect close to its minimum wall time while still parallelizing
22/// across different nodes (each batch gets its own semaphore).
23const PER_NODE_REFILL_CONCURRENCY: usize = 8;
24
25impl PoolManager {
26    /// Open idle connections at startup based on `min_connections` config.
27    pub async fn warm_up(self: &Arc<Self>) {
28        let shortfalls = self.shortfalls();
29        if shortfalls.is_empty() {
30            debug!("warm-up: nothing to do");
31            return;
32        }
33
34        let mut tasks = tokio::task::JoinSet::new();
35        for (key, n) in shortfalls {
36            let this = Arc::clone(self);
37            tasks.spawn(async move { this.spawn_connections(&key, n).await });
38        }
39
40        let mut opened = 0u32;
41        while let Some(result) = tasks.join_next().await {
42            match result {
43                Ok(n) => opened += n,
44                Err(e) => warn!("warm-up task panicked: {e}"),
45            }
46        }
47        debug!(opened, "warm-up complete");
48    }
49
50    /// Remove idle connections that exceed `max_lifetime` or whose socket is
51    /// dead, then apply `idle_timeout` only to connections in excess of
52    /// `min_connections` for that pool entry.
53    ///
54    /// `max_lifetime` and dead-socket detection always apply because those
55    /// represent unhealthy connections. `idle_timeout` represents excess
56    /// capacity that can be released — so it must respect the floor set by
57    /// `min_connections`, otherwise warm-up connections decay away even when
58    /// the user explicitly asked for a minimum.
59    ///
60    /// Returns the number of connections removed across all pool entries.
61    /// Normally invoked from [`PoolManager::run_scavenger`]; exposed so tests
62    /// can drive it directly without spinning up the full background task.
63    pub fn scavenge_idle(&self) -> u32 {
64        let cfg = self.config.load_full();
65        let mut freed: Vec<(PoolKey, u32)> = Vec::new();
66        let mut removed = 0u32;
67
68        {
69            let mut inner = self.inner.lock();
70
71            for (key, pool) in &mut inner.pools {
72                let pool_config = cfg.find_pool(&key.database).map(|(_, _, p)| p);
73                let idle_timeout =
74                    pool_config.map_or(std::time::Duration::from_mins(5), |p| p.idle_timeout);
75                let max_lifetime =
76                    pool_config.map_or(std::time::Duration::from_hours(1), |p| p.max_lifetime);
77
78                let before = pool.idle.len();
79
80                // Remove dead sockets and connections past max_lifetime
81                // unconditionally (regardless of min_connections floor).
82                pool.idle.retain(|idle| {
83                    idle.conn.created_at.elapsed() <= max_lifetime
84                        && crate::connections::sock::is_alive(idle.conn.framed.get_ref())
85                });
86
87                // Apply idle_timeout only to excess capacity above
88                // the min_connections floor.
89                let min_idle = self.min_for_key(&cfg, key) as usize;
90                let removable = pool.idle.len().saturating_sub(min_idle);
91                let mut dropped = 0usize;
92                pool.idle.retain(|idle| {
93                    if dropped >= removable {
94                        return true;
95                    }
96                    if idle.idle_since.elapsed() > idle_timeout {
97                        dropped += 1;
98                        return false;
99                    }
100                    true
101                });
102
103                let n = (before - pool.idle.len()) as u32;
104                if n > 0 {
105                    freed.push((key.clone(), n));
106                }
107                removed += n;
108            }
109
110            inner.waits.retain(|_, q| !q.waiters.is_empty());
111        }
112
113        // Removing idle connections frees physical capacity — wake
114        // any waiter that was blocked on node-level limits.
115        for (key, n) in &freed {
116            for _ in 0..*n {
117                self.wake_one_for_node(key);
118            }
119        }
120
121        removed
122    }
123
124    /// Enumerate every configured pool key whose connection count is below
125    /// its `min_for_key` floor, paired with the deficit. Walks the
126    /// configured `(cluster, database, user)` tuples and resolves the
127    /// expected primary and replica nodes from current topology — so the
128    /// result includes brand-new replicas that have no pool entry yet.
129    fn shortfalls(&self) -> Vec<(PoolKey, u32)> {
130        let cfg = self.config.load_full();
131        let inner = self.inner.lock();
132        let mut result = Vec::new();
133
134        for (cluster_name, cluster) in &cfg.cluster {
135            let primary = self.topology.primary(cluster_name);
136            let replicas = self.topology.replicas(cluster_name);
137
138            for (db_name, pool_config) in &cluster.pool {
139                for (user_name, user_config) in &pool_config.user {
140                    // Primary shortfall.
141                    if user_config.min_connections.primary > 0
142                        && let Some(ref primary_addr) = primary
143                    {
144                        let key = PoolKey {
145                            node: primary_addr.clone(),
146                            database: db_name.clone(),
147                            user: user_name.clone(),
148                        };
149                        let total = inner.pools.get(&key).map_or(0, |p| p.total() as u32);
150                        let want = user_config.min_connections.primary;
151                        if total < want {
152                            result.push((key, want - total));
153                        }
154                    }
155
156                    // Replica shortfall — per replica node, not distributed.
157                    if user_config.min_connections.replica > 0 {
158                        for replica_addr in &replicas {
159                            let key = PoolKey {
160                                node: replica_addr.clone(),
161                                database: db_name.clone(),
162                                user: user_name.clone(),
163                            };
164                            let total = inner.pools.get(&key).map_or(0, |p| p.total() as u32);
165                            let want = user_config.min_connections.replica;
166                            if total < want {
167                                result.push((key, want - total));
168                            }
169                        }
170                    }
171                }
172            }
173        }
174
175        result
176    }
177
178    /// Open up to `n` new connections for the given pool key in parallel
179    /// and insert each successful one into the idle queue. Returns the
180    /// number opened. Per-connection failures are logged and counted as
181    /// zero so a single bad node does not abort the rest of the batch.
182    ///
183    /// Wrapped in a `pool.refill` span so the per-connection `pool.connect`
184    /// spans nest under it in OTel traces.
185    #[tracing::instrument(name = "pool.refill", skip_all, fields(
186        server.address = %key.node,
187        db.namespace = %key.database,
188        user = %key.user,
189        count = n,
190        opened = tracing::field::Empty,
191        otel.status_code,
192    ))]
193    async fn spawn_connections(self: &Arc<Self>, key: &PoolKey, n: u32) -> u32 {
194        if n == 0 {
195            return 0;
196        }
197
198        // Resolve the upstream identity once for the batch through the
199        // resolver shared with checkout — refusal semantics for a
200        // missing database or user are identical on both paths.
201        // Failures are surfaced via warn, not by silently returning 0 —
202        // that would mask config drift.
203        let cfg = self.config.load_full();
204        let spec = match ConnectSpec::resolve(&cfg, &key.database, &key.user) {
205            Ok(spec) => spec,
206            Err(e) => {
207                warn!(
208                    database = %key.database,
209                    user = %key.user,
210                    %e,
211                    "refill: connect identity unresolved",
212                );
213                return 0;
214            }
215        };
216
217        // Cap in-flight opens to the same upstream node. PostgreSQL's
218        // postmaster accepts and forks backends serially — slamming it
219        // with tons of simultaneous connects queues every SCRAM handshake
220        // behind the fork loop, so each individual `pool.connect` span
221        // stretches to hundreds of milliseconds. Capping the batch keeps
222        // the per-connect wall time close to the minimum and gets the
223        // earliest connections into the idle queue sooner. This semaphore
224        // is local to one `spawn_connections` call, so different nodes
225        // still run fully in parallel.
226        let permits = Arc::new(tokio::sync::Semaphore::new(PER_NODE_REFILL_CONCURRENCY));
227
228        let parent_span = tracing::Span::current();
229        let mut tasks = tokio::task::JoinSet::new();
230        for _ in 0..n {
231            let this = Arc::clone(self);
232            let key = key.clone();
233            let spec = spec.clone();
234            let permits = Arc::clone(&permits);
235
236            tasks.spawn(
237                async move {
238                    // Acquire before connecting so permits throttle the
239                    // actual handshake work, not just scheduling.
240                    let _permit = permits
241                        .acquire_owned()
242                        .await
243                        .expect("refill semaphore is never closed");
244                    match this.connect_with_spec(&key.node, &spec).await {
245                        Ok(conn) => {
246                            {
247                                let mut inner = this.inner.lock();
248                                let pool = inner.pools.entry(key.clone()).or_default();
249                                pool.idle.push_back(IdleConn {
250                                    id: ConnId::next(),
251                                    conn,
252                                    idle_since: Instant::now(),
253                                });
254                            }
255                            // New idle connection — wake a waiter if
256                            // anyone is queued for this role.
257                            this.wake_one_for_node(&key);
258                            true
259                        }
260                        Err(e) => {
261                            warn!(
262                                database = key.database,
263                                user = key.user,
264                                node = key.node,
265                                %e,
266                                "refill connection failed",
267                            );
268                            false
269                        }
270                    }
271                }
272                .instrument(parent_span.clone()),
273            );
274        }
275
276        let mut opened = 0u32;
277        while let Some(result) = tasks.join_next().await {
278            match result {
279                Ok(true) => opened += 1,
280                Ok(false) => {}
281                Err(e) => warn!(%e, "refill task panicked"),
282            }
283        }
284        let span = tracing::Span::current();
285        span.record("opened", opened);
286        // Status is OK as long as at least one connection made it into the
287        // pool. A batch where every task failed is recorded as error.
288        if opened > 0 || n == 0 {
289            span.record("otel.status_code", "OK");
290        } else {
291            span.record("otel.status_code", "ERROR");
292        }
293        opened
294    }
295
296    /// Run the idle scavenger and refill loop at a fixed interval.
297    ///
298    /// Each tick prunes expired idle connections (preserving the
299    /// `min_connections` floor), then refills any pool entry below its
300    /// floor by fanning every `spawn_connections` call into a
301    /// `JoinSet` and awaiting them concurrently — matching
302    /// [`PoolManager::warm_up`]. Running them in series here would
303    /// let a single slow node blow most of the 30-second budget and
304    /// leave unrelated pools under their `min_connections` floor.
305    /// The per-tick state is logged at debug level so the loop is
306    /// visible without needing trace-level filters.
307    pub async fn run_scavenger(self: &Arc<Self>) {
308        let interval = std::time::Duration::from_secs(30);
309        loop {
310            tokio::time::sleep(interval).await;
311            let removed = self.scavenge_idle();
312            let shortfalls = self.shortfalls();
313            debug!(removed, refills = shortfalls.len(), "scavenger tick");
314            let mut tasks = tokio::task::JoinSet::new();
315            for (key, n) in shortfalls {
316                let this = Arc::clone(self);
317                tasks.spawn(async move { this.spawn_connections(&key, n).await });
318            }
319            while let Some(result) = tasks.join_next().await {
320                if let Err(e) = result {
321                    warn!(%e, "scavenger refill task panicked");
322                }
323            }
324        }
325    }
326
327    pub(in crate::pool) async fn reset_and_return(
328        self: &Arc<Self>,
329        key: PoolKey,
330        id: ConnId,
331        mut conn: ServerConn,
332    ) {
333        match reset_connection(&mut conn).await {
334            Ok(()) => {
335                let returned = {
336                    let mut inner = self.inner.lock();
337                    if let Some(pool) = inner.pools.get_mut(&key) {
338                        pool.resetting.remove(&id);
339                        pool.idle.push_back(IdleConn {
340                            id,
341                            conn,
342                            idle_since: Instant::now(),
343                        });
344                        true
345                    } else {
346                        false
347                    }
348                };
349                if returned {
350                    self.wake_one_for_node(&key);
351                }
352            }
353            Err(e) => {
354                debug!(
355                    node = key.node,
356                    database = key.database,
357                    user = key.user,
358                    %e,
359                    "reset failed, discarding connection"
360                );
361                {
362                    let mut inner = self.inner.lock();
363                    if let Some(pool) = inner.pools.get_mut(&key) {
364                        pool.resetting.remove(&id);
365                    }
366                }
367                self.wake_one_for_node(&key);
368            }
369        }
370    }
371}