halephant_core/pool/refill.rs
1//! Background connection management: warm-up, shortfall refills,
2//! scavenger ticks, and the reset-and-return path taken by
3//! [`crate::pool::types::ConnGuard::checkin`].
4
5use std::sync::Arc;
6use std::time::Instant;
7
8use tracing::{Instrument, debug, warn};
9
10use crate::connections::reset::reset_connection;
11use crate::connections::server::ServerConn;
12use crate::pool::PoolManager;
13use crate::pool::candidates::ConnectSpec;
14use crate::pool::types::{ConnId, IdleConn, PoolKey};
15
16/// Maximum in-flight connection opens per call to
17/// [`PoolManager::spawn_connections`]. Because PostgreSQL's postmaster
18/// accepts and forks backends serially, slamming a single node with many
19/// simultaneous connects queues every SCRAM handshake behind the fork
20/// loop and inflates per-connection latency. A small cap keeps each
21/// connect close to its minimum wall time while still parallelizing
22/// across different nodes (each batch gets its own semaphore).
23const PER_NODE_REFILL_CONCURRENCY: usize = 8;
24
25impl PoolManager {
26 /// Open idle connections at startup based on `min_connections` config.
27 pub async fn warm_up(self: &Arc<Self>) {
28 let shortfalls = self.shortfalls();
29 if shortfalls.is_empty() {
30 debug!("warm-up: nothing to do");
31 return;
32 }
33
34 let mut tasks = tokio::task::JoinSet::new();
35 for (key, n) in shortfalls {
36 let this = Arc::clone(self);
37 tasks.spawn(async move { this.spawn_connections(&key, n).await });
38 }
39
40 let mut opened = 0u32;
41 while let Some(result) = tasks.join_next().await {
42 match result {
43 Ok(n) => opened += n,
44 Err(e) => warn!("warm-up task panicked: {e}"),
45 }
46 }
47 debug!(opened, "warm-up complete");
48 }
49
50 /// Remove idle connections that exceed `max_lifetime` or whose socket is
51 /// dead, then apply `idle_timeout` only to connections in excess of
52 /// `min_connections` for that pool entry.
53 ///
54 /// `max_lifetime` and dead-socket detection always apply because those
55 /// represent unhealthy connections. `idle_timeout` represents excess
56 /// capacity that can be released — so it must respect the floor set by
57 /// `min_connections`, otherwise warm-up connections decay away even when
58 /// the user explicitly asked for a minimum.
59 ///
60 /// Returns the number of connections removed across all pool entries.
61 /// Normally invoked from [`PoolManager::run_scavenger`]; exposed so tests
62 /// can drive it directly without spinning up the full background task.
63 pub fn scavenge_idle(&self) -> u32 {
64 let cfg = self.config.load_full();
65 let mut freed: Vec<(PoolKey, u32)> = Vec::new();
66 let mut removed = 0u32;
67
68 {
69 let mut inner = self.inner.lock();
70
71 for (key, pool) in &mut inner.pools {
72 let pool_config = cfg.find_pool(&key.database).map(|(_, _, p)| p);
73 let idle_timeout =
74 pool_config.map_or(std::time::Duration::from_mins(5), |p| p.idle_timeout);
75 let max_lifetime =
76 pool_config.map_or(std::time::Duration::from_hours(1), |p| p.max_lifetime);
77
78 let before = pool.idle.len();
79
80 // Remove dead sockets and connections past max_lifetime
81 // unconditionally (regardless of min_connections floor).
82 pool.idle.retain(|idle| {
83 idle.conn.created_at.elapsed() <= max_lifetime
84 && crate::connections::sock::is_alive(idle.conn.framed.get_ref())
85 });
86
87 // Apply idle_timeout only to excess capacity above
88 // the min_connections floor.
89 let min_idle = self.min_for_key(&cfg, key) as usize;
90 let removable = pool.idle.len().saturating_sub(min_idle);
91 let mut dropped = 0usize;
92 pool.idle.retain(|idle| {
93 if dropped >= removable {
94 return true;
95 }
96 if idle.idle_since.elapsed() > idle_timeout {
97 dropped += 1;
98 return false;
99 }
100 true
101 });
102
103 let n = (before - pool.idle.len()) as u32;
104 if n > 0 {
105 freed.push((key.clone(), n));
106 }
107 removed += n;
108 }
109
110 inner.waits.retain(|_, q| !q.waiters.is_empty());
111 }
112
113 // Removing idle connections frees physical capacity — wake
114 // any waiter that was blocked on node-level limits.
115 for (key, n) in &freed {
116 for _ in 0..*n {
117 self.wake_one_for_node(key);
118 }
119 }
120
121 removed
122 }
123
124 /// Enumerate every configured pool key whose connection count is below
125 /// its `min_for_key` floor, paired with the deficit. Walks the
126 /// configured `(cluster, database, user)` tuples and resolves the
127 /// expected primary and replica nodes from current topology — so the
128 /// result includes brand-new replicas that have no pool entry yet.
129 fn shortfalls(&self) -> Vec<(PoolKey, u32)> {
130 let cfg = self.config.load_full();
131 let inner = self.inner.lock();
132 let mut result = Vec::new();
133
134 for (cluster_name, cluster) in &cfg.cluster {
135 let primary = self.topology.primary(cluster_name);
136 let replicas = self.topology.replicas(cluster_name);
137
138 for (db_name, pool_config) in &cluster.pool {
139 for (user_name, user_config) in &pool_config.user {
140 // Primary shortfall.
141 if user_config.min_connections.primary > 0
142 && let Some(ref primary_addr) = primary
143 {
144 let key = PoolKey {
145 node: primary_addr.clone(),
146 database: db_name.clone(),
147 user: user_name.clone(),
148 };
149 let total = inner.pools.get(&key).map_or(0, |p| p.total() as u32);
150 let want = user_config.min_connections.primary;
151 if total < want {
152 result.push((key, want - total));
153 }
154 }
155
156 // Replica shortfall — per replica node, not distributed.
157 if user_config.min_connections.replica > 0 {
158 for replica_addr in &replicas {
159 let key = PoolKey {
160 node: replica_addr.clone(),
161 database: db_name.clone(),
162 user: user_name.clone(),
163 };
164 let total = inner.pools.get(&key).map_or(0, |p| p.total() as u32);
165 let want = user_config.min_connections.replica;
166 if total < want {
167 result.push((key, want - total));
168 }
169 }
170 }
171 }
172 }
173 }
174
175 result
176 }
177
178 /// Open up to `n` new connections for the given pool key in parallel
179 /// and insert each successful one into the idle queue. Returns the
180 /// number opened. Per-connection failures are logged and counted as
181 /// zero so a single bad node does not abort the rest of the batch.
182 ///
183 /// Wrapped in a `pool.refill` span so the per-connection `pool.connect`
184 /// spans nest under it in OTel traces.
185 #[tracing::instrument(name = "pool.refill", skip_all, fields(
186 server.address = %key.node,
187 db.namespace = %key.database,
188 user = %key.user,
189 count = n,
190 opened = tracing::field::Empty,
191 otel.status_code,
192 ))]
193 async fn spawn_connections(self: &Arc<Self>, key: &PoolKey, n: u32) -> u32 {
194 if n == 0 {
195 return 0;
196 }
197
198 // Resolve the upstream identity once for the batch through the
199 // resolver shared with checkout — refusal semantics for a
200 // missing database or user are identical on both paths.
201 // Failures are surfaced via warn, not by silently returning 0 —
202 // that would mask config drift.
203 let cfg = self.config.load_full();
204 let spec = match ConnectSpec::resolve(&cfg, &key.database, &key.user) {
205 Ok(spec) => spec,
206 Err(e) => {
207 warn!(
208 database = %key.database,
209 user = %key.user,
210 %e,
211 "refill: connect identity unresolved",
212 );
213 return 0;
214 }
215 };
216
217 // Cap in-flight opens to the same upstream node. PostgreSQL's
218 // postmaster accepts and forks backends serially — slamming it
219 // with tons of simultaneous connects queues every SCRAM handshake
220 // behind the fork loop, so each individual `pool.connect` span
221 // stretches to hundreds of milliseconds. Capping the batch keeps
222 // the per-connect wall time close to the minimum and gets the
223 // earliest connections into the idle queue sooner. This semaphore
224 // is local to one `spawn_connections` call, so different nodes
225 // still run fully in parallel.
226 let permits = Arc::new(tokio::sync::Semaphore::new(PER_NODE_REFILL_CONCURRENCY));
227
228 let parent_span = tracing::Span::current();
229 let mut tasks = tokio::task::JoinSet::new();
230 for _ in 0..n {
231 let this = Arc::clone(self);
232 let key = key.clone();
233 let spec = spec.clone();
234 let permits = Arc::clone(&permits);
235
236 tasks.spawn(
237 async move {
238 // Acquire before connecting so permits throttle the
239 // actual handshake work, not just scheduling.
240 let _permit = permits
241 .acquire_owned()
242 .await
243 .expect("refill semaphore is never closed");
244 match this.connect_with_spec(&key.node, &spec).await {
245 Ok(conn) => {
246 {
247 let mut inner = this.inner.lock();
248 let pool = inner.pools.entry(key.clone()).or_default();
249 pool.idle.push_back(IdleConn {
250 id: ConnId::next(),
251 conn,
252 idle_since: Instant::now(),
253 });
254 }
255 // New idle connection — wake a waiter if
256 // anyone is queued for this role.
257 this.wake_one_for_node(&key);
258 true
259 }
260 Err(e) => {
261 warn!(
262 database = key.database,
263 user = key.user,
264 node = key.node,
265 %e,
266 "refill connection failed",
267 );
268 false
269 }
270 }
271 }
272 .instrument(parent_span.clone()),
273 );
274 }
275
276 let mut opened = 0u32;
277 while let Some(result) = tasks.join_next().await {
278 match result {
279 Ok(true) => opened += 1,
280 Ok(false) => {}
281 Err(e) => warn!(%e, "refill task panicked"),
282 }
283 }
284 let span = tracing::Span::current();
285 span.record("opened", opened);
286 // Status is OK as long as at least one connection made it into the
287 // pool. A batch where every task failed is recorded as error.
288 if opened > 0 || n == 0 {
289 span.record("otel.status_code", "OK");
290 } else {
291 span.record("otel.status_code", "ERROR");
292 }
293 opened
294 }
295
296 /// Run the idle scavenger and refill loop at a fixed interval.
297 ///
298 /// Each tick prunes expired idle connections (preserving the
299 /// `min_connections` floor), then refills any pool entry below its
300 /// floor by fanning every `spawn_connections` call into a
301 /// `JoinSet` and awaiting them concurrently — matching
302 /// [`PoolManager::warm_up`]. Running them in series here would
303 /// let a single slow node blow most of the 30-second budget and
304 /// leave unrelated pools under their `min_connections` floor.
305 /// The per-tick state is logged at debug level so the loop is
306 /// visible without needing trace-level filters.
307 pub async fn run_scavenger(self: &Arc<Self>) {
308 let interval = std::time::Duration::from_secs(30);
309 loop {
310 tokio::time::sleep(interval).await;
311 let removed = self.scavenge_idle();
312 let shortfalls = self.shortfalls();
313 debug!(removed, refills = shortfalls.len(), "scavenger tick");
314 let mut tasks = tokio::task::JoinSet::new();
315 for (key, n) in shortfalls {
316 let this = Arc::clone(self);
317 tasks.spawn(async move { this.spawn_connections(&key, n).await });
318 }
319 while let Some(result) = tasks.join_next().await {
320 if let Err(e) = result {
321 warn!(%e, "scavenger refill task panicked");
322 }
323 }
324 }
325 }
326
327 pub(in crate::pool) async fn reset_and_return(
328 self: &Arc<Self>,
329 key: PoolKey,
330 id: ConnId,
331 mut conn: ServerConn,
332 ) {
333 match reset_connection(&mut conn).await {
334 Ok(()) => {
335 let returned = {
336 let mut inner = self.inner.lock();
337 if let Some(pool) = inner.pools.get_mut(&key) {
338 pool.resetting.remove(&id);
339 pool.idle.push_back(IdleConn {
340 id,
341 conn,
342 idle_since: Instant::now(),
343 });
344 true
345 } else {
346 false
347 }
348 };
349 if returned {
350 self.wake_one_for_node(&key);
351 }
352 }
353 Err(e) => {
354 debug!(
355 node = key.node,
356 database = key.database,
357 user = key.user,
358 %e,
359 "reset failed, discarding connection"
360 );
361 {
362 let mut inner = self.inner.lock();
363 if let Some(pool) = inner.pools.get_mut(&key) {
364 pool.resetting.remove(&id);
365 }
366 }
367 self.wake_one_for_node(&key);
368 }
369 }
370 }
371}