Skip to main content

sqlmodel_pool/
lib.rs

1//! Connection pooling for SQLModel Rust using asupersync.
2//!
3//! `sqlmodel-pool` is the **connection lifecycle layer**. It provides a generic,
4//! budget-aware pool that integrates with structured concurrency and can wrap any
5//! `Connection` implementation.
6//!
7//! # Role In The Architecture
8//!
9//! - **Shared connection management**: reuse connections across tasks safely.
10//! - **Budget-aware acquisition**: respects `Cx` timeouts and cancellation.
11//! - **Health checks**: validates connections before handing them out.
12//! - **Metrics**: exposes stats for pool sizing and tuning.
13//!
14//! # Features
15//!
16//! - Generic over any `Connection` type
17//! - RAII-based connection return (connections returned on drop)
18//! - Timeout support via `Cx` context
19//! - Connection health validation
20//! - Idle and max lifetime tracking
21//! - Pool statistics
22//!
23//! # Example
24//!
25//! ```rust,ignore
26//! use sqlmodel_pool::{Pool, PoolConfig};
27//!
28//! // Create a pool
29//! let config = PoolConfig::new(10)
30//!     .min_connections(2)
31//!     .acquire_timeout(5000);
32//!
33//! let pool = Pool::new(config, || async {
34//!     // Factory function to create new connections
35//!     PgConnection::connect(&cx, &pg_config).await
36//! });
37//!
38//! // Acquire a connection
39//! let conn = pool.acquire(&cx).await?;
40//!
41//! // Use the connection (automatically returned to pool on drop)
42//! conn.query(&cx, "SELECT 1", &[]).await?;
43//! ```
44
45pub mod replica;
46pub use replica::{ReplicaPool, ReplicaStrategy};
47
48pub mod sharding;
49pub use sharding::{ModuloShardChooser, QueryHints, ShardChooser, ShardedPool, ShardedPoolStats};
50
51use std::collections::VecDeque;
52use std::future::Future;
53use std::sync::atomic::{AtomicU64, Ordering};
54use std::sync::{Arc, Condvar, Mutex, Weak};
55use std::time::{Duration, Instant};
56
57use asupersync::{
58    CancelReason, Cx, Outcome,
59    combinator::{Either, Select},
60    runtime::RuntimeBuilder,
61    sync::OnceCell,
62};
63use sqlmodel_core::error::{ConnectionError, ConnectionErrorKind, PoolError, PoolErrorKind};
64use sqlmodel_core::{Connection, Error};
65
66/// Connection pool configuration.
67#[derive(Debug, Clone)]
68pub struct PoolConfig {
69    /// Minimum number of connections to maintain
70    pub min_connections: usize,
71    /// Maximum number of connections allowed
72    pub max_connections: usize,
73    /// Connection idle timeout in milliseconds
74    pub idle_timeout_ms: u64,
75    /// Maximum time to wait for a connection in milliseconds
76    pub acquire_timeout_ms: u64,
77    /// Maximum lifetime of a connection in milliseconds
78    pub max_lifetime_ms: u64,
79    /// Test connections before giving them out
80    pub test_on_checkout: bool,
81    /// Test connections when returning them to the pool
82    pub test_on_return: bool,
83}
84
85impl Default for PoolConfig {
86    fn default() -> Self {
87        Self {
88            min_connections: 1,
89            max_connections: 10,
90            idle_timeout_ms: 600_000,   // 10 minutes
91            acquire_timeout_ms: 30_000, // 30 seconds
92            max_lifetime_ms: 1_800_000, // 30 minutes
93            test_on_checkout: true,
94            test_on_return: false,
95        }
96    }
97}
98
99impl PoolConfig {
100    /// Create a new pool configuration with the given max connections.
101    #[must_use]
102    pub fn new(max_connections: usize) -> Self {
103        Self {
104            max_connections,
105            ..Default::default()
106        }
107    }
108
109    /// Set minimum connections.
110    #[must_use]
111    pub fn min_connections(mut self, n: usize) -> Self {
112        self.min_connections = n;
113        self
114    }
115
116    /// Set idle timeout in milliseconds.
117    #[must_use]
118    pub fn idle_timeout(mut self, ms: u64) -> Self {
119        self.idle_timeout_ms = ms;
120        self
121    }
122
123    /// Set acquire timeout in milliseconds.
124    #[must_use]
125    pub fn acquire_timeout(mut self, ms: u64) -> Self {
126        self.acquire_timeout_ms = ms;
127        self
128    }
129
130    /// Set max lifetime in milliseconds.
131    #[must_use]
132    pub fn max_lifetime(mut self, ms: u64) -> Self {
133        self.max_lifetime_ms = ms;
134        self
135    }
136
137    /// Enable/disable test on checkout.
138    #[must_use]
139    pub fn test_on_checkout(mut self, enabled: bool) -> Self {
140        self.test_on_checkout = enabled;
141        self
142    }
143
144    /// Enable/disable test on return.
145    #[must_use]
146    pub fn test_on_return(mut self, enabled: bool) -> Self {
147        self.test_on_return = enabled;
148        self
149    }
150}
151
152/// Pool statistics.
153#[derive(Debug, Clone, Default)]
154pub struct PoolStats {
155    /// Total number of connections (active + idle)
156    pub total_connections: usize,
157    /// Number of idle connections
158    pub idle_connections: usize,
159    /// Number of active connections (currently in use)
160    pub active_connections: usize,
161    /// Number of pending acquire requests
162    pub pending_requests: usize,
163    /// Total number of connections created
164    pub connections_created: u64,
165    /// Total number of connections closed
166    pub connections_closed: u64,
167    /// Total number of successful acquires
168    pub acquires: u64,
169    /// Total number of acquire timeouts
170    pub timeouts: u64,
171}
172
173/// Metadata about a pooled connection.
174#[derive(Debug)]
175struct ConnectionMeta<C> {
176    /// The actual connection
177    conn: C,
178    /// When this connection was created
179    created_at: Instant,
180    /// When this connection was last used
181    last_used: Instant,
182}
183
184impl<C> ConnectionMeta<C> {
185    fn new(conn: C) -> Self {
186        let now = Instant::now();
187        Self {
188            conn,
189            created_at: now,
190            last_used: now,
191        }
192    }
193
194    fn touch(&mut self) {
195        self.last_used = Instant::now();
196    }
197
198    fn age(&self) -> Duration {
199        self.created_at.elapsed()
200    }
201
202    fn idle_time(&self) -> Duration {
203        self.last_used.elapsed()
204    }
205}
206
207/// Internal pool state shared between pool and connections.
208struct PoolInner<C: Connection> {
209    /// Pool configuration
210    config: PoolConfig,
211    /// Idle connections available for use
212    idle: VecDeque<ConnectionMeta<C>>,
213    /// Number of connections currently checked out
214    active_count: usize,
215    /// Total number of connections (idle + active)
216    total_count: usize,
217    /// Number of waiters in the queue
218    waiter_count: usize,
219    /// Whether the pool has been closed
220    closed: bool,
221}
222
223impl<C: Connection> PoolInner<C> {
224    fn new(config: PoolConfig) -> Self {
225        Self {
226            config,
227            idle: VecDeque::new(),
228            active_count: 0,
229            total_count: 0,
230            waiter_count: 0,
231            closed: false,
232        }
233    }
234
235    fn can_create_new(&self) -> bool {
236        !self.closed && self.total_count < self.config.max_connections
237    }
238
239    fn stats(&self) -> PoolStats {
240        PoolStats {
241            total_connections: self.total_count,
242            idle_connections: self.idle.len(),
243            active_connections: self.active_count,
244            pending_requests: self.waiter_count,
245            ..Default::default()
246        }
247    }
248}
249
250/// Shared state wrapper with condition variable for notification.
251struct PoolShared<C: Connection> {
252    /// Protected pool state
253    inner: Mutex<PoolInner<C>>,
254    /// Notifies waiters when connections become available
255    conn_available: Condvar,
256    /// One-shot latch initialized when the irreversible pool drain completes.
257    active_drained: OnceCell<()>,
258    /// Stable first teardown failure, retained so every current or later
259    /// drainer observes the same fail-closed lifecycle state.
260    retirement_failure: Mutex<Option<Arc<str>>>,
261    /// Statistics counters (atomic for lock-free reads)
262    connections_created: AtomicU64,
263    connections_closed: AtomicU64,
264    acquires: AtomicU64,
265    timeouts: AtomicU64,
266}
267
268impl<C: Connection> PoolShared<C> {
269    fn new(config: PoolConfig) -> Self {
270        Self {
271            inner: Mutex::new(PoolInner::new(config)),
272            conn_available: Condvar::new(),
273            active_drained: OnceCell::new(),
274            retirement_failure: Mutex::new(None),
275            connections_created: AtomicU64::new(0),
276            connections_closed: AtomicU64::new(0),
277            acquires: AtomicU64::new(0),
278            timeouts: AtomicU64::new(0),
279        }
280    }
281
282    /// Lock the inner mutex, recovering from poisoning for read-only access.
283    ///
284    /// A poisoned mutex occurs when a thread panicked while holding the lock.
285    /// The data inside is still valid for reading, so we recover by logging
286    /// and using `into_inner()` to get the guard.
287    ///
288    /// This should only be used for read-only operations where the data is
289    /// always valid regardless of whether a previous operation completed.
290    fn lock_or_recover(&self) -> std::sync::MutexGuard<'_, PoolInner<C>> {
291        self.inner.lock().unwrap_or_else(|poisoned| {
292            tracing::error!(
293                "Pool mutex poisoned; recovering for read-only access. \
294                 A thread panicked while holding the lock."
295            );
296            poisoned.into_inner()
297        })
298    }
299
300    /// Lock the inner mutex, returning an error if poisoned.
301    ///
302    /// Use this for mutation operations where the pool state may be inconsistent
303    /// after a panic. Unlike `lock_or_recover()`, this propagates the error
304    /// to the caller.
305    #[allow(clippy::result_large_err)] // Error type is large by design for rich diagnostics
306    fn lock_or_error(
307        &self,
308        operation: &'static str,
309    ) -> Result<std::sync::MutexGuard<'_, PoolInner<C>>, Error> {
310        self.inner
311            .lock()
312            .map_err(|_| Error::Pool(PoolError::poisoned(operation)))
313    }
314
315    /// Release one active slot that is leaving the pool permanently.
316    ///
317    /// The caller must not invoke this until any required driver close has
318    /// completed. Keeping the slot active through close is what makes
319    /// `Pool::close_and_drain` a resource-quiescence boundary instead of only
320    /// a bookkeeping boundary.
321    fn release_active_slot(&self, operation: &'static str) {
322        let mut accounting_underflow = false;
323        let (drained, notify_open_waiter) = match self.inner.lock() {
324            Ok(mut inner) => {
325                if inner.active_count == 0 || inner.total_count == 0 {
326                    tracing::error!(
327                        operation,
328                        active_count = inner.active_count,
329                        total_count = inner.total_count,
330                        "attempted to release an unaccounted pool slot"
331                    );
332                    accounting_underflow = true;
333                }
334                inner.active_count = inner.active_count.saturating_sub(1);
335                inner.total_count = inner.total_count.saturating_sub(1);
336                if accounting_underflow && inner.closed && inner.active_count == 0 {
337                    inner.total_count = 0;
338                }
339                (inner.closed && inner.active_count == 0, !inner.closed)
340            }
341            Err(poisoned) => {
342                tracing::error!(
343                    operation,
344                    "Pool mutex poisoned while releasing an active slot; \
345                     recovering to prevent stranded drain accounting"
346                );
347                let error = Error::Pool(PoolError::poisoned(operation));
348                self.record_retirement_failure(operation, &error);
349                let mut inner = poisoned.into_inner();
350                if inner.active_count == 0 || inner.total_count == 0 {
351                    accounting_underflow = true;
352                }
353                inner.active_count = inner.active_count.saturating_sub(1);
354                inner.total_count = inner.total_count.saturating_sub(1);
355                if accounting_underflow && inner.closed && inner.active_count == 0 {
356                    inner.total_count = 0;
357                }
358                (inner.closed && inner.active_count == 0, !inner.closed)
359            }
360        };
361
362        if accounting_underflow {
363            let error = Error::Custom(format!(
364                "pool accounting underflow while releasing active slot during {operation}"
365            ));
366            self.record_retirement_failure(operation, &error);
367        }
368        if notify_open_waiter {
369            self.conn_available.notify_one();
370        }
371        if drained {
372            // Pool closure is irreversible, so a one-shot latch exactly models
373            // the single transition to fully drained and wakes every waiter.
374            let _ = self.active_drained.set(());
375        }
376    }
377
378    fn record_retirement_failure(&self, context: &'static str, error: &Error) {
379        let message: Arc<str> = format!("{context}: {error}").into();
380        let mut failure = self.retirement_failure.lock().unwrap_or_else(|poisoned| {
381            tracing::error!(
382                context,
383                "Pool retirement-failure mutex poisoned; recovering"
384            );
385            poisoned.into_inner()
386        });
387        if failure.is_none() {
388            *failure = Some(message);
389        }
390    }
391
392    fn retirement_failure_error(&self) -> Option<Error> {
393        let failure = self
394            .retirement_failure
395            .lock()
396            .unwrap_or_else(|poisoned| {
397                tracing::error!("Pool retirement-failure mutex poisoned; recovering");
398                poisoned.into_inner()
399            })
400            .clone();
401        failure.map(|message| Error::Custom(format!("pool retirement failed: {message}")))
402    }
403}
404
405#[allow(clippy::result_large_err)] // Error is the crate's rich public error type.
406fn close_connection_blocking<C: Connection>(conn: C, context: &'static str) -> Result<(), Error> {
407    let runtime = match RuntimeBuilder::current_thread().build() {
408        Ok(runtime) => runtime,
409        Err(error) => {
410            tracing::warn!(
411                context,
412                error = %error,
413                "failed to build runtime while closing pooled connection"
414            );
415            drop(conn);
416            return Err(Error::Custom(format!(
417                "failed to build runtime while closing pooled connection: {error}"
418            )));
419        }
420    };
421    let cx = Cx::for_testing();
422    let result = runtime.block_on(async { conn.close_for_pool(&cx).await });
423    if let Err(error) = &result {
424        tracing::warn!(
425            context,
426            error = %error,
427            "failed to close pooled connection explicitly"
428        );
429    }
430    result
431}
432
433/// Owns an active/total pool slot while an asynchronous connection factory is
434/// in flight.
435///
436/// Dropping an `acquire` future must not strand its reserved slot forever:
437/// `close_and_drain` may already be waiting for that slot to retire.
438struct ActiveSlotGuard<C: Connection> {
439    pool: Arc<PoolShared<C>>,
440    armed: bool,
441}
442
443impl<C: Connection> ActiveSlotGuard<C> {
444    fn new(pool: Arc<PoolShared<C>>) -> Self {
445        Self { pool, armed: true }
446    }
447
448    fn disarm(&mut self) {
449        self.armed = false;
450    }
451
452    fn release(&mut self, operation: &'static str) {
453        if self.armed {
454            self.armed = false;
455            self.pool.release_active_slot(operation);
456        }
457    }
458}
459
460impl<C: Connection> Drop for ActiveSlotGuard<C> {
461    fn drop(&mut self) {
462        self.release("connection factory future drop");
463    }
464}
465
466/// Owns an active connection while checkout validation is in flight.
467///
468/// On hard cancellation the guard drops the connection resource before it
469/// releases the active slot, preserving the drain boundary without requiring
470/// asynchronous work from `Drop`.
471struct ActiveConnectionGuard<C: Connection> {
472    pool: Arc<PoolShared<C>>,
473    meta: Option<ConnectionMeta<C>>,
474    armed: bool,
475}
476
477enum RetirementOutcome {
478    Closed,
479    Cancelled(CancelReason),
480    Failed(Error),
481}
482
483impl<C: Connection> ActiveConnectionGuard<C> {
484    fn new(pool: Arc<PoolShared<C>>, meta: ConnectionMeta<C>) -> Self {
485        Self {
486            pool,
487            meta: Some(meta),
488            armed: true,
489        }
490    }
491
492    fn connection(&self) -> &C {
493        &self
494            .meta
495            .as_ref()
496            .expect("active connection guard already consumed")
497            .conn
498    }
499
500    fn into_meta(mut self) -> ConnectionMeta<C> {
501        self.armed = false;
502        self.meta
503            .take()
504            .expect("active connection guard already consumed")
505    }
506
507    fn release(&mut self, operation: &'static str) {
508        if self.armed {
509            self.armed = false;
510            self.pool.connections_closed.fetch_add(1, Ordering::Relaxed);
511            self.pool.release_active_slot(operation);
512        }
513    }
514
515    async fn close(mut self, cx: &Cx, context: &'static str) -> RetirementOutcome {
516        // `close_for_pool` consumes the connection. Keep `self` (the armed
517        // accounting guard) alive first, then declare the consuming future.
518        // Rust drops locals in reverse declaration order, so hard-dropping this
519        // async function drops the later close future and its owned connection
520        // before `self` releases the active/total slot.
521        let meta = self
522            .meta
523            .take()
524            .expect("active connection guard already consumed");
525        let close_future = Box::pin(meta.conn.close_for_pool(cx));
526        let cancellation_latch = OnceCell::<()>::new();
527        let cancellation_future = Box::pin(cancellation_latch.wait(cx));
528
529        // The close hook receives `cx`, but a driver may fail to observe it.
530        // Race it against a cancel-aware one-shot wait so cancellation always
531        // drops the owned close future and resource. This is an intentional
532        // loser drop: there is no task or obligation to drain after the
533        // connection-owning future itself has been destroyed.
534        let selected = Select::new(close_future, cancellation_future).await;
535        let outcome = match selected {
536            Ok(Either::Left(result)) => match result {
537                Ok(()) => RetirementOutcome::Closed,
538                Err(error) => {
539                    tracing::warn!(
540                        context,
541                        error = %error,
542                        "failed to close pooled connection explicitly"
543                    );
544                    self.pool.record_retirement_failure(context, &error);
545                    RetirementOutcome::Failed(error)
546                }
547            },
548            Ok(Either::Right(_)) => RetirementOutcome::Cancelled(
549                cx.cancel_reason()
550                    .unwrap_or_else(|| CancelReason::user("pool retirement cancelled")),
551            ),
552            Err(error) => {
553                tracing::error!(
554                    context,
555                    error = %error,
556                    "fresh pool retirement select completed inconsistently"
557                );
558                let error = Error::Custom(format!(
559                    "pool retirement select completed inconsistently: {error}"
560                ));
561                self.pool.record_retirement_failure(context, &error);
562                RetirementOutcome::Failed(error)
563            }
564        };
565        // Record any failure above before releasing the potentially-final slot:
566        // the latch publication is the synchronization point for all drainers.
567        self.release(context);
568        outcome
569    }
570}
571
572impl<C: Connection> Drop for ActiveConnectionGuard<C> {
573    fn drop(&mut self) {
574        if !self.armed {
575            return;
576        }
577        // Async Drop is impossible. Dropping the resource itself is the
578        // hard-cancellation fallback; do that before releasing its pool slot so
579        // a concurrent drainer cannot observe quiescence too early.
580        drop(self.meta.take());
581        self.release("checkout validation future drop");
582    }
583}
584
585/// A connection pool for database connections.
586///
587/// The pool manages a collection of connections, reusing them across
588/// requests to avoid the overhead of establishing new connections.
589///
590/// # Type Parameters
591///
592/// - `C`: The connection type, must implement `Connection`
593///
594/// # Cancellation
595///
596/// Pool operations respect cancellation via the `Cx` context:
597/// - `acquire` will return early if cancellation is requested
598/// - Connections are properly cleaned up on cancellation
599pub struct Pool<C: Connection> {
600    shared: Arc<PoolShared<C>>,
601}
602
603impl<C: Connection> Pool<C> {
604    /// Create a new connection pool with the given configuration.
605    #[must_use]
606    pub fn new(config: PoolConfig) -> Self {
607        Self {
608            shared: Arc::new(PoolShared::new(config)),
609        }
610    }
611
612    /// Get the pool configuration.
613    #[must_use]
614    pub fn config(&self) -> PoolConfig {
615        let inner = self.shared.lock_or_recover();
616        inner.config.clone()
617    }
618
619    /// Get the current pool statistics.
620    #[must_use]
621    pub fn stats(&self) -> PoolStats {
622        let inner = self.shared.lock_or_recover();
623        let mut stats = inner.stats();
624        stats.connections_created = self.shared.connections_created.load(Ordering::Relaxed);
625        stats.connections_closed = self.shared.connections_closed.load(Ordering::Relaxed);
626        stats.acquires = self.shared.acquires.load(Ordering::Relaxed);
627        stats.timeouts = self.shared.timeouts.load(Ordering::Relaxed);
628        stats
629    }
630
631    /// Check if the pool is at capacity.
632    #[must_use]
633    pub fn at_capacity(&self) -> bool {
634        let inner = self.shared.lock_or_recover();
635        inner.total_count >= inner.config.max_connections
636    }
637
638    /// Check if the pool has been closed.
639    #[must_use]
640    pub fn is_closed(&self) -> bool {
641        let inner = self.shared.lock_or_recover();
642        inner.closed
643    }
644
645    /// Acquire a connection from the pool.
646    ///
647    /// This method will:
648    /// 1. Return an idle connection if one is available
649    /// 2. Create a new connection if below capacity
650    /// 3. Wait for a connection to become available (up to timeout)
651    ///
652    /// # Errors
653    ///
654    /// Returns an error if:
655    /// - The pool is closed
656    /// - The acquire timeout is exceeded
657    /// - Cancellation is requested via the `Cx` context
658    /// - Connection validation fails (if `test_on_checkout` is enabled)
659    pub async fn acquire<F, Fut>(&self, cx: &Cx, factory: F) -> Outcome<PooledConnection<C>, Error>
660    where
661        F: Fn() -> Fut,
662        Fut: Future<Output = Outcome<C, Error>>,
663    {
664        let deadline = Instant::now() + Duration::from_millis(self.config().acquire_timeout_ms);
665        let test_on_checkout = self.config().test_on_checkout;
666        let max_lifetime = Duration::from_millis(self.config().max_lifetime_ms);
667        let idle_timeout = Duration::from_millis(self.config().idle_timeout_ms);
668
669        loop {
670            // Check cancellation
671            if cx.is_cancel_requested() {
672                return Outcome::Cancelled(CancelReason::user("pool acquire cancelled"));
673            }
674
675            // Check timeout
676            if Instant::now() >= deadline {
677                self.shared.timeouts.fetch_add(1, Ordering::Relaxed);
678                return Outcome::Err(Error::Pool(PoolError {
679                    kind: PoolErrorKind::Timeout,
680                    message: "acquire timeout: no connections available".to_string(),
681                    source: None,
682                }));
683            }
684
685            // Try to get an idle connection or determine if we can create new
686            let (action, retired) = {
687                let mut inner = match self.shared.lock_or_error("acquire") {
688                    Ok(guard) => guard,
689                    Err(e) => return Outcome::Err(e),
690                };
691                // Reserve before moving any idle entry into active retirement
692                // accounting. Every removed entry is immediately protected by
693                // an armed guard, so panic or hard cancellation cannot strand
694                // later entries in a raw vector.
695                let mut retired = Vec::with_capacity(inner.idle.len());
696
697                let action = if inner.closed {
698                    AcquireAction::PoolClosed
699                } else {
700                    // Try to get an idle connection
701                    let mut found_conn = None;
702                    while let Some(mut meta) = inner.idle.pop_front() {
703                        // Check if connection is too old
704                        if meta.age() > max_lifetime {
705                            inner.active_count += 1;
706                            retired
707                                .push(ActiveConnectionGuard::new(Arc::clone(&self.shared), meta));
708                            continue;
709                        }
710
711                        // Check if connection has been idle too long
712                        if meta.idle_time() > idle_timeout {
713                            inner.active_count += 1;
714                            retired
715                                .push(ActiveConnectionGuard::new(Arc::clone(&self.shared), meta));
716                            continue;
717                        }
718
719                        if !retired.is_empty() {
720                            // Retire removed connections before selecting or
721                            // reserving an active slot. Otherwise dropping the
722                            // acquire future during an async retirement close
723                            // would strand that selected slot forever.
724                            inner.idle.push_front(meta);
725                            break;
726                        }
727
728                        // Found a valid connection
729                        meta.touch();
730                        inner.active_count += 1;
731                        found_conn = Some(meta);
732                        break;
733                    }
734
735                    if !retired.is_empty() {
736                        AcquireAction::RetireAndRetry
737                    } else if let Some(meta) = found_conn {
738                        AcquireAction::ValidateExisting(meta)
739                    } else if inner.can_create_new() {
740                        // No idle connections, can we create new?
741                        inner.total_count += 1;
742                        inner.active_count += 1;
743                        AcquireAction::CreateNew
744                    } else {
745                        // Must wait
746                        inner.waiter_count += 1;
747                        AcquireAction::Wait
748                    }
749                };
750                (action, retired)
751            };
752
753            // Teardown may perform driver I/O. Keep it outside the pool mutex
754            // so one slow close cannot block returns, acquires, or shutdown.
755            for guard in retired {
756                match guard
757                    .close(cx, "expired pooled connection retirement")
758                    .await
759                {
760                    RetirementOutcome::Closed => {}
761                    RetirementOutcome::Cancelled(reason) => {
762                        return Outcome::Cancelled(reason);
763                    }
764                    RetirementOutcome::Failed(error) => return Outcome::Err(error),
765                }
766            }
767
768            match action {
769                AcquireAction::RetireAndRetry => {
770                    continue;
771                }
772                AcquireAction::PoolClosed => {
773                    return Outcome::Err(Error::Pool(PoolError {
774                        kind: PoolErrorKind::Closed,
775                        message: "pool has been closed".to_string(),
776                        source: None,
777                    }));
778                }
779                AcquireAction::ValidateExisting(meta) => {
780                    // Validate and wrap the connection (lock is released)
781                    return self.validate_and_wrap(cx, meta, test_on_checkout).await;
782                }
783                AcquireAction::CreateNew => {
784                    // Create new connection outside of lock
785                    let mut slot_guard = ActiveSlotGuard::new(Arc::clone(&self.shared));
786                    match factory().await {
787                        Outcome::Ok(conn) => {
788                            self.shared
789                                .connections_created
790                                .fetch_add(1, Ordering::Relaxed);
791                            let publish = match self.shared.lock_or_error("acquire_publish") {
792                                Ok(inner) if inner.closed => {
793                                    let retirement = ActiveConnectionGuard::new(
794                                        Arc::clone(&self.shared),
795                                        ConnectionMeta::new(conn),
796                                    );
797                                    slot_guard.disarm();
798                                    FactoryPublish::Retire {
799                                        guard: retirement,
800                                        error: Error::Pool(PoolError {
801                                            kind: PoolErrorKind::Closed,
802                                            message: "pool has been closed".to_string(),
803                                            source: None,
804                                        }),
805                                        context: "acquire publish after pool closure",
806                                    }
807                                }
808                                Ok(_inner) => {
809                                    // Disarming while still holding the pool
810                                    // mutex is the admission publication point.
811                                    // A close either precedes this check and
812                                    // rejects the connection, or follows it and
813                                    // waits for this active checkout.
814                                    self.shared.acquires.fetch_add(1, Ordering::Relaxed);
815                                    let meta = ConnectionMeta::new(conn);
816                                    slot_guard.disarm();
817                                    FactoryPublish::Published(PooledConnection::new(
818                                        meta,
819                                        Arc::downgrade(&self.shared),
820                                    ))
821                                }
822                                Err(error) => {
823                                    let retirement = ActiveConnectionGuard::new(
824                                        Arc::clone(&self.shared),
825                                        ConnectionMeta::new(conn),
826                                    );
827                                    slot_guard.disarm();
828                                    FactoryPublish::Retire {
829                                        guard: retirement,
830                                        error,
831                                        context: "acquire publish bookkeeping failure",
832                                    }
833                                }
834                            };
835                            match publish {
836                                FactoryPublish::Published(pooled) => {
837                                    return Outcome::Ok(pooled);
838                                }
839                                FactoryPublish::Retire {
840                                    guard,
841                                    error,
842                                    context,
843                                } => match guard.close(cx, context).await {
844                                    RetirementOutcome::Closed => return Outcome::Err(error),
845                                    RetirementOutcome::Cancelled(reason) => {
846                                        return Outcome::Cancelled(reason);
847                                    }
848                                    RetirementOutcome::Failed(close_error) => {
849                                        return Outcome::Err(close_error);
850                                    }
851                                },
852                            }
853                        }
854                        Outcome::Err(e) => {
855                            slot_guard.release("connection factory error");
856                            return Outcome::Err(e);
857                        }
858                        Outcome::Cancelled(reason) => {
859                            slot_guard.release("connection factory cancellation");
860                            return Outcome::Cancelled(reason);
861                        }
862                        Outcome::Panicked(info) => {
863                            slot_guard.release("connection factory panic");
864                            return Outcome::Panicked(info);
865                        }
866                    }
867                }
868                AcquireAction::Wait => {
869                    // Wait for a connection to become available
870                    let remaining = deadline.saturating_duration_since(Instant::now());
871                    if remaining.is_zero() {
872                        if let Ok(mut inner) = self.shared.lock_or_error("acquire_timeout") {
873                            inner.waiter_count -= 1;
874                        }
875                        self.shared.timeouts.fetch_add(1, Ordering::Relaxed);
876                        return Outcome::Err(Error::Pool(PoolError {
877                            kind: PoolErrorKind::Timeout,
878                            message: "acquire timeout: no connections available".to_string(),
879                            source: None,
880                        }));
881                    }
882
883                    // Wait with timeout (use shorter interval for cancellation checks)
884                    let wait_time = remaining.min(Duration::from_millis(100));
885                    {
886                        let inner = match self.shared.lock_or_error("acquire_wait") {
887                            Ok(guard) => guard,
888                            Err(e) => return Outcome::Err(e),
889                        };
890                        // wait_timeout can also return a poisoned error, handle it
891                        let _ = self
892                            .shared
893                            .conn_available
894                            .wait_timeout(inner, wait_time)
895                            .map_err(|_| {
896                                tracing::error!("Pool mutex poisoned during wait_timeout");
897                            });
898                    }
899
900                    // Decrement waiter count after waking
901                    {
902                        if let Ok(mut inner) = self.shared.lock_or_error("acquire_wake") {
903                            inner.waiter_count = inner.waiter_count.saturating_sub(1);
904                        }
905                    }
906
907                    // Loop back to try again
908                }
909            }
910        }
911    }
912
913    /// Validate a connection and wrap it in a PooledConnection.
914    async fn validate_and_wrap(
915        &self,
916        cx: &Cx,
917        meta: ConnectionMeta<C>,
918        test_on_checkout: bool,
919    ) -> Outcome<PooledConnection<C>, Error> {
920        let guard = ActiveConnectionGuard::new(Arc::clone(&self.shared), meta);
921        if test_on_checkout {
922            // Validate the connection
923            match guard.connection().ping(cx).await {
924                Outcome::Ok(()) => {
925                    self.shared.acquires.fetch_add(1, Ordering::Relaxed);
926                    Outcome::Ok(PooledConnection::new(
927                        guard.into_meta(),
928                        Arc::downgrade(&self.shared),
929                    ))
930                }
931                Outcome::Err(_) | Outcome::Cancelled(_) | Outcome::Panicked(_) => {
932                    // Connection is invalid. Keep it active until explicit
933                    // retirement completes so close-and-drain cannot return
934                    // while the driver still owns the resource.
935                    match guard
936                        .close(cx, "pooled connection checkout validation failure")
937                        .await
938                    {
939                        RetirementOutcome::Closed => {}
940                        RetirementOutcome::Cancelled(reason) => {
941                            return Outcome::Cancelled(reason);
942                        }
943                        RetirementOutcome::Failed(error) => return Outcome::Err(error),
944                    }
945                    // Return error - caller should retry
946                    Outcome::Err(Error::Connection(ConnectionError {
947                        kind: ConnectionErrorKind::Disconnected,
948                        message: "connection validation failed".to_string(),
949                        source: None,
950                    }))
951                }
952            }
953        } else {
954            self.shared.acquires.fetch_add(1, Ordering::Relaxed);
955            Outcome::Ok(PooledConnection::new(
956                guard.into_meta(),
957                Arc::downgrade(&self.shared),
958            ))
959        }
960    }
961
962    /// Remove every idle connection from admission and transfer it to active
963    /// retirement accounting.
964    ///
965    /// `total_count` deliberately remains unchanged until each retirement
966    /// guard has finished or been hard-dropped. This lets `close_and_drain`
967    /// treat driver teardown as part of the quiescence boundary.
968    fn begin_idle_retirement(&self) -> Vec<ActiveConnectionGuard<C>> {
969        match self.shared.inner.lock() {
970            Ok(mut inner) => {
971                let mut retired = Vec::with_capacity(inner.idle.len());
972                while let Some(meta) = inner.idle.pop_front() {
973                    inner.active_count += 1;
974                    retired.push(ActiveConnectionGuard::new(Arc::clone(&self.shared), meta));
975                }
976                retired
977            }
978            Err(_poisoned) => {
979                tracing::error!(
980                    "Pool mutex poisoned during idle retirement; \
981                     idle connections cannot be retired safely"
982                );
983                Vec::new()
984            }
985        }
986    }
987
988    /// Atomically close admission and transfer all idle inventory to active
989    /// retirement accounting.
990    fn begin_close(&self) -> Vec<ActiveConnectionGuard<C>> {
991        let retired = match self.shared.inner.lock() {
992            Ok(mut inner) => {
993                inner.closed = true;
994                let mut retired = Vec::with_capacity(inner.idle.len());
995                while let Some(meta) = inner.idle.pop_front() {
996                    inner.active_count += 1;
997                    retired.push(ActiveConnectionGuard::new(Arc::clone(&self.shared), meta));
998                }
999                retired
1000            }
1001            Err(poisoned) => {
1002                // Recover from poisoning - we still want to mark the pool as
1003                // closed and wake waiters even if counts may be inconsistent.
1004                tracing::error!(
1005                    "Pool mutex poisoned during close; attempting recovery. \
1006                     Pool state may be inconsistent."
1007                );
1008                let mut inner = poisoned.into_inner();
1009                inner.closed = true;
1010                let mut retired = Vec::with_capacity(inner.idle.len());
1011                while let Some(meta) = inner.idle.pop_front() {
1012                    inner.active_count += 1;
1013                    retired.push(ActiveConnectionGuard::new(Arc::clone(&self.shared), meta));
1014                }
1015                retired
1016            }
1017        };
1018
1019        // Wake all waiters so they see the pool is closed
1020        self.shared.conn_available.notify_all();
1021        retired
1022    }
1023
1024    fn close_retirements_blocking(
1025        &self,
1026        retired: Vec<ActiveConnectionGuard<C>>,
1027        context: &'static str,
1028    ) {
1029        for guard in retired {
1030            let runtime = match RuntimeBuilder::current_thread().build() {
1031                Ok(runtime) => runtime,
1032                Err(error) => {
1033                    tracing::warn!(
1034                        context,
1035                        error = %error,
1036                        "failed to build runtime while retiring pooled connection"
1037                    );
1038                    let error = Error::Custom(format!(
1039                        "failed to build runtime while retiring pooled connection: {error}"
1040                    ));
1041                    self.shared.record_retirement_failure(context, &error);
1042                    drop(guard);
1043                    continue;
1044                }
1045            };
1046            let cx = Cx::for_testing();
1047            let _ = runtime.block_on(guard.close(&cx, context));
1048        }
1049    }
1050
1051    /// Close all currently idle connections.
1052    ///
1053    /// If the pool mutex is poisoned, this logs an error and leaves the idle
1054    /// inventory untouched because its accounting cannot be mutated safely.
1055    pub fn clear_idle(&self) {
1056        let retired = self.begin_idle_retirement();
1057        self.close_retirements_blocking(retired, "pool clear_idle");
1058    }
1059
1060    /// Close the pool, preventing new connections and closing all idle connections.
1061    ///
1062    /// If the pool mutex is poisoned, this logs an error but still wakes waiters.
1063    pub fn close(&self) {
1064        let retired = self.begin_close();
1065        self.close_retirements_blocking(retired, "pool close");
1066
1067        // Closing an already-empty pool is itself the one and only drain
1068        // transition. Accounted retirements set the latch when their final
1069        // guard releases.
1070        if self.shared.lock_or_recover().active_count == 0 {
1071            let _ = self.shared.active_drained.set(());
1072        }
1073    }
1074
1075    /// Close the pool and wait for every pool-owned active connection to retire.
1076    ///
1077    /// Closing admission and removing idle inventory happen synchronously
1078    /// before this method first yields. Blocked acquirers are woken and observe
1079    /// [`PoolErrorKind::Closed`]. Connections already checked out are closed
1080    /// when returned; this future completes only after their explicit
1081    /// `close_for_pool` hooks finish and the active count reaches zero.
1082    ///
1083    /// The wait is cancellation- and deadline-aware through `cx`. Cancellation
1084    /// returns [`Outcome::Cancelled`] without reopening the pool: `closed`
1085    /// remains a one-way lifecycle transition, and a later caller may resume
1086    /// draining. Dropping this future has the same persistent-close property.
1087    ///
1088    /// Multiple concurrent drainers are supported by the pool's one-shot drain
1089    /// latch. Since pool closure is irreversible, there is no reopen generation
1090    /// whose notification could be confused with this drain cycle.
1091    ///
1092    /// Driver teardown failures are sticky and fail closed: the pool still
1093    /// retires every accounted resource, but this and every later drainer
1094    /// returns [`Outcome::Err`] after the active count reaches zero.
1095    ///
1096    /// A connection removed with [`PooledConnection::detach`] is caller-owned
1097    /// and is no longer part of pool accounting, so it is outside this drain
1098    /// guarantee.
1099    pub async fn close_and_drain(&self, cx: &Cx) -> Outcome<(), Error> {
1100        // The irreversible lifecycle transition and waiter wake happen before
1101        // any cancellation point. Idle inventory remains accounted as active
1102        // retirement work until each close hook or hard drop releases it.
1103        let retired = self.begin_close();
1104        let mut direct_failure = None;
1105        for guard in retired {
1106            match guard.close(cx, "pool close-and-drain").await {
1107                RetirementOutcome::Closed => {}
1108                RetirementOutcome::Cancelled(reason) => {
1109                    return Outcome::Cancelled(reason);
1110                }
1111                RetirementOutcome::Failed(error) => {
1112                    direct_failure.get_or_insert(error);
1113                }
1114            }
1115        }
1116
1117        let poisoned_failure = match self.shared.inner.lock() {
1118            Ok(inner) => {
1119                drop(inner);
1120                None
1121            }
1122            Err(poisoned) => {
1123                let error = Error::Pool(PoolError::poisoned("close_and_drain"));
1124                self.shared
1125                    .record_retirement_failure("close_and_drain", &error);
1126                drop(poisoned.into_inner());
1127                Some(error)
1128            }
1129        };
1130
1131        if self.shared.lock_or_recover().active_count == 0 {
1132            let _ = self.shared.active_drained.set(());
1133        }
1134
1135        if self.shared.active_drained.wait(cx).await.is_ok() {
1136            if let Some(error) = direct_failure {
1137                Outcome::Err(error)
1138            } else if let Some(error) = poisoned_failure {
1139                Outcome::Err(error)
1140            } else if let Some(error) = self.shared.retirement_failure_error() {
1141                Outcome::Err(error)
1142            } else {
1143                Outcome::Ok(())
1144            }
1145        } else {
1146            let reason = cx
1147                .cancel_reason()
1148                .unwrap_or_else(|| CancelReason::user("pool drain cancelled"));
1149            Outcome::Cancelled(reason)
1150        }
1151    }
1152
1153    /// Get the number of idle connections.
1154    #[must_use]
1155    pub fn idle_count(&self) -> usize {
1156        let inner = self.shared.lock_or_recover();
1157        inner.idle.len()
1158    }
1159
1160    /// Get the number of active connections.
1161    #[must_use]
1162    pub fn active_count(&self) -> usize {
1163        let inner = self.shared.lock_or_recover();
1164        inner.active_count
1165    }
1166
1167    /// Get the total number of connections.
1168    #[must_use]
1169    pub fn total_count(&self) -> usize {
1170        let inner = self.shared.lock_or_recover();
1171        inner.total_count
1172    }
1173}
1174
1175impl<C: Connection> Drop for Pool<C> {
1176    fn drop(&mut self) {
1177        self.close();
1178    }
1179}
1180
1181/// Action to take when acquiring a connection.
1182enum AcquireAction<C> {
1183    /// Expired idle connections were removed and must retire before retrying.
1184    RetireAndRetry,
1185    /// Pool is closed
1186    PoolClosed,
1187    /// Found an existing connection to validate
1188    ValidateExisting(ConnectionMeta<C>),
1189    /// Create a new connection
1190    CreateNew,
1191    /// Wait for a connection to become available
1192    Wait,
1193}
1194
1195enum FactoryPublish<C: Connection> {
1196    Published(PooledConnection<C>),
1197    Retire {
1198        guard: ActiveConnectionGuard<C>,
1199        error: Error,
1200        context: &'static str,
1201    },
1202}
1203
1204/// A connection borrowed from the pool.
1205///
1206/// When dropped, the connection is automatically returned to the pool.
1207/// The connection can be used via `Deref` and `DerefMut`.
1208pub struct PooledConnection<C: Connection> {
1209    /// The connection metadata (Some while held, None after return)
1210    meta: Option<ConnectionMeta<C>>,
1211    /// Weak reference to pool for returning
1212    pool: Weak<PoolShared<C>>,
1213}
1214
1215impl<C: Connection> PooledConnection<C> {
1216    fn new(meta: ConnectionMeta<C>, pool: Weak<PoolShared<C>>) -> Self {
1217        Self {
1218            meta: Some(meta),
1219            pool,
1220        }
1221    }
1222
1223    /// Detach this connection from the pool.
1224    ///
1225    /// The connection will not be returned to the pool when dropped.
1226    /// This is useful when you need to close a connection explicitly.
1227    pub fn detach(mut self) -> C {
1228        let conn = self.meta.take().expect("connection already detached").conn;
1229        if let Some(pool) = self.pool.upgrade() {
1230            pool.connections_closed.fetch_add(1, Ordering::Relaxed);
1231            pool.release_active_slot("pooled connection detach");
1232        }
1233        conn
1234    }
1235
1236    /// Get the age of this connection (time since creation).
1237    #[must_use]
1238    pub fn age(&self) -> Duration {
1239        self.meta.as_ref().map_or(Duration::ZERO, |m| m.age())
1240    }
1241
1242    /// Get the idle time of this connection (time since last use).
1243    #[must_use]
1244    pub fn idle_time(&self) -> Duration {
1245        self.meta.as_ref().map_or(Duration::ZERO, |m| m.idle_time())
1246    }
1247}
1248
1249impl<C: Connection> std::ops::Deref for PooledConnection<C> {
1250    type Target = C;
1251
1252    fn deref(&self) -> &Self::Target {
1253        &self
1254            .meta
1255            .as_ref()
1256            .expect("connection already returned to pool")
1257            .conn
1258    }
1259}
1260
1261impl<C: Connection> std::ops::DerefMut for PooledConnection<C> {
1262    fn deref_mut(&mut self) -> &mut Self::Target {
1263        &mut self
1264            .meta
1265            .as_mut()
1266            .expect("connection already returned to pool")
1267            .conn
1268    }
1269}
1270
1271impl<C: Connection> Drop for PooledConnection<C> {
1272    fn drop(&mut self) {
1273        if let Some(mut meta) = self.meta.take() {
1274            meta.touch(); // Update last used time
1275            if let Some(pool) = self.pool.upgrade() {
1276                // Return to pool - but if mutex is poisoned, do not panic in
1277                // Drop. Close the resource, then recover only enough accounting
1278                // to prevent a drain waiter from being stranded.
1279                let mut inner = match pool.inner.lock() {
1280                    Ok(guard) => guard,
1281                    Err(poisoned) => {
1282                        // Release the poisoned guard before the close hook and
1283                        // poison-aware accounting transition reacquire state.
1284                        drop(poisoned);
1285                        tracing::error!(
1286                            "Pool mutex poisoned during connection return; \
1287                             connection will be closed instead of returned. A thread panicked while holding the lock."
1288                        );
1289                        let context = "pooled connection drop poisoned";
1290                        if let Err(error) = close_connection_blocking(meta.conn, context) {
1291                            pool.record_retirement_failure(context, &error);
1292                        }
1293                        pool.connections_closed.fetch_add(1, Ordering::Relaxed);
1294                        pool.release_active_slot(context);
1295                        return;
1296                    }
1297                };
1298
1299                if inner.closed {
1300                    drop(inner);
1301                    let context = "pooled connection drop closed pool";
1302                    if let Err(error) = close_connection_blocking(meta.conn, context) {
1303                        pool.record_retirement_failure(context, &error);
1304                    }
1305                    pool.connections_closed.fetch_add(1, Ordering::Relaxed);
1306                    pool.release_active_slot("pooled connection drop closed pool");
1307                    return;
1308                }
1309
1310                // Check max lifetime
1311                let max_lifetime = Duration::from_millis(inner.config.max_lifetime_ms);
1312                if meta.age() > max_lifetime {
1313                    drop(inner);
1314                    let context = "pooled connection drop max lifetime";
1315                    if let Err(error) = close_connection_blocking(meta.conn, context) {
1316                        pool.record_retirement_failure(context, &error);
1317                    }
1318                    pool.connections_closed.fetch_add(1, Ordering::Relaxed);
1319                    pool.release_active_slot("pooled connection drop max lifetime");
1320                    return;
1321                }
1322
1323                inner.active_count -= 1;
1324                inner.idle.push_back(meta);
1325
1326                drop(inner);
1327                pool.conn_available.notify_one();
1328            } else {
1329                let _ = close_connection_blocking(meta.conn, "pooled connection drop missing pool");
1330            }
1331        }
1332    }
1333}
1334
1335impl<C: Connection + std::fmt::Debug> std::fmt::Debug for PooledConnection<C> {
1336    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1337        f.debug_struct("PooledConnection")
1338            .field("conn", &self.meta.as_ref().map(|m| &m.conn))
1339            .field("age", &self.age())
1340            .field("idle_time", &self.idle_time())
1341            .finish_non_exhaustive()
1342    }
1343}
1344
1345#[cfg(test)]
1346mod tests {
1347    use super::*;
1348    use asupersync::{Budget, Time};
1349    use sqlmodel_core::connection::{IsolationLevel, PreparedStatement, TransactionOps};
1350    use sqlmodel_core::{Row, Value};
1351    use std::pin::Pin;
1352    use std::sync::atomic::{AtomicBool, AtomicUsize};
1353    use std::task::{Context, Poll, Wake, Waker};
1354
1355    /// A mock connection for testing pool behavior.
1356    #[derive(Debug)]
1357    struct MockConnection {
1358        id: u32,
1359        ping_should_fail: Arc<AtomicBool>,
1360        /// Incremented each time the pool retires this connection via
1361        /// `close_for_pool` (as opposed to a caller-owned `close`).
1362        pool_close_calls: Arc<AtomicUsize>,
1363        /// When true, `close_for_pool` remains pending until its future is
1364        /// dropped. Used to prove slot guards across close await points.
1365        pool_close_pending: bool,
1366        /// When true, pool retirement returns a deterministic driver error.
1367        pool_close_should_fail: bool,
1368        /// Optional probe used to prove the pool mutex is not held while the
1369        /// retirement hook runs.
1370        pool_shared: Option<Weak<PoolShared<MockConnection>>>,
1371        pool_lock_was_free: Option<Arc<AtomicBool>>,
1372    }
1373
1374    impl MockConnection {
1375        fn new(id: u32) -> Self {
1376            Self {
1377                id,
1378                ping_should_fail: Arc::new(AtomicBool::new(false)),
1379                pool_close_calls: Arc::new(AtomicUsize::new(0)),
1380                pool_close_pending: false,
1381                pool_close_should_fail: false,
1382                pool_shared: None,
1383                pool_lock_was_free: None,
1384            }
1385        }
1386
1387        #[allow(dead_code)]
1388        fn with_ping_behavior(id: u32, should_fail: Arc<AtomicBool>) -> Self {
1389            Self {
1390                id,
1391                ping_should_fail: should_fail,
1392                pool_close_calls: Arc::new(AtomicUsize::new(0)),
1393                pool_close_pending: false,
1394                pool_close_should_fail: false,
1395                pool_shared: None,
1396                pool_lock_was_free: None,
1397            }
1398        }
1399
1400        fn with_pool_close_counter(id: u32, pool_close_calls: Arc<AtomicUsize>) -> Self {
1401            Self {
1402                id,
1403                ping_should_fail: Arc::new(AtomicBool::new(false)),
1404                pool_close_calls,
1405                pool_close_pending: false,
1406                pool_close_should_fail: false,
1407                pool_shared: None,
1408                pool_lock_was_free: None,
1409            }
1410        }
1411
1412        fn with_pool_close_probe(
1413            id: u32,
1414            pool_close_calls: Arc<AtomicUsize>,
1415            pool_shared: Weak<PoolShared<MockConnection>>,
1416            pool_lock_was_free: Arc<AtomicBool>,
1417        ) -> Self {
1418            Self {
1419                id,
1420                ping_should_fail: Arc::new(AtomicBool::new(false)),
1421                pool_close_calls,
1422                pool_close_pending: false,
1423                pool_close_should_fail: false,
1424                pool_shared: Some(pool_shared),
1425                pool_lock_was_free: Some(pool_lock_was_free),
1426            }
1427        }
1428
1429        fn with_pending_pool_close(id: u32, pool_close_calls: Arc<AtomicUsize>) -> Self {
1430            Self {
1431                id,
1432                ping_should_fail: Arc::new(AtomicBool::new(false)),
1433                pool_close_calls,
1434                pool_close_pending: true,
1435                pool_close_should_fail: false,
1436                pool_shared: None,
1437                pool_lock_was_free: None,
1438            }
1439        }
1440
1441        fn with_failing_pool_close(id: u32) -> Self {
1442            Self {
1443                id,
1444                ping_should_fail: Arc::new(AtomicBool::new(false)),
1445                pool_close_calls: Arc::new(AtomicUsize::new(0)),
1446                pool_close_pending: false,
1447                pool_close_should_fail: true,
1448                pool_shared: None,
1449                pool_lock_was_free: None,
1450            }
1451        }
1452    }
1453
1454    /// Manually released connection factory used to put `acquire` exactly
1455    /// across the pool-close publication fence without timing sleeps.
1456    struct GatedFactory {
1457        ready: Arc<AtomicBool>,
1458        conn: Option<MockConnection>,
1459    }
1460
1461    #[derive(Default)]
1462    struct WakeCounter {
1463        wakes: AtomicUsize,
1464    }
1465
1466    impl Wake for WakeCounter {
1467        fn wake(self: Arc<Self>) {
1468            self.wakes.fetch_add(1, Ordering::Relaxed);
1469        }
1470
1471        fn wake_by_ref(self: &Arc<Self>) {
1472            self.wakes.fetch_add(1, Ordering::Relaxed);
1473        }
1474    }
1475
1476    impl Future for GatedFactory {
1477        type Output = Outcome<MockConnection, Error>;
1478
1479        fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
1480            if self.ready.load(Ordering::Acquire) {
1481                Poll::Ready(Outcome::Ok(
1482                    self.conn
1483                        .take()
1484                        .expect("gated factory polled after completion"),
1485                ))
1486            } else {
1487                Poll::Pending
1488            }
1489        }
1490    }
1491
1492    /// Mock transaction for MockConnection.
1493    struct MockTx;
1494
1495    // These test doubles deliberately mirror the trait's async spelling; the
1496    // bodies are immediate because no real driver I/O occurs.
1497    #[allow(clippy::unused_async_trait_impl)]
1498    impl TransactionOps for MockTx {
1499        async fn query(&self, _cx: &Cx, _sql: &str, _params: &[Value]) -> Outcome<Vec<Row>, Error> {
1500            Outcome::Ok(vec![])
1501        }
1502
1503        async fn query_one(
1504            &self,
1505            _cx: &Cx,
1506            _sql: &str,
1507            _params: &[Value],
1508        ) -> Outcome<Option<Row>, Error> {
1509            Outcome::Ok(None)
1510        }
1511
1512        async fn execute(&self, _cx: &Cx, _sql: &str, _params: &[Value]) -> Outcome<u64, Error> {
1513            Outcome::Ok(0)
1514        }
1515
1516        async fn savepoint(&self, _cx: &Cx, _name: &str) -> Outcome<(), Error> {
1517            Outcome::Ok(())
1518        }
1519
1520        async fn rollback_to(&self, _cx: &Cx, _name: &str) -> Outcome<(), Error> {
1521            Outcome::Ok(())
1522        }
1523
1524        async fn release(&self, _cx: &Cx, _name: &str) -> Outcome<(), Error> {
1525            Outcome::Ok(())
1526        }
1527
1528        async fn commit(self, _cx: &Cx) -> Outcome<(), Error> {
1529            Outcome::Ok(())
1530        }
1531
1532        async fn rollback(self, _cx: &Cx) -> Outcome<(), Error> {
1533            Outcome::Ok(())
1534        }
1535    }
1536
1537    #[allow(clippy::unused_async_trait_impl)]
1538    impl Connection for MockConnection {
1539        type Tx<'conn> = MockTx;
1540
1541        async fn query(&self, _cx: &Cx, _sql: &str, _params: &[Value]) -> Outcome<Vec<Row>, Error> {
1542            Outcome::Ok(vec![])
1543        }
1544
1545        async fn query_one(
1546            &self,
1547            _cx: &Cx,
1548            _sql: &str,
1549            _params: &[Value],
1550        ) -> Outcome<Option<Row>, Error> {
1551            Outcome::Ok(None)
1552        }
1553
1554        async fn execute(&self, _cx: &Cx, _sql: &str, _params: &[Value]) -> Outcome<u64, Error> {
1555            Outcome::Ok(0)
1556        }
1557
1558        async fn insert(&self, _cx: &Cx, _sql: &str, _params: &[Value]) -> Outcome<i64, Error> {
1559            Outcome::Ok(0)
1560        }
1561
1562        async fn batch(
1563            &self,
1564            _cx: &Cx,
1565            _statements: &[(String, Vec<Value>)],
1566        ) -> Outcome<Vec<u64>, Error> {
1567            Outcome::Ok(vec![])
1568        }
1569
1570        async fn begin(&self, _cx: &Cx) -> Outcome<Self::Tx<'_>, Error> {
1571            Outcome::Ok(MockTx)
1572        }
1573
1574        async fn begin_with(
1575            &self,
1576            _cx: &Cx,
1577            _isolation: IsolationLevel,
1578        ) -> Outcome<Self::Tx<'_>, Error> {
1579            Outcome::Ok(MockTx)
1580        }
1581
1582        async fn prepare(&self, _cx: &Cx, _sql: &str) -> Outcome<PreparedStatement, Error> {
1583            Outcome::Ok(PreparedStatement::new(1, String::new(), 0))
1584        }
1585
1586        async fn query_prepared(
1587            &self,
1588            _cx: &Cx,
1589            _stmt: &PreparedStatement,
1590            _params: &[Value],
1591        ) -> Outcome<Vec<Row>, Error> {
1592            Outcome::Ok(vec![])
1593        }
1594
1595        async fn execute_prepared(
1596            &self,
1597            _cx: &Cx,
1598            _stmt: &PreparedStatement,
1599            _params: &[Value],
1600        ) -> Outcome<u64, Error> {
1601            Outcome::Ok(0)
1602        }
1603
1604        async fn ping(&self, _cx: &Cx) -> Outcome<(), Error> {
1605            if self.ping_should_fail.load(Ordering::Relaxed) {
1606                Outcome::Err(Error::Connection(ConnectionError {
1607                    kind: ConnectionErrorKind::Disconnected,
1608                    message: "mock ping failed".to_string(),
1609                    source: None,
1610                }))
1611            } else {
1612                Outcome::Ok(())
1613            }
1614        }
1615
1616        async fn close(self, _cx: &Cx) -> Result<(), Error> {
1617            Ok(())
1618        }
1619
1620        async fn close_for_pool(self, _cx: &Cx) -> Result<(), Error> {
1621            if let (Some(pool_shared), Some(pool_lock_was_free)) =
1622                (self.pool_shared.as_ref(), self.pool_lock_was_free.as_ref())
1623            {
1624                let mutex_is_available = pool_shared
1625                    .upgrade()
1626                    .is_none_or(|shared| shared.inner.try_lock().is_ok());
1627                pool_lock_was_free.store(mutex_is_available, Ordering::Relaxed);
1628            }
1629            self.pool_close_calls.fetch_add(1, Ordering::Relaxed);
1630            if self.pool_close_pending {
1631                std::future::pending::<()>().await;
1632            }
1633            if self.pool_close_should_fail {
1634                return Err(Error::Custom("mock pool close failure".to_string()));
1635            }
1636            Ok(())
1637        }
1638    }
1639
1640    #[test]
1641    fn test_config_default() {
1642        let config = PoolConfig::default();
1643        assert_eq!(config.min_connections, 1);
1644        assert_eq!(config.max_connections, 10);
1645        assert_eq!(config.idle_timeout_ms, 600_000);
1646        assert_eq!(config.acquire_timeout_ms, 30_000);
1647        assert_eq!(config.max_lifetime_ms, 1_800_000);
1648        assert!(config.test_on_checkout);
1649        assert!(!config.test_on_return);
1650    }
1651
1652    #[test]
1653    fn test_config_builder() {
1654        let config = PoolConfig::new(20)
1655            .min_connections(5)
1656            .idle_timeout(60_000)
1657            .acquire_timeout(5_000)
1658            .max_lifetime(300_000)
1659            .test_on_checkout(false)
1660            .test_on_return(true);
1661
1662        assert_eq!(config.min_connections, 5);
1663        assert_eq!(config.max_connections, 20);
1664        assert_eq!(config.idle_timeout_ms, 60_000);
1665        assert_eq!(config.acquire_timeout_ms, 5_000);
1666        assert_eq!(config.max_lifetime_ms, 300_000);
1667        assert!(!config.test_on_checkout);
1668        assert!(config.test_on_return);
1669    }
1670
1671    #[test]
1672    fn test_config_clone() {
1673        let config = PoolConfig::new(15).min_connections(3);
1674        let cloned = config.clone();
1675        assert_eq!(config.max_connections, cloned.max_connections);
1676        assert_eq!(config.min_connections, cloned.min_connections);
1677    }
1678
1679    #[test]
1680    fn test_stats_default() {
1681        let stats = PoolStats::default();
1682        assert_eq!(stats.total_connections, 0);
1683        assert_eq!(stats.idle_connections, 0);
1684        assert_eq!(stats.active_connections, 0);
1685        assert_eq!(stats.pending_requests, 0);
1686        assert_eq!(stats.connections_created, 0);
1687        assert_eq!(stats.connections_closed, 0);
1688        assert_eq!(stats.acquires, 0);
1689        assert_eq!(stats.timeouts, 0);
1690    }
1691
1692    #[test]
1693    fn test_stats_clone() {
1694        let stats = PoolStats {
1695            total_connections: 5,
1696            acquires: 100,
1697            ..Default::default()
1698        };
1699        let cloned = stats.clone();
1700        assert_eq!(stats.total_connections, cloned.total_connections);
1701        assert_eq!(stats.acquires, cloned.acquires);
1702    }
1703
1704    #[test]
1705    fn test_connection_meta_timing() {
1706        use std::thread;
1707
1708        // Create a dummy type for testing
1709        struct DummyConn;
1710
1711        let meta = ConnectionMeta::new(DummyConn);
1712        let initial_age = meta.age();
1713
1714        // Small sleep to ensure time passes
1715        thread::sleep(Duration::from_millis(10));
1716
1717        // Age should have increased
1718        assert!(meta.age() > initial_age);
1719        assert!(meta.idle_time() > Duration::ZERO);
1720    }
1721
1722    #[test]
1723    fn test_connection_meta_touch() {
1724        use std::thread;
1725
1726        struct DummyConn;
1727
1728        let mut meta = ConnectionMeta::new(DummyConn);
1729
1730        // Small sleep to build up some idle time
1731        thread::sleep(Duration::from_millis(10));
1732        let idle_before_touch = meta.idle_time();
1733        assert!(idle_before_touch > Duration::ZERO);
1734
1735        // Touch should reset idle time
1736        meta.touch();
1737        let idle_after_touch = meta.idle_time();
1738
1739        // After touch, idle time should be very small (less than before)
1740        assert!(idle_after_touch < idle_before_touch);
1741    }
1742
1743    #[test]
1744    fn test_pool_new() {
1745        let config = PoolConfig::new(5);
1746        let pool: Pool<MockConnection> = Pool::new(config);
1747
1748        // New pool should be empty
1749        assert_eq!(pool.idle_count(), 0);
1750        assert_eq!(pool.active_count(), 0);
1751        assert_eq!(pool.total_count(), 0);
1752        assert!(!pool.is_closed());
1753        assert!(!pool.at_capacity());
1754    }
1755
1756    #[test]
1757    fn test_pool_config() {
1758        let config = PoolConfig::new(7).min_connections(2);
1759        let pool: Pool<MockConnection> = Pool::new(config);
1760
1761        let retrieved_config = pool.config();
1762        assert_eq!(retrieved_config.max_connections, 7);
1763        assert_eq!(retrieved_config.min_connections, 2);
1764    }
1765
1766    #[test]
1767    fn test_pool_stats_initial() {
1768        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
1769
1770        let stats = pool.stats();
1771        assert_eq!(stats.total_connections, 0);
1772        assert_eq!(stats.idle_connections, 0);
1773        assert_eq!(stats.active_connections, 0);
1774        assert_eq!(stats.pending_requests, 0);
1775        assert_eq!(stats.connections_created, 0);
1776        assert_eq!(stats.connections_closed, 0);
1777        assert_eq!(stats.acquires, 0);
1778        assert_eq!(stats.timeouts, 0);
1779    }
1780
1781    #[test]
1782    fn test_pool_close() {
1783        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
1784
1785        assert!(!pool.is_closed());
1786        pool.close();
1787        assert!(pool.is_closed());
1788    }
1789
1790    #[test]
1791    fn test_close_and_drain_zero_active_completes_immediately() {
1792        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
1793        let runtime = RuntimeBuilder::current_thread()
1794            .build()
1795            .expect("build test runtime");
1796        let cx = Cx::for_testing();
1797
1798        let outcome = runtime.block_on(pool.close_and_drain(&cx));
1799
1800        assert!(matches!(outcome, Outcome::Ok(())));
1801        assert!(pool.is_closed());
1802        assert_eq!(pool.active_count(), 0);
1803        assert_eq!(pool.total_count(), 0);
1804    }
1805
1806    #[test]
1807    fn test_close_and_drain_surfaces_exact_idle_retirement_error() {
1808        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(1));
1809        {
1810            let mut inner = pool.shared.inner.lock().unwrap();
1811            inner.total_count = 1;
1812            inner.idle.push_back(ConnectionMeta::new(
1813                MockConnection::with_failing_pool_close(1),
1814            ));
1815        }
1816        let runtime = RuntimeBuilder::current_thread()
1817            .build()
1818            .expect("build test runtime");
1819        let cx = Cx::for_testing();
1820
1821        let outcome = runtime.block_on(pool.close_and_drain(&cx));
1822
1823        match outcome {
1824            Outcome::Err(Error::Custom(message)) => {
1825                assert_eq!(message, "mock pool close failure");
1826            }
1827            other => panic!("expected exact driver close error, got {other:?}"),
1828        }
1829        assert!(pool.is_closed());
1830        assert_eq!(pool.active_count(), 0);
1831        assert_eq!(pool.total_count(), 0);
1832
1833        let later_cx = Cx::for_testing();
1834        let later = runtime.block_on(pool.close_and_drain(&later_cx));
1835        match later {
1836            Outcome::Err(Error::Custom(message)) => {
1837                assert_eq!(
1838                    message,
1839                    "pool retirement failed: pool close-and-drain: mock pool close failure"
1840                );
1841            }
1842            other => panic!("expected persistent retirement error, got {other:?}"),
1843        }
1844    }
1845
1846    #[test]
1847    fn test_checked_out_retirement_error_reaches_every_drainer() {
1848        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(1));
1849        {
1850            let mut inner = pool.shared.inner.lock().unwrap();
1851            inner.total_count = 1;
1852            inner.active_count = 1;
1853        }
1854        let pooled = PooledConnection::new(
1855            ConnectionMeta::new(MockConnection::with_failing_pool_close(1)),
1856            Arc::downgrade(&pool.shared),
1857        );
1858        let first_cx = Cx::for_testing();
1859        let second_cx = Cx::for_testing();
1860        let mut first_drain = Box::pin(pool.close_and_drain(&first_cx));
1861        let mut second_drain = Box::pin(pool.close_and_drain(&second_cx));
1862        let mut task_cx = Context::from_waker(Waker::noop());
1863
1864        assert!(matches!(
1865            first_drain.as_mut().poll(&mut task_cx),
1866            Poll::Pending
1867        ));
1868        assert!(matches!(
1869            second_drain.as_mut().poll(&mut task_cx),
1870            Poll::Pending
1871        ));
1872
1873        drop(pooled);
1874
1875        let first_message = match first_drain.as_mut().poll(&mut task_cx) {
1876            Poll::Ready(Outcome::Err(Error::Custom(message))) => message,
1877            other => panic!("first drainer did not fail closed: {other:?}"),
1878        };
1879        let second_message = match second_drain.as_mut().poll(&mut task_cx) {
1880            Poll::Ready(Outcome::Err(Error::Custom(message))) => message,
1881            other => panic!("second drainer did not fail closed: {other:?}"),
1882        };
1883        assert_eq!(first_message, second_message);
1884        assert_eq!(
1885            first_message,
1886            "pool retirement failed: pooled connection drop closed pool: \
1887             mock pool close failure"
1888        );
1889        assert!(pool.is_closed());
1890        assert_eq!(pool.active_count(), 0);
1891        assert_eq!(pool.total_count(), 0);
1892    }
1893
1894    #[test]
1895    fn test_close_and_drain_waits_for_active_return_and_explicit_close() {
1896        let pool_close_calls = Arc::new(AtomicUsize::new(0));
1897        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(1));
1898        {
1899            let mut inner = pool.shared.inner.lock().unwrap();
1900            inner.total_count = 1;
1901            inner.active_count = 1;
1902        }
1903        let pooled = PooledConnection::new(
1904            ConnectionMeta::new(MockConnection::with_pool_close_counter(
1905                1,
1906                Arc::clone(&pool_close_calls),
1907            )),
1908            Arc::downgrade(&pool.shared),
1909        );
1910        let cx = Cx::for_testing();
1911        let mut drain = Box::pin(pool.close_and_drain(&cx));
1912        let mut task_cx = Context::from_waker(Waker::noop());
1913
1914        assert!(matches!(drain.as_mut().poll(&mut task_cx), Poll::Pending));
1915        assert!(pool.is_closed());
1916        assert_eq!(pool.active_count(), 1);
1917        assert_eq!(pool_close_calls.load(Ordering::Relaxed), 0);
1918
1919        drop(pooled);
1920
1921        assert_eq!(pool_close_calls.load(Ordering::Relaxed), 1);
1922        assert!(matches!(
1923            drain.as_mut().poll(&mut task_cx),
1924            Poll::Ready(Outcome::Ok(()))
1925        ));
1926        assert_eq!(pool.active_count(), 0);
1927        assert_eq!(pool.total_count(), 0);
1928    }
1929
1930    #[test]
1931    fn test_close_and_drain_multiple_handles_and_drainers_share_final_wake() {
1932        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(2));
1933        {
1934            let mut inner = pool.shared.inner.lock().unwrap();
1935            inner.total_count = 2;
1936            inner.active_count = 2;
1937        }
1938        let first = PooledConnection::new(
1939            ConnectionMeta::new(MockConnection::new(1)),
1940            Arc::downgrade(&pool.shared),
1941        );
1942        let second = PooledConnection::new(
1943            ConnectionMeta::new(MockConnection::new(2)),
1944            Arc::downgrade(&pool.shared),
1945        );
1946        let first_cx = Cx::for_testing();
1947        let second_cx = Cx::for_testing();
1948        let mut first_drain = Box::pin(pool.close_and_drain(&first_cx));
1949        let mut second_drain = Box::pin(pool.close_and_drain(&second_cx));
1950        let mut task_cx = Context::from_waker(Waker::noop());
1951
1952        assert!(matches!(
1953            first_drain.as_mut().poll(&mut task_cx),
1954            Poll::Pending
1955        ));
1956        assert!(matches!(
1957            second_drain.as_mut().poll(&mut task_cx),
1958            Poll::Pending
1959        ));
1960
1961        drop(first);
1962        assert_eq!(pool.active_count(), 1);
1963        assert!(matches!(
1964            first_drain.as_mut().poll(&mut task_cx),
1965            Poll::Pending
1966        ));
1967        assert!(matches!(
1968            second_drain.as_mut().poll(&mut task_cx),
1969            Poll::Pending
1970        ));
1971
1972        drop(second);
1973        assert_eq!(pool.active_count(), 0);
1974        assert!(matches!(
1975            first_drain.as_mut().poll(&mut task_cx),
1976            Poll::Ready(Outcome::Ok(()))
1977        ));
1978        assert!(matches!(
1979            second_drain.as_mut().poll(&mut task_cx),
1980            Poll::Ready(Outcome::Ok(()))
1981        ));
1982    }
1983
1984    #[test]
1985    fn test_close_and_drain_cancellation_keeps_pool_closed() {
1986        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(1));
1987        {
1988            let mut inner = pool.shared.inner.lock().unwrap();
1989            inner.total_count = 1;
1990            inner.active_count = 1;
1991        }
1992        let pooled = PooledConnection::new(
1993            ConnectionMeta::new(MockConnection::new(1)),
1994            Arc::downgrade(&pool.shared),
1995        );
1996        let cx = Cx::for_testing();
1997        let mut drain = Box::pin(pool.close_and_drain(&cx));
1998        let wake_counter = Arc::new(WakeCounter::default());
1999        let waker = Waker::from(Arc::clone(&wake_counter));
2000        let mut task_cx = Context::from_waker(&waker);
2001
2002        assert!(matches!(drain.as_mut().poll(&mut task_cx), Poll::Pending));
2003        cx.set_cancel_requested(true);
2004        assert!(
2005            wake_counter.wakes.load(Ordering::Relaxed) > 0,
2006            "Cx cancellation must wake the registered close-and-drain waiter"
2007        );
2008        assert!(matches!(
2009            drain.as_mut().poll(&mut task_cx),
2010            Poll::Ready(Outcome::Cancelled(_))
2011        ));
2012        assert!(pool.is_closed());
2013        assert_eq!(pool.active_count(), 1);
2014
2015        drop(drain);
2016        let resume_cx = Cx::for_testing();
2017        let mut resumed_drain = Box::pin(pool.close_and_drain(&resume_cx));
2018        let mut resumed_task_cx = Context::from_waker(Waker::noop());
2019        assert!(matches!(
2020            resumed_drain.as_mut().poll(&mut resumed_task_cx),
2021            Poll::Pending
2022        ));
2023        assert!(pool.is_closed());
2024
2025        drop(pooled);
2026        assert!(matches!(
2027            resumed_drain.as_mut().poll(&mut resumed_task_cx),
2028            Poll::Ready(Outcome::Ok(()))
2029        ));
2030        assert!(pool.is_closed());
2031        assert_eq!(pool.active_count(), 0);
2032    }
2033
2034    #[test]
2035    fn test_close_and_drain_expired_deadline_keeps_pool_closed() {
2036        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(1));
2037        {
2038            let mut inner = pool.shared.inner.lock().unwrap();
2039            inner.total_count = 1;
2040            inner.active_count = 1;
2041        }
2042        let pooled = PooledConnection::new(
2043            ConnectionMeta::new(MockConnection::new(1)),
2044            Arc::downgrade(&pool.shared),
2045        );
2046        let cx = Cx::for_testing_with_budget(Budget::new().with_deadline(Time::ZERO));
2047        let mut drain = Box::pin(pool.close_and_drain(&cx));
2048        let mut task_cx = Context::from_waker(Waker::noop());
2049
2050        assert!(matches!(
2051            drain.as_mut().poll(&mut task_cx),
2052            Poll::Ready(Outcome::Cancelled(_))
2053        ));
2054        assert!(pool.is_closed());
2055        assert_eq!(pool.active_count(), 1);
2056
2057        drop(drain);
2058        drop(pooled);
2059        assert_eq!(pool.active_count(), 0);
2060    }
2061
2062    #[test]
2063    fn test_dropped_in_flight_factory_releases_reserved_slot() {
2064        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(1));
2065        let cx = Cx::for_testing();
2066        let mut acquire = Box::pin(pool.acquire(&cx, || {
2067            std::future::pending::<Outcome<MockConnection, Error>>()
2068        }));
2069        let mut task_cx = Context::from_waker(Waker::noop());
2070
2071        assert!(matches!(acquire.as_mut().poll(&mut task_cx), Poll::Pending));
2072        assert_eq!(pool.active_count(), 1);
2073        assert_eq!(pool.total_count(), 1);
2074
2075        drop(acquire);
2076
2077        assert_eq!(pool.active_count(), 0);
2078        assert_eq!(pool.total_count(), 0);
2079    }
2080
2081    #[test]
2082    fn test_dropped_multi_expired_idle_close_releases_all_retirement_slots() {
2083        let pool_close_calls = Arc::new(AtomicUsize::new(0));
2084        let pool: Pool<MockConnection> =
2085            Pool::new(PoolConfig::new(3).max_lifetime(1).test_on_checkout(false));
2086        let mut first_expired = ConnectionMeta::new(MockConnection::with_pending_pool_close(
2087            1,
2088            Arc::clone(&pool_close_calls),
2089        ));
2090        first_expired.created_at = Instant::now()
2091            .checked_sub(Duration::from_secs(1))
2092            .expect("one second must fit before the current instant");
2093        let mut second_expired = ConnectionMeta::new(MockConnection::with_pool_close_counter(
2094            2,
2095            Arc::clone(&pool_close_calls),
2096        ));
2097        second_expired.created_at = Instant::now()
2098            .checked_sub(Duration::from_secs(1))
2099            .expect("one second must fit before the current instant");
2100        {
2101            let mut inner = pool.shared.inner.lock().unwrap();
2102            inner.total_count = 2;
2103            inner.idle.push_back(first_expired);
2104            inner.idle.push_back(second_expired);
2105        }
2106        let cx = Cx::for_testing();
2107        let mut acquire =
2108            Box::pin(pool.acquire(&cx, || async { Outcome::Ok(MockConnection::new(3)) }));
2109        let mut task_cx = Context::from_waker(Waker::noop());
2110
2111        assert!(matches!(acquire.as_mut().poll(&mut task_cx), Poll::Pending));
2112        assert_eq!(pool_close_calls.load(Ordering::Relaxed), 1);
2113        assert_eq!(pool.active_count(), 2);
2114        assert_eq!(pool.total_count(), 2);
2115
2116        let drain_cx = Cx::for_testing();
2117        let mut drain = Box::pin(pool.close_and_drain(&drain_cx));
2118        assert!(matches!(drain.as_mut().poll(&mut task_cx), Poll::Pending));
2119
2120        drop(acquire);
2121        assert!(matches!(
2122            drain.as_mut().poll(&mut task_cx),
2123            Poll::Ready(Outcome::Ok(()))
2124        ));
2125        assert!(pool.is_closed());
2126        assert_eq!(pool.active_count(), 0);
2127        assert_eq!(pool.total_count(), 0);
2128    }
2129
2130    #[test]
2131    fn test_dropped_pending_idle_drain_releases_resource_for_other_drainer() {
2132        let pool_close_calls = Arc::new(AtomicUsize::new(0));
2133        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(1));
2134        {
2135            let mut inner = pool.shared.inner.lock().unwrap();
2136            inner.total_count = 1;
2137            inner.idle.push_back(ConnectionMeta::new(
2138                MockConnection::with_pending_pool_close(1, Arc::clone(&pool_close_calls)),
2139            ));
2140        }
2141        let first_cx = Cx::for_testing();
2142        let second_cx = Cx::for_testing();
2143        let mut first_drain = Box::pin(pool.close_and_drain(&first_cx));
2144        let mut second_drain = Box::pin(pool.close_and_drain(&second_cx));
2145        let mut task_cx = Context::from_waker(Waker::noop());
2146
2147        assert!(matches!(
2148            first_drain.as_mut().poll(&mut task_cx),
2149            Poll::Pending
2150        ));
2151        assert!(pool.is_closed());
2152        assert_eq!(pool_close_calls.load(Ordering::Relaxed), 1);
2153        assert_eq!(pool.active_count(), 1);
2154        assert_eq!(pool.total_count(), 1);
2155
2156        assert!(matches!(
2157            second_drain.as_mut().poll(&mut task_cx),
2158            Poll::Pending
2159        ));
2160
2161        first_cx.set_cancel_requested(true);
2162        assert!(matches!(
2163            first_drain.as_mut().poll(&mut task_cx),
2164            Poll::Ready(Outcome::Cancelled(_))
2165        ));
2166        drop(first_drain);
2167
2168        assert_eq!(pool.active_count(), 0);
2169        assert_eq!(pool.total_count(), 0);
2170        assert!(matches!(
2171            second_drain.as_mut().poll(&mut task_cx),
2172            Poll::Ready(Outcome::Ok(()))
2173        ));
2174        assert!(pool.is_closed());
2175    }
2176
2177    #[test]
2178    fn test_dropped_validation_close_releases_armed_active_slot() {
2179        let pool_close_calls = Arc::new(AtomicUsize::new(0));
2180        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(1).test_on_checkout(true));
2181        let failed = MockConnection::with_pending_pool_close(1, Arc::clone(&pool_close_calls));
2182        failed.ping_should_fail.store(true, Ordering::Relaxed);
2183        {
2184            let mut inner = pool.shared.inner.lock().unwrap();
2185            inner.total_count = 1;
2186            inner.idle.push_back(ConnectionMeta::new(failed));
2187        }
2188        let cx = Cx::for_testing();
2189        let mut acquire =
2190            Box::pin(pool.acquire(&cx, || async { Outcome::Ok(MockConnection::new(2)) }));
2191        let mut task_cx = Context::from_waker(Waker::noop());
2192
2193        assert!(matches!(acquire.as_mut().poll(&mut task_cx), Poll::Pending));
2194        assert_eq!(pool_close_calls.load(Ordering::Relaxed), 1);
2195        assert_eq!(pool.active_count(), 1);
2196        assert_eq!(pool.total_count(), 1);
2197
2198        pool.close();
2199        drop(acquire);
2200
2201        assert_eq!(pool.active_count(), 0);
2202        assert_eq!(pool.total_count(), 0);
2203        let drain_cx = Cx::for_testing();
2204        let mut drain = Box::pin(pool.close_and_drain(&drain_cx));
2205        assert!(matches!(
2206            drain.as_mut().poll(&mut task_cx),
2207            Poll::Ready(Outcome::Ok(()))
2208        ));
2209    }
2210
2211    #[test]
2212    fn test_in_flight_factory_cannot_publish_after_close() {
2213        let pool_close_calls = Arc::new(AtomicUsize::new(0));
2214        let factory_ready = Arc::new(AtomicBool::new(false));
2215        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(1));
2216        let cx = Cx::for_testing();
2217        let mut acquire = Box::pin(pool.acquire(&cx, || GatedFactory {
2218            ready: Arc::clone(&factory_ready),
2219            conn: Some(MockConnection::with_pool_close_counter(
2220                1,
2221                Arc::clone(&pool_close_calls),
2222            )),
2223        }));
2224        let mut task_cx = Context::from_waker(Waker::noop());
2225
2226        assert!(matches!(acquire.as_mut().poll(&mut task_cx), Poll::Pending));
2227        assert_eq!(pool.active_count(), 1);
2228
2229        pool.close();
2230        factory_ready.store(true, Ordering::Release);
2231
2232        assert!(matches!(
2233            acquire.as_mut().poll(&mut task_cx),
2234            Poll::Ready(Outcome::Err(Error::Pool(PoolError {
2235                kind: PoolErrorKind::Closed,
2236                ..
2237            })))
2238        ));
2239        assert_eq!(pool_close_calls.load(Ordering::Relaxed), 1);
2240        assert_eq!(pool.active_count(), 0);
2241        assert_eq!(pool.total_count(), 0);
2242    }
2243
2244    #[test]
2245    fn test_close_wakes_blocked_acquirer_to_observe_closed_state() {
2246        use std::sync::mpsc;
2247        use std::thread;
2248
2249        let pool = Arc::new(Pool::<MockConnection>::new(PoolConfig::new(1)));
2250        {
2251            let mut inner = pool.shared.inner.lock().unwrap();
2252            inner.total_count = 1;
2253            inner.active_count = 1;
2254        }
2255        let pooled = PooledConnection::new(
2256            ConnectionMeta::new(MockConnection::new(1)),
2257            Arc::downgrade(&pool.shared),
2258        );
2259        let waiting_pool = Arc::clone(&pool);
2260        let (started_tx, started_rx) = mpsc::sync_channel(0);
2261        let waiter = thread::spawn(move || {
2262            let runtime = RuntimeBuilder::current_thread()
2263                .build()
2264                .expect("build waiter runtime");
2265            let cx = Cx::for_testing();
2266            started_tx.send(()).expect("signal waiter start");
2267            matches!(
2268                runtime.block_on(
2269                    waiting_pool.acquire(&cx, || async { Outcome::Ok(MockConnection::new(2)) })
2270                ),
2271                Outcome::Err(Error::Pool(PoolError {
2272                    kind: PoolErrorKind::Closed,
2273                    ..
2274                }))
2275            )
2276        });
2277        started_rx.recv().expect("waiter thread should start");
2278
2279        let mut observed_waiter = false;
2280        for _ in 0..100_000 {
2281            if pool.stats().pending_requests == 1 {
2282                observed_waiter = true;
2283                break;
2284            }
2285            thread::yield_now();
2286        }
2287
2288        pool.close();
2289        let observed_closed = waiter.join().expect("waiter thread should not panic");
2290        drop(pooled);
2291
2292        assert!(
2293            observed_waiter,
2294            "acquirer never registered as a pool waiter"
2295        );
2296        assert!(
2297            observed_closed,
2298            "blocked acquirer did not observe pool close"
2299        );
2300    }
2301
2302    #[test]
2303    fn test_pool_close_routes_through_close_for_pool() {
2304        let pool_close_calls = Arc::new(AtomicUsize::new(0));
2305        let pool_lock_was_free = Arc::new(AtomicBool::new(false));
2306        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(2));
2307
2308        // Seed one idle connection whose `close_for_pool` override records
2309        // the call, proving pool teardown uses the driver's pool-close path
2310        // rather than the ordinary `close`.
2311        {
2312            let mut inner = pool
2313                .shared
2314                .inner
2315                .lock()
2316                .expect("pool mutex should not be poisoned");
2317            inner.total_count = 1;
2318            inner
2319                .idle
2320                .push_back(ConnectionMeta::new(MockConnection::with_pool_close_probe(
2321                    1,
2322                    Arc::clone(&pool_close_calls),
2323                    Arc::downgrade(&pool.shared),
2324                    Arc::clone(&pool_lock_was_free),
2325                )));
2326        }
2327
2328        pool.close();
2329
2330        assert!(pool.is_closed());
2331        assert_eq!(pool_close_calls.load(Ordering::Relaxed), 1);
2332        assert!(pool_lock_was_free.load(Ordering::Relaxed));
2333    }
2334
2335    #[test]
2336    fn test_expired_idle_connection_routes_through_close_for_pool() {
2337        let pool_close_calls = Arc::new(AtomicUsize::new(0));
2338        let pool: Pool<MockConnection> =
2339            Pool::new(PoolConfig::new(2).max_lifetime(1).test_on_checkout(false));
2340        let mut expired = ConnectionMeta::new(MockConnection::with_pool_close_counter(
2341            1,
2342            Arc::clone(&pool_close_calls),
2343        ));
2344        expired.created_at = Instant::now()
2345            .checked_sub(Duration::from_secs(1))
2346            .expect("one second must fit before the current instant");
2347        {
2348            let mut inner = pool
2349                .shared
2350                .inner
2351                .lock()
2352                .expect("pool mutex should not be poisoned");
2353            inner.total_count = 1;
2354            inner.idle.push_back(expired);
2355        }
2356
2357        let runtime = RuntimeBuilder::current_thread()
2358            .build()
2359            .expect("build test runtime");
2360        let cx = Cx::for_testing();
2361        let acquired =
2362            runtime.block_on(pool.acquire(&cx, || async { Outcome::Ok(MockConnection::new(2)) }));
2363
2364        assert!(matches!(acquired, Outcome::Ok(_)));
2365        assert_eq!(pool_close_calls.load(Ordering::Relaxed), 1);
2366    }
2367
2368    #[test]
2369    fn test_failed_validation_routes_through_close_for_pool() {
2370        let pool_close_calls = Arc::new(AtomicUsize::new(0));
2371        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(2).test_on_checkout(true));
2372        let failed = MockConnection::with_pool_close_counter(1, Arc::clone(&pool_close_calls));
2373        failed.ping_should_fail.store(true, Ordering::Relaxed);
2374        {
2375            let mut inner = pool
2376                .shared
2377                .inner
2378                .lock()
2379                .expect("pool mutex should not be poisoned");
2380            inner.total_count = 1;
2381            inner.idle.push_back(ConnectionMeta::new(failed));
2382        }
2383
2384        let runtime = RuntimeBuilder::current_thread()
2385            .build()
2386            .expect("build test runtime");
2387        let cx = Cx::for_testing();
2388        let acquired =
2389            runtime.block_on(pool.acquire(&cx, || async { Outcome::Ok(MockConnection::new(2)) }));
2390
2391        assert!(matches!(acquired, Outcome::Err(_)));
2392        assert_eq!(pool_close_calls.load(Ordering::Relaxed), 1);
2393    }
2394
2395    #[test]
2396    fn test_pool_inner_can_create_new() {
2397        let mut inner = PoolInner::<MockConnection>::new(PoolConfig::new(3));
2398
2399        // Initially can create new
2400        assert!(inner.can_create_new());
2401
2402        // At capacity
2403        inner.total_count = 3;
2404        assert!(!inner.can_create_new());
2405
2406        // Below capacity again
2407        inner.total_count = 2;
2408        assert!(inner.can_create_new());
2409
2410        // Closed pool
2411        inner.closed = true;
2412        assert!(!inner.can_create_new());
2413    }
2414
2415    #[test]
2416    fn test_pool_inner_stats() {
2417        let mut inner = PoolInner::<MockConnection>::new(PoolConfig::new(10));
2418
2419        inner.total_count = 5;
2420        inner.active_count = 3;
2421        inner.waiter_count = 2;
2422        inner
2423            .idle
2424            .push_back(ConnectionMeta::new(MockConnection::new(1)));
2425        inner
2426            .idle
2427            .push_back(ConnectionMeta::new(MockConnection::new(2)));
2428
2429        let stats = inner.stats();
2430        assert_eq!(stats.total_connections, 5);
2431        assert_eq!(stats.idle_connections, 2);
2432        assert_eq!(stats.active_connections, 3);
2433        assert_eq!(stats.pending_requests, 2);
2434    }
2435
2436    #[test]
2437    fn test_pooled_connection_age_and_idle_time() {
2438        use std::thread;
2439
2440        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
2441
2442        // Properly initialize pool state as if acquire happened
2443        {
2444            let mut inner = pool.shared.inner.lock().unwrap();
2445            inner.total_count = 1;
2446            inner.active_count = 1;
2447        }
2448
2449        let meta = ConnectionMeta::new(MockConnection::new(1));
2450        let pooled = PooledConnection::new(meta, Arc::downgrade(&pool.shared));
2451
2452        // Should have some small positive age
2453        assert!(pooled.age() >= Duration::ZERO);
2454
2455        thread::sleep(Duration::from_millis(5));
2456        assert!(pooled.age() > Duration::ZERO);
2457    }
2458
2459    #[test]
2460    fn test_pooled_connection_detach() {
2461        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
2462
2463        // Manually add a connection to simulate acquire
2464        {
2465            let mut inner = pool.shared.inner.lock().unwrap();
2466            inner.total_count = 1;
2467            inner.active_count = 1;
2468        }
2469
2470        let meta = ConnectionMeta::new(MockConnection::new(42));
2471        let pooled = PooledConnection::new(meta, Arc::downgrade(&pool.shared));
2472
2473        // Verify counts before detach
2474        assert_eq!(pool.total_count(), 1);
2475        assert_eq!(pool.active_count(), 1);
2476
2477        // Detach returns the connection
2478        let conn = pooled.detach();
2479        assert_eq!(conn.id, 42);
2480
2481        // After detach, counts should be decremented
2482        assert_eq!(pool.total_count(), 0);
2483        assert_eq!(pool.active_count(), 0);
2484
2485        // connections_closed should be incremented
2486        let stats = pool.stats();
2487        assert_eq!(stats.connections_closed, 1);
2488    }
2489
2490    #[test]
2491    fn test_pooled_connection_drop_returns_to_pool() {
2492        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
2493
2494        // Manually set up pool state as if we acquired a connection
2495        {
2496            let mut inner = pool.shared.inner.lock().unwrap();
2497            inner.total_count = 1;
2498            inner.active_count = 1;
2499        }
2500
2501        let meta = ConnectionMeta::new(MockConnection::new(1));
2502        let pooled = PooledConnection::new(meta, Arc::downgrade(&pool.shared));
2503
2504        // While held, active=1, idle=0
2505        assert_eq!(pool.active_count(), 1);
2506        assert_eq!(pool.idle_count(), 0);
2507
2508        // Drop the connection
2509        drop(pooled);
2510
2511        // After drop, active=0, idle=1 (returned to pool)
2512        assert_eq!(pool.active_count(), 0);
2513        assert_eq!(pool.idle_count(), 1);
2514        assert_eq!(pool.total_count(), 1); // Total unchanged
2515    }
2516
2517    #[test]
2518    fn test_pooled_connection_drop_when_pool_closed() {
2519        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
2520
2521        // Set up pool state
2522        {
2523            let mut inner = pool.shared.inner.lock().unwrap();
2524            inner.total_count = 1;
2525            inner.active_count = 1;
2526        }
2527
2528        let meta = ConnectionMeta::new(MockConnection::new(1));
2529        let pooled = PooledConnection::new(meta, Arc::downgrade(&pool.shared));
2530
2531        // Close the pool while connection is out
2532        pool.close();
2533
2534        // Drop the connection
2535        drop(pooled);
2536
2537        // Connection should not be returned to idle (pool is closed)
2538        assert_eq!(pool.idle_count(), 0);
2539        assert_eq!(pool.active_count(), 0);
2540        assert_eq!(pool.total_count(), 0);
2541
2542        // Connection was closed
2543        assert_eq!(pool.stats().connections_closed, 1);
2544    }
2545
2546    #[test]
2547    fn test_pooled_connection_deref() {
2548        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
2549
2550        // Properly initialize pool state as if acquire happened
2551        {
2552            let mut inner = pool.shared.inner.lock().unwrap();
2553            inner.total_count = 1;
2554            inner.active_count = 1;
2555        }
2556
2557        let meta = ConnectionMeta::new(MockConnection::new(99));
2558        let pooled = PooledConnection::new(meta, Arc::downgrade(&pool.shared));
2559
2560        // Deref should give access to the connection's id
2561        assert_eq!(pooled.id, 99);
2562    }
2563
2564    #[test]
2565    fn test_pooled_connection_deref_mut() {
2566        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
2567
2568        // Properly initialize pool state as if acquire happened
2569        {
2570            let mut inner = pool.shared.inner.lock().unwrap();
2571            inner.total_count = 1;
2572            inner.active_count = 1;
2573        }
2574
2575        let meta = ConnectionMeta::new(MockConnection::new(1));
2576        let mut pooled = PooledConnection::new(meta, Arc::downgrade(&pool.shared));
2577
2578        // DerefMut should allow mutation
2579        pooled.id = 50;
2580        assert_eq!(pooled.id, 50);
2581    }
2582
2583    #[test]
2584    fn test_pooled_connection_debug() {
2585        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
2586
2587        // Properly initialize pool state as if acquire happened
2588        {
2589            let mut inner = pool.shared.inner.lock().unwrap();
2590            inner.total_count = 1;
2591            inner.active_count = 1;
2592        }
2593
2594        let meta = ConnectionMeta::new(MockConnection::new(1));
2595        let pooled = PooledConnection::new(meta, Arc::downgrade(&pool.shared));
2596
2597        let debug_str = format!("{:?}", pooled);
2598        assert!(debug_str.contains("PooledConnection"));
2599        assert!(debug_str.contains("age"));
2600    }
2601
2602    #[test]
2603    fn test_pool_at_capacity() {
2604        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(2));
2605
2606        assert!(!pool.at_capacity());
2607
2608        // Simulate connections being created
2609        {
2610            let mut inner = pool.shared.inner.lock().unwrap();
2611            inner.total_count = 1;
2612        }
2613        assert!(!pool.at_capacity());
2614
2615        {
2616            let mut inner = pool.shared.inner.lock().unwrap();
2617            inner.total_count = 2;
2618        }
2619        assert!(pool.at_capacity());
2620    }
2621
2622    #[test]
2623    fn test_acquire_action_enum() {
2624        // Verify the enum variants exist and can be pattern-matched
2625        let retire: AcquireAction<MockConnection> = AcquireAction::RetireAndRetry;
2626        assert!(matches!(retire, AcquireAction::RetireAndRetry));
2627
2628        let closed: AcquireAction<MockConnection> = AcquireAction::PoolClosed;
2629        assert!(matches!(closed, AcquireAction::PoolClosed));
2630
2631        let create: AcquireAction<MockConnection> = AcquireAction::CreateNew;
2632        assert!(matches!(create, AcquireAction::CreateNew));
2633
2634        let wait: AcquireAction<MockConnection> = AcquireAction::Wait;
2635        assert!(matches!(wait, AcquireAction::Wait));
2636
2637        let meta = ConnectionMeta::new(MockConnection::new(1));
2638        let validate: AcquireAction<MockConnection> = AcquireAction::ValidateExisting(meta);
2639        assert!(matches!(validate, AcquireAction::ValidateExisting(_)));
2640    }
2641
2642    #[test]
2643    fn test_pool_shared_atomic_counters() {
2644        let shared = PoolShared::<MockConnection>::new(PoolConfig::new(5));
2645
2646        // Initial values should be 0
2647        assert_eq!(shared.connections_created.load(Ordering::Relaxed), 0);
2648        assert_eq!(shared.connections_closed.load(Ordering::Relaxed), 0);
2649        assert_eq!(shared.acquires.load(Ordering::Relaxed), 0);
2650        assert_eq!(shared.timeouts.load(Ordering::Relaxed), 0);
2651
2652        // Test incrementing
2653        shared.connections_created.fetch_add(1, Ordering::Relaxed);
2654        shared.connections_closed.fetch_add(2, Ordering::Relaxed);
2655        shared.acquires.fetch_add(10, Ordering::Relaxed);
2656        shared.timeouts.fetch_add(3, Ordering::Relaxed);
2657
2658        assert_eq!(shared.connections_created.load(Ordering::Relaxed), 1);
2659        assert_eq!(shared.connections_closed.load(Ordering::Relaxed), 2);
2660        assert_eq!(shared.acquires.load(Ordering::Relaxed), 10);
2661        assert_eq!(shared.timeouts.load(Ordering::Relaxed), 3);
2662    }
2663
2664    #[test]
2665    fn test_pool_close_clears_idle() {
2666        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
2667
2668        // Add some idle connections
2669        {
2670            let mut inner = pool.shared.inner.lock().unwrap();
2671            inner.total_count = 3;
2672            inner
2673                .idle
2674                .push_back(ConnectionMeta::new(MockConnection::new(1)));
2675            inner
2676                .idle
2677                .push_back(ConnectionMeta::new(MockConnection::new(2)));
2678            inner
2679                .idle
2680                .push_back(ConnectionMeta::new(MockConnection::new(3)));
2681        }
2682
2683        assert_eq!(pool.idle_count(), 3);
2684        assert_eq!(pool.total_count(), 3);
2685
2686        pool.close();
2687
2688        // After close, idle connections should be cleared
2689        assert_eq!(pool.idle_count(), 0);
2690        assert_eq!(pool.total_count(), 0);
2691        assert!(pool.is_closed());
2692
2693        // connections_closed should reflect the 3 idle connections
2694        assert_eq!(pool.stats().connections_closed, 3);
2695    }
2696
2697    // ==================== Lock Poisoning Safety Tests ====================
2698    //
2699    // These tests verify that the pool correctly handles mutex poisoning,
2700    // which occurs when a thread panics while holding the lock.
2701    //
2702    // Tier 1 (mutations): Return Error if poisoned
2703    // Tier 2 (read-only): Recover and return valid data
2704    // Tier 3 (Drop): Log, close, and recover drain accounting (don't panic)
2705
2706    /// Helper to poison a pool's mutex by panicking while holding the lock.
2707    ///
2708    /// Returns the pool with a poisoned mutex.
2709    fn poison_pool_mutex() -> Pool<MockConnection> {
2710        use std::panic;
2711        use std::thread;
2712
2713        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
2714
2715        // Set up some valid state before poisoning
2716        {
2717            let mut inner = pool.shared.inner.lock().unwrap();
2718            inner.total_count = 2;
2719            inner.active_count = 1;
2720            inner
2721                .idle
2722                .push_back(ConnectionMeta::new(MockConnection::new(1)));
2723        }
2724
2725        // Spawn a thread that will panic while holding the lock
2726        let shared_clone = Arc::clone(&pool.shared);
2727        let handle = thread::spawn(move || {
2728            let _guard = shared_clone.inner.lock().unwrap();
2729            // Panic while holding the lock - this poisons the mutex
2730            panic!("intentional panic to poison mutex");
2731        });
2732
2733        // Wait for the thread to panic (ignore the panic result)
2734        let _ = handle.join();
2735
2736        // Verify the mutex is now poisoned
2737        assert!(pool.shared.inner.lock().is_err());
2738
2739        pool
2740    }
2741
2742    // -------------------- Tier 2: Read-Only Methods --------------------
2743
2744    #[test]
2745    fn test_config_after_poisoning_returns_valid_data() {
2746        let pool = poison_pool_mutex();
2747
2748        // config() should recover and return the configuration
2749        let config = pool.config();
2750        assert_eq!(config.max_connections, 5);
2751    }
2752
2753    #[test]
2754    fn test_stats_after_poisoning_returns_valid_data() {
2755        let pool = poison_pool_mutex();
2756
2757        // stats() should recover and return valid statistics
2758        let stats = pool.stats();
2759        // The state before poisoning was: total=2, active=1, idle=1
2760        assert_eq!(stats.total_connections, 2);
2761        assert_eq!(stats.active_connections, 1);
2762        assert_eq!(stats.idle_connections, 1);
2763    }
2764
2765    #[test]
2766    fn test_at_capacity_after_poisoning() {
2767        let pool = poison_pool_mutex();
2768
2769        // at_capacity() should recover and return correct value
2770        // Pool has 2 connections, max is 5, so not at capacity
2771        assert!(!pool.at_capacity());
2772    }
2773
2774    #[test]
2775    fn test_is_closed_after_poisoning() {
2776        let pool = poison_pool_mutex();
2777
2778        // is_closed() should recover and return correct value
2779        assert!(!pool.is_closed());
2780    }
2781
2782    #[test]
2783    fn test_idle_count_after_poisoning() {
2784        let pool = poison_pool_mutex();
2785
2786        // idle_count() should recover and return correct value
2787        assert_eq!(pool.idle_count(), 1);
2788    }
2789
2790    #[test]
2791    fn test_active_count_after_poisoning() {
2792        let pool = poison_pool_mutex();
2793
2794        // active_count() should recover and return correct value
2795        assert_eq!(pool.active_count(), 1);
2796    }
2797
2798    #[test]
2799    fn test_total_count_after_poisoning() {
2800        let pool = poison_pool_mutex();
2801
2802        // total_count() should recover and return correct value
2803        assert_eq!(pool.total_count(), 2);
2804    }
2805
2806    // -------------------- Tier 1: Mutation Methods --------------------
2807
2808    #[test]
2809    fn test_lock_or_error_returns_error_when_poisoned() {
2810        use std::thread;
2811
2812        let shared = Arc::new(PoolShared::<MockConnection>::new(PoolConfig::new(5)));
2813
2814        // Poison the mutex
2815        let shared_clone = Arc::clone(&shared);
2816        let handle = thread::spawn(move || {
2817            let _guard = shared_clone.inner.lock().unwrap();
2818            panic!("intentional panic to poison mutex");
2819        });
2820        let _ = handle.join();
2821
2822        // lock_or_error should return an error
2823        let result = shared.lock_or_error("test_operation");
2824
2825        // Verify it's a pool poisoning error
2826        match result {
2827            Err(Error::Pool(pool_err)) => {
2828                assert!(matches!(pool_err.kind, PoolErrorKind::Poisoned));
2829                assert!(pool_err.message.contains("poisoned"));
2830            }
2831            Err(other) => panic!("Expected Pool error, got: {:?}", other),
2832            Ok(_) => panic!("Expected error, got Ok"),
2833        }
2834    }
2835
2836    #[test]
2837    fn test_lock_or_recover_succeeds_when_poisoned() {
2838        use std::thread;
2839
2840        let shared = Arc::new(PoolShared::<MockConnection>::new(PoolConfig::new(5)));
2841
2842        // Set up some state
2843        {
2844            let mut inner = shared.inner.lock().unwrap();
2845            inner.total_count = 42;
2846        }
2847
2848        // Poison the mutex
2849        let shared_clone = Arc::clone(&shared);
2850        let handle = thread::spawn(move || {
2851            let _guard = shared_clone.inner.lock().unwrap();
2852            panic!("intentional panic to poison mutex");
2853        });
2854        let _ = handle.join();
2855
2856        // Verify mutex is poisoned
2857        assert!(shared.inner.lock().is_err());
2858
2859        // lock_or_recover should still succeed and provide access to data
2860        let inner = shared.lock_or_recover();
2861        assert_eq!(inner.total_count, 42);
2862    }
2863
2864    #[test]
2865    fn test_close_after_poisoning_recovers_and_closes() {
2866        let pool = poison_pool_mutex();
2867
2868        // close() should recover from poisoning and still close the pool
2869        pool.close();
2870
2871        // After close, the pool should be marked as closed
2872        assert!(pool.is_closed());
2873
2874        // Idle connections should be cleared
2875        assert_eq!(pool.idle_count(), 0);
2876    }
2877
2878    #[test]
2879    fn test_poisoned_pool_return_completes_drain_accounting() {
2880        use std::thread;
2881
2882        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(1));
2883        {
2884            let mut inner = pool.shared.inner.lock().unwrap();
2885            inner.total_count = 1;
2886            inner.active_count = 1;
2887        }
2888        let pooled = PooledConnection::new(
2889            ConnectionMeta::new(MockConnection::new(1)),
2890            Arc::downgrade(&pool.shared),
2891        );
2892
2893        let shared = Arc::clone(&pool.shared);
2894        let poisoner = thread::spawn(move || {
2895            let _guard = shared.inner.lock().unwrap();
2896            panic!("intentional panic to poison drain accounting");
2897        });
2898        let _ = poisoner.join();
2899
2900        let cx = Cx::for_testing();
2901        let mut drain = Box::pin(pool.close_and_drain(&cx));
2902        let mut task_cx = Context::from_waker(Waker::noop());
2903
2904        assert!(matches!(drain.as_mut().poll(&mut task_cx), Poll::Pending));
2905        assert!(pool.is_closed());
2906        assert_eq!(pool.active_count(), 1);
2907
2908        drop(pooled);
2909
2910        assert_eq!(pool.active_count(), 0);
2911        assert_eq!(pool.total_count(), 0);
2912        assert!(
2913            pool.shared.active_drained.get().is_some(),
2914            "poison-aware final release must publish the drain latch"
2915        );
2916        assert!(matches!(
2917            drain.as_mut().poll(&mut task_cx),
2918            Poll::Ready(Outcome::Err(Error::Pool(PoolError {
2919                kind: PoolErrorKind::Poisoned,
2920                ..
2921            })))
2922        ));
2923    }
2924
2925    // -------------------- Tier 3: Drop Safety --------------------
2926
2927    #[test]
2928    fn test_drop_pooled_connection_after_poisoning_does_not_panic() {
2929        use std::panic;
2930        use std::thread;
2931
2932        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
2933
2934        // Set up a connection that's "checked out"
2935        {
2936            let mut inner = pool.shared.inner.lock().unwrap();
2937            inner.total_count = 1;
2938            inner.active_count = 1;
2939        }
2940
2941        // Create a pooled connection
2942        let meta = ConnectionMeta::new(MockConnection::new(1));
2943        let pooled = PooledConnection::new(meta, Arc::downgrade(&pool.shared));
2944
2945        // Poison the mutex by panicking in another thread
2946        let shared_clone = Arc::clone(&pool.shared);
2947        let handle = thread::spawn(move || {
2948            let _guard = shared_clone.inner.lock().unwrap();
2949            panic!("intentional panic to poison mutex");
2950        });
2951        let _ = handle.join();
2952
2953        // Verify mutex is poisoned
2954        assert!(pool.shared.inner.lock().is_err());
2955
2956        // Drop the pooled connection - should NOT panic
2957        // The connection will be leaked, but that's the correct behavior
2958        let drop_result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
2959            drop(pooled);
2960        }));
2961
2962        // Dropping should not panic
2963        assert!(
2964            drop_result.is_ok(),
2965            "Dropping PooledConnection after mutex poisoning should not panic"
2966        );
2967    }
2968
2969    #[test]
2970    fn test_detach_after_poisoning_does_not_panic() {
2971        use std::panic;
2972        use std::thread;
2973
2974        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
2975
2976        // Set up a connection that's "checked out"
2977        {
2978            let mut inner = pool.shared.inner.lock().unwrap();
2979            inner.total_count = 1;
2980            inner.active_count = 1;
2981        }
2982
2983        // Create a pooled connection
2984        let meta = ConnectionMeta::new(MockConnection::new(42));
2985        let pooled = PooledConnection::new(meta, Arc::downgrade(&pool.shared));
2986
2987        // Poison the mutex
2988        let shared_clone = Arc::clone(&pool.shared);
2989        let handle = thread::spawn(move || {
2990            let _guard = shared_clone.inner.lock().unwrap();
2991            panic!("intentional panic to poison mutex");
2992        });
2993        let _ = handle.join();
2994
2995        // Verify mutex is poisoned
2996        assert!(pool.shared.inner.lock().is_err());
2997
2998        // Detach should not panic, even though it can't update counters
2999        let detach_result = panic::catch_unwind(panic::AssertUnwindSafe(|| pooled.detach()));
3000
3001        assert!(
3002            detach_result.is_ok(),
3003            "detach() after mutex poisoning should not panic"
3004        );
3005
3006        // Should still get the connection back
3007        let conn = detach_result.unwrap();
3008        assert_eq!(conn.id, 42);
3009    }
3010
3011    // -------------------- Integration: Pool Survives Thread Panic --------------------
3012
3013    #[test]
3014    fn test_pool_survives_thread_panic_during_acquire() {
3015        use std::thread;
3016
3017        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
3018        let pool_arc = Arc::new(pool);
3019
3020        // Simulate a thread that acquires, does work, then panics
3021        // The connection should be leaked but pool should remain usable for reads
3022        let pool_clone = Arc::clone(&pool_arc);
3023        let handle = thread::spawn(move || {
3024            // Manually simulate having acquired a connection
3025            {
3026                let mut inner = pool_clone.shared.inner.lock().unwrap();
3027                inner.total_count = 1;
3028                inner.active_count = 1;
3029            }
3030
3031            // Panic while holding the pool's internal mutex to simulate a poisoned lock.
3032            // This models an internal panic in pool bookkeeping, not user code.
3033            let _guard = pool_clone.shared.inner.lock().unwrap();
3034            panic!("simulated panic during database operation");
3035        });
3036
3037        // Wait for thread to panic
3038        let _ = handle.join();
3039
3040        // Pool's mutex is now poisoned, but read-only methods should still work
3041        assert_eq!(pool_arc.total_count(), 1);
3042        assert_eq!(pool_arc.config().max_connections, 5);
3043
3044        // Stats should be recoverable
3045        let stats = pool_arc.stats();
3046        assert_eq!(stats.total_connections, 1);
3047    }
3048
3049    #[test]
3050    fn test_pool_close_after_thread_panic() {
3051        use std::thread;
3052
3053        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
3054
3055        // Add some idle connections
3056        {
3057            let mut inner = pool.shared.inner.lock().unwrap();
3058            inner.total_count = 2;
3059            inner
3060                .idle
3061                .push_back(ConnectionMeta::new(MockConnection::new(1)));
3062            inner
3063                .idle
3064                .push_back(ConnectionMeta::new(MockConnection::new(2)));
3065        }
3066
3067        // Poison the mutex
3068        let shared_clone = Arc::clone(&pool.shared);
3069        let handle = thread::spawn(move || {
3070            let _guard = shared_clone.inner.lock().unwrap();
3071            panic!("intentional panic");
3072        });
3073        let _ = handle.join();
3074
3075        // close() should recover and still work
3076        pool.close();
3077
3078        // Pool should be closed and idle connections cleared
3079        assert!(pool.is_closed());
3080        assert_eq!(pool.idle_count(), 0);
3081    }
3082
3083    #[test]
3084    fn test_multiple_reads_after_poisoning() {
3085        let pool = poison_pool_mutex();
3086
3087        // Multiple read operations should all succeed
3088        for _ in 0..10 {
3089            let _ = pool.config();
3090            let _ = pool.stats();
3091            let _ = pool.at_capacity();
3092            let _ = pool.is_closed();
3093            let _ = pool.idle_count();
3094            let _ = pool.active_count();
3095            let _ = pool.total_count();
3096        }
3097
3098        // All reads should have recovered successfully
3099        assert_eq!(pool.total_count(), 2);
3100    }
3101
3102    #[test]
3103    fn test_waiters_count_after_poisoning() {
3104        use std::thread;
3105
3106        let pool: Pool<MockConnection> = Pool::new(PoolConfig::new(5));
3107
3108        // Set up waiter count
3109        {
3110            let mut inner = pool.shared.inner.lock().unwrap();
3111            inner.waiter_count = 3;
3112        }
3113
3114        // Poison the mutex
3115        let shared_clone = Arc::clone(&pool.shared);
3116        let handle = thread::spawn(move || {
3117            let _guard = shared_clone.inner.lock().unwrap();
3118            panic!("intentional panic");
3119        });
3120        let _ = handle.join();
3121
3122        // stats() should recover and show correct waiter count
3123        let stats = pool.stats();
3124        assert_eq!(stats.pending_requests, 3);
3125    }
3126}