#![allow(clippy::unwrap_used)]
#[path = "pool_test/mock_pg.rs"]
mod mock_pg;
use std::sync::Arc;
use arc_swap::ArcSwap;
use futures_util::{SinkExt, StreamExt};
use tokio::net::{TcpListener, TcpStream};
use tokio_util::codec::Framed;
use halephant_core::config::Config;
use halephant_core::config::cluster::pool::ListenMode;
use halephant_core::pool::PoolManager;
use halephant_core::pool::types::{CheckoutError, classify_checkout_error};
use halephant_core::proto::backend::BackendMessage;
use halephant_core::proto::codec::{BackendCodec, FrontendCodec};
use halephant_core::proto::frontend::FrontendMessage;
use halephant_core::proto::types::TransactionStatus;
use halephant_core::topology::{ClusterTopology, TopologyManager};
use super::test_client_guard;
use mock_pg::{MockBehavior, MockPg};
const TEST_CHECKOUT_TIMEOUT: &str = "50ms";
fn test_config(node_addr: &str) -> Config {
Config::parse(&format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
[cluster.test]
nodes = ["{node_addr}"]
[cluster.test.pool.testdb]
max_connections = {{ primary = 10 }}
[cluster.test.pool.testdb.user.testuser]
"#
))
.unwrap()
}
fn test_config_max(node_addr: &str, max: u32) -> Config {
Config::parse(&format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
[cluster.test]
nodes = ["{node_addr}"]
[cluster.test.pool.testdb]
max_connections = {{ primary = {max} }}
[cluster.test.pool.testdb.user.testuser]
"#
))
.unwrap()
}
fn test_config_with_replica(primary_addr: &str, replica_addr: &str) -> Config {
Config::parse(&format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
[cluster.test]
nodes = ["{primary_addr}", "{replica_addr}"]
[cluster.test.pool.testdb]
max_connections = {{ primary = 10, replica = 10 }}
[cluster.test.pool.testdb.user.testuser]
[cluster.test.pool.testdb.user.rouser]
max_connections = {{ replica = 10 }}
"#
))
.unwrap()
}
fn test_config_session(node_addr: &str) -> Config {
Config::parse(&format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
[cluster.test]
nodes = ["{node_addr}"]
[cluster.test.pool.testdb]
mode = "session"
max_connections = {{ primary = 10 }}
[cluster.test.pool.testdb.user.testuser]
"#
))
.unwrap()
}
async fn tcp_pair() -> (TcpStream, TcpStream) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let (client, server) = tokio::join!(TcpStream::connect(addr), async {
listener.accept().await.map(|(s, _)| s)
});
(client.unwrap(), server.unwrap())
}
fn seed_primary(topology: &TopologyManager, primary_addr: String) {
topology.set(
"test",
ClusterTopology {
primary: Some(primary_addr),
replicas: Vec::new(),
unreachable: Vec::new(),
},
);
}
fn setup_pool(mock: &MockPg) -> (Arc<PoolManager>, Arc<ArcSwap<Config>>) {
setup_pool_with_config(test_config(&mock.addr()), mock)
}
fn setup_pool_with_config(cfg: Config, mock: &MockPg) -> (Arc<PoolManager>, Arc<ArcSwap<Config>>) {
let config = Arc::new(ArcSwap::from_pointee(cfg));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary(&topology, mock.addr());
let pools = Arc::new(PoolManager::new(
Arc::clone(&config),
Arc::clone(&topology),
pgpass,
));
(pools, config)
}
fn spawn_tx_forward(
proxy_stream: TcpStream,
pools: Arc<PoolManager>,
database: &str,
user: &str,
read_only: bool,
) -> tokio::task::JoinHandle<anyhow::Result<()>> {
let db = database.to_owned();
let usr = user.to_owned();
tokio::spawn(async move {
let client = test_client_guard();
let mut proxy_client = Framed::new(proxy_stream, FrontendCodec::post_startup());
halephant_core::proxy::transaction::forward(
&mut proxy_client,
&pools,
&client,
&mut None,
ListenMode::Pin,
&db,
&usr,
read_only,
halephant_core::config::otel::QueryText::Off,
)
.await
})
}
async fn query_collect(
client: &mut Framed<TcpStream, BackendCodec>,
sql: &str,
) -> Vec<BackendMessage> {
client
.send(FrontendMessage::Query(sql.into()))
.await
.unwrap();
let mut msgs = Vec::new();
loop {
let msg = client.next().await.unwrap().unwrap();
let is_ready = matches!(msg, BackendMessage::ReadyForQuery(_));
msgs.push(msg);
if is_ready {
break;
}
}
msgs
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn basic_round_trip() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
let msgs = query_collect(&mut client, "SELECT 1").await;
assert!(
msgs.iter()
.any(|m| matches!(m, BackendMessage::RowDescription(_))),
"expected RowDescription in response"
);
assert!(
msgs.iter()
.any(|m| matches!(m, BackendMessage::CommandComplete(_))),
"expected CommandComplete in response"
);
assert!(
matches!(
msgs.last(),
Some(BackendMessage::ReadyForQuery(TransactionStatus::Idle))
),
"expected ReadyForQuery(Idle) as final message"
);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn connection_reuse() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
let _ = query_collect(&mut client, "SELECT 1").await;
assert_eq!(mock.connections(), 1, "first query opens one connection");
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
loop {
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
let before = mock.connections();
let _ = query_collect(&mut client, "SELECT 2").await;
if mock.connections() == before {
break;
}
assert!(
tokio::time::Instant::now() < deadline,
"connection was not reused within 2 s"
);
}
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn pool_exhaustion() {
let mock = MockPg::start(MockBehavior::Ok).await;
let config = Arc::new(ArcSwap::from_pointee(test_config_max(&mock.addr(), 1)));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
topology.refresh().await;
let pools = Arc::new(PoolManager::new(
Arc::clone(&config),
Arc::clone(&topology),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
let client = test_client_guard();
let guard = pools.checkout(&client, "testdb", "testuser", false).await;
assert!(guard.is_ok(), "first checkout should succeed");
let result = pools.checkout(&client, "testdb", "testuser", false).await;
assert!(result.is_err(), "second checkout should fail");
let err = result.err().unwrap();
assert!(
err.to_string().contains("timed out"),
"error should mention checkout timeout, got: {err}"
);
drop(guard);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn transaction_boundary() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
let msgs = query_collect(&mut client, "BEGIN").await;
assert!(
matches!(
msgs.last(),
Some(BackendMessage::ReadyForQuery(
TransactionStatus::InTransaction
))
),
"expected InTransaction after BEGIN"
);
let msgs = query_collect(&mut client, "SELECT 1").await;
assert!(
matches!(
msgs.last(),
Some(BackendMessage::ReadyForQuery(
TransactionStatus::InTransaction
))
),
"expected InTransaction during transaction"
);
let msgs = query_collect(&mut client, "COMMIT").await;
assert!(
matches!(
msgs.last(),
Some(BackendMessage::ReadyForQuery(TransactionStatus::Idle))
),
"expected Idle after COMMIT"
);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn server_close_mid_transaction() {
let mock = MockPg::start(MockBehavior::CloseOnQuery).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
client
.send(FrontendMessage::Query("SELECT 1".into()))
.await
.unwrap();
let mut got_error = false;
loop {
match client.next().await {
Some(Ok(BackendMessage::ErrorResponse(fields))) => {
let code = fields
.fields
.iter()
.find(|(k, _)| *k == b'C')
.map(|(_, v)| v.as_str());
assert_eq!(code, Some("08006"), "expected connection_failure SQLSTATE");
got_error = true;
}
Some(Ok(BackendMessage::ReadyForQuery(_))) | None => break, _ => {}
}
}
assert!(got_error, "expected synthesized ErrorResponse from proxy");
let result = handle.await.unwrap();
assert!(result.is_err());
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn server_close_after_startup() {
let mock = MockPg::start(MockBehavior::CloseAfterStartup).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
client
.send(FrontendMessage::Query("SELECT 1".into()))
.await
.unwrap();
while let Some(Ok(_)) = client.next().await {}
let result = handle.await.unwrap();
assert!(result.is_err());
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn read_only_routing() {
let primary = MockPg::start(MockBehavior::Ok).await;
let replica = MockPg::start_replica(MockBehavior::Ok).await;
let config = Arc::new(ArcSwap::from_pointee(test_config_with_replica(
&primary.addr(),
&replica.addr(),
)));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
topology.refresh().await;
let pools = Arc::new(PoolManager::new(
Arc::clone(&config),
Arc::clone(&topology),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
let client = test_client_guard();
let guard = pools.checkout(&client, "testdb", "testuser", true).await;
assert!(guard.is_ok(), "read-only checkout should succeed");
assert_eq!(
replica.connections(),
2, "read-only checkout should go to replica"
);
drop(guard);
let guard = pools.checkout(&client, "testdb", "testuser", false).await;
assert!(guard.is_ok(), "read-write checkout should succeed");
assert_eq!(
primary.connections(),
2, "read-write checkout should go to primary"
);
drop(guard);
primary.stop().await;
replica.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn read_only_user_routing() {
let primary = MockPg::start(MockBehavior::Ok).await;
let replica = MockPg::start_replica(MockBehavior::Ok).await;
let config = Arc::new(ArcSwap::from_pointee(test_config_with_replica(
&primary.addr(),
&replica.addr(),
)));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
topology.refresh().await;
let pools = Arc::new(PoolManager::new(
Arc::clone(&config),
Arc::clone(&topology),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "rouser", true);
let mut client = Framed::new(client_tcp, BackendCodec::new());
let _ = query_collect(&mut client, "SELECT 1").await;
assert_eq!(
replica.connections(),
2, "read-only user should route to replica"
);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
primary.stop().await;
replica.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn read_write_override_rejected() {
let primary = MockPg::start(MockBehavior::Ok).await;
let replica = MockPg::start_replica(MockBehavior::Ok).await;
let config = Arc::new(ArcSwap::from_pointee(test_config_with_replica(
&primary.addr(),
&replica.addr(),
)));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
topology.refresh().await;
let pools = Arc::new(PoolManager::new(
Arc::clone(&config),
Arc::clone(&topology),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", true);
let mut client = Framed::new(client_tcp, BackendCodec::new());
let msgs = query_collect(&mut client, "BEGIN READ WRITE").await;
let has_error = msgs.iter().any(|m| {
if let BackendMessage::ErrorResponse(fields) = m {
fields
.fields
.iter()
.any(|(k, v)| *k == b'C' && v == "25006")
} else {
false
}
});
assert!(has_error, "expected ErrorResponse with SQLSTATE 25006");
let msgs = query_collect(&mut client, "SELECT 1").await;
assert!(
matches!(
msgs.last(),
Some(BackendMessage::ReadyForQuery(TransactionStatus::Idle))
),
"proxy should still be functional after rejected override"
);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
primary.stop().await;
replica.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn begin_read_only_routes_to_replica() {
let primary = MockPg::start(MockBehavior::Ok).await;
let replica = MockPg::start_replica(MockBehavior::Ok).await;
let config = Arc::new(ArcSwap::from_pointee(test_config_with_replica(
&primary.addr(),
&replica.addr(),
)));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
topology.refresh().await;
let pools = Arc::new(PoolManager::new(
Arc::clone(&config),
Arc::clone(&topology),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
let primary_before = primary.connections();
let msgs = query_collect(&mut client, "BEGIN READ ONLY").await;
assert!(
matches!(
msgs.last(),
Some(BackendMessage::ReadyForQuery(
TransactionStatus::InTransaction
))
),
"expected InTransaction after BEGIN READ ONLY"
);
assert_eq!(
primary.connections(),
primary_before,
"BEGIN READ ONLY should not open a new primary connection"
);
assert!(
replica.connections() > 1,
"BEGIN READ ONLY should have opened a replica connection"
);
let _ = query_collect(&mut client, "COMMIT").await;
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
primary.stop().await;
replica.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn session_mode_forward() {
let mock = MockPg::start(MockBehavior::Ok).await;
let config = Arc::new(ArcSwap::from_pointee(test_config_session(&mock.addr())));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
topology.refresh().await;
let pools = Arc::new(PoolManager::new(
Arc::clone(&config),
Arc::clone(&topology),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
let checkout_client = test_client_guard();
let mut guard = pools
.checkout(&checkout_client, "testdb", "testuser", false)
.await
.unwrap();
let (client_tcp, proxy_tcp) = tcp_pair().await;
let mut proxy_client = Framed::new(proxy_tcp, FrontendCodec::post_startup());
let mut client = Framed::new(client_tcp, BackendCodec::new());
halephant_core::messages::send_post_auth_startup(&mut proxy_client, guard.conn())
.await
.unwrap();
loop {
let msg = client.next().await.unwrap().unwrap();
if matches!(msg, BackendMessage::ReadyForQuery(_)) {
break;
}
}
let pools_for_session = Arc::clone(&pools);
let handle = tokio::spawn(async move {
halephant_core::proxy::session::forward(&mut proxy_client, guard.conn(), &pools_for_session)
.await
});
let msgs = query_collect(&mut client, "SELECT 1").await;
assert!(
msgs.iter()
.any(|m| matches!(m, BackendMessage::CommandComplete(_))),
"expected CommandComplete in session mode"
);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn concurrent_checkouts() {
let mock = MockPg::start(MockBehavior::Ok).await;
let config = Arc::new(ArcSwap::from_pointee(test_config_max(&mock.addr(), 5)));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
topology.refresh().await;
let pools = Arc::new(PoolManager::new(
Arc::clone(&config),
Arc::clone(&topology),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
let mut handles = Vec::new();
for _ in 0..5 {
let pools = Arc::clone(&pools);
handles.push(tokio::spawn(async move {
let client = test_client_guard();
pools.checkout(&client, "testdb", "testuser", false).await
}));
}
let mut guards = Vec::new();
for h in handles {
guards.push(h.await.unwrap().unwrap());
}
assert_eq!(mock.connections(), 6);
let client = test_client_guard();
let result = pools.checkout(&client, "testdb", "testuser", false).await;
assert!(result.is_err(), "6th checkout should fail at max=5");
assert!(
result.err().unwrap().to_string().contains("timed out"),
"6th checkout should time out"
);
drop(guards);
mock.stop().await;
}
fn setup_scram_pool(mock: &MockPg, password: &str) -> (Arc<PoolManager>, Arc<ArcSwap<Config>>) {
let config = Arc::new(ArcSwap::from_pointee(test_config(&mock.addr())));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
seed_primary(&topology, mock.addr());
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(&format!(
"*:*:*:*:{password}"
)));
let pools = Arc::new(PoolManager::new(
Arc::clone(&config),
Arc::clone(&topology),
pgpass,
));
(pools, config)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scram_auth_success() {
let password = "testpassword";
let verifier = mock_pg::test_verifier(password);
let mock = MockPg::start(MockBehavior::ScramAuth(verifier)).await;
let (pools, _config) = setup_scram_pool(&mock, password);
let client = test_client_guard();
let guard = pools.checkout(&client, "testdb", "testuser", false).await;
assert!(
guard.is_ok(),
"SCRAM checkout should succeed: {:?}",
guard.err()
);
drop(guard);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scram_auth_wrong_password() {
let verifier = mock_pg::test_verifier("correct");
let mock = MockPg::start(MockBehavior::ScramAuth(verifier)).await;
let (pools, _config) = setup_scram_pool(&mock, "wrong");
let client = test_client_guard();
let result = pools.checkout(&client, "testdb", "testuser", false).await;
assert!(
result.is_err(),
"SCRAM checkout with wrong password should fail"
);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scram_auth_no_password() {
let verifier = mock_pg::test_verifier("password");
let mock = MockPg::start(MockBehavior::ScramAuth(verifier)).await;
let config = Arc::new(ArcSwap::from_pointee(test_config(&mock.addr())));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
seed_primary(&topology, mock.addr());
let pools = Arc::new(PoolManager::new(
Arc::clone(&config),
Arc::clone(&topology),
Arc::new(halephant_core::auth::pgpass::Pgpass::parse("")),
));
let client = test_client_guard();
let result = pools.checkout(&client, "testdb", "testuser", false).await;
assert!(result.is_err(), "checkout without password should fail");
let err = result.err().unwrap().to_string();
assert!(
err.contains("pgpass") || err.contains("password"),
"error should mention missing password: {err}"
);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scram_auth_query_round_trip() {
let password = "querypass";
let verifier = mock_pg::test_verifier(password);
let mock = MockPg::start(MockBehavior::ScramAuth(verifier)).await;
let (pools, _config) = setup_scram_pool(&mock, password);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
let msgs = query_collect(&mut client, "SELECT 1").await;
assert!(
msgs.iter()
.any(|m| matches!(m, BackendMessage::CommandComplete(_))),
"should get CommandComplete"
);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
mock.stop().await;
}
fn total_idle(pools: &PoolManager) -> u32 {
pools.pool_stats().iter().map(|(_, s)| s.idle).sum()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scavenger_preserves_min_connections_floor() {
let mock = MockPg::start(MockBehavior::Ok).await;
let toml = format!(
r#"
[cluster.test]
nodes = ["{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
idle_timeout = "50ms"
max_connections = {{ primary = 10 }}
[cluster.test.pool.testdb.user.testuser]
min_connections = {{ primary = 3 }}
"#,
mock.addr()
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary(&topology, mock.addr());
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
pools.warm_up().await;
assert_eq!(total_idle(&pools), 3, "warm-up should produce 3 idle");
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
pools.scavenge_idle();
assert_eq!(
total_idle(&pools),
3,
"scavenger must preserve min_connections.primary"
);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scavenger_releases_excess_above_floor() {
let mock = MockPg::start(MockBehavior::Ok).await;
let toml = format!(
r#"
[cluster.test]
nodes = ["{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
idle_timeout = "50ms"
max_connections = {{ primary = 10 }}
[cluster.test.pool.testdb.user.testuser]
min_connections = {{ primary = 2 }}
"#,
mock.addr()
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary(&topology, mock.addr());
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
pools.warm_up().await;
assert_eq!(total_idle(&pools), 2, "warm-up populates the floor");
let client = test_client_guard();
let mut guards = Vec::new();
for _ in 0..3 {
guards.push(
pools
.checkout(&client, "testdb", "testuser", false)
.await
.unwrap(),
);
}
for guard in guards {
guard.checkin();
}
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
while total_idle(&pools) < 3 {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
assert!(
tokio::time::Instant::now() < deadline,
"burst connections never returned to idle: idle={}",
total_idle(&pools)
);
}
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
pools.scavenge_idle();
assert_eq!(
total_idle(&pools),
2,
"scavenger should release excess down to the floor",
);
mock.stop().await;
}
fn seed_primary_with_replicas(
topology: &TopologyManager,
primary_addr: String,
replica_addrs: Vec<String>,
) {
topology.set(
"test",
ClusterTopology {
primary: Some(primary_addr),
replicas: replica_addrs,
unreachable: Vec::new(),
},
);
}
fn idle_at_node(pools: &PoolManager, node: &str) -> u32 {
pools
.pool_stats()
.iter()
.filter(|(k, _)| k.node == node)
.map(|(_, s)| s.idle)
.sum()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn warmup_per_node_min_with_two_replicas() {
let primary = MockPg::start(MockBehavior::Ok).await;
let replica_a = MockPg::start_replica(MockBehavior::Ok).await;
let replica_b = MockPg::start_replica(MockBehavior::Ok).await;
let toml = format!(
r#"
[cluster.test]
nodes = ["{}", "{}", "{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
max_connections = {{ primary = 10, replica = 10 }}
[cluster.test.pool.testdb.user.testuser]
min_connections = {{ primary = 0, replica = 3 }}
"#,
primary.addr(),
replica_a.addr(),
replica_b.addr(),
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary_with_replicas(
&topology,
primary.addr(),
vec![replica_a.addr(), replica_b.addr()],
);
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
pools.warm_up().await;
assert_eq!(
total_idle(&pools),
6,
"warm-up should produce 3 per replica = 6 total"
);
assert_eq!(idle_at_node(&pools, &replica_a.addr()), 3);
assert_eq!(idle_at_node(&pools, &replica_b.addr()), 3);
primary.stop().await;
replica_a.stop().await;
replica_b.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scavenger_preserves_per_replica_floor() {
let primary = MockPg::start(MockBehavior::Ok).await;
let replica_a = MockPg::start_replica(MockBehavior::Ok).await;
let replica_b = MockPg::start_replica(MockBehavior::Ok).await;
let toml = format!(
r#"
[cluster.test]
nodes = ["{}", "{}", "{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
idle_timeout = "50ms"
max_connections = {{ primary = 10, replica = 10 }}
[cluster.test.pool.testdb.user.testuser]
min_connections = {{ primary = 0, replica = 2 }}
"#,
primary.addr(),
replica_a.addr(),
replica_b.addr(),
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary_with_replicas(
&topology,
primary.addr(),
vec![replica_a.addr(), replica_b.addr()],
);
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
pools.warm_up().await;
assert_eq!(
total_idle(&pools),
4,
"warm-up should produce 2 per replica"
);
assert_eq!(idle_at_node(&pools, &replica_a.addr()), 2);
assert_eq!(idle_at_node(&pools, &replica_b.addr()), 2);
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
pools.scavenge_idle();
assert_eq!(
total_idle(&pools),
4,
"scavenger must preserve the floor on every replica"
);
assert_eq!(idle_at_node(&pools, &replica_a.addr()), 2);
assert_eq!(idle_at_node(&pools, &replica_b.addr()), 2);
primary.stop().await;
replica_a.stop().await;
replica_b.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn read_only_falls_back_to_primary_when_replicas_full() {
let primary = MockPg::start(MockBehavior::Ok).await;
let replica_a = MockPg::start_replica(MockBehavior::Ok).await;
let replica_b = MockPg::start_replica(MockBehavior::Ok).await;
let toml = format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
[cluster.test]
nodes = ["{}", "{}", "{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
max_connections = {{ primary = 10, replica = 2 }}
[cluster.test.pool.testdb.user.testuser]
"#,
primary.addr(),
replica_a.addr(),
replica_b.addr(),
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary_with_replicas(
&topology,
primary.addr(),
vec![replica_a.addr(), replica_b.addr()],
);
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
let client = test_client_guard();
let mut guards = Vec::new();
for _ in 0..4 {
let guard = pools
.checkout(&client, "testdb", "testuser", true)
.await
.expect("first 4 read-only checkouts should succeed");
guards.push(guard);
}
let guard5 = pools
.checkout(&client, "testdb", "testuser", true)
.await
.expect("5th read-only checkout should fall back to primary");
assert_eq!(
guard5.node(),
primary.addr(),
"fallback should route to the primary"
);
drop(guards);
drop(guard5);
primary.stop().await;
replica_a.stop().await;
replica_b.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn queue_wakes_waiter_on_checkin() {
let mock = MockPg::start(MockBehavior::Ok).await;
let toml = format!(
r#"
[server]
checkout_timeout = "5s"
[cluster.test]
nodes = ["{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
max_connections = {{ primary = 1 }}
[cluster.test.pool.testdb.user.testuser]
"#,
mock.addr()
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary(&topology, mock.addr());
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
let client1 = test_client_guard();
let guard1 = pools
.checkout(&client1, "testdb", "testuser", false)
.await
.expect("first checkout should succeed");
let pools2 = Arc::clone(&pools);
let waiter = tokio::spawn(async move {
let client2 = test_client_guard();
pools2.checkout(&client2, "testdb", "testuser", false).await
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
guard1.checkin();
let guard2 = tokio::time::timeout(std::time::Duration::from_secs(2), waiter)
.await
.expect("waiter did not wake within 2s")
.expect("waiter task panicked")
.expect("waiter checkout should succeed");
drop(guard2);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn queue_is_fifo() {
use std::sync::atomic::{AtomicU32, Ordering};
let mock = MockPg::start(MockBehavior::Ok).await;
let toml = format!(
r#"
[server]
checkout_timeout = "5s"
[cluster.test]
nodes = ["{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
max_connections = {{ primary = 1 }}
[cluster.test.pool.testdb.user.testuser]
"#,
mock.addr()
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary(&topology, mock.addr());
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
let client0 = test_client_guard();
let guard0 = pools
.checkout(&client0, "testdb", "testuser", false)
.await
.unwrap();
let order = Arc::new(AtomicU32::new(0));
let wait_for_depth = |target: u32| {
let pools = Arc::clone(&pools);
async move {
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
loop {
let depth: u32 = pools.queue_stats().iter().map(|q| q.depth).sum();
if depth >= target {
return;
}
assert!(
tokio::time::Instant::now() < deadline,
"wait queue never reached depth {target}"
);
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
}
}
};
let waiter_a = {
let pools = Arc::clone(&pools);
let order = Arc::clone(&order);
tokio::spawn(async move {
let client = test_client_guard();
let guard = pools
.checkout(&client, "testdb", "testuser", false)
.await
.unwrap();
let woke_at = order.fetch_add(1, Ordering::SeqCst);
(woke_at, guard)
})
};
wait_for_depth(1).await;
let waiter_b = {
let pools = Arc::clone(&pools);
let order = Arc::clone(&order);
tokio::spawn(async move {
let client = test_client_guard();
let guard = pools
.checkout(&client, "testdb", "testuser", false)
.await
.unwrap();
let woke_at = order.fetch_add(1, Ordering::SeqCst);
(woke_at, guard)
})
};
wait_for_depth(2).await;
guard0.checkin();
let (a_order, guard_a) = waiter_a.await.unwrap();
assert_eq!(a_order, 0, "waiter A (enqueued first) should wake first");
guard_a.checkin();
let (b_order, guard_b) = waiter_b.await.unwrap();
assert_eq!(b_order, 1, "waiter B (enqueued second) should wake second");
drop(guard_b);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn queue_timeout() {
let mock = MockPg::start(MockBehavior::Ok).await;
let toml = format!(
r#"
[server]
checkout_timeout = "50ms"
[cluster.test]
nodes = ["{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
max_connections = {{ primary = 1 }}
[cluster.test.pool.testdb.user.testuser]
"#,
mock.addr()
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary(&topology, mock.addr());
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
let client = test_client_guard();
let guard = pools
.checkout(&client, "testdb", "testuser", false)
.await
.unwrap();
let start = std::time::Instant::now();
let result = pools.checkout(&client, "testdb", "testuser", false).await;
let elapsed = start.elapsed();
assert!(result.is_err(), "second checkout should time out");
assert!(
elapsed >= std::time::Duration::from_millis(50),
"should have waited at least checkout_timeout before failing: {elapsed:?}"
);
assert!(
elapsed < std::time::Duration::from_millis(500),
"should have failed quickly after the timeout elapsed: {elapsed:?}"
);
assert!(result.err().unwrap().to_string().contains("timed out"));
drop(guard);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn queue_skips_cancelled_waiter() {
let mock = MockPg::start(MockBehavior::Ok).await;
let toml = format!(
r#"
[server]
checkout_timeout = "5s"
[cluster.test]
nodes = ["{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
max_connections = {{ primary = 1 }}
[cluster.test.pool.testdb.user.testuser]
"#,
mock.addr()
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary(&topology, mock.addr());
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
let client0 = test_client_guard();
let guard0 = pools
.checkout(&client0, "testdb", "testuser", false)
.await
.unwrap();
let pools_cancelled = Arc::clone(&pools);
let cancelled = tokio::spawn(async move {
let client = test_client_guard();
pools_cancelled
.checkout(&client, "testdb", "testuser", false)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
let pools_real = Arc::clone(&pools);
let real = tokio::spawn(async move {
let client = test_client_guard();
pools_real
.checkout(&client, "testdb", "testuser", false)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
cancelled.abort();
let _ = cancelled.await;
guard0.checkin();
let guard = tokio::time::timeout(std::time::Duration::from_secs(2), real)
.await
.expect("real waiter did not wake within 2s")
.expect("real waiter task panicked")
.expect("real waiter checkout should succeed");
drop(guard);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn queue_replica_is_shared() {
let primary = MockPg::start(MockBehavior::Ok).await;
let replica_a = MockPg::start_replica(MockBehavior::Ok).await;
let replica_b = MockPg::start_replica(MockBehavior::Ok).await;
let toml = format!(
r#"
[server]
checkout_timeout = "5s"
[cluster.test]
nodes = ["{}", "{}", "{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
max_connections = {{ primary = 10, replica = 1 }}
[cluster.test.pool.testdb.user.testuser]
"#,
primary.addr(),
replica_a.addr(),
replica_b.addr(),
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary_with_replicas(
&topology,
primary.addr(),
vec![replica_a.addr(), replica_b.addr()],
);
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
let client = test_client_guard();
let guard_a = pools
.checkout(&client, "testdb", "testuser", true)
.await
.unwrap();
let guard_b = pools
.checkout(&client, "testdb", "testuser", true)
.await
.unwrap();
let pools1 = Arc::clone(&pools);
let waiter1 = tokio::spawn(async move {
let c = test_client_guard();
pools1.checkout(&c, "testdb", "testuser", true).await
});
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
let pools2 = Arc::clone(&pools);
let waiter2 = tokio::spawn(async move {
let c = test_client_guard();
pools2.checkout(&c, "testdb", "testuser", true).await
});
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
guard_a.checkin();
let g1 = tokio::time::timeout(std::time::Duration::from_secs(2), waiter1)
.await
.expect("waiter1 did not wake within 2s")
.unwrap()
.expect("waiter1 checkout should succeed");
guard_b.checkin();
let g2 = tokio::time::timeout(std::time::Duration::from_secs(2), waiter2)
.await
.expect("waiter2 did not wake within 2s")
.unwrap()
.expect("waiter2 checkout should succeed");
drop(g1);
drop(g2);
primary.stop().await;
replica_a.stop().await;
replica_b.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn shutdown_wakes_queued_waiters() {
let mock = MockPg::start(MockBehavior::Ok).await;
let toml = format!(
r#"
[server]
checkout_timeout = "10s"
[cluster.test]
nodes = ["{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
max_connections = {{ primary = 1 }}
[cluster.test.pool.testdb.user.testuser]
"#,
mock.addr()
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary(&topology, mock.addr());
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
let client0 = test_client_guard();
let _guard0 = pools
.checkout(&client0, "testdb", "testuser", false)
.await
.unwrap();
let pools_waiter = Arc::clone(&pools);
let waiter = tokio::spawn(async move {
let client = test_client_guard();
pools_waiter
.checkout(&client, "testdb", "testuser", false)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let start = std::time::Instant::now();
pools.shutdown();
let result = tokio::time::timeout(std::time::Duration::from_secs(2), waiter)
.await
.expect("waiter did not wake within 2s of shutdown")
.expect("waiter task panicked");
let elapsed = start.elapsed();
assert!(
elapsed < std::time::Duration::from_millis(500),
"shutdown should wake waiters quickly, took: {elapsed:?}"
);
let err = result
.err()
.expect("waiter should fail with shutting-down error");
assert!(
err.to_string().contains("shutting down"),
"error should mention shutdown, got: {err}"
);
assert!(
matches!(
err.downcast_ref::<CheckoutError>(),
Some(CheckoutError::ShuttingDown)
),
"shutdown should downcast to CheckoutError::ShuttingDown"
);
assert_eq!(classify_checkout_error(&err), "shutting_down");
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn waiting_for_is_populated_during_wait() {
use halephant_core::clients::{ClientRegistry, ClientState, WaitRole};
let mock = MockPg::start(MockBehavior::Ok).await;
let toml = format!(
r#"
[server]
checkout_timeout = "5s"
[cluster.test]
nodes = ["{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
max_connections = {{ primary = 1 }}
[cluster.test.pool.testdb.user.testuser]
"#,
mock.addr()
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary(&topology, mock.addr());
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
let client0 = test_client_guard();
let guard0 = pools
.checkout(&client0, "testdb", "testuser", false)
.await
.unwrap();
let registry = Arc::new(ClientRegistry::new());
let waiter_registry = Arc::clone(®istry);
let pools_waiter = Arc::clone(&pools);
let waiter = tokio::spawn(async move {
let client = waiter_registry.register("127.0.0.1:0".parse().unwrap());
let result = pools_waiter
.checkout(&client, "testdb", "testuser", false)
.await;
(client, result)
});
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
let snap = loop {
let snap = registry.snapshot();
if let Some(entry) = snap.first()
&& entry.state == ClientState::Waiting
{
break snap;
}
assert!(
tokio::time::Instant::now() < deadline,
"waiter never transitioned to Waiting"
);
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
};
let entry = &snap[0];
let target = entry
.waiting_for
.as_ref()
.expect("waiting_for should be populated during wait");
assert_eq!(target.database, "testdb");
assert_eq!(target.user, "testuser");
assert_eq!(target.role, WaitRole::Primary);
guard0.checkin();
let (held_client, result) = tokio::time::timeout(std::time::Duration::from_secs(2), waiter)
.await
.expect("waiter did not wake")
.expect("waiter task panicked");
let guard = result.expect("waiter checkout should succeed");
let after = registry.snapshot();
let entry = after
.iter()
.find(|e| e.id == held_client.id())
.expect("entry still present");
assert_eq!(entry.state, ClientState::Idle);
assert!(
entry.waiting_for.is_none(),
"waiting_for should be cleared after wakeup"
);
drop(guard);
drop(held_client);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn per_pool_checkout_timeout_overrides_server_default() {
let mock = MockPg::start(MockBehavior::Ok).await;
let toml = format!(
r#"
[server]
checkout_timeout = "30s"
[cluster.test]
nodes = ["{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
max_connections = {{ primary = 1 }}
checkout_timeout = "50ms"
[cluster.test.pool.testdb.user.testuser]
"#,
mock.addr()
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary(&topology, mock.addr());
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
let client = test_client_guard();
let _guard = pools
.checkout(&client, "testdb", "testuser", false)
.await
.unwrap();
let start = std::time::Instant::now();
let result = pools.checkout(&client, "testdb", "testuser", false).await;
let elapsed = start.elapsed();
assert!(result.is_err(), "second checkout should time out");
assert!(
elapsed < std::time::Duration::from_millis(500),
"pool override should short-circuit the 30s server default, took: {elapsed:?}"
);
assert!(result.err().unwrap().to_string().contains("timed out"));
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn checkout_timeout_error_is_typed() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool_with_config(test_config_max(&mock.addr(), 1), &mock);
let client = test_client_guard();
let _guard = pools
.checkout(&client, "testdb", "testuser", false)
.await
.unwrap();
let err = pools
.checkout(&client, "testdb", "testuser", false)
.await
.err()
.expect("second checkout should time out at max=1");
assert!(
matches!(
err.downcast_ref::<CheckoutError>(),
Some(CheckoutError::Timeout { .. })
),
"timeout should downcast to CheckoutError::Timeout, got: {err}"
);
assert!(err.to_string().contains("timed out"));
assert_eq!(classify_checkout_error(&err), "checkout_timeout");
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn upstream_shutdown_error_is_a_connect_failure() {
let mock = MockPg::start(MockBehavior::ErrorOnStartup {
code: "57P03".into(),
message: "the database system is shutting down".into(),
})
.await;
let (pools, _config) = setup_pool(&mock);
let client = test_client_guard();
let err = pools
.checkout(&client, "testdb", "testuser", false)
.await
.err()
.expect("checkout against a shutting-down upstream should fail");
assert!(
err.to_string().contains("shutting down"),
"expected the upstream error text to surface, got: {err}"
);
assert!(
err.downcast_ref::<CheckoutError>().is_none(),
"an upstream connect failure must not be typed as CheckoutError"
);
assert_eq!(classify_checkout_error(&err), "connect_failed");
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unconfigured_user_never_reaches_upstream() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let client = test_client_guard();
let err = pools
.checkout(&client, "testdb", "ghost", false)
.await
.err()
.expect("unconfigured user should be refused");
assert!(
matches!(
err.downcast_ref::<CheckoutError>(),
Some(CheckoutError::UserNotConfigured { .. })
),
"refusal should downcast to CheckoutError::UserNotConfigured, got: {err}"
);
assert_eq!(
mock.connections(),
0,
"refusal must not open an upstream connection"
);
mock.stop().await;
}
fn test_config_alias(node_addr: &str, min_primary: u32) -> Config {
Config::parse(&format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
[cluster.test]
nodes = ["{node_addr}"]
[cluster.test.pool.testdb]
max_connections = {{ primary = 10 }}
[cluster.test.pool.testdb.user.app]
alias = "app_rw"
min_connections = {{ primary = {min_primary} }}
"#
))
.unwrap()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn checkout_connects_as_configured_alias() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool_with_config(test_config_alias(&mock.addr(), 0), &mock);
let client = test_client_guard();
let guard = pools
.checkout(&client, "testdb", "app", false)
.await
.expect("checkout for the aliased user should succeed");
assert_eq!(mock.received_startup_users(), vec!["app_rw".to_owned()]);
drop(guard);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn warm_up_connects_as_configured_alias() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool_with_config(test_config_alias(&mock.addr(), 2), &mock);
pools.warm_up().await;
assert_eq!(
mock.received_startup_users(),
vec!["app_rw".to_owned(), "app_rw".to_owned()],
"every warm-up connection should authenticate as the alias"
);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn deallocate_named_is_intercepted() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
client
.send(FrontendMessage::Parse(
halephant_core::proto::frontend::Parse {
name: "my_stmt".into(),
query: "SELECT 1".into(),
param_types: vec![],
},
))
.await
.unwrap();
client
.send(FrontendMessage::Bind(
halephant_core::proto::frontend::Bind {
portal: String::new(),
statement: "my_stmt".into(),
param_formats: vec![],
params: vec![],
result_formats: vec![],
},
))
.await
.unwrap();
client
.send(FrontendMessage::Execute(
halephant_core::proto::frontend::Execute {
portal: String::new(),
max_rows: 0,
},
))
.await
.unwrap();
client.send(FrontendMessage::Sync).await.unwrap();
loop {
let msg = client.next().await.unwrap().unwrap();
if matches!(msg, BackendMessage::ReadyForQuery(_)) {
break;
}
}
let msgs = query_collect(&mut client, "DEALLOCATE my_stmt").await;
assert!(
msgs.iter().any(|m| matches!(
m,
BackendMessage::CommandComplete(tag) if tag == "DEALLOCATE"
)),
"expected CommandComplete(\"DEALLOCATE\") in response: {msgs:?}"
);
assert!(
matches!(
msgs.last(),
Some(BackendMessage::ReadyForQuery(TransactionStatus::Idle))
),
"expected ReadyForQuery(Idle) as final message: {msgs:?}"
);
let received = mock.received_queries();
assert!(
!received
.iter()
.any(|q| q.to_ascii_uppercase().contains("DEALLOCATE")),
"mock should not have received any DEALLOCATE query, got: {received:?}"
);
drop(client);
let _ = handle.await;
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn deallocate_all_is_intercepted() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
for name in ["stmt_a", "stmt_b"] {
client
.send(FrontendMessage::Parse(
halephant_core::proto::frontend::Parse {
name: name.into(),
query: format!("SELECT {name}"),
param_types: vec![],
},
))
.await
.unwrap();
client
.send(FrontendMessage::Bind(
halephant_core::proto::frontend::Bind {
portal: String::new(),
statement: name.into(),
param_formats: vec![],
params: vec![],
result_formats: vec![],
},
))
.await
.unwrap();
client
.send(FrontendMessage::Execute(
halephant_core::proto::frontend::Execute {
portal: String::new(),
max_rows: 0,
},
))
.await
.unwrap();
client.send(FrontendMessage::Sync).await.unwrap();
loop {
let msg = client.next().await.unwrap().unwrap();
if matches!(msg, BackendMessage::ReadyForQuery(_)) {
break;
}
}
}
let msgs = query_collect(&mut client, "DEALLOCATE ALL").await;
assert!(
msgs.iter().any(|m| matches!(
m,
BackendMessage::CommandComplete(tag) if tag == "DEALLOCATE"
)),
"expected CommandComplete(\"DEALLOCATE\") in response: {msgs:?}"
);
let received = mock.received_queries();
assert!(
!received
.iter()
.any(|q| q.to_ascii_uppercase().contains("DEALLOCATE")),
"mock should not have received any DEALLOCATE query, got: {received:?}"
);
drop(client);
let _ = handle.await;
mock.stop().await;
}
async fn extended_query(client: &mut Framed<TcpStream, BackendCodec>, name: &str, query: &str) {
client
.send(FrontendMessage::Parse(
halephant_core::proto::frontend::Parse {
name: name.into(),
query: query.into(),
param_types: vec![],
},
))
.await
.unwrap();
client
.send(FrontendMessage::Bind(
halephant_core::proto::frontend::Bind {
portal: String::new(),
statement: name.into(),
param_formats: vec![],
params: vec![],
result_formats: vec![],
},
))
.await
.unwrap();
client
.send(FrontendMessage::Execute(
halephant_core::proto::frontend::Execute {
portal: String::new(),
max_rows: 0,
},
))
.await
.unwrap();
client.send(FrontendMessage::Sync).await.unwrap();
loop {
let msg = client.next().await.unwrap().unwrap();
if matches!(msg, BackendMessage::ReadyForQuery(_)) {
break;
}
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn server_lru_eviction_sends_close() {
let mock = MockPg::start(MockBehavior::Ok).await;
let cfg = Config::parse(&format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
max_prepared_statements = 2
[cluster.test]
nodes = ["{}"]
[cluster.test.pool.testdb.user.testuser]
"#,
mock.addr()
))
.unwrap();
let (pools, _config) = setup_pool_with_config(cfg, &mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
query_collect(&mut client, "BEGIN").await;
extended_query(&mut client, "s1", "SELECT 1").await;
extended_query(&mut client, "s2", "SELECT 2").await;
extended_query(&mut client, "s3", "SELECT 3").await;
query_collect(&mut client, "COMMIT").await;
let closes = mock.received_closes();
assert!(
!closes.is_empty(),
"expected at least one Close for LRU eviction, got none"
);
let parses = mock.received_parses();
assert_eq!(
parses.len(),
3,
"expected 3 Parse messages on server, got {parses:?}"
);
drop(client);
let _ = handle.await;
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unnamed_reuse_across_transactions() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
extended_query(&mut client, "", "SELECT 1").await;
client
.send(FrontendMessage::Bind(
halephant_core::proto::frontend::Bind {
portal: String::new(),
statement: String::new(),
param_formats: vec![],
params: vec![],
result_formats: vec![],
},
))
.await
.unwrap();
client
.send(FrontendMessage::Execute(
halephant_core::proto::frontend::Execute {
portal: String::new(),
max_rows: 0,
},
))
.await
.unwrap();
client.send(FrontendMessage::Sync).await.unwrap();
let mut saw_complete = false;
loop {
let msg = client.next().await.unwrap().unwrap();
if matches!(msg, BackendMessage::CommandComplete(_)) {
saw_complete = true;
}
assert!(
!matches!(msg, BackendMessage::ErrorResponse(_)),
"Bind on unnamed in new transaction should not fail"
);
if matches!(msg, BackendMessage::ReadyForQuery(_)) {
break;
}
}
assert!(saw_complete, "expected CommandComplete from unnamed reuse");
drop(client);
let _ = handle.await;
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unnamed_replacement_releases_old_canonical() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
extended_query(&mut client, "", "SELECT 1").await;
extended_query(&mut client, "", "SELECT 2").await;
let parses = mock.received_parses();
assert_eq!(parses.len(), 2, "expected 2 Parse messages: {parses:?}");
assert_ne!(parses[0], parses[1], "canonical names should differ");
client
.send(FrontendMessage::Bind(
halephant_core::proto::frontend::Bind {
portal: String::new(),
statement: String::new(),
param_formats: vec![],
params: vec![],
result_formats: vec![],
},
))
.await
.unwrap();
client
.send(FrontendMessage::Execute(
halephant_core::proto::frontend::Execute {
portal: String::new(),
max_rows: 0,
},
))
.await
.unwrap();
client.send(FrontendMessage::Sync).await.unwrap();
loop {
let msg = client.next().await.unwrap().unwrap();
assert!(
!matches!(msg, BackendMessage::ErrorResponse(_)),
"Bind on replaced unnamed should succeed"
);
if matches!(msg, BackendMessage::ReadyForQuery(_)) {
break;
}
}
let all_parses = mock.received_parses();
let last_parse = all_parses.last().unwrap();
assert_eq!(last_parse, &parses[1]);
drop(client);
let _ = handle.await;
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn client_disconnect_releases_all_prepared() {
use halephant_core::proxy::prepared::canonical_for_test;
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
extended_query(&mut client, "s1", "SELECT 1").await;
extended_query(&mut client, "s2", "SELECT 2").await;
extended_query(&mut client, "", "SELECT 3").await;
let c1 = canonical_for_test("SELECT 1", &[]);
let c2 = canonical_for_test("SELECT 2", &[]);
let c3 = canonical_for_test("SELECT 3", &[]);
assert!(pools.has_prepared(&c1), "SELECT 1 should be in store");
assert!(pools.has_prepared(&c2), "SELECT 2 should be in store");
assert!(pools.has_prepared(&c3), "SELECT 3 should be in store");
drop(client);
let _ = handle.await;
assert!(!pools.has_prepared(&c1), "SELECT 1 should be released");
assert!(!pools.has_prepared(&c2), "SELECT 2 should be released");
assert!(!pools.has_prepared(&c3), "SELECT 3 should be released");
mock.stop().await;
}
#[derive(Debug, PartialEq, Eq)]
enum Tag {
Parse,
Bind,
Command,
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn pipelined_autocommit_batches_each_get_a_transaction() {
let mock = MockPg::start_slow(MockBehavior::Ok, std::time::Duration::from_millis(20)).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
for (name, query) in [("a", "SELECT 1"), ("b", "SELECT 2")] {
client
.feed(FrontendMessage::Parse(
halephant_core::proto::frontend::Parse {
name: name.into(),
query: query.into(),
param_types: vec![],
},
))
.await
.unwrap();
client
.feed(FrontendMessage::Bind(
halephant_core::proto::frontend::Bind {
portal: String::new(),
statement: name.into(),
param_formats: vec![],
params: vec![],
result_formats: vec![],
},
))
.await
.unwrap();
client
.feed(FrontendMessage::Execute(
halephant_core::proto::frontend::Execute {
portal: String::new(),
max_rows: 0,
},
))
.await
.unwrap();
client.feed(FrontendMessage::Sync).await.unwrap();
}
SinkExt::flush(&mut client).await.unwrap();
let counted = tokio::time::timeout(std::time::Duration::from_secs(5), async {
let mut completes = 0;
let mut readies = 0;
while readies < 2 {
match client.next().await {
Some(Ok(BackendMessage::CommandComplete(_))) => completes += 1,
Some(Ok(BackendMessage::ReadyForQuery(_))) => readies += 1,
Some(Ok(BackendMessage::ErrorResponse(e))) => {
panic!("unexpected error in pipelined batch: {e:?}")
}
Some(Ok(_)) => {}
_ => break,
}
}
(completes, readies)
})
.await
.expect("timed out — batch 2's responses were stranded (read-ahead across Sync)");
assert_eq!(
counted.1, 2,
"expected one ReadyForQuery per pipelined batch"
);
assert_eq!(
counted.0, 2,
"expected one CommandComplete per pipelined batch"
);
client.send(FrontendMessage::Terminate).await.unwrap();
let _ = handle.await;
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn pipelined_statements_preserve_all_responses() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
for (name, query) in [("s1", "SELECT 1"), ("s2", "SELECT 2")] {
client
.feed(FrontendMessage::Parse(
halephant_core::proto::frontend::Parse {
name: name.into(),
query: query.into(),
param_types: vec![],
},
))
.await
.unwrap();
client
.feed(FrontendMessage::Bind(
halephant_core::proto::frontend::Bind {
portal: String::new(),
statement: name.into(),
param_formats: vec![],
params: vec![],
result_formats: vec![],
},
))
.await
.unwrap();
client
.feed(FrontendMessage::Execute(
halephant_core::proto::frontend::Execute {
portal: String::new(),
max_rows: 0,
},
))
.await
.unwrap();
}
client.send(FrontendMessage::Sync).await.unwrap();
let mut msgs = Vec::new();
loop {
let m = client.next().await.unwrap().unwrap();
let done = matches!(m, BackendMessage::ReadyForQuery(_));
msgs.push(m);
if done {
break;
}
}
let count = |want: &Tag| {
msgs.iter()
.filter(|m| match m {
BackendMessage::ParseComplete => want == &Tag::Parse,
BackendMessage::BindComplete => want == &Tag::Bind,
BackendMessage::CommandComplete(_) => want == &Tag::Command,
_ => false,
})
.count()
};
assert_eq!(
count(&Tag::Command),
2,
"both statements' CommandComplete must reach the client: {msgs:?}"
);
assert_eq!(
count(&Tag::Bind),
2,
"both statements' BindComplete must reach the client: {msgs:?}"
);
assert_eq!(
count(&Tag::Parse),
2,
"both statements' ParseComplete must reach the client: {msgs:?}"
);
let order: Vec<Tag> = msgs
.iter()
.filter_map(|m| match m {
BackendMessage::ParseComplete => Some(Tag::Parse),
BackendMessage::BindComplete => Some(Tag::Bind),
BackendMessage::CommandComplete(_) => Some(Tag::Command),
_ => None,
})
.collect();
assert_eq!(
order,
vec![
Tag::Parse,
Tag::Bind,
Tag::Command,
Tag::Parse,
Tag::Bind,
Tag::Command
],
"responses must be in per-statement protocol order: {msgs:?}"
);
drop(client);
let _ = handle.await;
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn reprepare_and_eviction_replies_are_hidden_from_client() {
let mock = MockPg::start(MockBehavior::Ok).await;
let cfg = Config::parse(&format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
max_prepared_statements = 1
[cluster.test]
nodes = ["{}"]
[cluster.test.pool.testdb.user.testuser]
"#,
mock.addr()
))
.unwrap();
let (pools, _config) = setup_pool_with_config(cfg, &mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
query_collect(&mut client, "BEGIN").await;
extended_query(&mut client, "s1", "SELECT 1").await; extended_query(&mut client, "s2", "SELECT 2").await;
client
.feed(FrontendMessage::Bind(
halephant_core::proto::frontend::Bind {
portal: String::new(),
statement: "s1".into(),
param_formats: vec![],
params: vec![],
result_formats: vec![],
},
))
.await
.unwrap();
client
.feed(FrontendMessage::Execute(
halephant_core::proto::frontend::Execute {
portal: String::new(),
max_rows: 0,
},
))
.await
.unwrap();
client.send(FrontendMessage::Sync).await.unwrap();
let mut msgs = Vec::new();
loop {
let m = client.next().await.unwrap().unwrap();
let done = matches!(m, BackendMessage::ReadyForQuery(_));
msgs.push(m);
if done {
break;
}
}
assert!(
msgs.iter()
.any(|m| matches!(m, BackendMessage::BindComplete)),
"client should see BindComplete for its Bind: {msgs:?}"
);
assert!(
msgs.iter()
.any(|m| matches!(m, BackendMessage::CommandComplete(_))),
"client should see CommandComplete for its Execute: {msgs:?}"
);
assert!(
!msgs
.iter()
.any(|m| matches!(m, BackendMessage::ParseComplete)),
"injected re-prepare ParseComplete must not leak to the client: {msgs:?}"
);
assert!(
!msgs
.iter()
.any(|m| matches!(m, BackendMessage::CloseComplete)),
"injected eviction CloseComplete must not leak to the client: {msgs:?}"
);
drop(client);
let _ = handle.await;
mock.stop().await;
}
async fn parse_only(
client: &mut Framed<TcpStream, BackendCodec>,
name: &str,
query: &str,
) -> Vec<BackendMessage> {
client
.feed(FrontendMessage::Parse(
halephant_core::proto::frontend::Parse {
name: name.into(),
query: query.into(),
param_types: vec![],
},
))
.await
.unwrap();
client.send(FrontendMessage::Sync).await.unwrap();
let mut msgs = Vec::new();
loop {
let m = client.next().await.unwrap().unwrap();
let done = matches!(m, BackendMessage::ReadyForQuery(_));
msgs.push(m);
if done {
break;
}
}
msgs
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn failed_parse_does_not_leave_stale_cache_entry() {
use halephant_core::proxy::prepared::canonical_for_test;
let mock = MockPg::start(MockBehavior::Ok).await;
let cfg = Config::parse(&format!(
r#"
[server]
checkout_timeout = "5s"
[cluster.test]
nodes = ["{}"]
[cluster.test.pool.testdb]
max_connections = {{ primary = 1 }}
[cluster.test.pool.testdb.user.testuser]
"#,
mock.addr()
))
.unwrap();
let (pools, _config) = setup_pool_with_config(cfg, &mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
let tx1 = parse_only(&mut client, "s1", "SELECT INVALID").await;
assert!(
tx1.iter()
.any(|m| matches!(m, BackendMessage::ErrorResponse(_))),
"tx1 should surface the backend Parse error: {tx1:?}"
);
let tx2 = parse_only(&mut client, "s2", "SELECT INVALID").await;
assert!(
tx2.iter()
.any(|m| matches!(m, BackendMessage::ErrorResponse(_))),
"tx2 must re-validate on the backend, not falsely succeed from a stale cache entry: {tx2:?}"
);
let canon = canonical_for_test("SELECT INVALID", &[]);
let forwarded = mock
.received_parses()
.iter()
.filter(|n| **n == canon)
.count();
assert_eq!(
forwarded,
2,
"the rejected Parse should be re-forwarded on reuse, not skipped: {:?}",
mock.received_parses()
);
client.send(FrontendMessage::Terminate).await.unwrap();
let _ = handle.await;
mock.stop().await;
}
async fn session_prepare(
guard: &mut halephant_core::pool::ConnGuard,
pools: &Arc<PoolManager>,
name: &str,
query: &str,
) -> Vec<BackendMessage> {
let (client_tcp, proxy_tcp) = tcp_pair().await;
let mut proxy = Framed::new(proxy_tcp, FrontendCodec::post_startup());
let mut client = Framed::new(client_tcp, BackendCodec::new());
let session = halephant_core::proxy::session::forward(&mut proxy, guard.conn(), pools);
let driver = async {
client
.feed(FrontendMessage::Parse(
halephant_core::proto::frontend::Parse {
name: name.into(),
query: query.into(),
param_types: vec![],
},
))
.await
.unwrap();
client.send(FrontendMessage::Sync).await.unwrap();
let mut msgs = Vec::new();
loop {
let m = client.next().await.unwrap().unwrap();
let done = matches!(m, BackendMessage::ReadyForQuery(_));
msgs.push(m);
if done {
break;
}
}
drop(client); msgs
};
let (session_res, msgs) = tokio::join!(session, driver);
session_res.unwrap();
msgs
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn session_mode_reused_backend_avoids_duplicate_prepared() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool_with_config(test_config_session(&mock.addr()), &mock);
let cg = test_client_guard();
let mut guard = pools
.checkout(&cg, "testdb", "testuser", false)
.await
.unwrap();
let a = session_prepare(&mut guard, &pools, "_pg3_0", "SELECT 1").await;
assert!(
!a.iter()
.any(|m| matches!(m, BackendMessage::ErrorResponse(_))),
"client A's prepare should succeed: {a:?}"
);
let b = session_prepare(&mut guard, &pools, "_pg3_0", "SELECT 2").await;
assert!(
!b.iter()
.any(|m| matches!(m, BackendMessage::ErrorResponse(_))),
"client B must not collide on the reused backend's prepared name: {b:?}"
);
assert!(
b.iter().any(|m| matches!(m, BackendMessage::ParseComplete)),
"client B should see a ParseComplete: {b:?}"
);
drop(guard);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn user_max_connections_enforced() {
let mock = MockPg::start(MockBehavior::Ok).await;
let cfg = Config::parse(&format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
[cluster.test]
nodes = ["{}"]
[cluster.test.pool.testdb]
max_connections = {{ primary = 10 }}
[cluster.test.pool.testdb.user.userA]
max_connections = {{ primary = 2 }}
[cluster.test.pool.testdb.user.userB]
"#,
mock.addr()
))
.unwrap();
let (pools, _config) = setup_pool_with_config(cfg, &mock);
let client_a = test_client_guard();
let g1 = pools
.checkout(&client_a, "testdb", "userA", false)
.await
.expect("userA checkout 1");
let g2 = pools
.checkout(&client_a, "testdb", "userA", false)
.await
.expect("userA checkout 2");
let err = pools.checkout(&client_a, "testdb", "userA", false).await;
assert!(
err.is_err(),
"userA checkout 3 should fail (per-user max = 2)"
);
let client_b = test_client_guard();
let g3 = pools
.checkout(&client_b, "testdb", "userB", false)
.await
.expect("userB should succeed despite userA being full");
drop((g1, g2, g3));
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn pool_max_connections_aggregate_across_users() {
let mock = MockPg::start(MockBehavior::Ok).await;
let cfg = Config::parse(&format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
[cluster.test]
nodes = ["{}"]
[cluster.test.pool.testdb]
max_connections = {{ primary = 3 }}
[cluster.test.pool.testdb.user.userA]
[cluster.test.pool.testdb.user.userB]
"#,
mock.addr()
))
.unwrap();
let (pools, _config) = setup_pool_with_config(cfg, &mock);
let client = test_client_guard();
let g1 = pools
.checkout(&client, "testdb", "userA", false)
.await
.expect("checkout 1");
let g2 = pools
.checkout(&client, "testdb", "userB", false)
.await
.expect("checkout 2");
let g3 = pools
.checkout(&client, "testdb", "userA", false)
.await
.expect("checkout 3");
let err = pools.checkout(&client, "testdb", "userB", false).await;
assert!(
err.is_err(),
"checkout 4 should fail (pool max = 3, 3 already active)"
);
drop((g1, g2, g3));
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn user_limit_within_pool_limit() {
let mock = MockPg::start(MockBehavior::Ok).await;
let cfg = Config::parse(&format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
[cluster.test]
nodes = ["{}"]
[cluster.test.pool.testdb]
max_connections = {{ primary = 4 }}
[cluster.test.pool.testdb.user.userA]
max_connections = {{ primary = 1 }}
[cluster.test.pool.testdb.user.userB]
max_connections = {{ primary = 3 }}
"#,
mock.addr()
))
.unwrap();
let (pools, _config) = setup_pool_with_config(cfg, &mock);
let client = test_client_guard();
let ga = pools
.checkout(&client, "testdb", "userA", false)
.await
.expect("userA checkout 1");
let err_a = pools.checkout(&client, "testdb", "userA", false).await;
assert!(
err_a.is_err(),
"userA checkout 2 should fail (user max = 1)"
);
let gb1 = pools
.checkout(&client, "testdb", "userB", false)
.await
.expect("userB checkout 1");
let gb2 = pools
.checkout(&client, "testdb", "userB", false)
.await
.expect("userB checkout 2");
let gb3 = pools
.checkout(&client, "testdb", "userB", false)
.await
.expect("userB checkout 3");
let err_b = pools.checkout(&client, "testdb", "userB", false).await;
assert!(
err_b.is_err(),
"userB checkout 4 should fail (pool max = 4, all used)"
);
drop((ga, gb1, gb2, gb3));
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cross_user_wake_on_pool_level_checkin() {
let mock = MockPg::start(MockBehavior::Ok).await;
let cfg = Config::parse(&format!(
r#"
[server]
checkout_timeout = "2s"
[cluster.test]
nodes = ["{}"]
[cluster.test.pool.testdb]
max_connections = {{ primary = 1 }}
[cluster.test.pool.testdb.user.userA]
[cluster.test.pool.testdb.user.userB]
"#,
mock.addr()
))
.unwrap();
let (pools, _config) = setup_pool_with_config(cfg, &mock);
let client_a = test_client_guard();
let guard_a = pools
.checkout(&client_a, "testdb", "userA", false)
.await
.expect("userA checkout should succeed");
let pools2 = Arc::clone(&pools);
let userb_handle = tokio::spawn(async move {
let client_b = test_client_guard();
pools2.checkout(&client_b, "testdb", "userB", false).await
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
guard_a.checkin();
let result = tokio::time::timeout(std::time::Duration::from_secs(1), userb_handle)
.await
.expect("userB should be woken within 1s")
.expect("task should not panic");
assert!(
result.is_ok(),
"userB checkout should succeed after userA checkin, got: {:?}",
result.err()
);
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn read_only_user_does_not_fall_back_to_primary() {
let primary = MockPg::start(MockBehavior::Ok).await;
let replica = MockPg::start_replica(MockBehavior::Ok).await;
let toml = format!(
r#"
[server]
checkout_timeout = "{TEST_CHECKOUT_TIMEOUT}"
[cluster.test]
nodes = ["{}", "{}"]
admin_user = "testuser"
[cluster.test.pool.testdb]
max_connections = {{ primary = 10, replica = 2 }}
[cluster.test.pool.testdb.user.rouser]
max_connections = {{ replica = 2 }}
"#,
primary.addr(),
replica.addr(),
);
let config = Arc::new(ArcSwap::from_pointee(Config::parse(&toml).unwrap()));
let pgpass = Arc::new(halephant_core::auth::pgpass::Pgpass::parse(""));
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
seed_primary_with_replicas(&topology, primary.addr(), vec![replica.addr()]);
let pools = Arc::new(PoolManager::new(Arc::clone(&config), topology, pgpass));
let client = test_client_guard();
let g1 = pools
.checkout(&client, "testdb", "rouser", true)
.await
.expect("checkout 1");
let g2 = pools
.checkout(&client, "testdb", "rouser", true)
.await
.expect("checkout 2");
let result = pools.checkout(&client, "testdb", "rouser", true).await;
assert!(
result.is_err(),
"read-only user should NOT fall back to primary"
);
drop((g1, g2));
primary.stop().await;
replica.stop().await;
}
async fn feed_extended_batch(
client: &mut Framed<TcpStream, BackendCodec>,
name: &str,
query: &str,
) {
client
.feed(FrontendMessage::Parse(
halephant_core::proto::frontend::Parse {
name: name.into(),
query: query.into(),
param_types: vec![],
},
))
.await
.unwrap();
client
.feed(FrontendMessage::Bind(
halephant_core::proto::frontend::Bind {
portal: String::new(),
statement: name.into(),
param_formats: vec![],
params: vec![],
result_formats: vec![],
},
))
.await
.unwrap();
client
.feed(FrontendMessage::Execute(
halephant_core::proto::frontend::Execute {
portal: String::new(),
max_rows: 0,
},
))
.await
.unwrap();
client.feed(FrontendMessage::Sync).await.unwrap();
}
async fn collect_until_ready(client: &mut Framed<TcpStream, BackendCodec>) -> Vec<BackendMessage> {
let mut msgs = Vec::new();
loop {
let msg = client.next().await.unwrap().unwrap();
let done = matches!(msg, BackendMessage::ReadyForQuery(_));
msgs.push(msg);
if done {
return msgs;
}
}
}
async fn wait_for_idle(pools: &PoolManager, want: u32) {
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
while total_idle(pools) < want {
assert!(
tokio::time::Instant::now() < deadline,
"connection was not reset and returned to the idle pool within 2 s"
);
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn pipelined_explicit_transaction_batches_flow_on_one_backend() {
let mock = MockPg::start_slow(MockBehavior::Ok, std::time::Duration::from_millis(20)).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
client
.feed(FrontendMessage::Query("BEGIN".into()))
.await
.unwrap();
feed_extended_batch(&mut client, "s1", "SELECT 1").await;
feed_extended_batch(&mut client, "s2", "SELECT 2").await;
client
.feed(FrontendMessage::Query("COMMIT".into()))
.await
.unwrap();
SinkExt::flush(&mut client).await.unwrap();
let msgs = tokio::time::timeout(std::time::Duration::from_secs(5), async {
let mut msgs = Vec::new();
loop {
let msg = client.next().await.unwrap().unwrap();
let done = matches!(msg, BackendMessage::ReadyForQuery(TransactionStatus::Idle));
msgs.push(msg);
if done {
break;
}
}
msgs
})
.await
.expect("timed out — an in-transaction pipelined batch was stranded");
let readies: Vec<TransactionStatus> = msgs
.iter()
.filter_map(|m| match m {
BackendMessage::ReadyForQuery(s) => Some(*s),
_ => None,
})
.collect();
assert_eq!(
readies,
vec![
TransactionStatus::InTransaction, TransactionStatus::InTransaction, TransactionStatus::InTransaction, TransactionStatus::Idle, ],
"expected one ReadyForQuery per batch with in-transaction statuses: {msgs:?}"
);
let completes = msgs
.iter()
.filter(|m| matches!(m, BackendMessage::CommandComplete(_)))
.count();
assert_eq!(
completes, 4,
"BEGIN, both Executes, and COMMIT must each complete: {msgs:?}"
);
let parses = msgs
.iter()
.filter(|m| matches!(m, BackendMessage::ParseComplete))
.count();
assert_eq!(
parses, 2,
"both ParseComplete must reach the client: {msgs:?}"
);
assert_eq!(
mock.connections(),
1,
"the explicit transaction must hold a single backend"
);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn set_via_query_is_reset_on_checkin() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
let _ = query_collect(&mut client, "SET application_name TO 'trackme'").await;
wait_for_idle(&pools, 1).await;
let received = mock.received_queries();
assert!(
received
.iter()
.any(|q| q.contains("RESET application_name")),
"checkin must RESET the session-SET variable, got: {received:?}"
);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn set_via_parse_is_not_tracked_for_reset() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
extended_query(
&mut client,
"set_stmt",
"SET application_name TO 'untracked'",
)
.await;
wait_for_idle(&pools, 1).await;
let received = mock.received_queries();
assert!(
!received
.iter()
.any(|q| q.contains("RESET application_name")),
"a Parse-carried SET must not be tracked for reset, got: {received:?}"
);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn deallocate_via_parse_is_forwarded_not_intercepted() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool(&mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
feed_extended_batch(&mut client, "d1", "DEALLOCATE ALL").await;
SinkExt::flush(&mut client).await.unwrap();
let msgs = collect_until_ready(&mut client).await;
assert!(
msgs.iter()
.any(|m| matches!(m, BackendMessage::ParseComplete)),
"the DEALLOCATE Parse must draw a ParseComplete: {msgs:?}"
);
assert!(
msgs.iter()
.any(|m| matches!(m, BackendMessage::BindComplete)),
"the batch must complete through the backend: {msgs:?}"
);
assert!(
!msgs
.iter()
.any(|m| matches!(m, BackendMessage::CommandComplete(tag) if tag == "DEALLOCATE")),
"extended-protocol DEALLOCATE must not draw the synthetic simple-query response: {msgs:?}"
);
assert_eq!(
mock.received_parses().len(),
1,
"the DEALLOCATE Parse must reach the backend"
);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn pin_mode_listen_pins_connection_until_unlisten_star() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool_with_config(test_config_max(&mock.addr(), 1), &mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
let msgs = query_collect(&mut client, "LISTEN pinchan").await;
assert!(
matches!(
msgs.last(),
Some(BackendMessage::ReadyForQuery(TransactionStatus::Idle))
),
"LISTEN should complete with ReadyForQuery(Idle): {msgs:?}"
);
assert!(
mock.received_queries()
.iter()
.any(|q| q.contains("LISTEN pinchan")),
"pin mode must forward the LISTEN to the backend"
);
let other = test_client_guard();
let result = pools.checkout(&other, "testdb", "testuser", false).await;
assert!(
result.is_err(),
"pinned connection must not be handed to another client"
);
let msgs = query_collect(&mut client, "DEALLOCATE bogus").await;
assert!(
msgs.iter()
.any(|m| matches!(m, BackendMessage::CommandComplete(tag) if tag == "DEALLOCATE")),
"DEALLOCATE must be intercepted with a synthetic response: {msgs:?}"
);
assert!(
matches!(
msgs.last(),
Some(BackendMessage::ReadyForQuery(TransactionStatus::Idle))
),
"the synthetic status must mirror the last server status: {msgs:?}"
);
assert!(
!mock
.received_queries()
.iter()
.any(|q| q.to_ascii_uppercase().contains("DEALLOCATE")),
"the intercepted DEALLOCATE must not reach the backend"
);
let _ = query_collect(&mut client, "UNLISTEN *").await;
wait_for_idle(&pools, 1).await;
let guard = pools
.checkout(&other, "testdb", "testuser", false)
.await
.expect("released connection must be checkoutable after UNLISTEN *");
drop(guard);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn pin_mode_listen_mid_transaction_pins_connection() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool_with_config(test_config_max(&mock.addr(), 1), &mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
let _ = query_collect(&mut client, "BEGIN").await;
let _ = query_collect(&mut client, "LISTEN midchan").await;
let msgs = query_collect(&mut client, "COMMIT").await;
assert!(
matches!(
msgs.last(),
Some(BackendMessage::ReadyForQuery(TransactionStatus::Idle))
),
"COMMIT should return the session to idle: {msgs:?}"
);
let other = test_client_guard();
let result = pools.checkout(&other, "testdb", "testuser", false).await;
assert!(
result.is_err(),
"mid-transaction LISTEN must pin the connection past COMMIT"
);
let _ = query_collect(&mut client, "UNLISTEN *").await;
wait_for_idle(&pools, 1).await;
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
mock.stop().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn pin_mode_listen_via_parse_pins_connection() {
let mock = MockPg::start(MockBehavior::Ok).await;
let (pools, _config) = setup_pool_with_config(test_config_max(&mock.addr(), 1), &mock);
let (client_tcp, proxy_tcp) = tcp_pair().await;
let handle = spawn_tx_forward(proxy_tcp, Arc::clone(&pools), "testdb", "testuser", false);
let mut client = Framed::new(client_tcp, BackendCodec::new());
extended_query(&mut client, "l1", "LISTEN extchan").await;
let other = test_client_guard();
let result = pools.checkout(&other, "testdb", "testuser", false).await;
assert!(
result.is_err(),
"Parse-carried LISTEN must pin the connection"
);
client.send(FrontendMessage::Terminate).await.unwrap();
handle.await.unwrap().unwrap();
wait_for_idle(&pools, 1).await;
mock.stop().await;
}