use std::{
borrow::Cow,
fmt,
future::Future,
time::{Duration, Instant},
};
use sqlx::{
ConnectOptions, PgPool, Postgres, Transaction,
migrate::{MigrateError, Migrator},
pool::PoolConnection,
postgres::{PgConnectOptions, PgPoolOptions},
};
use thiserror::Error;
use crate::{
result::{ClassifyError, ErrorDescriptor, ErrorKind},
secret::ExposeSecret,
};
use super::{
PostgresConnectionConfig,
config::{
DEFAULT_POSTGRES_ADVISORY_LOCK_TIMEOUT, DEFAULT_POSTGRES_MIGRATION_TIMEOUT,
PostgresBootstrapConfig, PostgresPoolConfig, PostgresStartupRetryConfig,
},
diagnostics::{
PostgresBootstrapError, PostgresOperation, PostgresPoolError, diagnose_migration_error,
diagnose_migration_lock_timeout, diagnose_operation_timeout, diagnose_pool_error,
diagnose_sqlx_error,
},
identity::{
connect_options as parse_connect_options, database_name,
schema_maintenance_advisory_lock_key,
},
};
pub fn connect_options(
config: &PostgresConnectionConfig,
) -> Result<PgConnectOptions, PostgresPoolError> {
let mut options = parse_connect_options(config.database_url.expose_secret())?;
for (name, timeout) in [
("statement_timeout", config.statement_timeout),
(
"idle_in_transaction_session_timeout",
config.idle_in_transaction_session_timeout,
),
] {
if let Some(timeout) = timeout {
options = options.options([(name, timeout.as_millis().to_string())]);
}
}
Ok(options)
}
pub async fn connect_pool(
connection: &PostgresConnectionConfig,
config: PostgresPoolConfig,
) -> Result<PgPool, PostgresPoolError> {
Ok(pool_options(config)
.connect_with(connect_options(connection)?)
.await?)
}
pub fn connect_lazy_pool(
connection: &PostgresConnectionConfig,
config: PostgresPoolConfig,
) -> Result<PgPool, PostgresPoolError> {
Ok(pool_options(config).connect_lazy_with(connect_options(connection)?))
}
#[must_use]
pub fn pool_options(config: PostgresPoolConfig) -> PgPoolOptions {
PgPoolOptions::new()
.max_connections(config.max_connections)
.min_connections(config.min_connections)
.acquire_timeout(config.acquire_timeout)
.test_before_acquire(config.test_before_acquire)
.idle_timeout(config.idle_timeout)
.max_lifetime(config.max_lifetime)
}
pub async fn bootstrap_database(
config: PostgresBootstrapConfig,
migrator: Option<&Migrator>,
) -> Result<PgPool, PostgresBootstrapError> {
let migrator = if config.migrate_on_startup {
Some(migrator.ok_or(PostgresBootstrapError::MissingMigrator)?)
} else {
None
};
let database_url = config.connection.database_url.expose_secret();
if config.create_on_startup {
tracing::info!(
database = %display_database_name(database_url),
"ensuring PostgreSQL database exists"
);
ensure_database_exists_with_retry(
database_url,
config.ensure_database_timeout,
config.startup_retry,
)
.await
.map_err(|source| PostgresBootstrapError::EnsureDatabase {
diagnosis: diagnose_sqlx_error(
PostgresOperation::EnsureDatabase,
database_url,
&source,
),
source: Box::new(source),
})?;
}
let pool =
connect_pool_with_retry(&config.connection, config.pool_config, config.startup_retry)
.await
.map_err(|source| PostgresBootstrapError::Connect {
diagnosis: diagnose_pool_error(PostgresOperation::Connect, database_url, &source),
source: Box::new(source),
})?;
if let Some(migrator) = migrator {
tracing::info!(
database = %display_database_name(database_url),
"running PostgreSQL database migrations"
);
if let Err(error) = run_migrations_for_database(
&pool,
migrator,
database_name(database_url).as_deref(),
config.migration_lock_key,
config.migration_lock_timeout,
config.migration_timeout,
)
.await
{
return Err(bootstrap_migration_run_error(database_url, error));
}
}
Ok(pool)
}
pub async fn ensure_database_exists(
database_url: &str,
timeout: Duration,
) -> Result<(), sqlx::Error> {
ensure_database_exists_with_retry(
database_url,
timeout,
PostgresStartupRetryConfig::disabled(),
)
.await
}
async fn ensure_database_with_options(
options: sqlx::postgres::PgConnectOptions,
) -> Result<(), sqlx::Error> {
let database = options
.get_database()
.unwrap_or(options.get_username())
.to_owned();
let options = options.database(if database == "postgres" {
"template1"
} else {
"postgres"
});
let mut connection = options.connect().await?;
let name_fits: bool = sqlx::query_scalar(
"SELECT octet_length($1::text) <= current_setting('max_identifier_length')::int4",
)
.bind(&database)
.fetch_one(&mut connection)
.await?;
if !name_fits {
return Err(sqlx::Error::Configuration(
"database name exceeds the server identifier limit".into(),
));
}
let exists = "SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = $1)";
if sqlx::query_scalar::<_, bool>(exists)
.bind(&database)
.fetch_one(&mut connection)
.await?
{
return Ok(());
}
let create = format!("CREATE DATABASE \"{}\"", database.replace('"', "\"\""));
match sqlx::query(sqlx::AssertSqlSafe(create))
.execute(&mut connection)
.await
{
Ok(_) => Ok(()),
Err(error) => match sqlx::query_scalar::<_, bool>(exists)
.bind(&database)
.fetch_one(&mut connection)
.await
{
Ok(true) => Ok(()),
Ok(false) | Err(_) => Err(error),
},
}
}
pub async fn ensure_database_exists_with_retry(
database_url: &str,
timeout: Duration,
retry: PostgresStartupRetryConfig,
) -> Result<(), sqlx::Error> {
let options = parse_connect_options(database_url)?;
let deadline = tokio::time::Instant::now()
.checked_add(timeout)
.filter(|_| !timeout.is_zero())
.ok_or_else(|| {
sqlx::Error::Configuration(
"database creation timeout must be nonzero and fit the runtime timer range".into(),
)
})?;
tokio::time::timeout_at(
deadline,
retry_async(
database_url,
retry,
PostgresOperation::EnsureDatabase,
|| ensure_database_with_options(options.clone()),
),
)
.await
.map_err(|_| {
sqlx::Error::Io(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"PostgreSQL database creation deadline elapsed",
))
})?
}
pub async fn run_migrations(
pool: &PgPool,
migrator: &Migrator,
) -> Result<(), PostgresMigrationRunError> {
run_migrations_with_timeouts(
pool,
migrator,
None,
DEFAULT_POSTGRES_ADVISORY_LOCK_TIMEOUT,
DEFAULT_POSTGRES_MIGRATION_TIMEOUT,
)
.await
}
#[derive(Error)]
#[non_exhaustive]
pub enum PostgresMigrationRunError {
#[error("PostgreSQL schema-maintenance lock operation failed for migrations")]
Lock(#[source] sqlx::Error),
#[error("PostgreSQL schema-maintenance lock operation timed out after {timeout:?}")]
LockTimeout {
timeout: Duration,
},
#[error("failed to run PostgreSQL migrations")]
Migration(#[source] Box<MigrateError>),
#[error("timed out while running PostgreSQL migrations after {timeout:?}")]
MigrationTimeout {
timeout: Duration,
},
}
impl fmt::Debug for PostgresMigrationRunError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("PostgresMigrationRunError")
.field("error_code", &self.error_code())
.field("source", &"[REDACTED]")
.finish()
}
}
impl ClassifyError for PostgresMigrationRunError {
fn error_kind(&self) -> ErrorKind {
match self {
Self::LockTimeout { .. } | Self::MigrationTimeout { .. } => ErrorKind::Timeout,
Self::Lock(_) | Self::Migration(_) => ErrorKind::Dependency,
}
}
fn error_code(&self) -> crate::result::ErrorCode {
crate::result::ErrorCode::internal(match self {
Self::Lock(_) => "postgres_migration_lock_failed",
Self::LockTimeout { .. } => "postgres_migration_lock_timeout",
Self::Migration(_) => "postgres_migration_failed",
Self::MigrationTimeout { .. } => "postgres_migration_timeout",
})
}
fn error_descriptor(&self) -> ErrorDescriptor {
let retryable = match self {
Self::Lock(source) => {
diagnose_sqlx_error(PostgresOperation::MigrationLock, "", source).retryable()
}
Self::Migration(source) => diagnose_migration_error("", source).retryable(),
Self::LockTimeout { .. } | Self::MigrationTimeout { .. } => true,
};
ErrorDescriptor::new(self.error_kind(), self.error_code(), self.to_string())
.with_retryable(retryable)
}
}
impl From<PostgresMigrationRunError> for PostgresBootstrapError {
fn from(error: PostgresMigrationRunError) -> Self {
bootstrap_migration_run_error("", error)
}
}
fn bootstrap_migration_run_error(
database_url: &str,
error: PostgresMigrationRunError,
) -> PostgresBootstrapError {
match error {
PostgresMigrationRunError::Lock(source) => PostgresBootstrapError::MigrationLock {
diagnosis: diagnose_sqlx_error(PostgresOperation::MigrationLock, database_url, &source),
source: Box::new(source),
},
PostgresMigrationRunError::LockTimeout { timeout } => {
PostgresBootstrapError::MigrationLockTimeout {
diagnosis: diagnose_migration_lock_timeout(database_url, timeout),
timeout,
}
}
PostgresMigrationRunError::Migration(source) => PostgresBootstrapError::Migration {
diagnosis: diagnose_migration_error(database_url, &source),
source,
},
PostgresMigrationRunError::MigrationTimeout { timeout } => {
PostgresBootstrapError::MigrationTimeout {
diagnosis: diagnose_operation_timeout(
PostgresOperation::Migrate,
database_url,
timeout,
),
timeout,
}
}
}
}
pub async fn run_migrations_with_timeouts(
pool: &PgPool,
migrator: &Migrator,
additional_lock_key: Option<i64>,
lock_timeout: Duration,
migration_timeout: Duration,
) -> Result<(), PostgresMigrationRunError> {
run_migrations_for_database(
pool,
migrator,
None,
additional_lock_key,
lock_timeout,
migration_timeout,
)
.await
}
async fn run_migrations_for_database(
pool: &PgPool,
migrator: &Migrator,
known_database_name: Option<&str>,
additional_lock_key: Option<i64>,
lock_timeout: Duration,
migration_timeout: Duration,
) -> Result<(), PostgresMigrationRunError> {
if migration_timeout.is_zero() || Instant::now().checked_add(migration_timeout).is_none() {
return Err(PostgresMigrationRunError::MigrationTimeout {
timeout: migration_timeout,
});
}
let mut lock_guard = acquire_schema_maintenance_session_lock(
pool,
known_database_name,
additional_lock_key,
lock_timeout,
)
.await
.map_err(map_migration_lock_error)?;
let migration_result = tokio::time::timeout(
migration_timeout,
migrator.run(&mut **lock_guard.connection()),
)
.await;
match migration_result {
Ok(Ok(())) => lock_guard
.release_with_timeout(lock_timeout)
.await
.map_err(map_migration_lock_error),
Ok(Err(source)) => Err(PostgresMigrationRunError::Migration(Box::new(source))),
Err(_) => Err(PostgresMigrationRunError::MigrationTimeout {
timeout: migration_timeout,
}),
}
}
fn map_migration_lock_error(error: SessionAdvisoryLockError) -> PostgresMigrationRunError {
match error {
SessionAdvisoryLockError::Sqlx(source) => PostgresMigrationRunError::Lock(source),
SessionAdvisoryLockError::Timeout { timeout } => {
PostgresMigrationRunError::LockTimeout { timeout }
}
}
}
#[derive(Debug)]
pub(super) enum SessionAdvisoryLockError {
Sqlx(sqlx::Error),
Timeout { timeout: Duration },
}
pub(super) struct SessionAdvisoryLockGuard {
connection: PoolConnection<Postgres>,
lock_keys: Vec<i64>,
armed: bool,
}
impl SessionAdvisoryLockGuard {
fn new(connection: PoolConnection<Postgres>) -> Self {
Self {
connection,
lock_keys: Vec::with_capacity(2),
armed: false,
}
}
pub(super) fn connection(&mut self) -> &mut PoolConnection<Postgres> {
&mut self.connection
}
pub(super) async fn release_with_timeout(
mut self,
timeout: Duration,
) -> Result<(), SessionAdvisoryLockError> {
let started = Instant::now();
while let Some(lock_key) = self.lock_keys.pop() {
let remaining = remaining_timeout(started, timeout)?;
let result: Result<bool, sqlx::Error> = tokio::time::timeout(
remaining,
sqlx::query_scalar("SELECT pg_advisory_unlock($1)")
.bind(lock_key)
.fetch_one(&mut *self.connection),
)
.await
.map_err(|_| SessionAdvisoryLockError::Timeout { timeout })?;
match result {
Ok(true) => {}
Ok(false) => {
return Err(SessionAdvisoryLockError::Sqlx(sqlx::Error::Protocol(
"PostgreSQL advisory lock ownership was lost before release".to_owned(),
)));
}
Err(error) => return Err(SessionAdvisoryLockError::Sqlx(error)),
}
}
self.armed = false;
Ok(())
}
}
impl Drop for SessionAdvisoryLockGuard {
fn drop(&mut self) {
if self.armed {
self.connection.close_on_drop();
}
}
}
pub(super) async fn acquire_schema_maintenance_session_lock(
pool: &PgPool,
known_database_name: Option<&str>,
additional_lock_key: Option<i64>,
timeout: Duration,
) -> Result<SessionAdvisoryLockGuard, SessionAdvisoryLockError> {
if timeout.is_zero() || Instant::now().checked_add(timeout).is_none() {
return Err(SessionAdvisoryLockError::Timeout { timeout });
}
let started = Instant::now();
let remaining = remaining_timeout(started, timeout)?;
let connection = tokio::time::timeout(remaining, pool.acquire())
.await
.map_err(|_| SessionAdvisoryLockError::Timeout { timeout })?
.map_err(SessionAdvisoryLockError::Sqlx)?;
let mut guard = SessionAdvisoryLockGuard::new(connection);
let remaining = remaining_timeout(started, timeout)?;
guard.armed = true;
let resolved_database_name: String = tokio::time::timeout(
remaining,
sqlx::query_scalar("SELECT current_database()").fetch_one(&mut **guard.connection()),
)
.await
.map_err(|_| SessionAdvisoryLockError::Timeout { timeout })?
.map_err(SessionAdvisoryLockError::Sqlx)?;
if known_database_name.is_some_and(|name| name != resolved_database_name) {
return Err(SessionAdvisoryLockError::Sqlx(sqlx::Error::Configuration(
"configured database name does not match the connected database".into(),
)));
}
guard.armed = false;
let lock_keys = schema_maintenance_lock_keys(&resolved_database_name, additional_lock_key);
acquire_locks_on_guard(guard, &lock_keys, started, timeout).await
}
pub(super) fn schema_maintenance_lock_keys(
database_name: &str,
additional_lock_key: Option<i64>,
) -> Vec<i64> {
let maintenance_key = schema_maintenance_advisory_lock_key(database_name);
let mut lock_keys = Vec::with_capacity(2);
lock_keys.push(maintenance_key);
if let Some(additional_key) = additional_lock_key
&& additional_key != maintenance_key
{
lock_keys.push(additional_key);
}
lock_keys
}
async fn acquire_locks_on_guard(
mut guard: SessionAdvisoryLockGuard,
lock_keys: &[i64],
started: Instant,
timeout: Duration,
) -> Result<SessionAdvisoryLockGuard, SessionAdvisoryLockError> {
for &lock_key in lock_keys {
loop {
let remaining = remaining_timeout(started, timeout)?;
guard.armed = true;
let acquired = tokio::time::timeout(
remaining,
sqlx::query_scalar("SELECT pg_try_advisory_lock($1)")
.bind(lock_key)
.fetch_one(&mut **guard.connection()),
)
.await
.map_err(|_| SessionAdvisoryLockError::Timeout { timeout })?
.map_err(SessionAdvisoryLockError::Sqlx)?;
if acquired {
guard.lock_keys.push(lock_key);
break;
}
guard.armed = !guard.lock_keys.is_empty();
let remaining = remaining_timeout(started, timeout)?;
tokio::time::sleep(remaining.min(Duration::from_millis(50))).await;
}
}
Ok(guard)
}
fn remaining_timeout(
started: Instant,
timeout: Duration,
) -> Result<Duration, SessionAdvisoryLockError> {
timeout
.checked_sub(started.elapsed())
.filter(|remaining| !remaining.is_zero())
.ok_or(SessionAdvisoryLockError::Timeout { timeout })
}
pub async fn ping_pool(pool: &PgPool) -> Result<(), sqlx::Error> {
sqlx::query("SELECT 1").execute(pool).await?;
Ok(())
}
pub async fn connect_pool_with_retry(
connection: &PostgresConnectionConfig,
config: PostgresPoolConfig,
retry: PostgresStartupRetryConfig,
) -> Result<PgPool, PostgresPoolError> {
let database_url = connection.database_url.expose_secret();
let options = connect_options(connection)?;
retry_async(database_url, retry, PostgresOperation::Connect, || {
pool_options(config).connect_with(options.clone())
})
.await
.map_err(PostgresPoolError::Sqlx)
}
pub async fn check_pool_health(pool: &PgPool) -> PostgresHealth {
let started = Instant::now();
match ping_pool(pool).await {
Ok(()) => PostgresHealth {
status: PostgresHealthStatus::Ready,
latency: started.elapsed(),
},
Err(_) => PostgresHealth {
status: PostgresHealthStatus::Unavailable,
latency: started.elapsed(),
},
}
}
pub(super) const POSTGRES_HEALTH_ERROR: &str = "database health check failed";
fn display_database_name(database_url: &str) -> Cow<'_, str> {
database_name(database_url)
.map(Cow::Owned)
.unwrap_or(Cow::Borrowed("<unknown>"))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum PostgresHealthStatus {
Ready,
Unavailable,
}
#[derive(Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct PostgresHealth {
pub(super) status: PostgresHealthStatus,
pub(super) latency: Duration,
}
impl fmt::Debug for PostgresHealth {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("PostgresHealth")
.field("status", &self.status)
.field("latency", &self.latency)
.field("has_error", &!self.ready())
.finish()
}
}
impl PostgresHealth {
#[must_use]
pub const fn status(&self) -> PostgresHealthStatus {
self.status
}
#[must_use]
pub const fn latency(&self) -> Duration {
self.latency
}
#[must_use]
pub fn error(&self) -> Option<&str> {
(!self.ready()).then_some(POSTGRES_HEALTH_ERROR)
}
#[must_use]
pub const fn ready(&self) -> bool {
matches!(self.status, PostgresHealthStatus::Ready)
}
}
async fn retry_async<T, F, Fut>(
database_url: &str,
retry: PostgresStartupRetryConfig,
operation: PostgresOperation,
mut run: F,
) -> Result<T, sqlx::Error>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, sqlx::Error>>,
{
let max_attempts = retry.max_attempts;
let mut attempt = 1_u32;
let mut delay = retry.initial_delay;
loop {
match run().await {
Ok(value) => return Ok(value),
Err(error)
if attempt >= max_attempts
|| !diagnose_sqlx_error(operation, database_url, &error).retryable() =>
{
return Err(error);
}
Err(_) => {
let retry_delay = if retry.jitter {
crate::jitter::retry_delay(delay)
} else {
delay
};
tracing::warn!(
attempt,
max_attempts,
retry_after_ms = retry_delay.as_millis(),
operation = operation.as_str(),
"PostgreSQL startup operation failed; retrying"
);
tokio::time::sleep(retry_delay).await;
attempt += 1;
delay = delay.saturating_mul(2).min(retry.max_delay);
}
}
}
}
pub type PostgresTransaction<'a> = Transaction<'a, Postgres>;
pub async fn acquire_advisory_transaction_lock(
transaction: &mut PostgresTransaction<'_>,
key: i64,
) -> Result<(), sqlx::Error> {
sqlx::query("SELECT pg_advisory_xact_lock($1)")
.bind(key)
.execute(&mut **transaction)
.await?;
Ok(())
}
pub async fn acquire_shared_advisory_transaction_lock(
transaction: &mut PostgresTransaction<'_>,
key: i64,
) -> Result<(), sqlx::Error> {
sqlx::query("SELECT pg_advisory_xact_lock_shared($1)")
.bind(key)
.execute(&mut **transaction)
.await?;
Ok(())
}
pub async fn try_acquire_advisory_transaction_lock(
transaction: &mut PostgresTransaction<'_>,
key: i64,
) -> Result<bool, sqlx::Error> {
sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1)")
.bind(key)
.fetch_one(&mut **transaction)
.await
}
#[cfg(test)]
mod retry_tests {
use super::*;
#[tokio::test]
async fn lock_release_timeout_closes_the_session_and_preserves_timeout_classification() {
let required = std::env::var("FOOTING_REQUIRE_POSTGRES_TESTS").as_deref() == Ok("1");
let database_url = match std::env::var("FOOTING_TEST_POSTGRES_URL") {
Ok(url) => url,
Err(error) => {
assert!(
!required,
"FOOTING_REQUIRE_POSTGRES_TESTS=1 requires FOOTING_TEST_POSTGRES_URL: {error}"
);
return;
}
};
let pool = PgPoolOptions::new()
.max_connections(1)
.acquire_timeout(Duration::from_secs(5))
.connect(&database_url)
.await
.unwrap();
let additional_key =
super::super::advisory_lock_key("footing.test", "lock-release-timeout");
for additional_key in [None, Some(additional_key)] {
let mut guard = acquire_schema_maintenance_session_lock(
&pool,
None,
additional_key,
Duration::from_secs(5),
)
.await
.unwrap();
let held_keys = guard.lock_keys.clone();
assert_eq!(
held_keys.len(),
if additional_key.is_some() { 2 } else { 1 }
);
let previous_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
.fetch_one(&mut **guard.connection())
.await
.unwrap();
let error = guard
.release_with_timeout(Duration::ZERO)
.await
.unwrap_err();
let mut replacement = pool.acquire().await.unwrap();
let replacement_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
.fetch_one(&mut *replacement)
.await
.unwrap();
assert_ne!(replacement_pid, previous_pid);
for key in held_keys {
tokio::time::timeout(
Duration::from_secs(5),
sqlx::query("SELECT pg_advisory_lock($1)")
.bind(key)
.execute(&mut *replacement),
)
.await
.expect("closing the previous session must release every held lock")
.unwrap();
let unlocked: bool = sqlx::query_scalar("SELECT pg_advisory_unlock($1)")
.bind(key)
.fetch_one(&mut *replacement)
.await
.unwrap();
assert!(unlocked);
}
drop(replacement);
assert!(
matches!(
error,
SessionAdvisoryLockError::Timeout { timeout } if timeout.is_zero()
),
"unexpected release error: {error:?}"
);
let migration_error = map_migration_lock_error(error);
assert_eq!(migration_error.error_kind(), ErrorKind::Timeout);
assert_eq!(
migration_error.error_code().as_str(),
"postgres_migration_lock_timeout"
);
let bootstrap_error = PostgresBootstrapError::from(migration_error);
assert_eq!(bootstrap_error.error_kind(), ErrorKind::Timeout);
assert_eq!(
bootstrap_error.diagnosis().unwrap().kind(),
super::super::PostgresDiagnosticKind::MigrationLockUnavailable
);
}
pool.close().await;
}
#[tokio::test(start_paused = true)]
async fn jitter_preserves_attempt_limits_error_classification_and_exponential_progression() {
let retry = super::super::PostgresStartupRetrySettings::default()
.with_max_attempts(5)
.with_initial_delay(Duration::from_secs(1))
.with_max_delay(Duration::from_secs(3))
.with_jitter(true)
.build()
.unwrap();
for permanent in [true, false] {
let mut starts = Vec::new();
let result: Result<(), _> = retry_async(
"postgres://localhost/app",
retry,
PostgresOperation::Connect,
|| {
starts.push(tokio::time::Instant::now());
std::future::ready(Err(if permanent {
sqlx::Error::Configuration("invalid configuration".into())
} else {
sqlx::Error::Io(std::io::ErrorKind::ConnectionRefused.into())
}))
},
)
.await;
assert!(result.is_err());
assert_eq!(starts.len(), if permanent { 1 } else { 5 });
for (pair, base_secs) in starts.windows(2).zip([1, 2, 3, 3]) {
let base = Duration::from_secs(base_secs);
let gap = pair[1] - pair[0];
let upper = base + Duration::from_millis(1);
assert!(
(base / 2..=upper).contains(&gap),
"gap: {gap:?}, base: {base:?}"
);
}
}
}
#[tokio::test(start_paused = true)]
async fn permanent_startup_failures_do_not_retry_but_transient_failures_do() {
let retry = super::super::PostgresStartupRetrySettings::default()
.with_max_attempts(3)
.build()
.unwrap();
for permanent in [true, false] {
let mut attempts = 0;
let start = tokio::time::Instant::now();
let result: Result<(), _> = retry_async(
"postgres://localhost/app",
retry,
PostgresOperation::Connect,
|| {
attempts += 1;
std::future::ready(Err(if permanent {
sqlx::Error::Configuration("invalid configuration".into())
} else {
sqlx::Error::Io(std::io::ErrorKind::ConnectionRefused.into())
}))
},
)
.await;
assert!(result.is_err());
assert_eq!(attempts, if permanent { 1 } else { 3 });
assert_eq!(
start.elapsed(),
if permanent {
Duration::ZERO
} else {
Duration::from_millis(750)
}
);
}
}
#[tokio::test(start_paused = true)]
async fn server_restarts_retry_and_recover_but_authentication_and_permissions_do_not() {
use super::super::{PostgresDiagnosticKind, tests::TestDatabaseError};
let retry = super::super::PostgresStartupRetrySettings::default()
.with_max_attempts(3)
.build()
.unwrap();
for code in ["57P01", "57P02", "57P03", "28P01", "28000", "42501"] {
let transient = code.starts_with("57");
let mut attempts = 0;
let started = tokio::time::Instant::now();
let result = retry_async(
"postgres://localhost/app",
retry,
PostgresOperation::EnsureDatabase,
|| {
attempts += 1;
let result = if attempts == 1 {
let error = sqlx::Error::Database(Box::new(TestDatabaseError {
code,
message: "private server error detail",
}));
let diagnostic = diagnose_sqlx_error(
PostgresOperation::EnsureDatabase,
"postgres://localhost/app",
&error,
);
assert_eq!(diagnostic.retryable(), transient);
if transient {
assert_eq!(
diagnostic.kind(),
PostgresDiagnosticKind::ConnectionUnavailable
);
}
Err(error)
} else {
Ok(())
};
std::future::ready(result)
},
)
.await;
assert_eq!(result.is_ok(), transient);
assert_eq!(attempts, if transient { 2 } else { 1 });
assert_eq!(
started.elapsed(),
if transient {
retry.initial_delay
} else {
Duration::ZERO
}
);
}
}
#[tokio::test(start_paused = true)]
async fn invalid_url_is_not_retried() {
let retry = super::super::PostgresStartupRetrySettings::default()
.with_max_attempts(3)
.build()
.unwrap();
let start = tokio::time::Instant::now();
assert!(
super::super::PostgresConnectionSettings::new("sqlite::memory:")
.build()
.is_err()
);
assert!(
ensure_database_exists_with_retry("sqlite::memory:", Duration::from_secs(1), retry)
.await
.is_err()
);
assert_eq!(start.elapsed(), Duration::ZERO);
}
#[tokio::test]
async fn database_creation_deadline_covers_a_stalled_handshake_and_all_retries() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!(
"postgres://app:secret@{}/app?sslmode=disable",
listener.local_addr().unwrap()
);
let server = tokio::spawn(async move {
let (_socket, _) = listener.accept().await.unwrap();
std::future::pending::<()>().await;
});
let retry = super::super::PostgresStartupRetrySettings::default()
.with_max_attempts(3)
.build()
.unwrap();
let error = tokio::time::timeout(
Duration::from_secs(2),
ensure_database_exists_with_retry(&url, Duration::from_millis(30), retry),
)
.await
.expect("the complete operation must be bounded")
.unwrap_err();
assert!(
matches!(error, sqlx::Error::Io(ref io) if io.kind() == std::io::ErrorKind::TimedOut)
);
server.abort();
}
#[tokio::test(start_paused = true)]
async fn creation_deadline_includes_retry_delay() {
let socket = tokio::net::TcpSocket::new_v4().unwrap();
socket.bind("127.0.0.1:0".parse().unwrap()).unwrap();
let url = format!(
"postgres://app@{}/app?sslmode=disable",
socket.local_addr().unwrap()
);
let retry = super::super::PostgresStartupRetrySettings::default()
.with_max_attempts(3)
.build()
.unwrap();
let started = tokio::time::Instant::now();
let error = ensure_database_exists_with_retry(&url, Duration::from_millis(30), retry)
.await
.unwrap_err();
assert!(
matches!(error, sqlx::Error::Io(ref io) if io.kind() == std::io::ErrorKind::TimedOut)
);
assert_eq!(started.elapsed(), Duration::from_millis(30));
}
}