use crate::context::{
AuthStateSnapshot, ContextInner, DjogiContext, NESTED_ATOMIC_CANCELLED_POISON_REASON,
};
use crate::pg::connection::PgConnection;
use crate::pg::pool::DjogiPool;
use crate::{DbError, DjogiError};
use futures::FutureExt;
use std::future::Future;
use std::panic::{AssertUnwindSafe, resume_unwind};
use std::pin::Pin;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
static JITTER_COUNTER: AtomicU64 = AtomicU64::new(0);
pub type AtomicFuture<'a, R> = Pin<Box<dyn Future<Output = Result<R, DjogiError>> + Send + 'a>>;
enum TopLevelAtomicOwner {
Connection(PgConnection),
Context(DjogiContext),
}
impl TopLevelAtomicOwner {
fn conn_mut(&mut self) -> &mut PgConnection {
match self {
TopLevelAtomicOwner::Connection(conn) => conn,
TopLevelAtomicOwner::Context(ctx) => match ctx.inner_mut() {
ContextInner::Transaction(conn) => conn,
ContextInner::Pool(_) => {
unreachable!("top-level atomic owner should never hold a pool-backed context",)
}
},
}
}
fn detach(self) {
match self {
TopLevelAtomicOwner::Connection(conn) => conn.detach(),
TopLevelAtomicOwner::Context(ctx) => ctx.detach_transaction_connection(),
}
}
}
struct TopLevelAtomicGuard {
owner: Option<TopLevelAtomicOwner>,
clean: bool,
scope: &'static str,
}
impl TopLevelAtomicGuard {
fn from_connection(conn: PgConnection, scope: &'static str) -> Self {
Self {
owner: Some(TopLevelAtomicOwner::Connection(conn)),
clean: false,
scope,
}
}
fn conn_mut(&mut self) -> &mut PgConnection {
self.owner
.as_mut()
.expect("top-level atomic guard owns the connection until Drop")
.conn_mut()
}
fn promote_to_context<F>(&mut self, build_ctx: F)
where
F: FnOnce(PgConnection) -> DjogiContext,
{
let owner = self
.owner
.take()
.expect("promote_to_context requires an owned connection");
let conn = match owner {
TopLevelAtomicOwner::Connection(conn) => conn,
TopLevelAtomicOwner::Context(_) => {
unreachable!("top-level atomic owner already promoted to DjogiContext")
}
};
self.owner = Some(TopLevelAtomicOwner::Context(build_ctx(conn)));
}
fn tx_ctx_mut(&mut self) -> &mut DjogiContext {
match self
.owner
.as_mut()
.expect("top-level atomic guard owns the transaction context until Drop")
{
TopLevelAtomicOwner::Connection(_) => {
unreachable!("transaction context requested before BEGIN completed")
}
TopLevelAtomicOwner::Context(ctx) => ctx,
}
}
async fn commit(&mut self) -> Result<(), DjogiError> {
let result = self.tx_ctx_mut().commit_in_place().await;
if result.is_ok()
|| (matches!(&result, Err(DjogiError::TransactionPoisoned { .. }))
&& !self.tx_ctx_mut().is_transaction_poisoned())
{
self.clean = true;
}
result
}
async fn rollback(&mut self) -> Result<(), DjogiError> {
self.tx_ctx_mut().rollback_in_place().await?;
self.clean = true;
Ok(())
}
}
impl Drop for TopLevelAtomicGuard {
fn drop(&mut self) {
if let Some(owner) = self.owner.take() {
if self.clean {
drop(owner);
} else {
tracing::warn!(
scope = self.scope,
"djogi::transaction::atomic::dirty_detach: detaching dirty \
top-level transaction connection on drop",
);
owner.detach();
}
}
}
}
struct PoolContextAtomicGuard<'a> {
parent_ctx: &'a mut DjogiContext,
inner: TopLevelAtomicGuard,
}
impl<'a> PoolContextAtomicGuard<'a> {
fn new(parent_ctx: &'a mut DjogiContext, conn: PgConnection) -> Self {
Self {
parent_ctx,
inner: TopLevelAtomicGuard::from_connection(conn, "pool_ctx"),
}
}
fn conn_mut(&mut self) -> &mut PgConnection {
self.inner.conn_mut()
}
fn promote_to_context(&mut self) {
let sassi = std::sync::Arc::clone(&self.parent_ctx.sassi);
let auth = self.parent_ctx.auth.clone();
let tenant_scope_suppressed = self.parent_ctx.tenant_scope_suppressed;
self.inner.promote_to_context(|conn| {
let mut tx_ctx = DjogiContext::from_connection_with_sassi(conn, sassi);
tx_ctx.auth = auth;
tx_ctx.tenant_scope_suppressed = tenant_scope_suppressed;
tx_ctx
});
}
fn tx_ctx_mut(&mut self) -> &mut DjogiContext {
self.inner.tx_ctx_mut()
}
async fn commit(&mut self) -> Result<(), DjogiError> {
self.inner.commit().await
}
async fn rollback(&mut self) -> Result<(), DjogiError> {
self.inner.rollback().await
}
fn propagate_success_to_parent(&mut self) {
let auth_after = self.tx_ctx_mut().auth.clone();
let tenant_scope_suppressed_after = self.tx_ctx_mut().tenant_scope_suppressed;
self.parent_ctx.auth = auth_after;
self.parent_ctx.tenant_scope_suppressed = tenant_scope_suppressed_after;
clear_pool_context_transaction_trackers(self.parent_ctx);
}
fn clear_parent_trackers(&mut self) {
clear_pool_context_transaction_trackers(self.parent_ctx);
}
}
impl Drop for PoolContextAtomicGuard<'_> {
fn drop(&mut self) {
if !self.inner.clean {
clear_pool_context_transaction_trackers(self.parent_ctx);
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum IsolationLevel {
ReadCommitted,
RepeatableRead,
Serializable,
}
impl IsolationLevel {
pub const fn as_sql_keyword(self) -> &'static str {
match self {
IsolationLevel::ReadCommitted => "READ COMMITTED",
IsolationLevel::RepeatableRead => "REPEATABLE READ",
IsolationLevel::Serializable => "SERIALIZABLE",
}
}
}
impl std::fmt::Display for IsolationLevel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_sql_keyword())
}
}
fn begin_with_isolation_sql(level: IsolationLevel) -> String {
format!("BEGIN ISOLATION LEVEL {}", level.as_sql_keyword())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RetryableErrorClasses {
lock_conflict: bool,
db_lock_conflict: bool,
pool_timeout: bool,
}
impl Default for RetryableErrorClasses {
fn default() -> Self {
Self {
lock_conflict: true,
db_lock_conflict: true,
pool_timeout: true,
}
}
}
impl RetryableErrorClasses {
pub fn all() -> Self {
Self::default()
}
pub fn with_lock_conflict(mut self, enabled: bool) -> Self {
self.lock_conflict = enabled;
self
}
pub fn with_db_lock_conflict(mut self, enabled: bool) -> Self {
self.db_lock_conflict = enabled;
self
}
pub fn with_pool_timeout(mut self, enabled: bool) -> Self {
self.pool_timeout = enabled;
self
}
fn is_retryable(self, error: &DjogiError) -> bool {
match error {
DjogiError::LockConflict(_) => self.lock_conflict,
DjogiError::Db(db_error) => {
self.db_lock_conflict && crate::error::is_lock_error(db_error)
}
DjogiError::PoolTimeout { .. } => self.pool_timeout,
_ => false,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TransactionRetryBackoff {
lock_conflict_initial_delay: Duration,
pool_timeout_initial_delay: Duration,
max_delay: Duration,
jitter: Duration,
retryable_error_classes: RetryableErrorClasses,
}
impl Default for TransactionRetryBackoff {
fn default() -> Self {
Self {
lock_conflict_initial_delay: Duration::from_millis(5),
pool_timeout_initial_delay: Duration::from_millis(50),
max_delay: Duration::from_secs(1),
jitter: Duration::from_millis(10),
retryable_error_classes: RetryableErrorClasses::default(),
}
}
}
impl TransactionRetryBackoff {
pub fn new() -> Self {
Self::default()
}
pub fn none() -> Self {
Self {
lock_conflict_initial_delay: Duration::ZERO,
pool_timeout_initial_delay: Duration::ZERO,
max_delay: Duration::ZERO,
jitter: Duration::ZERO,
retryable_error_classes: RetryableErrorClasses::default(),
}
}
pub fn with_lock_conflict_initial_delay(mut self, delay: Duration) -> Self {
self.lock_conflict_initial_delay = delay;
self
}
pub fn with_pool_timeout_initial_delay(mut self, delay: Duration) -> Self {
self.pool_timeout_initial_delay = delay;
self
}
pub fn with_max_delay(mut self, delay: Duration) -> Self {
self.max_delay = delay;
self
}
pub fn with_jitter(mut self, jitter: Duration) -> Self {
self.jitter = jitter;
self
}
pub fn with_retryable_error_classes(mut self, classes: RetryableErrorClasses) -> Self {
self.retryable_error_classes = classes;
self
}
pub fn retryable_error_classes(self) -> RetryableErrorClasses {
self.retryable_error_classes
}
pub fn base_delay_for_retry(
self,
error: &DjogiError,
completed_attempt: u32,
) -> Option<Duration> {
if !self.should_retry(error) {
return None;
}
let base = match error {
DjogiError::PoolTimeout { .. } => self.pool_timeout_initial_delay,
_ => self.lock_conflict_initial_delay,
};
let exponent = completed_attempt.saturating_sub(1).min(31);
let factor = 1_u32.checked_shl(exponent).unwrap_or(u32::MAX);
Some(base.saturating_mul(factor).min(self.max_delay))
}
fn delay_for_retry(self, error: &DjogiError, completed_attempt: u32) -> Option<Duration> {
let base = self.base_delay_for_retry(error, completed_attempt)?;
if self.jitter.is_zero() {
return Some(base);
}
let jitter = jitter_duration(self.jitter);
Some(base.saturating_add(jitter))
}
fn should_retry(self, error: &DjogiError) -> bool {
self.retryable_error_classes.is_retryable(error)
}
}
fn jitter_duration(max_jitter: Duration) -> Duration {
let max_nanos = max_jitter.as_nanos();
if max_nanos == 0 {
return Duration::ZERO;
}
nanos_to_duration(mixed_jitter_nanos(jitter_seed(), max_nanos))
}
fn jitter_seed() -> u128 {
let now_nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0);
let counter = JITTER_COUNTER.fetch_add(0x9E37_79B9_7F4A_7C15, Ordering::Relaxed);
now_nanos ^ ((counter as u128) << 64) ^ counter as u128
}
fn mixed_jitter_nanos(seed: u128, max_nanos: u128) -> u128 {
if max_nanos == 0 {
return 0;
}
let lo = splitmix64(seed as u64);
let hi = splitmix64((seed >> 64) as u64 ^ 0xD1B5_4A32_D192_ED03);
let mixed = ((hi as u128) << 64) | lo as u128;
mixed % (max_nanos + 1)
}
fn splitmix64(mut value: u64) -> u64 {
value = value.wrapping_add(0x9E37_79B9_7F4A_7C15);
value = (value ^ (value >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
value = (value ^ (value >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
value ^ (value >> 31)
}
fn nanos_to_duration(nanos: u128) -> Duration {
const NANOS_PER_SEC: u128 = 1_000_000_000;
Duration::new(
(nanos / NANOS_PER_SEC) as u64,
(nanos % NANOS_PER_SEC) as u32,
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum DeferScope {
All,
Named(&'static [&'static str]),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ConstraintMode {
Deferred,
Immediate,
}
impl ConstraintMode {
pub(crate) const fn as_sql_keyword(self) -> &'static str {
match self {
ConstraintMode::Deferred => "DEFERRED",
ConstraintMode::Immediate => "IMMEDIATE",
}
}
}
mod sealed {
pub trait Sealed {}
impl Sealed for &crate::pg::pool::DjogiPool {}
impl Sealed for &mut crate::DjogiContext {}
}
#[doc(hidden)]
pub trait IntoAtomicScope: sealed::Sealed {
fn run_atomic<F, R>(
self,
closure: F,
) -> impl std::future::Future<Output = Result<R, DjogiError>> + Send
where
R: Send + 'static,
F: for<'a> FnOnce(&'a mut DjogiContext) -> AtomicFuture<'a, R> + Send;
fn run_atomic_with<F, R>(
self,
level: IsolationLevel,
closure: F,
) -> impl std::future::Future<Output = Result<R, DjogiError>> + Send
where
R: Send + 'static,
F: for<'a> FnOnce(&'a mut DjogiContext) -> AtomicFuture<'a, R> + Send;
}
impl IntoAtomicScope for &DjogiPool {
async fn run_atomic<F, R>(self, closure: F) -> Result<R, DjogiError>
where
R: Send + 'static,
F: for<'a> FnOnce(&'a mut DjogiContext) -> AtomicFuture<'a, R> + Send,
{
run_pool_atomic_inner(self, "BEGIN", closure).await
}
async fn run_atomic_with<F, R>(self, level: IsolationLevel, closure: F) -> Result<R, DjogiError>
where
R: Send + 'static,
F: for<'a> FnOnce(&'a mut DjogiContext) -> AtomicFuture<'a, R> + Send,
{
let begin_sql = begin_with_isolation_sql(level);
run_pool_atomic_inner(self, &begin_sql, closure).await
}
}
async fn run_pool_atomic_inner<F, R>(
pool: &DjogiPool,
begin_sql: &str,
closure: F,
) -> Result<R, DjogiError>
where
R: Send + 'static,
F: for<'a> FnOnce(&'a mut DjogiContext) -> AtomicFuture<'a, R> + Send,
{
let conn = pool.get().await?;
let mut guard = TopLevelAtomicGuard::from_connection(conn, "pool");
guard.conn_mut().batch_execute(begin_sql).await?;
guard.promote_to_context(DjogiContext::from_connection);
let result = AssertUnwindSafe(closure(guard.tx_ctx_mut()))
.catch_unwind()
.await;
match result {
Ok(Ok(value)) => {
guard.commit().await?;
Ok(value)
}
Ok(Err(err)) => {
if let Err(rb_err) = guard.rollback().await {
tracing::error!(
error = ?rb_err,
"atomic: rollback after closure Err failed; returning closure err",
);
}
Err(err)
}
Err(panic_payload) => {
if let Err(rb_err) = guard.rollback().await {
tracing::error!(
error = ?rb_err,
"atomic: rollback after closure panic failed; resuming panic",
);
}
resume_unwind(panic_payload);
}
}
}
async fn run_pool_context_atomic<F, R>(
ctx: &mut DjogiContext,
begin_sql: &str,
closure: F,
) -> Result<R, DjogiError>
where
R: Send + 'static,
F: for<'a> FnOnce(&'a mut DjogiContext) -> AtomicFuture<'a, R> + Send,
{
let pool = ctx.pool().cloned().ok_or_else(|| {
DjogiError::Db(DbError::other(
"atomic(&mut ctx, ...) expected a pool-backed context",
))
})?;
let conn = pool.get().await?;
let mut guard = PoolContextAtomicGuard::new(ctx, conn);
guard.conn_mut().batch_execute(begin_sql).await?;
guard.promote_to_context();
let result = AssertUnwindSafe(closure(guard.tx_ctx_mut()))
.catch_unwind()
.await;
match result {
Ok(Ok(value)) => match guard.commit().await {
Ok(()) => {
guard.propagate_success_to_parent();
Ok(value)
}
Err(err @ DjogiError::TransactionPoisoned { .. }) => {
guard.clear_parent_trackers();
Err(err)
}
Err(err) => Err(err),
},
Ok(Err(err)) => {
if let Err(rb_err) = guard.rollback().await {
tracing::error!(
error = ?rb_err,
"atomic: rollback after closure Err failed; returning closure err",
);
}
guard.clear_parent_trackers();
Err(err)
}
Err(panic_payload) => {
if let Err(rb_err) = guard.rollback().await {
tracing::error!(
error = ?rb_err,
"atomic: rollback after closure panic failed; resuming panic",
);
}
guard.clear_parent_trackers();
resume_unwind(panic_payload);
}
}
}
fn clear_pool_context_transaction_trackers(ctx: &mut DjogiContext) {
ctx.tenant_set = false;
ctx.applied_tenant_id = None;
}
struct NestedAtomicCancellationGuard {
ctx: *mut DjogiContext,
callbacks_before: usize,
auth_snapshot: AuthStateSnapshot,
depth_incremented: bool,
armed: bool,
}
unsafe impl Send for NestedAtomicCancellationGuard {}
impl NestedAtomicCancellationGuard {
fn armed(
ctx: &mut DjogiContext,
callbacks_before: usize,
auth_snapshot: AuthStateSnapshot,
) -> Self {
Self {
ctx: ctx as *mut DjogiContext,
callbacks_before,
auth_snapshot,
depth_incremented: true,
armed: true,
}
}
fn disarm(&mut self) {
self.armed = false;
}
fn restore_parent_state(&mut self) {
let callbacks_before = self.callbacks_before;
let auth_snapshot = self.auth_snapshot.clone();
let ctx = unsafe { &mut *self.ctx };
ctx.truncate_on_commit_queue(callbacks_before);
ctx.restore_auth_state(auth_snapshot);
}
}
impl Drop for NestedAtomicCancellationGuard {
fn drop(&mut self) {
if !self.armed {
return;
}
self.restore_parent_state();
let ctx = unsafe { &mut *self.ctx };
if self.depth_incremented {
ctx.decrement_savepoint_depth();
self.depth_incremented = false;
}
ctx.poison_transaction(NESTED_ATOMIC_CANCELLED_POISON_REASON);
}
}
impl IntoAtomicScope for &mut DjogiContext {
async fn run_atomic<F, R>(self, closure: F) -> Result<R, DjogiError>
where
R: Send + 'static,
F: for<'a> FnOnce(&'a mut DjogiContext) -> AtomicFuture<'a, R> + Send,
{
if matches!(self.inner_mut(), ContextInner::Pool(_)) {
return run_pool_context_atomic(self, "BEGIN", closure).await;
}
run_nested_savepoint_atomic(self, closure).await
}
async fn run_atomic_with<F, R>(self, level: IsolationLevel, closure: F) -> Result<R, DjogiError>
where
R: Send + 'static,
F: for<'a> FnOnce(&'a mut DjogiContext) -> AtomicFuture<'a, R> + Send,
{
match self.inner_mut() {
ContextInner::Pool(_) => {
let begin_sql = begin_with_isolation_sql(level);
run_pool_context_atomic(self, &begin_sql, closure).await
}
ContextInner::Transaction(_) => {
Err(DjogiError::IsolationLevelOnNestedScope { requested: level })
}
}
}
}
async fn run_nested_savepoint_atomic<F, R>(
ctx: &mut DjogiContext,
closure: F,
) -> Result<R, DjogiError>
where
R: Send + 'static,
F: for<'a> FnOnce(&'a mut DjogiContext) -> AtomicFuture<'a, R> + Send,
{
debug_assert!(
matches!(ctx.inner_mut(), ContextInner::Transaction(_)),
"run_nested_savepoint_atomic invoked on a non-transaction inner",
);
let callbacks_before = ctx.on_commit_queue_len();
let auth_snapshot = ctx.snapshot_auth_state();
ctx.increment_savepoint_depth();
let depth = ctx.savepoint_depth();
let savepoint_name = format!("sp_{depth}");
let mut cancel_guard =
NestedAtomicCancellationGuard::armed(ctx, callbacks_before, auth_snapshot.clone());
let sp_sql = format!("SAVEPOINT {savepoint_name}");
let push_result = {
let push_future = match ctx.inner_mut() {
ContextInner::Transaction(conn) => conn.batch_execute(&sp_sql),
ContextInner::Pool(_) => unreachable!("debug_assert above rules this out"),
};
push_future.await
};
if let Err(e) = push_result {
ctx.decrement_savepoint_depth();
cancel_guard.disarm();
return Err(e);
}
let inner_result = {
let inner_future = AssertUnwindSafe(closure(ctx)).catch_unwind();
inner_future.await
};
match inner_result {
Ok(Ok(value)) => {
let release_sql = format!("RELEASE SAVEPOINT {savepoint_name}");
let release_res = {
let release_future = match ctx.inner_mut() {
ContextInner::Transaction(conn) => conn.batch_execute(&release_sql),
ContextInner::Pool(_) => unreachable!(),
};
release_future.await
};
ctx.decrement_savepoint_depth();
cancel_guard.disarm();
release_res?;
Ok(value)
}
Ok(Err(err)) => {
ctx.truncate_on_commit_queue(callbacks_before);
ctx.restore_auth_state(auth_snapshot.clone());
let rb_sql = format!("ROLLBACK TO SAVEPOINT {savepoint_name}");
let rel_sql = format!("RELEASE SAVEPOINT {savepoint_name}");
let rollback_error = {
let rollback_future = ctx.run_rollback_to_release(&rb_sql, &rel_sql);
rollback_future.await
};
if let Some(rb_err) = rollback_error {
tracing::error!(
error = ?rb_err,
"atomic: ROLLBACK TO SAVEPOINT after closure Err failed; \
returning closure err",
);
}
ctx.decrement_savepoint_depth();
cancel_guard.disarm();
Err(err)
}
Err(panic_payload) => {
ctx.truncate_on_commit_queue(callbacks_before);
ctx.restore_auth_state(auth_snapshot);
let rb_sql = format!("ROLLBACK TO SAVEPOINT {savepoint_name}");
let rel_sql = format!("RELEASE SAVEPOINT {savepoint_name}");
let rollback_error = {
let rollback_future = ctx.run_rollback_to_release(&rb_sql, &rel_sql);
rollback_future.await
};
if let Some(rb_err) = rollback_error {
tracing::error!(
error = ?rb_err,
"atomic: ROLLBACK TO SAVEPOINT after closure panic failed; \
resuming panic",
);
}
ctx.decrement_savepoint_depth();
cancel_guard.disarm();
resume_unwind(panic_payload);
}
}
}
pub async fn atomic<S, F, R>(scope: S, closure: F) -> Result<R, DjogiError>
where
S: IntoAtomicScope,
R: Send + 'static,
F: for<'a> FnOnce(&'a mut DjogiContext) -> AtomicFuture<'a, R> + Send,
{
scope.run_atomic(closure).await
}
pub async fn atomic_with<S, F, R>(
level: IsolationLevel,
scope: S,
closure: F,
) -> Result<R, DjogiError>
where
S: IntoAtomicScope,
R: Send + 'static,
F: for<'a> FnOnce(&'a mut DjogiContext) -> AtomicFuture<'a, R> + Send,
{
scope.run_atomic_with(level, closure).await
}
pub async fn retry_on_conflict<F, R>(
ctx: &mut DjogiContext,
attempts: u32,
mut closure: F,
) -> Result<R, DjogiError>
where
F: AsyncFnMut(&mut DjogiContext) -> Result<R, DjogiError>,
{
debug_assert!(attempts >= 1, "attempts must be >= 1");
let mut attempt: u32 = 0;
loop {
attempt = attempt.saturating_add(1);
match closure(ctx).await {
Ok(value) => return Ok(value),
Err(e) => {
let retryable = e.is_transient();
if retryable && attempt < attempts {
tracing::debug!(
attempt,
attempts,
"retry_on_conflict: retryable lock error; retrying",
);
continue;
}
return Err(e);
}
}
}
}
pub async fn retry_on_conflict_with_backoff<F, R>(
ctx: &mut DjogiContext,
attempts: u32,
policy: TransactionRetryBackoff,
mut closure: F,
) -> Result<R, DjogiError>
where
F: AsyncFnMut(&mut DjogiContext) -> Result<R, DjogiError>,
{
debug_assert!(attempts >= 1, "attempts must be >= 1");
let mut attempt: u32 = 0;
loop {
attempt = attempt.saturating_add(1);
match closure(ctx).await {
Ok(value) => return Ok(value),
Err(e) => {
let retryable = policy.should_retry(&e);
if retryable && attempt < attempts {
let delay = policy
.delay_for_retry(&e, attempt)
.unwrap_or(Duration::ZERO);
tracing::debug!(
attempt,
attempts,
delay_ms = delay.as_millis(),
error_kind = retry_error_kind(&e),
"retry_on_conflict_with_backoff: retryable transaction error; sleeping before retry",
);
if !delay.is_zero() {
tokio::time::sleep(delay).await;
}
continue;
}
return Err(e);
}
}
}
}
fn retry_error_kind(error: &DjogiError) -> &'static str {
match error {
DjogiError::PoolTimeout { .. } => "pool_timeout",
DjogiError::LockConflict(_) => "lock_conflict",
DjogiError::Db(_) => "database_transient",
_ => "transient",
}
}
impl DjogiContext {
pub(crate) fn on_commit_queue_len(&self) -> usize {
self.on_commit_len()
}
pub(crate) fn truncate_on_commit_queue(&mut self, new_len: usize) {
self.on_commit_truncate(new_len);
}
async fn run_rollback_to_release(&mut self, rb_sql: &str, rel_sql: &str) -> Option<DjogiError> {
match self.inner_mut() {
ContextInner::Transaction(conn) => {
if let Err(e) = conn.batch_execute(rb_sql).await {
return Some(e);
}
if let Err(e) = conn.batch_execute(rel_sql).await {
return Some(e);
}
None
}
ContextInner::Pool(_) => Some(DjogiError::Db(DbError::other(
"run_rollback_to_release called on a pool-backed context; \
this is a framework-invariant bug",
))),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering as AtomicOrdering};
#[test]
fn isolation_level_keywords_match_postgres_grammar() {
assert_eq!(
IsolationLevel::ReadCommitted.as_sql_keyword(),
"READ COMMITTED"
);
assert_eq!(
IsolationLevel::RepeatableRead.as_sql_keyword(),
"REPEATABLE READ"
);
assert_eq!(
IsolationLevel::Serializable.as_sql_keyword(),
"SERIALIZABLE"
);
}
#[test]
fn isolation_level_display_matches_keyword() {
assert_eq!(IsolationLevel::ReadCommitted.to_string(), "READ COMMITTED");
assert_eq!(
IsolationLevel::RepeatableRead.to_string(),
"REPEATABLE READ"
);
assert_eq!(IsolationLevel::Serializable.to_string(), "SERIALIZABLE");
}
#[test]
fn begin_with_isolation_sql_composes_with_keyword() {
assert_eq!(
begin_with_isolation_sql(IsolationLevel::ReadCommitted),
"BEGIN ISOLATION LEVEL READ COMMITTED",
);
assert_eq!(
begin_with_isolation_sql(IsolationLevel::RepeatableRead),
"BEGIN ISOLATION LEVEL REPEATABLE READ",
);
assert_eq!(
begin_with_isolation_sql(IsolationLevel::Serializable),
"BEGIN ISOLATION LEVEL SERIALIZABLE",
);
}
#[test]
fn default_retry_backoff_treats_pool_timeout_as_stronger_pressure_signal() {
let policy = TransactionRetryBackoff::default().with_jitter(Duration::ZERO);
let lock_error = DjogiError::LockConflict(DbError::other("synthetic lock conflict"));
let pool_error = DjogiError::PoolTimeout { phase: "wait" };
let lock_delay = policy.base_delay_for_retry(&lock_error, 1).unwrap();
let pool_delay = policy.base_delay_for_retry(&pool_error, 1).unwrap();
assert!(
pool_delay > lock_delay,
"PoolTimeout retry delay must exceed lock-conflict delay by default"
);
}
#[test]
fn retry_backoff_is_exponential_and_capped() {
let policy = TransactionRetryBackoff::none()
.with_lock_conflict_initial_delay(Duration::from_millis(10))
.with_max_delay(Duration::from_millis(25));
let error = DjogiError::LockConflict(DbError::other("synthetic lock conflict"));
assert_eq!(
policy.base_delay_for_retry(&error, 1),
Some(Duration::from_millis(10)),
);
assert_eq!(
policy.base_delay_for_retry(&error, 2),
Some(Duration::from_millis(20)),
);
assert_eq!(
policy.base_delay_for_retry(&error, 3),
Some(Duration::from_millis(25)),
);
}
#[test]
fn retry_backoff_ignores_terminal_errors() {
let policy = TransactionRetryBackoff::default();
let error = DjogiError::Validation("not retryable".to_string());
assert_eq!(policy.base_delay_for_retry(&error, 1), None);
}
#[test]
fn retry_backoff_retry_classes_can_disable_pool_timeout() {
let policy = TransactionRetryBackoff::default().with_retryable_error_classes(
RetryableErrorClasses::default().with_pool_timeout(false),
);
let lock_error = DjogiError::LockConflict(DbError::other("synthetic lock conflict"));
let pool_error = DjogiError::PoolTimeout { phase: "wait" };
assert!(
policy.base_delay_for_retry(&lock_error, 1).is_some(),
"lock conflicts remain retryable with the default class set",
);
assert_eq!(
policy.base_delay_for_retry(&pool_error, 1),
None,
"pool timeout retries must be policy-controlled and can be disabled",
);
}
#[test]
fn retry_backoff_jitter_can_span_multiple_seconds() {
let max_nanos = Duration::from_secs(2).as_nanos();
let mut saw_above_one_second = false;
for seed in 0_u128..128 {
let jitter = mixed_jitter_nanos(seed, max_nanos);
assert!(
jitter <= max_nanos,
"jitter must not exceed configured max, got {jitter} > {max_nanos}",
);
saw_above_one_second |= jitter > Duration::from_secs(1).as_nanos();
}
assert!(
saw_above_one_second,
"jitter mixing must not be capped to the subsecond range",
);
}
async fn retry_helper_test_context() -> DjogiContext {
let pool = DjogiPool::builder("postgres://localhost/_djogi_unreachable")
.max_size(1)
.build()
.await
.expect("pool construction should not connect until checkout");
DjogiContext::from_pool(pool)
}
#[tokio::test]
async fn retry_on_conflict_with_backoff_returns_first_try_success_without_retry() {
let mut ctx = retry_helper_test_context().await;
let calls = Arc::new(AtomicU32::new(0));
let observed_calls = calls.clone();
let value = retry_on_conflict_with_backoff(
&mut ctx,
3,
TransactionRetryBackoff::none(),
async move |_| {
calls.fetch_add(1, AtomicOrdering::SeqCst);
Ok::<_, DjogiError>(42)
},
)
.await
.expect("first attempt should succeed");
assert_eq!(value, 42);
assert_eq!(observed_calls.load(AtomicOrdering::SeqCst), 1);
}
#[tokio::test]
async fn retry_on_conflict_with_backoff_short_circuits_terminal_error() {
let mut ctx = retry_helper_test_context().await;
let calls = Arc::new(AtomicU32::new(0));
let observed_calls = calls.clone();
let err = retry_on_conflict_with_backoff(
&mut ctx,
5,
TransactionRetryBackoff::none(),
async move |_| {
calls.fetch_add(1, AtomicOrdering::SeqCst);
Err::<(), _>(DjogiError::Validation("terminal".to_string()))
},
)
.await
.expect_err("terminal error must surface immediately");
assert!(matches!(err, DjogiError::Validation(ref msg) if msg == "terminal"));
assert_eq!(observed_calls.load(AtomicOrdering::SeqCst), 1);
}
#[tokio::test]
async fn retry_on_conflict_with_backoff_recovers_after_retryable_error() {
let mut ctx = retry_helper_test_context().await;
let calls = Arc::new(AtomicU32::new(0));
let observed_calls = calls.clone();
let value = retry_on_conflict_with_backoff(
&mut ctx,
3,
TransactionRetryBackoff::none(),
async move |_| {
let completed = calls.fetch_add(1, AtomicOrdering::SeqCst);
if completed == 0 {
Err(DjogiError::PoolTimeout { phase: "wait" })
} else {
Ok(completed + 1)
}
},
)
.await
.expect("second attempt should recover");
assert_eq!(value, 2);
assert_eq!(observed_calls.load(AtomicOrdering::SeqCst), 2);
}
#[tokio::test]
async fn retry_on_conflict_with_backoff_policy_can_disable_pool_timeout_retries() {
let mut ctx = retry_helper_test_context().await;
let calls = Arc::new(AtomicU32::new(0));
let observed_calls = calls.clone();
let policy = TransactionRetryBackoff::none().with_retryable_error_classes(
RetryableErrorClasses::default().with_pool_timeout(false),
);
let err = retry_on_conflict_with_backoff(&mut ctx, 5, policy, async move |_| {
calls.fetch_add(1, AtomicOrdering::SeqCst);
Err::<(), _>(DjogiError::PoolTimeout { phase: "wait" })
})
.await
.expect_err("PoolTimeout should surface immediately when disabled");
assert!(matches!(err, DjogiError::PoolTimeout { phase: "wait" }));
assert_eq!(
observed_calls.load(AtomicOrdering::SeqCst),
1,
"retry class policy must prevent incidental pool-timeout retries",
);
}
#[test]
fn constraint_mode_keywords_match_postgres_grammar() {
assert_eq!(ConstraintMode::Deferred.as_sql_keyword(), "DEFERRED");
assert_eq!(ConstraintMode::Immediate.as_sql_keyword(), "IMMEDIATE");
}
#[test]
fn defer_scope_all_is_value_equal() {
assert_eq!(DeferScope::All, DeferScope::All);
}
#[test]
fn defer_scope_named_compares_by_slice_contents() {
static A: &[&str] = &["posts_author_id_fkey"];
static B: &[&str] = &["posts_author_id_fkey"];
assert_eq!(DeferScope::Named(A), DeferScope::Named(B));
}
}