use std::time::SystemTime;
use deadpool_postgres::Object;
use parking_lot::Mutex;
use super::joint::sweep_lock_key;
use super::{PgJointTx, PostgresBackend};
use crate::gc::{EngineFleetState, PinRow};
use crate::pg_common::{detached_batch_execute, err, pg_err, quote_identifier, time_to_nanos};
use crate::{Error, PinState};
impl PostgresBackend {
pub(crate) fn register_liveness_source(&self, reference: &str) -> Result<(), Error> {
let quoted = quote_table_reference(reference)?;
let mut sources = self.liveness_sources.lock();
if !sources.contains("ed) {
sources.push(quoted);
}
Ok(())
}
pub(crate) fn unregister_liveness_source(&self, reference: &str) -> Result<bool, Error> {
let quoted = quote_table_reference(reference)?;
let mut sources = self.liveness_sources.lock();
let before = sources.len();
sources.retain(|held| *held != quoted);
Ok(sources.len() != before)
}
}
fn quote_table_reference(reference: &str) -> Result<String, Error> {
let mut quoted = String::new();
for part in reference.split('.') {
if part.is_empty() {
return Err(Error::Io(format!(
"postgres: invalid liveness table reference {reference:?}"
)));
}
if !quoted.is_empty() {
quoted.push('.');
}
quoted.push_str("e_identifier(part));
}
Ok(quoted)
}
impl EngineFleetState for PostgresBackend {
type JointTx = PgJointTx;
async fn begin_joint_tx(&self) -> Result<PgJointTx, Error> {
let client = self.pool.get().await.map_err(err)?;
client.batch_execute("BEGIN").await.map_err(pg_err)?;
Ok(PgJointTx {
client: Some(client),
blobs_table: self.table.clone(),
pins_table: self.pins_table.clone(),
#[cfg(test)]
fail_commit: self.fail_joint_commit,
})
}
async fn joint_put(&self, tx: &mut PgJointTx, key: &[u8], value: &[u8]) -> Result<(), Error> {
let sql = format!(
"INSERT INTO {} (key, value, size) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING",
tx.blobs_table
);
let client = tx.client()?;
let stmt = client.prepare_cached(&sql).await.map_err(pg_err)?;
let size = value.len() as i64;
client
.execute(&stmt, &[&key, &value, &size])
.await
.map_err(pg_err)?;
Ok(())
}
async fn pin_commit(&self, mut tx: PgJointTx, entries: &[PinRow]) -> Result<(), Error> {
let Some(client) = tx.client.take() else {
return Err(Error::Io("postgres: joint tx already consumed".to_owned()));
};
#[cfg(test)]
if tx.fail_commit {
client.batch_execute("ROLLBACK").await.map_err(pg_err)?;
return Err(Error::Io(
"engine joint commit failure (postgres, injected)".to_owned(),
));
}
let sql = format!(
"INSERT INTO {} (key, owner, expiry) VALUES ($1, $2, $3) \
ON CONFLICT (key, owner) DO UPDATE SET expiry = excluded.expiry",
tx.pins_table
);
let stmt = match client.prepare_cached(&sql).await {
Ok(stmt) => stmt,
Err(e) => {
let _ = client.batch_execute("ROLLBACK").await;
return Err(err(e));
}
};
for entry in entries {
let expiry = time_to_nanos(entry.expiry);
match client
.execute(&stmt, &[&entry.key, &entry.owner, &expiry])
.await
{
Ok(_) => {}
Err(e) => {
let _ = client.batch_execute("ROLLBACK").await;
return Err(err(e));
}
}
}
match client.batch_execute("COMMIT").await {
Ok(_) => Ok(()),
Err(e) => {
let _ = client.batch_execute("ROLLBACK").await;
Err(err(e))
}
}
}
async fn pin_renew(&self, key: &[u8], owner: &[u8]) -> Result<bool, Error> {
let client = self.pool.get().await.map_err(err)?;
let now = SystemTime::now();
let successor = now
.checked_add(self.pin_ttl)
.ok_or_else(|| Error::Io("postgres: pin expiry overflow".to_owned()))?;
let successor_nanos = time_to_nanos(successor);
let now_nanos = time_to_nanos(now);
let sql = format!(
"UPDATE {} SET expiry = $1 WHERE key = $2 AND owner = $3 AND expiry > $4",
self.pins_table
);
let stmt = client.prepare_cached(&sql).await.map_err(pg_err)?;
let updated = client
.execute(&stmt, &[&successor_nanos, &key, &owner, &now_nanos])
.await
.map_err(pg_err)?;
Ok(updated == 1)
}
async fn pin_check(&self, key: &[u8]) -> Result<PinState, Error> {
let client = self.pool.get().await.map_err(err)?;
let now_nanos = time_to_nanos(SystemTime::now());
let sql = format!(
"SELECT EXISTS(SELECT 1 FROM {} WHERE key = $1 AND expiry > $2)",
self.pins_table
);
let stmt = client.prepare_cached(&sql).await.map_err(pg_err)?;
let present = client
.query_one(&stmt, &[&key, &now_nanos])
.await
.map_err(pg_err)?
.try_get::<_, bool>(0)
.map_err(pg_err)?;
Ok(if present {
PinState::Protected
} else {
PinState::Unprotected
})
}
async fn pin_reap(&self, now: SystemTime) -> Result<u64, Error> {
let client = self.pool.get().await.map_err(err)?;
let now_nanos = time_to_nanos(now);
let sql = format!("DELETE FROM {} WHERE expiry <= $1", self.pins_table);
let stmt = client.prepare_cached(&sql).await.map_err(pg_err)?;
client.execute(&stmt, &[&now_nanos]).await.map_err(pg_err)
}
async fn arbitrate(&self, key: &[u8]) -> Result<bool, Error> {
let sources = self.liveness_sources.lock().clone();
let client = self.pool.get().await.map_err(err)?;
client.batch_execute("BEGIN").await.map_err(pg_err)?;
let folded: Result<bool, Error> = async {
let now_nanos = time_to_nanos(SystemTime::now());
let pins_sql = format!(
"SELECT 1 FROM {} WHERE key = $1 AND expiry > $2 FOR UPDATE LIMIT 1",
self.pins_table
);
let stmt = client.prepare_cached(&pins_sql).await.map_err(pg_err)?;
let mut protected = client
.query_opt(&stmt, &[&key, &now_nanos])
.await
.map_err(pg_err)?
.is_some();
if !protected {
for table in &sources {
let sql = format!("SELECT 1 FROM {table} WHERE key = $1 FOR UPDATE LIMIT 1");
let liveness_stmt = client.prepare_cached(&sql).await.map_err(pg_err)?;
let live = client
.query_opt(&liveness_stmt, &[&key])
.await
.map_err(pg_err)?
.is_some();
if live {
protected = true;
break;
}
}
}
client.batch_execute("COMMIT").await.map_err(pg_err)?;
Ok(protected)
}
.await;
if folded.is_err() {
let _ = client.batch_execute("ROLLBACK").await;
}
folded
}
async fn sweep_lock_acquire(&self) -> Result<bool, Error> {
{
let mut lease = self.lease.lock();
if lease.as_ref().is_some_and(|client| !client.is_closed()) {
return Ok(false);
}
*lease = None;
}
let client = self.pool.get().await.map_err(err)?;
let guard = LeaseUnlockGuard::new(client, &self.schema);
let key = sweep_lock_key(&self.schema);
let session = guard
.session()
.ok_or_else(|| Error::Io("postgres: lease checkout already consumed".to_owned()))?;
let stmt = session
.prepare_cached("SELECT pg_try_advisory_lock($1)")
.await
.map_err(pg_err)?;
let held = session
.query_one(&stmt, &[&key])
.await
.map_err(pg_err)?
.try_get::<_, bool>(0)
.map_err(pg_err)?;
if held {
#[cfg(test)]
crate::postgres::testing::lease_cancel_window_park(&self.schema).await;
guard.into_hold(&self.lease);
} else {
guard.release_unheld();
}
Ok(held)
}
async fn sweep_lock_release(&self) -> Result<(), Error> {
let held = self.lease.lock().take();
let Some(client) = held else {
return Ok(());
};
let guard = LeaseUnlockGuard::new(client, &self.schema);
#[cfg(test)]
crate::postgres::testing::lease_cancel_window_park(&self.schema).await;
guard.unlock(sweep_lock_key(&self.schema)).await;
Ok(())
}
}
struct LeaseUnlockGuard {
client: Option<Object>,
schema: String,
}
impl LeaseUnlockGuard {
fn new(client: Object, schema: &str) -> Self {
Self {
client: Some(client),
schema: schema.to_owned(),
}
}
fn session(&self) -> Option<&Object> {
self.client.as_ref()
}
fn into_hold(self, lease: &Mutex<Option<Object>>) {
let mut this = self;
*lease.lock() = this.client.take();
}
fn release_unheld(mut self) {
let _ = self.client.take();
}
async fn unlock(self, key: i64) {
let mut this = self;
if let Some(client) = this.client.take() {
let _ = client
.batch_execute(&format!("SELECT pg_advisory_unlock({key})"))
.await;
}
}
}
impl Drop for LeaseUnlockGuard {
fn drop(&mut self) {
if let Some(client) = self.client.take() {
let key = sweep_lock_key(&self.schema);
detached_batch_execute(client, format!("SELECT pg_advisory_unlock({key})"));
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use crate::pg_common::time_to_nanos;
use crate::postgres::pool::{DEFAULT_PIN_TTL, DEFAULT_PIN_TTL_SECS};
use crate::postgres::testing::{
arc_backend, at, harness_dsn, harness_gc, harness_with_ttl, pin_row, pin_rows,
raw_connection, schema_tables, CONTENT,
};
#[tokio::test]
async fn pin_renew_never_resurrects_an_expired_or_absent_row() {
let Some(backend) = harness_gc("renew").await else {
return;
};
let key = b"\x01renew-key".to_vec();
let owner = b"\x01instance-id".to_vec();
assert!(
!backend.pin_renew(&key, &owner).await.unwrap(),
"absent row: no renewal, no creation"
);
assert_eq!(pin_rows(&backend).await, vec![], "absent stays absent");
let live = SystemTime::now() + Duration::from_secs(30);
{
let client = backend.pool.get().await.unwrap();
let sql = format!(
"INSERT INTO {} (key, owner, expiry) VALUES ($1, $2, $3)",
backend.pins_table
);
client
.execute(sql.as_str(), &[&key, &owner, &time_to_nanos(live)])
.await
.unwrap();
}
assert!(
backend.pin_renew(&key, &owner).await.unwrap(),
"live row renews"
);
let after = pin_rows(&backend).await;
let expected =
time_to_nanos(SystemTime::now()) + DEFAULT_PIN_TTL_SECS as i64 * 1_000_000_000;
let got = after[0].2;
assert!(
(got - expected).abs() < 3_000_000_000,
"renewed to the successor time now + {DEFAULT_PIN_TTL_SECS}s: {got} vs {expected}"
);
let expired_nanos = time_to_nanos(at(-1_000_000_000));
{
let client = backend.pool.get().await.unwrap();
let sql = format!(
"UPDATE {} SET expiry = $1 WHERE key = $2 AND owner = $3",
backend.pins_table
);
client
.execute(sql.as_str(), &[&expired_nanos, &key, &owner])
.await
.unwrap();
}
assert!(
!backend.pin_renew(&key, &owner).await.unwrap(),
"expired row: never resurrected"
);
assert_eq!(
pin_rows(&backend).await,
vec![(key, owner, expired_nanos)],
"the expired row's expiry is untouched"
);
}
#[tokio::test]
async fn pin_reap_reaps_exactly_the_expired_set() {
let Some(backend) = harness_gc("reap").await else {
return;
};
let now = SystemTime::now();
let expired_solo = pin_row(
b"\x01reap-expired-solo",
b"\x01owner-a",
now - Duration::from_secs(1),
);
let expired_pair = pin_row(
b"\x01reap-pair-key",
b"\x01owner-b",
now - Duration::from_secs(300),
);
let live_pair = pin_row(
b"\x01reap-pair-key",
b"\x01owner-c",
now + Duration::from_secs(30),
);
let live_solo = pin_row(
b"\x01reap-live-solo",
b"\x01owner-d",
now + Duration::from_secs(400),
);
{
let mut tx = backend.begin_joint_tx().await.unwrap();
EngineFleetState::joint_put(&*backend, &mut tx, &expired_solo.key, CONTENT)
.await
.unwrap();
backend
.pin_commit(
tx,
&[
expired_solo.clone(),
expired_pair.clone(),
live_pair.clone(),
],
)
.await
.unwrap();
}
{
let tx = backend.begin_joint_tx().await.unwrap();
backend
.pin_commit(tx, std::slice::from_ref(&live_solo))
.await
.unwrap();
}
let reaped = EngineFleetState::pin_reap(&*backend, now).await.unwrap();
assert_eq!(
reaped, 2,
"exactly the expired rows reclaimed: the solo and the pair-key's expired owner"
);
let after = pin_rows(&backend).await;
let mut expected: Vec<(Vec<u8>, Vec<u8>, i64)> = vec![
(
live_pair.key.clone(),
live_pair.owner.clone(),
time_to_nanos(live_pair.expiry),
),
(
live_solo.key.clone(),
live_solo.owner.clone(),
time_to_nanos(live_solo.expiry),
),
];
expected.sort();
assert_eq!(
after, expected,
"un-expired rows survive untouched — including the live row over the reaped key"
);
let second = EngineFleetState::pin_reap(&*backend, now).await.unwrap();
assert_eq!(
second, 0,
"a second reap of the same instant reclaims nothing"
);
assert_eq!(pin_rows(&backend).await, expected, "no further change");
}
#[tokio::test]
async fn pin_renew_renews_to_the_constructor_carried_ttl() {
let Some(backend) = harness_with_ttl("renew-ttl", Duration::from_secs(3600)).await else {
return;
};
let key = b"\x01renew-ttl-key".to_vec();
let owner = b"\x0ainstance-id".to_vec();
let live = SystemTime::now() + Duration::from_secs(30);
{
let client = backend.pool.get().await.unwrap();
let sql = format!(
"INSERT INTO {} (key, owner, expiry) VALUES ($1, $2, $3)",
backend.pins_table
);
client
.execute(sql.as_str(), &[&key, &owner, &time_to_nanos(live)])
.await
.unwrap();
}
assert!(
backend.pin_renew(&key, &owner).await.unwrap(),
"live row renews"
);
let after = pin_rows(&backend).await;
let expected = time_to_nanos(SystemTime::now()) + 3600i64 * 1_000_000_000;
let got = after[0].2;
assert!(
(got - expected).abs() < 3_000_000_000,
"renewed to the constructor-carried successor time now + 3600s: {got} vs {expected}"
);
}
#[tokio::test]
async fn pin_check_classifies_the_two_variants() {
let Some(backend) = harness_gc("check").await else {
return;
};
let key = b"\x01check-key".to_vec();
let owner = b"\x01instance-id".to_vec();
let other_owner = b"\x02instance-id".to_vec();
assert_eq!(
backend.pin_check(&key).await.unwrap(),
PinState::Unprotected,
"no rows: Unprotected"
);
{
let client = backend.pool.get().await.unwrap();
let sql = format!(
"INSERT INTO {} (key, owner, expiry) VALUES ($1, $2, $3)",
backend.pins_table
);
client
.execute(
sql.as_str(),
&[&key, &owner, &time_to_nanos(at(-5_000_000_000))],
)
.await
.unwrap();
}
assert_eq!(
backend.pin_check(&key).await.unwrap(),
PinState::Unprotected,
"only an expired row: Unprotected (abandoned pins never protect)"
);
{
let client = backend.pool.get().await.unwrap();
let sql = format!(
"INSERT INTO {} (key, owner, expiry) VALUES ($1, $2, $3)",
backend.pins_table
);
client
.execute(
sql.as_str(),
&[
&key,
&other_owner,
&time_to_nanos(SystemTime::now() + DEFAULT_PIN_TTL),
],
)
.await
.unwrap();
}
assert_eq!(
backend.pin_check(&key).await.unwrap(),
PinState::Protected,
"any owner's live row protects (protection composes by row)"
);
}
#[tokio::test]
async fn arbitrate_folds_pins_and_registered_liveness_tables() {
let Some(backend) = harness_gc("arbitrate").await else {
return;
};
let key = b"\x01arbitrate-key".to_vec();
let owner = b"\x01instance-id".to_vec();
let embedder_table = format!("{}.alive_keys", backend.schema);
{
let client = backend.pool.get().await.unwrap();
let sql = format!(
"CREATE TABLE {} (key bytea PRIMARY KEY, seq bigint)",
quote_table_reference(&embedder_table).unwrap()
);
client.execute(sql.as_str(), &[]).await.unwrap();
}
backend.register_liveness_source(&embedder_table).unwrap();
backend.register_liveness_source(&embedder_table).unwrap();
let tables = schema_tables(&backend).await;
assert_eq!(
tables,
vec!["alive_keys", "blobs", "gc_pins", "gc_window"],
"the embedder's table exists only because the test (the embedder) created it"
);
assert!(
!backend.arbitrate(&key).await.unwrap(),
"nothing protects an absent key"
);
{
let client = backend.pool.get().await.unwrap();
let sql = format!(
"INSERT INTO {} (key, seq) VALUES ($1, $2)",
quote_table_reference(&embedder_table).unwrap()
);
client.execute(sql.as_str(), &[&key, &1_i64]).await.unwrap();
}
assert!(
backend.arbitrate(&key).await.unwrap(),
"a registered liveness row protects the key"
);
{
let client = backend.pool.get().await.unwrap();
client
.execute(
&format!(
"DELETE FROM {} WHERE key = $1",
quote_table_reference(&embedder_table).unwrap()
),
&[&key],
)
.await
.unwrap();
}
assert!(
!backend.arbitrate(&key).await.unwrap(),
"liveness row gone, no pin rows: unprotected"
);
{
let client = backend.pool.get().await.unwrap();
let sql = format!(
"INSERT INTO {} (key, owner, expiry) VALUES ($1, $2, $3)",
backend.pins_table
);
client
.execute(
sql.as_str(),
&[&key, &owner, &time_to_nanos(at(-1_000_000_000))],
)
.await
.unwrap();
}
assert!(
!backend.arbitrate(&key).await.unwrap(),
"an expired pin row is abandoned: it protects nothing"
);
{
let client = backend.pool.get().await.unwrap();
let sql = format!(
"UPDATE {} SET expiry = $1 WHERE key = $2 AND owner = $3",
backend.pins_table
);
client
.execute(
sql.as_str(),
&[
&time_to_nanos(SystemTime::now() + DEFAULT_PIN_TTL),
&key,
&owner,
],
)
.await
.unwrap();
}
assert!(
backend.arbitrate(&key).await.unwrap(),
"a live pin row protects the key"
);
}
#[tokio::test]
async fn arbitrate_finger_lock_blocks_a_concurrent_renewal() {
let Some(backend) = harness_gc("finger-lock").await else {
return;
};
let key = b"\x01finger-lock-key".to_vec();
let owner = b"\x01instance-id".to_vec();
{
let client = backend.pool.get().await.unwrap();
let sql = format!(
"INSERT INTO {} (key, owner, expiry) VALUES ($1, $2, $3)",
backend.pins_table
);
client
.execute(
sql.as_str(),
&[
&key,
&owner,
&time_to_nanos(SystemTime::now() + DEFAULT_PIN_TTL),
],
)
.await
.unwrap();
}
let client = backend.pool.get().await.unwrap();
client.batch_execute("BEGIN").await.unwrap();
let lock_sql = format!(
"SELECT expiry FROM {} WHERE key = $1 AND owner = $2 FOR UPDATE",
backend.pins_table
);
let stmt = client.prepare(&lock_sql).await.unwrap();
let locked = client.query(&stmt, &[&key, &owner]).await.unwrap().len();
assert_eq!(locked, 1, "the raw session holds the finger-lock");
let (renewed_tx, mut renewed_rx) = tokio::sync::oneshot::channel();
tokio::spawn(async move {
let renewed = backend.pin_renew(&key, &owner).await;
let _ = renewed_tx.send(renewed);
});
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
assert!(
matches!(
renewed_rx.try_recv(),
Err(tokio::sync::oneshot::error::TryRecvError::Empty)
),
"the renewal is still blocked on the finger-lock"
);
client.batch_execute("COMMIT").await.unwrap();
assert!(
renewed_rx.await.unwrap().unwrap(),
"renew lands once the holder releases"
);
}
#[tokio::test]
async fn cancelled_acquire_cannot_strand_the_advisory_lock() {
let Some(backend) = harness_gc("lease-cancel-acquire").await else {
return;
};
let acquirer = arc_backend(&backend);
let dsn = harness_dsn().unwrap();
let (raw, _conn) = raw_connection(&dsn).await;
let key = sweep_lock_key(&backend.schema);
crate::postgres::testing::arm_lease_cancel_window(&backend.schema);
let task = tokio::spawn({
let acquirer = Arc::clone(&acquirer);
async move { acquirer.sweep_lock_acquire().await }
});
crate::postgres::testing::wait_for_advisory(
&raw,
key,
true,
"the acquire granted server-side, parked in the window",
)
.await;
task.abort();
let _ = task.await;
crate::postgres::testing::disarm_lease_cancel_window();
crate::postgres::testing::wait_for_advisory(
&raw,
key,
false,
"the guard's spawned unlock cleared the granted lock",
)
.await;
assert!(
acquirer.lease.lock().is_none(),
"the cancelled acquire stored no lease record"
);
assert!(
backend.sweep_lock_acquire().await.unwrap(),
"the backend acquires again — nothing stranded"
);
backend.sweep_lock_release().await.unwrap();
}
#[tokio::test]
async fn cancelled_release_cannot_strand_the_advisory_lock() {
let Some(backend) = harness_gc("lease-cancel-release").await else {
return;
};
let releaser = arc_backend(&backend);
assert!(releaser.sweep_lock_acquire().await.unwrap(), "held");
let dsn = harness_dsn().unwrap();
let (raw, _conn) = raw_connection(&dsn).await;
let key = sweep_lock_key(&backend.schema);
crate::postgres::testing::arm_lease_cancel_window(&backend.schema);
let task = tokio::spawn({
let releaser = Arc::clone(&releaser);
async move { releaser.sweep_lock_release().await }
});
let mut taken = false;
for _ in 0..100 {
if releaser.lease.lock().is_none() {
taken = true;
break;
}
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}
assert!(taken, "the release took the checkout out of the record");
crate::postgres::testing::wait_for_advisory(
&raw,
key,
true,
"the parked release has not unlocked yet",
)
.await;
task.abort();
let _ = task.await;
crate::postgres::testing::disarm_lease_cancel_window();
crate::postgres::testing::wait_for_advisory(
&raw,
key,
false,
"the guard's spawned unlock cleared the parked release's lock",
)
.await;
assert!(
backend.sweep_lock_acquire().await.unwrap(),
"the backend acquires again — nothing stranded"
);
backend.sweep_lock_release().await.unwrap();
}
}