use std::{future::Future, time::Duration};
use runlimit_core::{
BatchDecision, BatchError, Check, CounterKey, Decision, Denial, validate_batch,
};
use sha2::{Digest, Sha256};
use sqlx::{
Acquire, PgPool, Postgres, Row, Transaction,
migrate::{MigrateError, Migrator},
pool::PoolConnection,
postgres::{PgRow, types::PgInterval},
types::chrono::{DateTime, Utc},
};
use thiserror::Error;
use tokio::time::{Instant, timeout_at};
pub static MIGRATOR: Migrator = sqlx::migrate!("./migrations");
const ADVISORY_LOCK_DOMAIN: &[u8] = b"runlimit/postgres-advisory-lock/v1\0";
const SERVER_TIMEOUT_GRACE: Duration = Duration::from_millis(25);
const SET_LOCAL_TIMEOUTS_SQL: &str = r"
SELECT
set_config('statement_timeout', $1, true),
set_config('lock_timeout', $2, true)
";
const BATCH_ADVISORY_LOCK_SQL: &str = r"
WITH RECURSIVE acquired(position, locked) AS (
SELECT 1, pg_advisory_xact_lock($1[1])
WHERE cardinality($1::BIGINT[]) > 0
UNION ALL
SELECT
acquired.position + 1,
pg_advisory_xact_lock($1[acquired.position + 1])
FROM acquired
WHERE acquired.position < cardinality($1::BIGINT[])
)
SELECT count(*)
FROM acquired
";
const BATCH_ROW_LOCK_SQL: &str = r"
WITH input_keys AS (
SELECT *
FROM unnest(
$1::BYTEA[],
$2::BYTEA[]
) WITH ORDINALITY AS keys(
config_fingerprint,
subject_key,
input_position
)
),
lock_order AS (
SELECT *
FROM unnest($3::BIGINT[])
WITH ORDINALITY AS positions(input_position, lock_position)
),
ordered_keys AS (
SELECT
input_keys.config_fingerprint,
input_keys.subject_key,
lock_order.lock_position
FROM input_keys
INNER JOIN lock_order
ON lock_order.input_position = input_keys.input_position
ORDER BY lock_order.lock_position
)
SELECT 1
FROM ordered_keys
INNER JOIN runlimit_fixed_windows AS windows
ON windows.config_fingerprint = ordered_keys.config_fingerprint
AND windows.subject_key = ordered_keys.subject_key
ORDER BY ordered_keys.lock_position
FOR UPDATE OF windows
";
const BATCH_PREFLIGHT_SQL: &str = r"
WITH input AS (
SELECT *
FROM unnest(
$1::BYTEA[],
$2::BYTEA[],
$3::BIGINT[],
$4::BIGINT[]
) WITH ORDINALITY AS checks(
config_fingerprint,
subject_key,
cost,
quota_limit,
input_position
)
),
sample AS (
SELECT clock_timestamp() AS database_now
),
first_denial AS (
SELECT
input.input_position - 1 AS input_index,
windows.window_expires_at
FROM input
CROSS JOIN sample
INNER JOIN runlimit_fixed_windows AS windows
ON windows.config_fingerprint = input.config_fingerprint
AND windows.subject_key = input.subject_key
WHERE
windows.window_expires_at > sample.database_now
AND windows.used > input.quota_limit - input.cost
ORDER BY input.input_position
LIMIT 1
)
SELECT
sample.database_now,
clock_timestamp() AS response_now,
first_denial.input_index,
first_denial.window_expires_at
FROM sample
LEFT JOIN first_denial ON TRUE
";
const BATCH_UPSERT_SQL: &str = r"
WITH input AS (
SELECT *
FROM unnest(
$1::TEXT[],
$2::TEXT[],
$3::BYTEA[],
$4::BYTEA[],
$5::INTERVAL[],
$6::BIGINT[],
$7::BIGINT[]
) WITH ORDINALITY AS checks(
policy_id,
scope_id,
config_fingerprint,
subject_key,
window_size,
cost,
quota_limit,
input_position
)
),
upserted AS (
INSERT INTO runlimit_fixed_windows (
policy_id,
scope_id,
config_fingerprint,
subject_key,
window_started_at,
window_expires_at,
used
)
SELECT
policy_id,
scope_id,
config_fingerprint,
subject_key,
$8,
$8 + window_size,
cost
FROM input
ON CONFLICT (config_fingerprint, subject_key)
DO UPDATE SET
policy_id = EXCLUDED.policy_id,
scope_id = EXCLUDED.scope_id,
window_started_at = CASE
WHEN runlimit_fixed_windows.window_expires_at <= $8 THEN $8
ELSE runlimit_fixed_windows.window_started_at
END,
window_expires_at = CASE
WHEN runlimit_fixed_windows.window_expires_at <= $8
THEN $8 + (EXCLUDED.window_expires_at - EXCLUDED.window_started_at)
ELSE runlimit_fixed_windows.window_expires_at
END,
used = CASE
WHEN runlimit_fixed_windows.window_expires_at <= $8 THEN EXCLUDED.used
ELSE runlimit_fixed_windows.used + EXCLUDED.used
END
WHERE
runlimit_fixed_windows.window_expires_at <= $8
OR runlimit_fixed_windows.used
<= (
SELECT quota_limit - cost
FROM input
WHERE
input.config_fingerprint = EXCLUDED.config_fingerprint
AND input.subject_key = EXCLUDED.subject_key
)
RETURNING
config_fingerprint,
subject_key,
used,
window_expires_at
),
response AS (
SELECT clock_timestamp() AS response_now
FROM upserted
HAVING count(*) >= 0
)
SELECT
input.input_position - 1 AS input_index,
upserted.used,
upserted.window_expires_at,
response.response_now
FROM input
INNER JOIN upserted
ON upserted.config_fingerprint = input.config_fingerprint
AND upserted.subject_key = input.subject_key
CROSS JOIN response
ORDER BY input.input_position
";
const CLEANUP_SQL: &str = include_str!("cleanup_expired.sql");
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct PostgresConfig {
max_batch_size: usize,
operation_timeout: Duration,
}
impl PostgresConfig {
pub const DEFAULT_MAX_BATCH_SIZE: usize = 32;
pub const DEFAULT_OPERATION_TIMEOUT: Duration = Duration::from_secs(3);
pub const MAXIMUM_OPERATION_TIMEOUT: Duration = Duration::from_secs(60);
pub const fn new() -> Self {
Self {
max_batch_size: Self::DEFAULT_MAX_BATCH_SIZE,
operation_timeout: Self::DEFAULT_OPERATION_TIMEOUT,
}
}
pub fn with_max_batch_size(mut self, maximum: usize) -> Result<Self, PostgresConfigError> {
if maximum == 0 {
return Err(PostgresConfigError::ZeroBatchSize);
}
self.max_batch_size = maximum;
Ok(self)
}
pub fn with_operation_timeout(
mut self,
timeout: Duration,
) -> Result<Self, PostgresConfigError> {
if timeout.is_zero() {
return Err(PostgresConfigError::ZeroOperationTimeout);
}
if timeout > Self::MAXIMUM_OPERATION_TIMEOUT {
return Err(PostgresConfigError::OperationTimeoutTooLong {
actual: timeout,
maximum: Self::MAXIMUM_OPERATION_TIMEOUT,
});
}
self.operation_timeout = timeout;
Ok(self)
}
pub const fn max_batch_size(&self) -> usize {
self.max_batch_size
}
pub const fn operation_timeout(self) -> Duration {
self.operation_timeout
}
}
impl Default for PostgresConfig {
fn default() -> Self {
Self::new()
}
}
#[derive(Clone, Copy, Debug, Error, Eq, PartialEq)]
pub enum PostgresConfigError {
#[error("max_batch_size must be greater than zero")]
ZeroBatchSize,
#[error("PostgreSQL rate-limit operation timeout must be greater than zero")]
ZeroOperationTimeout,
#[error("PostgreSQL rate-limit operation timeout {actual:?} exceeds maximum {maximum:?}")]
OperationTimeoutTooLong {
actual: Duration,
maximum: Duration,
},
}
#[derive(Clone, Debug)]
pub struct PostgresLimiter {
pool: PgPool,
config: PostgresConfig,
}
impl PostgresLimiter {
pub const fn new(pool: PgPool) -> Self {
Self {
pool,
config: PostgresConfig::new(),
}
}
pub const fn with_config(pool: PgPool, config: PostgresConfig) -> Self {
Self { pool, config }
}
pub const fn pool(&self) -> &PgPool {
&self.pool
}
pub const fn config(&self) -> PostgresConfig {
self.config
}
pub async fn migrate(&self) -> Result<(), MigrateError> {
let connection = self.pool.acquire().await?;
let mut guarded_connection = ConnectionCancellationGuard::new(connection);
let mut migrator = sqlx::migrate!("./migrations");
migrator.set_ignore_missing(true);
let result = migrator
.run_direct(&mut **guarded_connection.connection())
.await;
if result.is_ok() {
guarded_connection.disarm();
}
result
}
pub async fn check(&self, check: &Check<'_>) -> Result<Decision, CheckError> {
single_decision_from_batch(self.check_all(std::slice::from_ref(check)).await?)
}
pub async fn check_all(&self, checks: &[Check<'_>]) -> Result<BatchDecision, CheckError> {
validate_batch(checks, self.config.max_batch_size())?;
if checks.is_empty() {
return Ok(BatchDecision::Allowed(Vec::new()));
}
let input = BatchSqlInput::from_checks(checks);
let deadline = Instant::now() + self.config.operation_timeout();
let connection = acquire_check_connection(&self.pool, deadline).await?;
let mut guarded_connection = ConnectionCancellationGuard::new(connection);
let result =
run_check_transaction(guarded_connection.connection(), &input, checks, deadline).await;
if !result
.as_ref()
.is_err_and(CheckRunError::client_timeout_won)
{
guarded_connection.disarm();
}
result.map_err(CheckRunError::into_public)
}
pub async fn cleanup_expired(&self, maximum_rows: u32) -> Result<u64, MaintenanceError> {
if maximum_rows == 0 {
return Ok(0);
}
let deadline = Instant::now() + self.config.operation_timeout();
let connection = acquire_maintenance_connection(&self.pool, deadline).await?;
let mut guarded_connection = ConnectionCancellationGuard::new(connection);
let result =
run_cleanup_transaction(guarded_connection.connection(), maximum_rows, deadline).await;
if !result
.as_ref()
.is_err_and(MaintenanceRunError::client_timeout_won)
{
guarded_connection.disarm();
}
result.map_err(MaintenanceRunError::into_public)
}
}
#[derive(Debug, Error)]
pub enum CheckError {
#[error(transparent)]
InvalidBatch(#[from] BatchError),
#[error("PostgreSQL rate-limit check failed before commit; quota was not consumed")]
DefinitelyNotConsumed(#[source] sqlx::Error),
#[error("PostgreSQL rate-limit commit outcome is unknown; quota may have been consumed")]
CommitOutcomeUnknown(#[source] sqlx::Error),
#[error("PostgreSQL rate-limit check timed out while {operation}; quota was not consumed")]
TimedOutBeforeCommit {
operation: &'static str,
},
#[error("PostgreSQL rate-limit commit timed out; quota may have been consumed")]
CommitTimedOut,
#[error("PostgreSQL rate-limit storage invariant failed: {0}")]
StorageInvariant(&'static str),
#[error(
"PostgreSQL rate-limit single-check response invariant failed after rollback; quota was not consumed"
)]
ResponseInvariant,
#[error(
"PostgreSQL rate-limit single-check response invariant failed after commit; quota was consumed"
)]
CommittedResponseInvariant,
}
impl CheckError {
pub const fn may_have_consumed_quota(&self) -> bool {
matches!(
self,
Self::CommitOutcomeUnknown(_) | Self::CommitTimedOut | Self::CommittedResponseInvariant
)
}
}
#[derive(Debug, Error)]
pub enum MaintenanceError {
#[error("PostgreSQL expired-window cleanup failed before commit; no rows were removed")]
Database(#[source] sqlx::Error),
#[error("PostgreSQL expired-window cleanup timed out while {operation}; no rows were removed")]
TimedOutBeforeCommit {
operation: &'static str,
},
#[error(
"PostgreSQL expired-window cleanup commit outcome is unknown; rows may have been removed"
)]
CommitOutcomeUnknown(#[source] sqlx::Error),
#[error("PostgreSQL expired-window cleanup commit timed out; rows may have been removed")]
CommitTimedOut,
}
impl MaintenanceError {
pub const fn may_have_removed_rows(&self) -> bool {
matches!(self, Self::CommitOutcomeUnknown(_) | Self::CommitTimedOut)
}
}
fn single_decision_from_batch(batch: BatchDecision) -> Result<Decision, CheckError> {
batch
.try_into_single_decision()
.map_err(|invalid| match invalid {
BatchDecision::Allowed(_) => CheckError::CommittedResponseInvariant,
BatchDecision::Denied { .. } => CheckError::ResponseInvariant,
})
}
struct ConnectionCancellationGuard {
connection: PoolConnection<Postgres>,
armed: bool,
}
impl ConnectionCancellationGuard {
const fn new(connection: PoolConnection<Postgres>) -> Self {
Self {
connection,
armed: true,
}
}
const fn connection(&mut self) -> &mut PoolConnection<Postgres> {
&mut self.connection
}
const fn disarm(&mut self) {
self.armed = false;
}
}
impl Drop for ConnectionCancellationGuard {
fn drop(&mut self) {
if self.armed {
self.connection.close_on_drop();
}
}
}
#[derive(Debug)]
enum CheckRunError {
Public(CheckError),
ClientTimedOutBeforeCommit { operation: &'static str },
ClientCommitTimedOut,
}
impl CheckRunError {
const fn client_timeout_won(&self) -> bool {
matches!(
self,
Self::ClientTimedOutBeforeCommit { .. } | Self::ClientCommitTimedOut
)
}
fn into_public(self) -> CheckError {
match self {
Self::Public(error) => error,
Self::ClientTimedOutBeforeCommit { operation } => {
CheckError::TimedOutBeforeCommit { operation }
}
Self::ClientCommitTimedOut => CheckError::CommitTimedOut,
}
}
}
impl From<CheckError> for CheckRunError {
fn from(error: CheckError) -> Self {
Self::Public(error)
}
}
#[derive(Debug)]
enum MaintenanceRunError {
Public(MaintenanceError),
ClientTimedOutBeforeCommit { operation: &'static str },
ClientCommitTimedOut,
}
impl MaintenanceRunError {
const fn client_timeout_won(&self) -> bool {
matches!(
self,
Self::ClientTimedOutBeforeCommit { .. } | Self::ClientCommitTimedOut
)
}
fn into_public(self) -> MaintenanceError {
match self {
Self::Public(error) => error,
Self::ClientTimedOutBeforeCommit { operation } => {
MaintenanceError::TimedOutBeforeCommit { operation }
}
Self::ClientCommitTimedOut => MaintenanceError::CommitTimedOut,
}
}
}
#[derive(Debug)]
enum PendingBatchOutcome {
Allowed(Vec<PendingAllowance>),
Denied { index: usize, denial: PendingDenial },
}
#[derive(Debug)]
struct BatchSqlInput {
policy_ids: Vec<String>,
scope_ids: Vec<String>,
fingerprints: Vec<Vec<u8>>,
subjects: Vec<Vec<u8>>,
lock_input_positions: Vec<i64>,
advisory_lock_ids: Vec<i64>,
windows: Vec<PgInterval>,
costs: Vec<i64>,
limits: Vec<i64>,
}
impl BatchSqlInput {
fn from_checks(checks: &[Check<'_>]) -> Self {
let counter_keys = checks.iter().map(Check::counter_key).collect::<Vec<_>>();
let mut ordered_indices = (0..checks.len()).collect::<Vec<_>>();
ordered_indices.sort_unstable_by_key(|index| counter_keys[*index]);
let mut input = Self {
policy_ids: Vec::with_capacity(checks.len()),
scope_ids: Vec::with_capacity(checks.len()),
fingerprints: Vec::with_capacity(checks.len()),
subjects: Vec::with_capacity(checks.len()),
lock_input_positions: ordered_indices
.into_iter()
.map(|index| {
i64::try_from(index + 1)
.expect("bounded batch input positions fit PostgreSQL BIGINT")
})
.collect(),
advisory_lock_ids: counter_keys.iter().copied().map(advisory_lock_id).collect(),
windows: Vec::with_capacity(checks.len()),
costs: Vec::with_capacity(checks.len()),
limits: Vec::with_capacity(checks.len()),
};
input.advisory_lock_ids.sort_unstable();
input.advisory_lock_ids.dedup();
for (check, counter_key) in checks.iter().zip(counter_keys) {
let policy = check.policy();
input.policy_ids.push(policy.id().as_str().to_owned());
input.scope_ids.push(policy.scope().as_str().to_owned());
input
.fingerprints
.push(counter_key.fingerprint().as_bytes().to_vec());
input
.subjects
.push(counter_key.subject().as_bytes().to_vec());
input.windows.push(
PgInterval::try_from(policy.window())
.expect("core policy windows fit PostgreSQL INTERVAL exactly"),
);
input.costs.push(database_integer(check.cost()));
input.limits.push(database_integer(policy.limit()));
}
input
}
}
#[derive(Debug)]
struct BatchPreflight {
database_now: DateTime<Utc>,
response_now: DateTime<Utc>,
denial: Option<(usize, PendingDenial)>,
}
#[derive(Debug)]
struct PendingAllowance {
limit: u64,
remaining: u64,
reset_from_sample: Duration,
}
impl PendingAllowance {
fn finish(self, authoritative_elapsed: Duration) -> Decision {
Decision::allowed(
self.limit,
self.remaining,
self.reset_from_sample.saturating_sub(authoritative_elapsed),
)
}
}
#[derive(Debug)]
struct PendingDenial {
limit: u64,
retry_from_sample: Duration,
}
impl PendingDenial {
fn finish(self, authoritative_elapsed: Duration) -> Denial {
Denial::QuotaExceeded {
limit: self.limit,
retry_after: self.retry_from_sample.saturating_sub(authoritative_elapsed),
}
}
}
async fn acquire_check_connection(
pool: &PgPool,
deadline: Instant,
) -> Result<PoolConnection<Postgres>, CheckError> {
timeout_at(deadline, pool.acquire())
.await
.map_err(|_| CheckError::TimedOutBeforeCommit {
operation: "acquiring database connection",
})?
.map_err(CheckError::DefinitelyNotConsumed)
}
async fn run_check_transaction(
connection: &mut PoolConnection<Postgres>,
input: &BatchSqlInput,
checks: &[Check<'_>],
deadline: Instant,
) -> Result<BatchDecision, CheckRunError> {
let mut transaction =
check_before_commit(deadline, "beginning transaction", connection.begin()).await?;
acquire_advisory_locks(&mut transaction, &input.advisory_lock_ids, deadline).await?;
acquire_existing_row_locks(&mut transaction, input, deadline).await?;
let (pending, authoritative_elapsed) =
execute_batch(&mut transaction, input, checks, deadline).await?;
match pending {
PendingBatchOutcome::Denied { index, denial } => {
set_check_server_timeouts(&mut transaction, deadline, "rolling back denied batch")
.await?;
check_before_commit(
deadline,
"rolling back denied batch",
transaction.rollback(),
)
.await?;
Ok(BatchDecision::Denied {
index,
denial: denial.finish(authoritative_elapsed),
})
}
PendingBatchOutcome::Allowed(allowances) => {
set_check_server_timeouts(&mut transaction, deadline, "committing counter batch")
.await?;
commit_check(deadline, transaction).await?;
Ok(BatchDecision::Allowed(
allowances
.into_iter()
.map(|allowance| allowance.finish(authoritative_elapsed))
.collect(),
))
}
}
}
async fn check_before_commit<T, F>(
deadline: Instant,
operation: &'static str,
future: F,
) -> Result<T, CheckRunError>
where
F: Future<Output = Result<T, sqlx::Error>>,
{
timeout_at(deadline, future)
.await
.map_err(|_| CheckRunError::ClientTimedOutBeforeCommit { operation })?
.map_err(|error| CheckRunError::Public(map_check_database_error(error, operation)))
}
async fn commit_check(
deadline: Instant,
transaction: Transaction<'_, Postgres>,
) -> Result<(), CheckRunError> {
if Instant::now() >= deadline {
return Err(CheckRunError::Public(CheckError::TimedOutBeforeCommit {
operation: "starting commit",
}));
}
timeout_at(deadline, transaction.commit())
.await
.map_err(|_| CheckRunError::ClientCommitTimedOut)?
.map_err(|error| CheckRunError::Public(CheckError::CommitOutcomeUnknown(error)))
}
fn map_check_database_error(error: sqlx::Error, operation: &'static str) -> CheckError {
if is_server_timeout(&error) {
CheckError::TimedOutBeforeCommit { operation }
} else {
CheckError::DefinitelyNotConsumed(error)
}
}
fn is_server_timeout(error: &sqlx::Error) -> bool {
let sqlx::Error::Database(database_error) = error else {
return false;
};
matches!(database_error.code().as_deref(), Some("57014" | "55P03"))
}
fn remaining_server_timeout_settings(deadline: Instant) -> Option<(String, String)> {
let remaining = deadline.checked_duration_since(Instant::now())?;
let remaining_millis = remaining.as_millis();
let statement_millis = remaining_millis
.saturating_sub(SERVER_TIMEOUT_GRACE.as_millis())
.max(1);
let lock_millis = statement_millis.saturating_sub(1).max(1);
Some((format!("{statement_millis}ms"), format!("{lock_millis}ms")))
}
async fn set_check_server_timeouts(
transaction: &mut Transaction<'_, Postgres>,
deadline: Instant,
operation: &'static str,
) -> Result<(), CheckRunError> {
let (statement_timeout, lock_timeout) = remaining_server_timeout_settings(deadline).ok_or(
CheckRunError::Public(CheckError::TimedOutBeforeCommit { operation }),
)?;
check_before_commit(
deadline,
operation,
sqlx::query(SET_LOCAL_TIMEOUTS_SQL)
.bind(statement_timeout)
.bind(lock_timeout)
.execute(&mut **transaction),
)
.await
.map(|_| ())
}
async fn acquire_advisory_locks(
transaction: &mut Transaction<'_, Postgres>,
advisory_lock_ids: &[i64],
deadline: Instant,
) -> Result<(), CheckRunError> {
set_check_server_timeouts(transaction, deadline, "acquiring logical key lock").await?;
check_before_commit(deadline, "acquiring logical key lock", async {
sqlx::query(BATCH_ADVISORY_LOCK_SQL)
.bind(advisory_lock_ids)
.execute(&mut **transaction)
.await
.map(|_| ())
})
.await
}
fn advisory_lock_id(counter_key: CounterKey) -> i64 {
let mut digest = Sha256::new();
digest.update(ADVISORY_LOCK_DOMAIN);
digest.update(counter_key.to_bytes());
let digest = digest.finalize();
let mut id_bytes = [0_u8; 8];
let id_length = id_bytes.len();
id_bytes.copy_from_slice(&digest[..id_length]);
i64::from_be_bytes(id_bytes)
}
async fn acquire_existing_row_locks(
transaction: &mut Transaction<'_, Postgres>,
input: &BatchSqlInput,
deadline: Instant,
) -> Result<(), CheckRunError> {
set_check_server_timeouts(transaction, deadline, "acquiring counter row lock").await?;
check_before_commit(deadline, "acquiring counter row lock", async {
sqlx::query(BATCH_ROW_LOCK_SQL)
.bind(input.fingerprints.as_slice())
.bind(input.subjects.as_slice())
.bind(input.lock_input_positions.as_slice())
.execute(&mut **transaction)
.await
.map(|_| ())
})
.await
}
async fn acquire_maintenance_connection(
pool: &PgPool,
deadline: Instant,
) -> Result<PoolConnection<Postgres>, MaintenanceError> {
timeout_at(deadline, pool.acquire())
.await
.map_err(|_| MaintenanceError::TimedOutBeforeCommit {
operation: "acquiring database connection",
})?
.map_err(MaintenanceError::Database)
}
async fn run_cleanup_transaction(
connection: &mut PoolConnection<Postgres>,
maximum_rows: u32,
deadline: Instant,
) -> Result<u64, MaintenanceRunError> {
let mut transaction = maintenance_before_commit(
deadline,
"beginning cleanup transaction",
connection.begin(),
)
.await?;
set_maintenance_server_timeouts(&mut transaction, deadline, "deleting expired windows").await?;
let result = maintenance_before_commit(
deadline,
"deleting expired windows",
sqlx::query(CLEANUP_SQL)
.bind(i64::from(maximum_rows))
.execute(&mut *transaction),
)
.await?;
let rows_affected = result.rows_affected();
set_maintenance_server_timeouts(
&mut transaction,
deadline,
"committing expired-window cleanup",
)
.await?;
commit_maintenance(deadline, transaction).await?;
Ok(rows_affected)
}
async fn maintenance_before_commit<T, F>(
deadline: Instant,
operation: &'static str,
future: F,
) -> Result<T, MaintenanceRunError>
where
F: Future<Output = Result<T, sqlx::Error>>,
{
timeout_at(deadline, future)
.await
.map_err(|_| MaintenanceRunError::ClientTimedOutBeforeCommit { operation })?
.map_err(|error| {
let public = if is_server_timeout(&error) {
MaintenanceError::TimedOutBeforeCommit { operation }
} else {
MaintenanceError::Database(error)
};
MaintenanceRunError::Public(public)
})
}
async fn set_maintenance_server_timeouts(
transaction: &mut Transaction<'_, Postgres>,
deadline: Instant,
operation: &'static str,
) -> Result<(), MaintenanceRunError> {
let (statement_timeout, lock_timeout) = remaining_server_timeout_settings(deadline).ok_or(
MaintenanceRunError::Public(MaintenanceError::TimedOutBeforeCommit { operation }),
)?;
maintenance_before_commit(
deadline,
operation,
sqlx::query(SET_LOCAL_TIMEOUTS_SQL)
.bind(statement_timeout)
.bind(lock_timeout)
.execute(&mut **transaction),
)
.await
.map(|_| ())
}
async fn commit_maintenance(
deadline: Instant,
transaction: Transaction<'_, Postgres>,
) -> Result<(), MaintenanceRunError> {
if Instant::now() >= deadline {
return Err(MaintenanceRunError::Public(
MaintenanceError::TimedOutBeforeCommit {
operation: "starting expired-window cleanup commit",
},
));
}
timeout_at(deadline, transaction.commit())
.await
.map_err(|_| MaintenanceRunError::ClientCommitTimedOut)?
.map_err(|error| MaintenanceRunError::Public(MaintenanceError::CommitOutcomeUnknown(error)))
}
fn authoritative_elapsed(start: DateTime<Utc>, end: DateTime<Utc>) -> Duration {
end.signed_duration_since(start)
.to_std()
.unwrap_or(Duration::ZERO)
}
async fn execute_batch(
transaction: &mut Transaction<'_, Postgres>,
input: &BatchSqlInput,
checks: &[Check<'_>],
deadline: Instant,
) -> Result<(PendingBatchOutcome, Duration), CheckRunError> {
let preflight = preflight_batch(transaction, input, checks, deadline).await?;
if let Some((index, denial)) = preflight.denial {
return Ok((
PendingBatchOutcome::Denied { index, denial },
authoritative_elapsed(preflight.database_now, preflight.response_now),
));
}
let (allowances, response_now) =
upsert_batch(transaction, input, checks, preflight.database_now, deadline).await?;
Ok((
PendingBatchOutcome::Allowed(allowances),
authoritative_elapsed(preflight.database_now, response_now),
))
}
async fn preflight_batch(
transaction: &mut Transaction<'_, Postgres>,
input: &BatchSqlInput,
checks: &[Check<'_>],
deadline: Instant,
) -> Result<BatchPreflight, CheckRunError> {
set_check_server_timeouts(transaction, deadline, "preflighting counter batch").await?;
let preflight_row = check_before_commit(
deadline,
"preflighting counter batch",
sqlx::query(BATCH_PREFLIGHT_SQL)
.bind(input.fingerprints.as_slice())
.bind(input.subjects.as_slice())
.bind(input.costs.as_slice())
.bind(input.limits.as_slice())
.fetch_one(&mut **transaction),
)
.await?;
let database_now: DateTime<Utc> = preflight_row
.try_get("database_now")
.map_err(CheckError::DefinitelyNotConsumed)?;
let preflight_response_now: DateTime<Utc> = preflight_row
.try_get("response_now")
.map_err(CheckError::DefinitelyNotConsumed)?;
let denied_index: Option<i64> = preflight_row
.try_get("input_index")
.map_err(CheckError::DefinitelyNotConsumed)?;
let denied_expiry: Option<DateTime<Utc>> = preflight_row
.try_get("window_expires_at")
.map_err(CheckError::DefinitelyNotConsumed)?;
let denial = match (denied_index, denied_expiry) {
(Some(input_index), Some(expires_at)) => {
let input_index = usize::try_from(input_index).map_err(|_| {
CheckError::StorageInvariant("batch preflight returned an invalid input index")
})?;
let check = checks.get(input_index).ok_or(CheckError::StorageInvariant(
"batch preflight returned an out-of-range input index",
))?;
Some((
input_index,
PendingDenial {
limit: check.policy().limit(),
retry_from_sample: duration_until(expires_at, database_now)?,
},
))
}
(None, None) => None,
_ => {
return Err(CheckError::StorageInvariant(
"batch preflight returned an incomplete denial",
)
.into());
}
};
Ok(BatchPreflight {
database_now,
response_now: preflight_response_now,
denial,
})
}
async fn upsert_batch(
transaction: &mut Transaction<'_, Postgres>,
input: &BatchSqlInput,
checks: &[Check<'_>],
database_now: DateTime<Utc>,
deadline: Instant,
) -> Result<(Vec<PendingAllowance>, DateTime<Utc>), CheckRunError> {
set_check_server_timeouts(transaction, deadline, "updating counter batch").await?;
let rows = check_before_commit(
deadline,
"updating counter batch",
sqlx::query(BATCH_UPSERT_SQL)
.bind(input.policy_ids.as_slice())
.bind(input.scope_ids.as_slice())
.bind(input.fingerprints.as_slice())
.bind(input.subjects.as_slice())
.bind(input.windows.as_slice())
.bind(input.costs.as_slice())
.bind(input.limits.as_slice())
.bind(database_now)
.fetch_all(&mut **transaction),
)
.await?;
if rows.is_empty() {
return Err(
CheckError::StorageInvariant("allowed batch update returned no decisions").into(),
);
}
let response_now: DateTime<Utc> = rows[0]
.try_get("response_now")
.map_err(CheckError::DefinitelyNotConsumed)?;
let mut allowances = Vec::with_capacity(checks.len());
for (output_position, row) in rows.iter().enumerate() {
let input_index: i64 = row
.try_get("input_index")
.map_err(CheckError::DefinitelyNotConsumed)?;
let input_index = usize::try_from(input_index).map_err(|_| {
CheckError::StorageInvariant("batch evaluation returned an invalid input index")
})?;
let Some(check) = checks.get(input_index) else {
return Err(CheckError::StorageInvariant(
"batch evaluation returned an out-of-range input index",
)
.into());
};
let expires_at = read_expiry(row)?;
let remaining_from_sample = duration_until(expires_at, database_now)?;
if input_index != output_position {
return Err(CheckError::StorageInvariant(
"allowed batch decisions were not returned in caller order",
)
.into());
}
let used = read_used(row)?;
let remaining =
check
.policy()
.limit()
.checked_sub(used)
.ok_or(CheckError::StorageInvariant(
"stored usage exceeds its policy limit",
))?;
allowances.push(PendingAllowance {
limit: check.policy().limit(),
remaining,
reset_from_sample: remaining_from_sample,
});
}
if allowances.len() != checks.len() {
return Err(CheckError::StorageInvariant(
"allowed batch returned an incomplete decision set",
)
.into());
}
Ok((allowances, response_now))
}
fn database_integer(value: u64) -> i64 {
i64::try_from(value)
.expect("core policy and check validation keep database integers within i64")
}
fn read_used(row: &PgRow) -> Result<u64, CheckError> {
let used: i64 = row
.try_get("used")
.map_err(CheckError::DefinitelyNotConsumed)?;
u64::try_from(used).map_err(|_| CheckError::StorageInvariant("stored usage is negative"))
}
fn read_expiry(row: &PgRow) -> Result<DateTime<Utc>, CheckError> {
row.try_get("window_expires_at")
.map_err(CheckError::DefinitelyNotConsumed)
}
fn duration_until(
expires_at: DateTime<Utc>,
database_now: DateTime<Utc>,
) -> Result<Duration, CheckError> {
expires_at
.signed_duration_since(database_now)
.to_std()
.map_err(|_| CheckError::StorageInvariant("stored window is already expired"))
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use super::*;
use runlimit_core::{
FixedWindowPolicy, MAX_LIMIT, MAX_WINDOW, MAX_WINDOW_MILLIS, PolicyId, ScopeId, SubjectKey,
};
fn policy(id: &str, scope: &str) -> FixedWindowPolicy {
FixedWindowPolicy::new(
PolicyId::new(id).expect("valid policy identifier"),
ScopeId::new(scope).expect("valid scope identifier"),
10,
Duration::from_secs(60),
)
.expect("valid policy")
}
fn key(byte: u8) -> SubjectKey {
SubjectKey::from_digest([byte; 32])
}
#[test]
fn portable_core_boundaries_fit_database_values_exactly() {
let policy = FixedWindowPolicy::new(
PolicyId::new("portable.maximum").unwrap(),
ScopeId::new("subject").unwrap(),
MAX_LIMIT,
MAX_WINDOW,
)
.unwrap();
let check = Check::with_cost(&policy, key(1), MAX_LIMIT).unwrap();
assert_eq!(database_integer(policy.limit()), i64::MAX);
assert_eq!(database_integer(check.cost()), i64::MAX);
let interval = PgInterval::try_from(policy.window()).unwrap();
assert_eq!(interval.months, 0);
assert_eq!(interval.days, 0);
assert_eq!(
interval.microseconds,
i64::try_from(MAX_WINDOW_MILLIS * 1_000).unwrap()
);
}
#[test]
fn row_lock_permutation_orders_complete_counter_keys() {
let alpha_client = policy("alpha", "client");
let alpha_identity = policy("alpha", "identity");
let beta_client = policy("beta", "client");
let checks = [
Check::new(&beta_client, key(0)),
Check::new(&alpha_identity, key(0)),
Check::new(&alpha_client, key(1)),
Check::new(&alpha_client, key(0)),
];
let input = BatchSqlInput::from_checks(&checks);
let locked_keys = input
.lock_input_positions
.iter()
.map(|position| checks[usize::try_from(*position - 1).unwrap()].counter_key())
.collect::<Vec<_>>();
assert!(locked_keys.windows(2).all(|pair| pair[0] < pair[1]));
}
#[test]
fn advisory_lock_protocol_has_a_golden_vector() {
let policy = policy("login", "client");
let check = Check::new(&policy, key(7));
assert_eq!(
advisory_lock_id(check.counter_key()),
4_549_358_434_381_602_087
);
}
#[test]
fn response_metadata_subtracts_only_authoritative_elapsed_time() {
let allowed = PendingAllowance {
limit: 10,
remaining: 4,
reset_from_sample: Duration::from_millis(250),
}
.finish(Duration::from_millis(80));
let denied = PendingDenial {
limit: 10,
retry_from_sample: Duration::from_millis(150),
}
.finish(Duration::from_millis(80));
assert_eq!(allowed.reset_after(), Some(Duration::from_millis(170)));
assert_eq!(denied.retry_after(), Some(Duration::from_millis(70)));
assert_eq!(denied.retry_after_seconds(), Some(1));
}
#[test]
fn configuration_bounds_batches_and_deadlines() {
let defaults = PostgresConfig::new();
assert_eq!(defaults.max_batch_size(), 32);
assert_eq!(defaults.operation_timeout(), Duration::from_secs(3));
assert_eq!(
defaults.with_max_batch_size(0),
Err(PostgresConfigError::ZeroBatchSize)
);
assert_eq!(
defaults.with_operation_timeout(Duration::ZERO),
Err(PostgresConfigError::ZeroOperationTimeout)
);
assert_eq!(
defaults.with_operation_timeout(Duration::from_secs(61)),
Err(PostgresConfigError::OperationTimeoutTooLong {
actual: Duration::from_secs(61),
maximum: Duration::from_secs(60),
})
);
}
#[tokio::test]
async fn duplicate_storage_keys_fail_before_connecting() {
let alpha = policy("alpha", "client");
let beta = policy("beta", "client");
let checks = [
Check::new(&beta, key(7)),
Check::new(&beta, key(7)),
Check::new(&alpha, key(8)),
Check::new(&alpha, key(8)),
];
let pool = sqlx::postgres::PgPoolOptions::new()
.connect_lazy("postgresql://runlimit:runlimit@127.0.0.1:1/runlimit")
.expect("syntactically valid database URL");
let limiter = PostgresLimiter::new(pool);
let error = limiter
.check_all(&checks)
.await
.expect_err("duplicate keys must be rejected");
assert!(matches!(
error,
CheckError::InvalidBatch(BatchError::DuplicateKey {
first_index: 0,
duplicate_index: 1,
})
));
}
#[tokio::test]
async fn oversized_batch_fails_before_connecting() {
let first_policy = policy("login", "client");
let second_policy = policy("signup", "client");
let checks = [
Check::new(&first_policy, key(1)),
Check::new(&second_policy, key(2)),
];
let pool = sqlx::postgres::PgPoolOptions::new()
.connect_lazy("postgresql://runlimit:runlimit@127.0.0.1:1/runlimit")
.expect("syntactically valid database URL");
let config = PostgresConfig::new()
.with_max_batch_size(1)
.expect("one is a valid batch limit");
let limiter = PostgresLimiter::with_config(pool, config);
let error = limiter
.check_all(&checks)
.await
.expect_err("oversized batch must be rejected");
assert!(matches!(
error,
CheckError::InvalidBatch(BatchError::BatchTooLarge {
actual: 2,
maximum: 1,
})
));
}
#[test]
fn error_reports_consumption_uncertainty() {
let definite = CheckError::DefinitelyNotConsumed(sqlx::Error::RowNotFound);
let unknown = CheckError::CommitOutcomeUnknown(sqlx::Error::RowNotFound);
let rolled_back_invariant = CheckError::ResponseInvariant;
let committed_invariant = CheckError::CommittedResponseInvariant;
let cleanup_definite = MaintenanceError::Database(sqlx::Error::RowNotFound);
let cleanup_unknown = MaintenanceError::CommitOutcomeUnknown(sqlx::Error::RowNotFound);
assert!(!definite.may_have_consumed_quota());
assert!(unknown.may_have_consumed_quota());
assert!(CheckError::CommitTimedOut.may_have_consumed_quota());
assert!(!rolled_back_invariant.may_have_consumed_quota());
assert!(committed_invariant.may_have_consumed_quota());
assert!(!cleanup_definite.may_have_removed_rows());
assert!(cleanup_unknown.may_have_removed_rows());
assert!(MaintenanceError::CommitTimedOut.may_have_removed_rows());
}
#[test]
fn malformed_single_responses_preserve_consumption_status() {
let denial = Denial::QuotaExceeded {
limit: 1,
retry_after: Duration::from_secs(1),
};
let committed = single_decision_from_batch(BatchDecision::Allowed(Vec::new()))
.expect_err("malformed allowed response follows a confirmed commit");
let rolled_back = single_decision_from_batch(BatchDecision::Denied { index: 1, denial })
.expect_err("malformed denied response follows a rollback");
assert!(matches!(committed, CheckError::CommittedResponseInvariant));
assert!(committed.may_have_consumed_quota());
assert!(matches!(rolled_back, CheckError::ResponseInvariant));
assert!(!rolled_back.may_have_consumed_quota());
}
#[test]
fn cleanup_query_is_bounded_and_nonblocking() {
assert_eq!(CLEANUP_SQL.matches("clock_timestamp()").count(), 1);
assert!(CLEANUP_SQL.contains("AS MATERIALIZED"));
assert!(CLEANUP_SQL.contains("SELECT sampled_at\n FROM authoritative_time"));
assert!(!CLEANUP_SQL.contains("window_expires_at <= clock_timestamp()"));
assert!(CLEANUP_SQL.contains("LIMIT $1"));
assert!(CLEANUP_SQL.contains("FOR UPDATE SKIP LOCKED"));
}
}