Skip to main content

runlimit_postgres/
lib.rs

1//! Replica-safe `PostgreSQL` storage for Runlimit fixed-window policies.
2//!
3//! [`PostgresLimiter`] implements the same anchored fixed-window model as the
4//! in-memory backend: the first admitted check starts a window, and subsequent
5//! checks share that window until it expires. `PostgreSQL`'s clock is the sole
6//! time authority, so callers on different hosts do not need synchronized
7//! clocks. Atomic batches use set-based lock, preflight, and update phases, so
8//! their number of SQL phases does not grow with the batch size.
9//!
10//! # Installation
11//!
12//! Call [`PostgresLimiter::migrate`] before serving traffic. It can share
13//! `SQLx`'s `_sqlx_migrations` table with application migrations only when every
14//! participating [`Migrator`] configures
15//! [`Migrator::set_ignore_missing`] to `true`.
16//! A strict application migrator will reject Runlimit's migration version as
17//! unknown. The migration creates `runlimit_fixed_windows` in the connection's
18//! default schema.
19//!
20//! [`MIGRATOR`] is the raw bundled migrator with `SQLx`'s strict defaults. It is
21//! intended for an exclusively managed migration history and connection. It
22//! does not add [`PostgresLimiter::migrate`]'s protection against returning a
23//! possibly session-locked connection to a pool after an error or cancellation.
24//! Applications that retain a strict host migrator should vendor the bundled
25//! Runlimit SQL as an application-owned migration instead of invoking either
26//! Runlimit migrator against the shared history.
27//!
28//! # Failure semantics
29//!
30//! Pre-commit database failures and pre-commit timeouts mean the transaction
31//! did not commit. A
32//! [`CheckError::CommitOutcomeUnknown`] or [`CheckError::CommitTimedOut`] error
33//! means `PostgreSQL` may have committed even though the client did not receive
34//! confirmation. [`CheckError::CommittedResponseInvariant`] means the commit
35//! was confirmed but its response was unusable. Fail closed in all three cases,
36//! and do not blindly retry a check that may already have consumed quota.
37//! Dropping a check or cleanup future prevents the backend from reporting its
38//! outcome; if cancellation races with commit, callers must likewise treat the
39//! outcome as unknown. Prefer this backend's configured deadline to a shorter
40//! outer timeout.
41//!
42//! The advisory-lock `v1` domain and derivation are a cross-replica protocol.
43//! They must not be changed in place: replicas from a rolling deployment must
44//! calculate identical lock IDs for the same persistent counter. `PostgreSQL`'s
45//! advisory-lock namespace is shared by every application in the same database.
46//! An unrelated or deliberately held matching `bigint` lock therefore blocks
47//! the affected Runlimit key until the operation deadline; use a trusted
48//! database boundary and restrict advisory-lock privileges for unrelated roles.
49//!
50//! # Integration tests
51//!
52//! The database tests are opt-in:
53//!
54//! ```text
55//! RUNLIMIT_POSTGRES_TEST_DATABASE_URL=postgresql:///runlimit_test \
56//! cargo test -p runlimit-postgres --test postgres -- --ignored
57//! ```
58
59use std::{future::Future, time::Duration};
60
61use runlimit_core::{
62    BatchDecision, BatchError, Check, CounterKey, Decision, Denial, validate_batch,
63};
64use sha2::{Digest, Sha256};
65use sqlx::{
66    Acquire, PgPool, Postgres, Row, Transaction,
67    migrate::{MigrateError, Migrator},
68    pool::PoolConnection,
69    postgres::{PgRow, types::PgInterval},
70    types::chrono::{DateTime, Utc},
71};
72use thiserror::Error;
73use tokio::time::{Instant, timeout_at};
74
75/// Raw bundled migrations required by [`PostgresLimiter`].
76///
77/// This value retains `SQLx`'s strict missing-version behavior and is suitable
78/// only for an exclusively managed migration history and connection. For a
79/// pooled application that shares `_sqlx_migrations`, prefer
80/// [`PostgresLimiter::migrate`] and configure every other participating
81/// [`Migrator`] to set [`Migrator::set_ignore_missing`] to `true`.
82pub static MIGRATOR: Migrator = sqlx::migrate!("./migrations");
83
84const ADVISORY_LOCK_DOMAIN: &[u8] = b"runlimit/postgres-advisory-lock/v1\0";
85const SERVER_TIMEOUT_GRACE: Duration = Duration::from_millis(25);
86
87const SET_LOCAL_TIMEOUTS_SQL: &str = r"
88SELECT
89    set_config('statement_timeout', $1, true),
90    set_config('lock_timeout', $2, true)
91";
92
93const BATCH_ADVISORY_LOCK_SQL: &str = r"
94WITH RECURSIVE acquired(position, locked) AS (
95    SELECT 1, pg_advisory_xact_lock($1[1])
96    WHERE cardinality($1::BIGINT[]) > 0
97
98    UNION ALL
99
100    SELECT
101        acquired.position + 1,
102        pg_advisory_xact_lock($1[acquired.position + 1])
103    FROM acquired
104    WHERE acquired.position < cardinality($1::BIGINT[])
105)
106SELECT count(*)
107FROM acquired
108";
109
110const BATCH_ROW_LOCK_SQL: &str = r"
111WITH input_keys AS (
112    SELECT *
113    FROM unnest(
114        $1::BYTEA[],
115        $2::BYTEA[]
116    ) WITH ORDINALITY AS keys(
117        config_fingerprint,
118        subject_key,
119        input_position
120    )
121),
122lock_order AS (
123    SELECT *
124    FROM unnest($3::BIGINT[])
125        WITH ORDINALITY AS positions(input_position, lock_position)
126),
127ordered_keys AS (
128    SELECT
129        input_keys.config_fingerprint,
130        input_keys.subject_key,
131        lock_order.lock_position
132    FROM input_keys
133    INNER JOIN lock_order
134        ON lock_order.input_position = input_keys.input_position
135    ORDER BY lock_order.lock_position
136)
137SELECT 1
138FROM ordered_keys
139INNER JOIN runlimit_fixed_windows AS windows
140    ON windows.config_fingerprint = ordered_keys.config_fingerprint
141    AND windows.subject_key = ordered_keys.subject_key
142ORDER BY ordered_keys.lock_position
143FOR UPDATE OF windows
144";
145
146const BATCH_PREFLIGHT_SQL: &str = r"
147WITH input AS (
148    SELECT *
149    FROM unnest(
150        $1::BYTEA[],
151        $2::BYTEA[],
152        $3::BIGINT[],
153        $4::BIGINT[]
154    ) WITH ORDINALITY AS checks(
155        config_fingerprint,
156        subject_key,
157        cost,
158        quota_limit,
159        input_position
160    )
161),
162sample AS (
163    SELECT clock_timestamp() AS database_now
164),
165first_denial AS (
166    SELECT
167        input.input_position - 1 AS input_index,
168        windows.window_expires_at
169    FROM input
170    CROSS JOIN sample
171    INNER JOIN runlimit_fixed_windows AS windows
172        ON windows.config_fingerprint = input.config_fingerprint
173        AND windows.subject_key = input.subject_key
174    WHERE
175        windows.window_expires_at > sample.database_now
176        AND windows.used > input.quota_limit - input.cost
177    ORDER BY input.input_position
178    LIMIT 1
179)
180SELECT
181    sample.database_now,
182    clock_timestamp() AS response_now,
183    first_denial.input_index,
184    first_denial.window_expires_at
185FROM sample
186LEFT JOIN first_denial ON TRUE
187";
188
189const BATCH_UPSERT_SQL: &str = r"
190WITH input AS (
191    SELECT *
192    FROM unnest(
193        $1::TEXT[],
194        $2::TEXT[],
195        $3::BYTEA[],
196        $4::BYTEA[],
197        $5::INTERVAL[],
198        $6::BIGINT[],
199        $7::BIGINT[]
200    ) WITH ORDINALITY AS checks(
201        policy_id,
202        scope_id,
203        config_fingerprint,
204        subject_key,
205        window_size,
206        cost,
207        quota_limit,
208        input_position
209    )
210),
211upserted AS (
212    INSERT INTO runlimit_fixed_windows (
213        policy_id,
214        scope_id,
215        config_fingerprint,
216        subject_key,
217        window_started_at,
218        window_expires_at,
219        used
220    )
221    SELECT
222        policy_id,
223        scope_id,
224        config_fingerprint,
225        subject_key,
226        $8,
227        $8 + window_size,
228        cost
229    FROM input
230    ON CONFLICT (config_fingerprint, subject_key)
231    DO UPDATE SET
232        policy_id = EXCLUDED.policy_id,
233        scope_id = EXCLUDED.scope_id,
234        window_started_at = CASE
235            WHEN runlimit_fixed_windows.window_expires_at <= $8 THEN $8
236            ELSE runlimit_fixed_windows.window_started_at
237        END,
238        window_expires_at = CASE
239            WHEN runlimit_fixed_windows.window_expires_at <= $8
240                THEN $8 + (EXCLUDED.window_expires_at - EXCLUDED.window_started_at)
241            ELSE runlimit_fixed_windows.window_expires_at
242        END,
243        used = CASE
244            WHEN runlimit_fixed_windows.window_expires_at <= $8 THEN EXCLUDED.used
245            ELSE runlimit_fixed_windows.used + EXCLUDED.used
246        END
247    WHERE
248        runlimit_fixed_windows.window_expires_at <= $8
249        OR runlimit_fixed_windows.used
250            <= (
251                SELECT quota_limit - cost
252                FROM input
253                WHERE
254                    input.config_fingerprint = EXCLUDED.config_fingerprint
255                    AND input.subject_key = EXCLUDED.subject_key
256            )
257    RETURNING
258        config_fingerprint,
259        subject_key,
260        used,
261        window_expires_at
262),
263response AS (
264    SELECT clock_timestamp() AS response_now
265    FROM upserted
266    HAVING count(*) >= 0
267)
268SELECT
269    input.input_position - 1 AS input_index,
270    upserted.used,
271    upserted.window_expires_at,
272    response.response_now
273FROM input
274INNER JOIN upserted
275    ON upserted.config_fingerprint = input.config_fingerprint
276    AND upserted.subject_key = input.subject_key
277CROSS JOIN response
278ORDER BY input.input_position
279";
280
281const CLEANUP_SQL: &str = include_str!("cleanup_expired.sql");
282
283/// Runtime bounds for `PostgreSQL` admission work.
284#[derive(Clone, Copy, Debug, Eq, PartialEq)]
285pub struct PostgresConfig {
286    max_batch_size: usize,
287    operation_timeout: Duration,
288}
289
290impl PostgresConfig {
291    /// Default maximum number of checks in one atomic batch.
292    pub const DEFAULT_MAX_BATCH_SIZE: usize = 32;
293
294    /// Default total deadline for one check or cleanup operation.
295    pub const DEFAULT_OPERATION_TIMEOUT: Duration = Duration::from_secs(3);
296
297    /// Largest accepted online-operation deadline.
298    pub const MAXIMUM_OPERATION_TIMEOUT: Duration = Duration::from_secs(60);
299
300    /// Returns the conservative defaults suitable for an online admission
301    /// path.
302    pub const fn new() -> Self {
303        Self {
304            max_batch_size: Self::DEFAULT_MAX_BATCH_SIZE,
305            operation_timeout: Self::DEFAULT_OPERATION_TIMEOUT,
306        }
307    }
308
309    /// Sets the maximum number of checks that may retain locks in one batch.
310    ///
311    /// # Errors
312    ///
313    /// Returns an error if `maximum` is zero.
314    pub fn with_max_batch_size(mut self, maximum: usize) -> Result<Self, PostgresConfigError> {
315        if maximum == 0 {
316            return Err(PostgresConfigError::ZeroBatchSize);
317        }
318        self.max_batch_size = maximum;
319        Ok(self)
320    }
321
322    /// Sets the total deadline for a check, batch, or cleanup statement.
323    ///
324    /// The deadline covers pool acquisition, lock waits, statements, rollback,
325    /// and commit. It is intentionally not applied to schema migrations.
326    ///
327    /// # Errors
328    ///
329    /// Returns an error if `timeout` is zero or exceeds 60 seconds.
330    pub fn with_operation_timeout(
331        mut self,
332        timeout: Duration,
333    ) -> Result<Self, PostgresConfigError> {
334        if timeout.is_zero() {
335            return Err(PostgresConfigError::ZeroOperationTimeout);
336        }
337        if timeout > Self::MAXIMUM_OPERATION_TIMEOUT {
338            return Err(PostgresConfigError::OperationTimeoutTooLong {
339                actual: timeout,
340                maximum: Self::MAXIMUM_OPERATION_TIMEOUT,
341            });
342        }
343        self.operation_timeout = timeout;
344        Ok(self)
345    }
346
347    /// Returns the maximum accepted batch length.
348    pub const fn max_batch_size(&self) -> usize {
349        self.max_batch_size
350    }
351
352    /// Returns the total deadline applied to an operation.
353    pub const fn operation_timeout(self) -> Duration {
354        self.operation_timeout
355    }
356}
357
358impl Default for PostgresConfig {
359    fn default() -> Self {
360        Self::new()
361    }
362}
363
364/// Invalid `PostgreSQL` backend configuration.
365#[derive(Clone, Copy, Debug, Error, Eq, PartialEq)]
366pub enum PostgresConfigError {
367    /// A zero maximum batch size would reject every nonempty batch.
368    #[error("max_batch_size must be greater than zero")]
369    ZeroBatchSize,
370    /// A zero deadline cannot permit database work.
371    #[error("PostgreSQL rate-limit operation timeout must be greater than zero")]
372    ZeroOperationTimeout,
373    /// An admission deadline longer than one minute is not operationally
374    /// bounded enough for this backend.
375    #[error("PostgreSQL rate-limit operation timeout {actual:?} exceeds maximum {maximum:?}")]
376    OperationTimeoutTooLong {
377        /// Supplied timeout.
378        actual: Duration,
379        /// Largest accepted timeout.
380        maximum: Duration,
381    },
382}
383
384/// A replica-safe rate limiter backed by a `PostgreSQL` pool.
385///
386/// Cloning this value is cheap because [`PgPool`] is internally reference
387/// counted.
388#[derive(Clone, Debug)]
389pub struct PostgresLimiter {
390    pool: PgPool,
391    config: PostgresConfig,
392}
393
394impl PostgresLimiter {
395    /// Creates a limiter using `pool`.
396    ///
397    /// Call [`Self::migrate`] during application startup before admitting
398    /// requests.
399    pub const fn new(pool: PgPool) -> Self {
400        Self {
401            pool,
402            config: PostgresConfig::new(),
403        }
404    }
405
406    /// Creates a limiter using an explicit runtime configuration.
407    pub const fn with_config(pool: PgPool, config: PostgresConfig) -> Self {
408        Self { pool, config }
409    }
410
411    /// Returns the underlying connection pool.
412    pub const fn pool(&self) -> &PgPool {
413        &self.pool
414    }
415
416    /// Returns the backend runtime configuration.
417    pub const fn config(&self) -> PostgresConfig {
418        self.config
419    }
420
421    /// Applies the bundled Runlimit database migrations.
422    ///
423    /// Unrelated successful versions in `_sqlx_migrations` are ignored, which
424    /// permits a shared migration history only when every other participating
425    /// `SQLx` migrator also sets [`Migrator::set_ignore_missing`] to `true`.
426    /// Known Runlimit versions still receive `SQLx`'s checksum validation.
427    /// Applications that keep a strict host migrator should instead vendor the
428    /// bundled SQL into their application-owned migration stream.
429    ///
430    /// Migration locking is session scoped. An acquired connection is returned
431    /// to the pool only after a successful run and unlock; an error or
432    /// cancellation closes it instead.
433    ///
434    /// # Errors
435    ///
436    /// Returns an error if migration metadata cannot be read or `PostgreSQL`
437    /// rejects a migration.
438    pub async fn migrate(&self) -> Result<(), MigrateError> {
439        let connection = self.pool.acquire().await?;
440        let mut guarded_connection = ConnectionCancellationGuard::new(connection);
441        let mut migrator = sqlx::migrate!("./migrations");
442        migrator.set_ignore_missing(true);
443
444        let result = migrator
445            .run_direct(&mut **guarded_connection.connection())
446            .await;
447        if result.is_ok() {
448            guarded_connection.disarm();
449        }
450        result
451    }
452
453    /// Checks and, when allowed, consumes one fixed-window quota.
454    ///
455    /// A denied decision does not change the stored counter.
456    ///
457    /// # Errors
458    ///
459    /// Returns [`CheckError::DefinitelyNotConsumed`] for failures before a
460    /// successful commit and [`CheckError::CommitOutcomeUnknown`] when commit
461    /// confirmation is unavailable. An internal single-response invariant
462    /// failure reports whether the batch was rolled back or committed.
463    pub async fn check(&self, check: &Check<'_>) -> Result<Decision, CheckError> {
464        single_decision_from_batch(self.check_all(std::slice::from_ref(check)).await?)
465    }
466
467    /// Atomically checks and consumes every quota in `checks`.
468    ///
469    /// Rows are acquired in a deterministic storage-key order to avoid
470    /// deadlocks between overlapping batches. Returned allowed decisions retain
471    /// the caller's input order. Repeated occurrences of the same storage key
472    /// are rejected before a connection is acquired because their independent
473    /// decision metadata would be ambiguous.
474    ///
475    /// If any check is denied, the entire transaction is rolled back and the
476    /// returned denial index is the earliest denied item in caller order.
477    /// Passing an empty slice succeeds without acquiring a database connection.
478    ///
479    /// # Errors
480    ///
481    /// Database failures before commit are
482    /// [`CheckError::DefinitelyNotConsumed`]. Any commit error is
483    /// conservatively classified as [`CheckError::CommitOutcomeUnknown`].
484    pub async fn check_all(&self, checks: &[Check<'_>]) -> Result<BatchDecision, CheckError> {
485        validate_batch(checks, self.config.max_batch_size())?;
486        if checks.is_empty() {
487            return Ok(BatchDecision::Allowed(Vec::new()));
488        }
489
490        // Materialize every SQL array and both lock orders before reserving a
491        // pool slot. Every database phase then consumes the exact same key
492        // arrays, preventing identity drift between locking and mutation.
493        let input = BatchSqlInput::from_checks(checks);
494        let deadline = Instant::now() + self.config.operation_timeout();
495        let connection = acquire_check_connection(&self.pool, deadline).await?;
496        let mut guarded_connection = ConnectionCancellationGuard::new(connection);
497        let result =
498            run_check_transaction(guarded_connection.connection(), &input, checks, deadline).await;
499        if !result
500            .as_ref()
501            .is_err_and(CheckRunError::client_timeout_won)
502        {
503            guarded_connection.disarm();
504        }
505        result.map_err(CheckRunError::into_public)
506    }
507
508    /// Deletes at most `maximum_rows` expired windows.
509    ///
510    /// Cleanup uses `FOR UPDATE SKIP LOCKED`, so it does not wait behind active
511    /// checks and cannot remove a row concurrently being renewed. A zero limit
512    /// performs no database work.
513    ///
514    /// # Errors
515    ///
516    /// Returns an error if `PostgreSQL` rejects the maintenance transaction or
517    /// it exceeds [`PostgresConfig::operation_timeout`]. A commit error or
518    /// commit timeout means rows may have been removed; inspect
519    /// [`MaintenanceError::may_have_removed_rows`]. Cleanup only targets
520    /// already-expired rows and never consumes quota.
521    pub async fn cleanup_expired(&self, maximum_rows: u32) -> Result<u64, MaintenanceError> {
522        if maximum_rows == 0 {
523            return Ok(0);
524        }
525
526        let deadline = Instant::now() + self.config.operation_timeout();
527        let connection = acquire_maintenance_connection(&self.pool, deadline).await?;
528        let mut guarded_connection = ConnectionCancellationGuard::new(connection);
529        let result =
530            run_cleanup_transaction(guarded_connection.connection(), maximum_rows, deadline).await;
531        if !result
532            .as_ref()
533            .is_err_and(MaintenanceRunError::client_timeout_won)
534        {
535            guarded_connection.disarm();
536        }
537        result.map_err(MaintenanceRunError::into_public)
538    }
539}
540
541/// Failure from a quota check.
542#[derive(Debug, Error)]
543pub enum CheckError {
544    /// The atomic batch violated a backend-independent structural requirement.
545    #[error(transparent)]
546    InvalidBatch(#[from] BatchError),
547
548    /// `PostgreSQL` did not commit the transaction, so no quota was consumed.
549    #[error("PostgreSQL rate-limit check failed before commit; quota was not consumed")]
550    DefinitelyNotConsumed(#[source] sqlx::Error),
551
552    /// Commit confirmation was lost, so quota may or may not have been consumed.
553    #[error("PostgreSQL rate-limit commit outcome is unknown; quota may have been consumed")]
554    CommitOutcomeUnknown(#[source] sqlx::Error),
555
556    /// The total deadline elapsed before commit started.
557    #[error("PostgreSQL rate-limit check timed out while {operation}; quota was not consumed")]
558    TimedOutBeforeCommit {
559        /// Database phase that exhausted the deadline.
560        operation: &'static str,
561    },
562
563    /// The total deadline elapsed after commit started.
564    #[error("PostgreSQL rate-limit commit timed out; quota may have been consumed")]
565    CommitTimedOut,
566
567    /// Persisted state violated an invariant guaranteed by the migration.
568    #[error("PostgreSQL rate-limit storage invariant failed: {0}")]
569    StorageInvariant(&'static str),
570
571    /// A rolled-back single check produced malformed response metadata.
572    #[error(
573        "PostgreSQL rate-limit single-check response invariant failed after rollback; quota was not consumed"
574    )]
575    ResponseInvariant,
576
577    /// A committed single check produced malformed response metadata.
578    #[error(
579        "PostgreSQL rate-limit single-check response invariant failed after commit; quota was consumed"
580    )]
581    CommittedResponseInvariant,
582}
583
584impl CheckError {
585    /// Reports whether the failed operation may have consumed quota.
586    pub const fn may_have_consumed_quota(&self) -> bool {
587        matches!(
588            self,
589            Self::CommitOutcomeUnknown(_) | Self::CommitTimedOut | Self::CommittedResponseInvariant
590        )
591    }
592}
593
594/// Failure from bounded expired-window cleanup.
595#[derive(Debug, Error)]
596pub enum MaintenanceError {
597    /// `PostgreSQL` did not commit the cleanup transaction, so no rows were
598    /// removed.
599    #[error("PostgreSQL expired-window cleanup failed before commit; no rows were removed")]
600    Database(#[source] sqlx::Error),
601    /// The total deadline elapsed before cleanup commit started.
602    #[error("PostgreSQL expired-window cleanup timed out while {operation}; no rows were removed")]
603    TimedOutBeforeCommit {
604        /// Database phase that exhausted the deadline.
605        operation: &'static str,
606    },
607    /// Commit confirmation was lost, so rows may or may not have been removed.
608    #[error(
609        "PostgreSQL expired-window cleanup commit outcome is unknown; rows may have been removed"
610    )]
611    CommitOutcomeUnknown(#[source] sqlx::Error),
612    /// The total deadline elapsed after cleanup commit started.
613    #[error("PostgreSQL expired-window cleanup commit timed out; rows may have been removed")]
614    CommitTimedOut,
615}
616
617impl MaintenanceError {
618    /// Reports whether the failed cleanup may have committed row removals.
619    pub const fn may_have_removed_rows(&self) -> bool {
620        matches!(self, Self::CommitOutcomeUnknown(_) | Self::CommitTimedOut)
621    }
622}
623
624fn single_decision_from_batch(batch: BatchDecision) -> Result<Decision, CheckError> {
625    batch
626        .try_into_single_decision()
627        .map_err(|invalid| match invalid {
628            BatchDecision::Allowed(_) => CheckError::CommittedResponseInvariant,
629            BatchDecision::Denied { .. } => CheckError::ResponseInvariant,
630        })
631}
632
633struct ConnectionCancellationGuard {
634    connection: PoolConnection<Postgres>,
635    armed: bool,
636}
637
638impl ConnectionCancellationGuard {
639    const fn new(connection: PoolConnection<Postgres>) -> Self {
640        Self {
641            connection,
642            armed: true,
643        }
644    }
645
646    const fn connection(&mut self) -> &mut PoolConnection<Postgres> {
647        &mut self.connection
648    }
649
650    const fn disarm(&mut self) {
651        self.armed = false;
652    }
653}
654
655impl Drop for ConnectionCancellationGuard {
656    fn drop(&mut self) {
657        if self.armed {
658            // Dropping an SQLx query future does not send a PostgreSQL cancel
659            // request. Close the socket instead of returning a potentially
660            // busy connection to the pool when either Runlimit's deadline or
661            // cancellation of the outer limiter future wins the race.
662            self.connection.close_on_drop();
663        }
664    }
665}
666
667#[derive(Debug)]
668enum CheckRunError {
669    Public(CheckError),
670    ClientTimedOutBeforeCommit { operation: &'static str },
671    ClientCommitTimedOut,
672}
673
674impl CheckRunError {
675    const fn client_timeout_won(&self) -> bool {
676        matches!(
677            self,
678            Self::ClientTimedOutBeforeCommit { .. } | Self::ClientCommitTimedOut
679        )
680    }
681
682    fn into_public(self) -> CheckError {
683        match self {
684            Self::Public(error) => error,
685            Self::ClientTimedOutBeforeCommit { operation } => {
686                CheckError::TimedOutBeforeCommit { operation }
687            }
688            Self::ClientCommitTimedOut => CheckError::CommitTimedOut,
689        }
690    }
691}
692
693impl From<CheckError> for CheckRunError {
694    fn from(error: CheckError) -> Self {
695        Self::Public(error)
696    }
697}
698
699#[derive(Debug)]
700enum MaintenanceRunError {
701    Public(MaintenanceError),
702    ClientTimedOutBeforeCommit { operation: &'static str },
703    ClientCommitTimedOut,
704}
705
706impl MaintenanceRunError {
707    const fn client_timeout_won(&self) -> bool {
708        matches!(
709            self,
710            Self::ClientTimedOutBeforeCommit { .. } | Self::ClientCommitTimedOut
711        )
712    }
713
714    fn into_public(self) -> MaintenanceError {
715        match self {
716            Self::Public(error) => error,
717            Self::ClientTimedOutBeforeCommit { operation } => {
718                MaintenanceError::TimedOutBeforeCommit { operation }
719            }
720            Self::ClientCommitTimedOut => MaintenanceError::CommitTimedOut,
721        }
722    }
723}
724
725#[derive(Debug)]
726enum PendingBatchOutcome {
727    Allowed(Vec<PendingAllowance>),
728    Denied { index: usize, denial: PendingDenial },
729}
730
731#[derive(Debug)]
732struct BatchSqlInput {
733    policy_ids: Vec<String>,
734    scope_ids: Vec<String>,
735    fingerprints: Vec<Vec<u8>>,
736    subjects: Vec<Vec<u8>>,
737    lock_input_positions: Vec<i64>,
738    advisory_lock_ids: Vec<i64>,
739    windows: Vec<PgInterval>,
740    costs: Vec<i64>,
741    limits: Vec<i64>,
742}
743
744impl BatchSqlInput {
745    fn from_checks(checks: &[Check<'_>]) -> Self {
746        let counter_keys = checks.iter().map(Check::counter_key).collect::<Vec<_>>();
747        let mut ordered_indices = (0..checks.len()).collect::<Vec<_>>();
748        ordered_indices.sort_unstable_by_key(|index| counter_keys[*index]);
749
750        let mut input = Self {
751            policy_ids: Vec::with_capacity(checks.len()),
752            scope_ids: Vec::with_capacity(checks.len()),
753            fingerprints: Vec::with_capacity(checks.len()),
754            subjects: Vec::with_capacity(checks.len()),
755            lock_input_positions: ordered_indices
756                .into_iter()
757                .map(|index| {
758                    i64::try_from(index + 1)
759                        .expect("bounded batch input positions fit PostgreSQL BIGINT")
760                })
761                .collect(),
762            advisory_lock_ids: counter_keys.iter().copied().map(advisory_lock_id).collect(),
763            windows: Vec::with_capacity(checks.len()),
764            costs: Vec::with_capacity(checks.len()),
765            limits: Vec::with_capacity(checks.len()),
766        };
767        input.advisory_lock_ids.sort_unstable();
768        input.advisory_lock_ids.dedup();
769
770        for (check, counter_key) in checks.iter().zip(counter_keys) {
771            let policy = check.policy();
772            input.policy_ids.push(policy.id().as_str().to_owned());
773            input.scope_ids.push(policy.scope().as_str().to_owned());
774            input
775                .fingerprints
776                .push(counter_key.fingerprint().as_bytes().to_vec());
777            input
778                .subjects
779                .push(counter_key.subject().as_bytes().to_vec());
780            input.windows.push(
781                PgInterval::try_from(policy.window())
782                    .expect("core policy windows fit PostgreSQL INTERVAL exactly"),
783            );
784            input.costs.push(database_integer(check.cost()));
785            input.limits.push(database_integer(policy.limit()));
786        }
787        input
788    }
789}
790
791#[derive(Debug)]
792struct BatchPreflight {
793    database_now: DateTime<Utc>,
794    response_now: DateTime<Utc>,
795    denial: Option<(usize, PendingDenial)>,
796}
797
798#[derive(Debug)]
799struct PendingAllowance {
800    limit: u64,
801    remaining: u64,
802    reset_from_sample: Duration,
803}
804
805impl PendingAllowance {
806    fn finish(self, authoritative_elapsed: Duration) -> Decision {
807        Decision::allowed(
808            self.limit,
809            self.remaining,
810            self.reset_from_sample.saturating_sub(authoritative_elapsed),
811        )
812    }
813}
814
815#[derive(Debug)]
816struct PendingDenial {
817    limit: u64,
818    retry_from_sample: Duration,
819}
820
821impl PendingDenial {
822    fn finish(self, authoritative_elapsed: Duration) -> Denial {
823        Denial::QuotaExceeded {
824            limit: self.limit,
825            retry_after: self.retry_from_sample.saturating_sub(authoritative_elapsed),
826        }
827    }
828}
829
830async fn acquire_check_connection(
831    pool: &PgPool,
832    deadline: Instant,
833) -> Result<PoolConnection<Postgres>, CheckError> {
834    timeout_at(deadline, pool.acquire())
835        .await
836        .map_err(|_| CheckError::TimedOutBeforeCommit {
837            operation: "acquiring database connection",
838        })?
839        .map_err(CheckError::DefinitelyNotConsumed)
840}
841
842async fn run_check_transaction(
843    connection: &mut PoolConnection<Postgres>,
844    input: &BatchSqlInput,
845    checks: &[Check<'_>],
846    deadline: Instant,
847) -> Result<BatchDecision, CheckRunError> {
848    let mut transaction =
849        check_before_commit(deadline, "beginning transaction", connection.begin()).await?;
850
851    // Advisory locks cover logical keys that do not have rows yet. Their
852    // stable numeric IDs are sorted independently from exact storage keys so
853    // deliberately colliding batches cannot acquire them in opposite orders.
854    acquire_advisory_locks(&mut transaction, &input.advisory_lock_ids, deadline).await?;
855
856    // Existing rows may be held by cleanup or a transaction predating the
857    // advisory-lock protocol. Wait for all row locks before sampling database
858    // time, otherwise a contended request could evaluate against stale time.
859    acquire_existing_row_locks(&mut transaction, input, deadline).await?;
860
861    let (pending, authoritative_elapsed) =
862        execute_batch(&mut transaction, input, checks, deadline).await?;
863
864    match pending {
865        PendingBatchOutcome::Denied { index, denial } => {
866            set_check_server_timeouts(&mut transaction, deadline, "rolling back denied batch")
867                .await?;
868            check_before_commit(
869                deadline,
870                "rolling back denied batch",
871                transaction.rollback(),
872            )
873            .await?;
874            Ok(BatchDecision::Denied {
875                index,
876                denial: denial.finish(authoritative_elapsed),
877            })
878        }
879        PendingBatchOutcome::Allowed(allowances) => {
880            set_check_server_timeouts(&mut transaction, deadline, "committing counter batch")
881                .await?;
882            commit_check(deadline, transaction).await?;
883            Ok(BatchDecision::Allowed(
884                allowances
885                    .into_iter()
886                    .map(|allowance| allowance.finish(authoritative_elapsed))
887                    .collect(),
888            ))
889        }
890    }
891}
892
893async fn check_before_commit<T, F>(
894    deadline: Instant,
895    operation: &'static str,
896    future: F,
897) -> Result<T, CheckRunError>
898where
899    F: Future<Output = Result<T, sqlx::Error>>,
900{
901    timeout_at(deadline, future)
902        .await
903        .map_err(|_| CheckRunError::ClientTimedOutBeforeCommit { operation })?
904        .map_err(|error| CheckRunError::Public(map_check_database_error(error, operation)))
905}
906
907async fn commit_check(
908    deadline: Instant,
909    transaction: Transaction<'_, Postgres>,
910) -> Result<(), CheckRunError> {
911    if Instant::now() >= deadline {
912        return Err(CheckRunError::Public(CheckError::TimedOutBeforeCommit {
913            operation: "starting commit",
914        }));
915    }
916
917    timeout_at(deadline, transaction.commit())
918        .await
919        .map_err(|_| CheckRunError::ClientCommitTimedOut)?
920        .map_err(|error| CheckRunError::Public(CheckError::CommitOutcomeUnknown(error)))
921}
922
923fn map_check_database_error(error: sqlx::Error, operation: &'static str) -> CheckError {
924    if is_server_timeout(&error) {
925        CheckError::TimedOutBeforeCommit { operation }
926    } else {
927        CheckError::DefinitelyNotConsumed(error)
928    }
929}
930
931fn is_server_timeout(error: &sqlx::Error) -> bool {
932    let sqlx::Error::Database(database_error) = error else {
933        return false;
934    };
935    matches!(database_error.code().as_deref(), Some("57014" | "55P03"))
936}
937
938fn remaining_server_timeout_settings(deadline: Instant) -> Option<(String, String)> {
939    let remaining = deadline.checked_duration_since(Instant::now())?;
940    let remaining_millis = remaining.as_millis();
941    let statement_millis = remaining_millis
942        .saturating_sub(SERVER_TIMEOUT_GRACE.as_millis())
943        .max(1);
944    let lock_millis = statement_millis.saturating_sub(1).max(1);
945    Some((format!("{statement_millis}ms"), format!("{lock_millis}ms")))
946}
947
948async fn set_check_server_timeouts(
949    transaction: &mut Transaction<'_, Postgres>,
950    deadline: Instant,
951    operation: &'static str,
952) -> Result<(), CheckRunError> {
953    let (statement_timeout, lock_timeout) = remaining_server_timeout_settings(deadline).ok_or(
954        CheckRunError::Public(CheckError::TimedOutBeforeCommit { operation }),
955    )?;
956    check_before_commit(
957        deadline,
958        operation,
959        sqlx::query(SET_LOCAL_TIMEOUTS_SQL)
960            .bind(statement_timeout)
961            .bind(lock_timeout)
962            .execute(&mut **transaction),
963    )
964    .await
965    .map(|_| ())
966}
967
968async fn acquire_advisory_locks(
969    transaction: &mut Transaction<'_, Postgres>,
970    advisory_lock_ids: &[i64],
971    deadline: Instant,
972) -> Result<(), CheckRunError> {
973    set_check_server_timeouts(transaction, deadline, "acquiring logical key lock").await?;
974    check_before_commit(deadline, "acquiring logical key lock", async {
975        sqlx::query(BATCH_ADVISORY_LOCK_SQL)
976            .bind(advisory_lock_ids)
977            .execute(&mut **transaction)
978            .await
979            .map(|_| ())
980    })
981    .await
982}
983
984fn advisory_lock_id(counter_key: CounterKey) -> i64 {
985    let mut digest = Sha256::new();
986    digest.update(ADVISORY_LOCK_DOMAIN);
987    digest.update(counter_key.to_bytes());
988    let digest = digest.finalize();
989    let mut id_bytes = [0_u8; 8];
990    let id_length = id_bytes.len();
991    id_bytes.copy_from_slice(&digest[..id_length]);
992    i64::from_be_bytes(id_bytes)
993}
994
995async fn acquire_existing_row_locks(
996    transaction: &mut Transaction<'_, Postgres>,
997    input: &BatchSqlInput,
998    deadline: Instant,
999) -> Result<(), CheckRunError> {
1000    set_check_server_timeouts(transaction, deadline, "acquiring counter row lock").await?;
1001    check_before_commit(deadline, "acquiring counter row lock", async {
1002        sqlx::query(BATCH_ROW_LOCK_SQL)
1003            .bind(input.fingerprints.as_slice())
1004            .bind(input.subjects.as_slice())
1005            .bind(input.lock_input_positions.as_slice())
1006            .execute(&mut **transaction)
1007            .await
1008            .map(|_| ())
1009    })
1010    .await
1011}
1012
1013async fn acquire_maintenance_connection(
1014    pool: &PgPool,
1015    deadline: Instant,
1016) -> Result<PoolConnection<Postgres>, MaintenanceError> {
1017    timeout_at(deadline, pool.acquire())
1018        .await
1019        .map_err(|_| MaintenanceError::TimedOutBeforeCommit {
1020            operation: "acquiring database connection",
1021        })?
1022        .map_err(MaintenanceError::Database)
1023}
1024
1025async fn run_cleanup_transaction(
1026    connection: &mut PoolConnection<Postgres>,
1027    maximum_rows: u32,
1028    deadline: Instant,
1029) -> Result<u64, MaintenanceRunError> {
1030    let mut transaction = maintenance_before_commit(
1031        deadline,
1032        "beginning cleanup transaction",
1033        connection.begin(),
1034    )
1035    .await?;
1036    set_maintenance_server_timeouts(&mut transaction, deadline, "deleting expired windows").await?;
1037    let result = maintenance_before_commit(
1038        deadline,
1039        "deleting expired windows",
1040        sqlx::query(CLEANUP_SQL)
1041            .bind(i64::from(maximum_rows))
1042            .execute(&mut *transaction),
1043    )
1044    .await?;
1045    let rows_affected = result.rows_affected();
1046
1047    set_maintenance_server_timeouts(
1048        &mut transaction,
1049        deadline,
1050        "committing expired-window cleanup",
1051    )
1052    .await?;
1053    commit_maintenance(deadline, transaction).await?;
1054    Ok(rows_affected)
1055}
1056
1057async fn maintenance_before_commit<T, F>(
1058    deadline: Instant,
1059    operation: &'static str,
1060    future: F,
1061) -> Result<T, MaintenanceRunError>
1062where
1063    F: Future<Output = Result<T, sqlx::Error>>,
1064{
1065    timeout_at(deadline, future)
1066        .await
1067        .map_err(|_| MaintenanceRunError::ClientTimedOutBeforeCommit { operation })?
1068        .map_err(|error| {
1069            let public = if is_server_timeout(&error) {
1070                MaintenanceError::TimedOutBeforeCommit { operation }
1071            } else {
1072                MaintenanceError::Database(error)
1073            };
1074            MaintenanceRunError::Public(public)
1075        })
1076}
1077
1078async fn set_maintenance_server_timeouts(
1079    transaction: &mut Transaction<'_, Postgres>,
1080    deadline: Instant,
1081    operation: &'static str,
1082) -> Result<(), MaintenanceRunError> {
1083    let (statement_timeout, lock_timeout) = remaining_server_timeout_settings(deadline).ok_or(
1084        MaintenanceRunError::Public(MaintenanceError::TimedOutBeforeCommit { operation }),
1085    )?;
1086    maintenance_before_commit(
1087        deadline,
1088        operation,
1089        sqlx::query(SET_LOCAL_TIMEOUTS_SQL)
1090            .bind(statement_timeout)
1091            .bind(lock_timeout)
1092            .execute(&mut **transaction),
1093    )
1094    .await
1095    .map(|_| ())
1096}
1097
1098async fn commit_maintenance(
1099    deadline: Instant,
1100    transaction: Transaction<'_, Postgres>,
1101) -> Result<(), MaintenanceRunError> {
1102    if Instant::now() >= deadline {
1103        return Err(MaintenanceRunError::Public(
1104            MaintenanceError::TimedOutBeforeCommit {
1105                operation: "starting expired-window cleanup commit",
1106            },
1107        ));
1108    }
1109
1110    timeout_at(deadline, transaction.commit())
1111        .await
1112        .map_err(|_| MaintenanceRunError::ClientCommitTimedOut)?
1113        .map_err(|error| MaintenanceRunError::Public(MaintenanceError::CommitOutcomeUnknown(error)))
1114}
1115
1116fn authoritative_elapsed(start: DateTime<Utc>, end: DateTime<Utc>) -> Duration {
1117    end.signed_duration_since(start)
1118        .to_std()
1119        .unwrap_or(Duration::ZERO)
1120}
1121
1122async fn execute_batch(
1123    transaction: &mut Transaction<'_, Postgres>,
1124    input: &BatchSqlInput,
1125    checks: &[Check<'_>],
1126    deadline: Instant,
1127) -> Result<(PendingBatchOutcome, Duration), CheckRunError> {
1128    let preflight = preflight_batch(transaction, input, checks, deadline).await?;
1129    if let Some((index, denial)) = preflight.denial {
1130        return Ok((
1131            PendingBatchOutcome::Denied { index, denial },
1132            authoritative_elapsed(preflight.database_now, preflight.response_now),
1133        ));
1134    }
1135    let (allowances, response_now) =
1136        upsert_batch(transaction, input, checks, preflight.database_now, deadline).await?;
1137    Ok((
1138        PendingBatchOutcome::Allowed(allowances),
1139        authoritative_elapsed(preflight.database_now, response_now),
1140    ))
1141}
1142
1143async fn preflight_batch(
1144    transaction: &mut Transaction<'_, Postgres>,
1145    input: &BatchSqlInput,
1146    checks: &[Check<'_>],
1147    deadline: Instant,
1148) -> Result<BatchPreflight, CheckRunError> {
1149    set_check_server_timeouts(transaction, deadline, "preflighting counter batch").await?;
1150    let preflight_row = check_before_commit(
1151        deadline,
1152        "preflighting counter batch",
1153        sqlx::query(BATCH_PREFLIGHT_SQL)
1154            .bind(input.fingerprints.as_slice())
1155            .bind(input.subjects.as_slice())
1156            .bind(input.costs.as_slice())
1157            .bind(input.limits.as_slice())
1158            .fetch_one(&mut **transaction),
1159    )
1160    .await?;
1161
1162    let database_now: DateTime<Utc> = preflight_row
1163        .try_get("database_now")
1164        .map_err(CheckError::DefinitelyNotConsumed)?;
1165    let preflight_response_now: DateTime<Utc> = preflight_row
1166        .try_get("response_now")
1167        .map_err(CheckError::DefinitelyNotConsumed)?;
1168    let denied_index: Option<i64> = preflight_row
1169        .try_get("input_index")
1170        .map_err(CheckError::DefinitelyNotConsumed)?;
1171    let denied_expiry: Option<DateTime<Utc>> = preflight_row
1172        .try_get("window_expires_at")
1173        .map_err(CheckError::DefinitelyNotConsumed)?;
1174
1175    let denial = match (denied_index, denied_expiry) {
1176        (Some(input_index), Some(expires_at)) => {
1177            let input_index = usize::try_from(input_index).map_err(|_| {
1178                CheckError::StorageInvariant("batch preflight returned an invalid input index")
1179            })?;
1180            let check = checks.get(input_index).ok_or(CheckError::StorageInvariant(
1181                "batch preflight returned an out-of-range input index",
1182            ))?;
1183            Some((
1184                input_index,
1185                PendingDenial {
1186                    limit: check.policy().limit(),
1187                    retry_from_sample: duration_until(expires_at, database_now)?,
1188                },
1189            ))
1190        }
1191        (None, None) => None,
1192        _ => {
1193            return Err(CheckError::StorageInvariant(
1194                "batch preflight returned an incomplete denial",
1195            )
1196            .into());
1197        }
1198    };
1199
1200    Ok(BatchPreflight {
1201        database_now,
1202        response_now: preflight_response_now,
1203        denial,
1204    })
1205}
1206
1207async fn upsert_batch(
1208    transaction: &mut Transaction<'_, Postgres>,
1209    input: &BatchSqlInput,
1210    checks: &[Check<'_>],
1211    database_now: DateTime<Utc>,
1212    deadline: Instant,
1213) -> Result<(Vec<PendingAllowance>, DateTime<Utc>), CheckRunError> {
1214    set_check_server_timeouts(transaction, deadline, "updating counter batch").await?;
1215    let rows = check_before_commit(
1216        deadline,
1217        "updating counter batch",
1218        sqlx::query(BATCH_UPSERT_SQL)
1219            .bind(input.policy_ids.as_slice())
1220            .bind(input.scope_ids.as_slice())
1221            .bind(input.fingerprints.as_slice())
1222            .bind(input.subjects.as_slice())
1223            .bind(input.windows.as_slice())
1224            .bind(input.costs.as_slice())
1225            .bind(input.limits.as_slice())
1226            .bind(database_now)
1227            .fetch_all(&mut **transaction),
1228    )
1229    .await?;
1230
1231    if rows.is_empty() {
1232        return Err(
1233            CheckError::StorageInvariant("allowed batch update returned no decisions").into(),
1234        );
1235    }
1236    let response_now: DateTime<Utc> = rows[0]
1237        .try_get("response_now")
1238        .map_err(CheckError::DefinitelyNotConsumed)?;
1239
1240    let mut allowances = Vec::with_capacity(checks.len());
1241    for (output_position, row) in rows.iter().enumerate() {
1242        let input_index: i64 = row
1243            .try_get("input_index")
1244            .map_err(CheckError::DefinitelyNotConsumed)?;
1245        let input_index = usize::try_from(input_index).map_err(|_| {
1246            CheckError::StorageInvariant("batch evaluation returned an invalid input index")
1247        })?;
1248        let Some(check) = checks.get(input_index) else {
1249            return Err(CheckError::StorageInvariant(
1250                "batch evaluation returned an out-of-range input index",
1251            )
1252            .into());
1253        };
1254        let expires_at = read_expiry(row)?;
1255        let remaining_from_sample = duration_until(expires_at, database_now)?;
1256
1257        if input_index != output_position {
1258            return Err(CheckError::StorageInvariant(
1259                "allowed batch decisions were not returned in caller order",
1260            )
1261            .into());
1262        }
1263        let used = read_used(row)?;
1264        let remaining =
1265            check
1266                .policy()
1267                .limit()
1268                .checked_sub(used)
1269                .ok_or(CheckError::StorageInvariant(
1270                    "stored usage exceeds its policy limit",
1271                ))?;
1272        allowances.push(PendingAllowance {
1273            limit: check.policy().limit(),
1274            remaining,
1275            reset_from_sample: remaining_from_sample,
1276        });
1277    }
1278
1279    if allowances.len() != checks.len() {
1280        return Err(CheckError::StorageInvariant(
1281            "allowed batch returned an incomplete decision set",
1282        )
1283        .into());
1284    }
1285    Ok((allowances, response_now))
1286}
1287
1288fn database_integer(value: u64) -> i64 {
1289    i64::try_from(value)
1290        .expect("core policy and check validation keep database integers within i64")
1291}
1292
1293fn read_used(row: &PgRow) -> Result<u64, CheckError> {
1294    let used: i64 = row
1295        .try_get("used")
1296        .map_err(CheckError::DefinitelyNotConsumed)?;
1297    u64::try_from(used).map_err(|_| CheckError::StorageInvariant("stored usage is negative"))
1298}
1299
1300fn read_expiry(row: &PgRow) -> Result<DateTime<Utc>, CheckError> {
1301    row.try_get("window_expires_at")
1302        .map_err(CheckError::DefinitelyNotConsumed)
1303}
1304
1305fn duration_until(
1306    expires_at: DateTime<Utc>,
1307    database_now: DateTime<Utc>,
1308) -> Result<Duration, CheckError> {
1309    expires_at
1310        .signed_duration_since(database_now)
1311        .to_std()
1312        .map_err(|_| CheckError::StorageInvariant("stored window is already expired"))
1313}
1314
1315#[cfg(test)]
1316mod tests {
1317    use std::time::Duration;
1318
1319    use super::*;
1320    use runlimit_core::{
1321        FixedWindowPolicy, MAX_LIMIT, MAX_WINDOW, MAX_WINDOW_MILLIS, PolicyId, ScopeId, SubjectKey,
1322    };
1323
1324    fn policy(id: &str, scope: &str) -> FixedWindowPolicy {
1325        FixedWindowPolicy::new(
1326            PolicyId::new(id).expect("valid policy identifier"),
1327            ScopeId::new(scope).expect("valid scope identifier"),
1328            10,
1329            Duration::from_secs(60),
1330        )
1331        .expect("valid policy")
1332    }
1333
1334    fn key(byte: u8) -> SubjectKey {
1335        SubjectKey::from_digest([byte; 32])
1336    }
1337
1338    #[test]
1339    fn portable_core_boundaries_fit_database_values_exactly() {
1340        let policy = FixedWindowPolicy::new(
1341            PolicyId::new("portable.maximum").unwrap(),
1342            ScopeId::new("subject").unwrap(),
1343            MAX_LIMIT,
1344            MAX_WINDOW,
1345        )
1346        .unwrap();
1347        let check = Check::with_cost(&policy, key(1), MAX_LIMIT).unwrap();
1348
1349        assert_eq!(database_integer(policy.limit()), i64::MAX);
1350        assert_eq!(database_integer(check.cost()), i64::MAX);
1351        let interval = PgInterval::try_from(policy.window()).unwrap();
1352        assert_eq!(interval.months, 0);
1353        assert_eq!(interval.days, 0);
1354        assert_eq!(
1355            interval.microseconds,
1356            i64::try_from(MAX_WINDOW_MILLIS * 1_000).unwrap()
1357        );
1358    }
1359
1360    #[test]
1361    fn row_lock_permutation_orders_complete_counter_keys() {
1362        let alpha_client = policy("alpha", "client");
1363        let alpha_identity = policy("alpha", "identity");
1364        let beta_client = policy("beta", "client");
1365        let checks = [
1366            Check::new(&beta_client, key(0)),
1367            Check::new(&alpha_identity, key(0)),
1368            Check::new(&alpha_client, key(1)),
1369            Check::new(&alpha_client, key(0)),
1370        ];
1371
1372        let input = BatchSqlInput::from_checks(&checks);
1373        let locked_keys = input
1374            .lock_input_positions
1375            .iter()
1376            .map(|position| checks[usize::try_from(*position - 1).unwrap()].counter_key())
1377            .collect::<Vec<_>>();
1378
1379        assert!(locked_keys.windows(2).all(|pair| pair[0] < pair[1]));
1380    }
1381
1382    #[test]
1383    fn advisory_lock_protocol_has_a_golden_vector() {
1384        let policy = policy("login", "client");
1385        let check = Check::new(&policy, key(7));
1386
1387        assert_eq!(
1388            advisory_lock_id(check.counter_key()),
1389            4_549_358_434_381_602_087
1390        );
1391    }
1392
1393    #[test]
1394    fn response_metadata_subtracts_only_authoritative_elapsed_time() {
1395        let allowed = PendingAllowance {
1396            limit: 10,
1397            remaining: 4,
1398            reset_from_sample: Duration::from_millis(250),
1399        }
1400        .finish(Duration::from_millis(80));
1401        let denied = PendingDenial {
1402            limit: 10,
1403            retry_from_sample: Duration::from_millis(150),
1404        }
1405        .finish(Duration::from_millis(80));
1406
1407        assert_eq!(allowed.reset_after(), Some(Duration::from_millis(170)));
1408        assert_eq!(denied.retry_after(), Some(Duration::from_millis(70)));
1409        assert_eq!(denied.retry_after_seconds(), Some(1));
1410    }
1411
1412    #[test]
1413    fn configuration_bounds_batches_and_deadlines() {
1414        let defaults = PostgresConfig::new();
1415        assert_eq!(defaults.max_batch_size(), 32);
1416        assert_eq!(defaults.operation_timeout(), Duration::from_secs(3));
1417        assert_eq!(
1418            defaults.with_max_batch_size(0),
1419            Err(PostgresConfigError::ZeroBatchSize)
1420        );
1421        assert_eq!(
1422            defaults.with_operation_timeout(Duration::ZERO),
1423            Err(PostgresConfigError::ZeroOperationTimeout)
1424        );
1425        assert_eq!(
1426            defaults.with_operation_timeout(Duration::from_secs(61)),
1427            Err(PostgresConfigError::OperationTimeoutTooLong {
1428                actual: Duration::from_secs(61),
1429                maximum: Duration::from_secs(60),
1430            })
1431        );
1432    }
1433
1434    #[tokio::test]
1435    async fn duplicate_storage_keys_fail_before_connecting() {
1436        let alpha = policy("alpha", "client");
1437        let beta = policy("beta", "client");
1438        let checks = [
1439            Check::new(&beta, key(7)),
1440            Check::new(&beta, key(7)),
1441            Check::new(&alpha, key(8)),
1442            Check::new(&alpha, key(8)),
1443        ];
1444        let pool = sqlx::postgres::PgPoolOptions::new()
1445            .connect_lazy("postgresql://runlimit:runlimit@127.0.0.1:1/runlimit")
1446            .expect("syntactically valid database URL");
1447        let limiter = PostgresLimiter::new(pool);
1448
1449        let error = limiter
1450            .check_all(&checks)
1451            .await
1452            .expect_err("duplicate keys must be rejected");
1453
1454        assert!(matches!(
1455            error,
1456            CheckError::InvalidBatch(BatchError::DuplicateKey {
1457                first_index: 0,
1458                duplicate_index: 1,
1459            })
1460        ));
1461    }
1462
1463    #[tokio::test]
1464    async fn oversized_batch_fails_before_connecting() {
1465        let first_policy = policy("login", "client");
1466        let second_policy = policy("signup", "client");
1467        let checks = [
1468            Check::new(&first_policy, key(1)),
1469            Check::new(&second_policy, key(2)),
1470        ];
1471        let pool = sqlx::postgres::PgPoolOptions::new()
1472            .connect_lazy("postgresql://runlimit:runlimit@127.0.0.1:1/runlimit")
1473            .expect("syntactically valid database URL");
1474        let config = PostgresConfig::new()
1475            .with_max_batch_size(1)
1476            .expect("one is a valid batch limit");
1477        let limiter = PostgresLimiter::with_config(pool, config);
1478
1479        let error = limiter
1480            .check_all(&checks)
1481            .await
1482            .expect_err("oversized batch must be rejected");
1483
1484        assert!(matches!(
1485            error,
1486            CheckError::InvalidBatch(BatchError::BatchTooLarge {
1487                actual: 2,
1488                maximum: 1,
1489            })
1490        ));
1491    }
1492
1493    #[test]
1494    fn error_reports_consumption_uncertainty() {
1495        let definite = CheckError::DefinitelyNotConsumed(sqlx::Error::RowNotFound);
1496        let unknown = CheckError::CommitOutcomeUnknown(sqlx::Error::RowNotFound);
1497        let rolled_back_invariant = CheckError::ResponseInvariant;
1498        let committed_invariant = CheckError::CommittedResponseInvariant;
1499        let cleanup_definite = MaintenanceError::Database(sqlx::Error::RowNotFound);
1500        let cleanup_unknown = MaintenanceError::CommitOutcomeUnknown(sqlx::Error::RowNotFound);
1501
1502        assert!(!definite.may_have_consumed_quota());
1503        assert!(unknown.may_have_consumed_quota());
1504        assert!(CheckError::CommitTimedOut.may_have_consumed_quota());
1505        assert!(!rolled_back_invariant.may_have_consumed_quota());
1506        assert!(committed_invariant.may_have_consumed_quota());
1507        assert!(!cleanup_definite.may_have_removed_rows());
1508        assert!(cleanup_unknown.may_have_removed_rows());
1509        assert!(MaintenanceError::CommitTimedOut.may_have_removed_rows());
1510    }
1511
1512    #[test]
1513    fn malformed_single_responses_preserve_consumption_status() {
1514        let denial = Denial::QuotaExceeded {
1515            limit: 1,
1516            retry_after: Duration::from_secs(1),
1517        };
1518
1519        let committed = single_decision_from_batch(BatchDecision::Allowed(Vec::new()))
1520            .expect_err("malformed allowed response follows a confirmed commit");
1521        let rolled_back = single_decision_from_batch(BatchDecision::Denied { index: 1, denial })
1522            .expect_err("malformed denied response follows a rollback");
1523
1524        assert!(matches!(committed, CheckError::CommittedResponseInvariant));
1525        assert!(committed.may_have_consumed_quota());
1526        assert!(matches!(rolled_back, CheckError::ResponseInvariant));
1527        assert!(!rolled_back.may_have_consumed_quota());
1528    }
1529
1530    #[test]
1531    fn cleanup_query_is_bounded_and_nonblocking() {
1532        assert_eq!(CLEANUP_SQL.matches("clock_timestamp()").count(), 1);
1533        assert!(CLEANUP_SQL.contains("AS MATERIALIZED"));
1534        assert!(CLEANUP_SQL.contains("SELECT sampled_at\n        FROM authoritative_time"));
1535        assert!(!CLEANUP_SQL.contains("window_expires_at <= clock_timestamp()"));
1536        assert!(CLEANUP_SQL.contains("LIMIT $1"));
1537        assert!(CLEANUP_SQL.contains("FOR UPDATE SKIP LOCKED"));
1538    }
1539}