1use std::{future::Future, time::Duration};
60
61use runlimit_core::{
62 BatchDecision, BatchError, Check, CounterKey, Decision, Denial, validate_batch,
63};
64use sha2::{Digest, Sha256};
65use sqlx::{
66 Acquire, PgPool, Postgres, Row, Transaction,
67 migrate::{MigrateError, Migrator},
68 pool::PoolConnection,
69 postgres::{PgRow, types::PgInterval},
70 types::chrono::{DateTime, Utc},
71};
72use thiserror::Error;
73use tokio::time::{Instant, timeout_at};
74
75pub static MIGRATOR: Migrator = sqlx::migrate!("./migrations");
83
84const ADVISORY_LOCK_DOMAIN: &[u8] = b"runlimit/postgres-advisory-lock/v1\0";
85const SERVER_TIMEOUT_GRACE: Duration = Duration::from_millis(25);
86
87const SET_LOCAL_TIMEOUTS_SQL: &str = r"
88SELECT
89 set_config('statement_timeout', $1, true),
90 set_config('lock_timeout', $2, true)
91";
92
93const BATCH_ADVISORY_LOCK_SQL: &str = r"
94WITH RECURSIVE acquired(position, locked) AS (
95 SELECT 1, pg_advisory_xact_lock($1[1])
96 WHERE cardinality($1::BIGINT[]) > 0
97
98 UNION ALL
99
100 SELECT
101 acquired.position + 1,
102 pg_advisory_xact_lock($1[acquired.position + 1])
103 FROM acquired
104 WHERE acquired.position < cardinality($1::BIGINT[])
105)
106SELECT count(*)
107FROM acquired
108";
109
110const BATCH_ROW_LOCK_SQL: &str = r"
111WITH input_keys AS (
112 SELECT *
113 FROM unnest(
114 $1::BYTEA[],
115 $2::BYTEA[]
116 ) WITH ORDINALITY AS keys(
117 config_fingerprint,
118 subject_key,
119 input_position
120 )
121),
122lock_order AS (
123 SELECT *
124 FROM unnest($3::BIGINT[])
125 WITH ORDINALITY AS positions(input_position, lock_position)
126),
127ordered_keys AS (
128 SELECT
129 input_keys.config_fingerprint,
130 input_keys.subject_key,
131 lock_order.lock_position
132 FROM input_keys
133 INNER JOIN lock_order
134 ON lock_order.input_position = input_keys.input_position
135 ORDER BY lock_order.lock_position
136)
137SELECT 1
138FROM ordered_keys
139INNER JOIN runlimit_fixed_windows AS windows
140 ON windows.config_fingerprint = ordered_keys.config_fingerprint
141 AND windows.subject_key = ordered_keys.subject_key
142ORDER BY ordered_keys.lock_position
143FOR UPDATE OF windows
144";
145
146const BATCH_PREFLIGHT_SQL: &str = r"
147WITH input AS (
148 SELECT *
149 FROM unnest(
150 $1::BYTEA[],
151 $2::BYTEA[],
152 $3::BIGINT[],
153 $4::BIGINT[]
154 ) WITH ORDINALITY AS checks(
155 config_fingerprint,
156 subject_key,
157 cost,
158 quota_limit,
159 input_position
160 )
161),
162sample AS (
163 SELECT clock_timestamp() AS database_now
164),
165first_denial AS (
166 SELECT
167 input.input_position - 1 AS input_index,
168 windows.window_expires_at
169 FROM input
170 CROSS JOIN sample
171 INNER JOIN runlimit_fixed_windows AS windows
172 ON windows.config_fingerprint = input.config_fingerprint
173 AND windows.subject_key = input.subject_key
174 WHERE
175 windows.window_expires_at > sample.database_now
176 AND windows.used > input.quota_limit - input.cost
177 ORDER BY input.input_position
178 LIMIT 1
179)
180SELECT
181 sample.database_now,
182 clock_timestamp() AS response_now,
183 first_denial.input_index,
184 first_denial.window_expires_at
185FROM sample
186LEFT JOIN first_denial ON TRUE
187";
188
189const BATCH_UPSERT_SQL: &str = r"
190WITH input AS (
191 SELECT *
192 FROM unnest(
193 $1::TEXT[],
194 $2::TEXT[],
195 $3::BYTEA[],
196 $4::BYTEA[],
197 $5::INTERVAL[],
198 $6::BIGINT[],
199 $7::BIGINT[]
200 ) WITH ORDINALITY AS checks(
201 policy_id,
202 scope_id,
203 config_fingerprint,
204 subject_key,
205 window_size,
206 cost,
207 quota_limit,
208 input_position
209 )
210),
211upserted AS (
212 INSERT INTO runlimit_fixed_windows (
213 policy_id,
214 scope_id,
215 config_fingerprint,
216 subject_key,
217 window_started_at,
218 window_expires_at,
219 used
220 )
221 SELECT
222 policy_id,
223 scope_id,
224 config_fingerprint,
225 subject_key,
226 $8,
227 $8 + window_size,
228 cost
229 FROM input
230 ON CONFLICT (config_fingerprint, subject_key)
231 DO UPDATE SET
232 policy_id = EXCLUDED.policy_id,
233 scope_id = EXCLUDED.scope_id,
234 window_started_at = CASE
235 WHEN runlimit_fixed_windows.window_expires_at <= $8 THEN $8
236 ELSE runlimit_fixed_windows.window_started_at
237 END,
238 window_expires_at = CASE
239 WHEN runlimit_fixed_windows.window_expires_at <= $8
240 THEN $8 + (EXCLUDED.window_expires_at - EXCLUDED.window_started_at)
241 ELSE runlimit_fixed_windows.window_expires_at
242 END,
243 used = CASE
244 WHEN runlimit_fixed_windows.window_expires_at <= $8 THEN EXCLUDED.used
245 ELSE runlimit_fixed_windows.used + EXCLUDED.used
246 END
247 WHERE
248 runlimit_fixed_windows.window_expires_at <= $8
249 OR runlimit_fixed_windows.used
250 <= (
251 SELECT quota_limit - cost
252 FROM input
253 WHERE
254 input.config_fingerprint = EXCLUDED.config_fingerprint
255 AND input.subject_key = EXCLUDED.subject_key
256 )
257 RETURNING
258 config_fingerprint,
259 subject_key,
260 used,
261 window_expires_at
262),
263response AS (
264 SELECT clock_timestamp() AS response_now
265 FROM upserted
266 HAVING count(*) >= 0
267)
268SELECT
269 input.input_position - 1 AS input_index,
270 upserted.used,
271 upserted.window_expires_at,
272 response.response_now
273FROM input
274INNER JOIN upserted
275 ON upserted.config_fingerprint = input.config_fingerprint
276 AND upserted.subject_key = input.subject_key
277CROSS JOIN response
278ORDER BY input.input_position
279";
280
281const CLEANUP_SQL: &str = include_str!("cleanup_expired.sql");
282
283#[derive(Clone, Copy, Debug, Eq, PartialEq)]
285pub struct PostgresConfig {
286 max_batch_size: usize,
287 operation_timeout: Duration,
288}
289
290impl PostgresConfig {
291 pub const DEFAULT_MAX_BATCH_SIZE: usize = 32;
293
294 pub const DEFAULT_OPERATION_TIMEOUT: Duration = Duration::from_secs(3);
296
297 pub const MAXIMUM_OPERATION_TIMEOUT: Duration = Duration::from_secs(60);
299
300 pub const fn new() -> Self {
303 Self {
304 max_batch_size: Self::DEFAULT_MAX_BATCH_SIZE,
305 operation_timeout: Self::DEFAULT_OPERATION_TIMEOUT,
306 }
307 }
308
309 pub fn with_max_batch_size(mut self, maximum: usize) -> Result<Self, PostgresConfigError> {
315 if maximum == 0 {
316 return Err(PostgresConfigError::ZeroBatchSize);
317 }
318 self.max_batch_size = maximum;
319 Ok(self)
320 }
321
322 pub fn with_operation_timeout(
331 mut self,
332 timeout: Duration,
333 ) -> Result<Self, PostgresConfigError> {
334 if timeout.is_zero() {
335 return Err(PostgresConfigError::ZeroOperationTimeout);
336 }
337 if timeout > Self::MAXIMUM_OPERATION_TIMEOUT {
338 return Err(PostgresConfigError::OperationTimeoutTooLong {
339 actual: timeout,
340 maximum: Self::MAXIMUM_OPERATION_TIMEOUT,
341 });
342 }
343 self.operation_timeout = timeout;
344 Ok(self)
345 }
346
347 pub const fn max_batch_size(&self) -> usize {
349 self.max_batch_size
350 }
351
352 pub const fn operation_timeout(self) -> Duration {
354 self.operation_timeout
355 }
356}
357
358impl Default for PostgresConfig {
359 fn default() -> Self {
360 Self::new()
361 }
362}
363
364#[derive(Clone, Copy, Debug, Error, Eq, PartialEq)]
366pub enum PostgresConfigError {
367 #[error("max_batch_size must be greater than zero")]
369 ZeroBatchSize,
370 #[error("PostgreSQL rate-limit operation timeout must be greater than zero")]
372 ZeroOperationTimeout,
373 #[error("PostgreSQL rate-limit operation timeout {actual:?} exceeds maximum {maximum:?}")]
376 OperationTimeoutTooLong {
377 actual: Duration,
379 maximum: Duration,
381 },
382}
383
384#[derive(Clone, Debug)]
389pub struct PostgresLimiter {
390 pool: PgPool,
391 config: PostgresConfig,
392}
393
394impl PostgresLimiter {
395 pub const fn new(pool: PgPool) -> Self {
400 Self {
401 pool,
402 config: PostgresConfig::new(),
403 }
404 }
405
406 pub const fn with_config(pool: PgPool, config: PostgresConfig) -> Self {
408 Self { pool, config }
409 }
410
411 pub const fn pool(&self) -> &PgPool {
413 &self.pool
414 }
415
416 pub const fn config(&self) -> PostgresConfig {
418 self.config
419 }
420
421 pub async fn migrate(&self) -> Result<(), MigrateError> {
439 let connection = self.pool.acquire().await?;
440 let mut guarded_connection = ConnectionCancellationGuard::new(connection);
441 let mut migrator = sqlx::migrate!("./migrations");
442 migrator.set_ignore_missing(true);
443
444 let result = migrator
445 .run_direct(&mut **guarded_connection.connection())
446 .await;
447 if result.is_ok() {
448 guarded_connection.disarm();
449 }
450 result
451 }
452
453 pub async fn check(&self, check: &Check<'_>) -> Result<Decision, CheckError> {
464 single_decision_from_batch(self.check_all(std::slice::from_ref(check)).await?)
465 }
466
467 pub async fn check_all(&self, checks: &[Check<'_>]) -> Result<BatchDecision, CheckError> {
485 validate_batch(checks, self.config.max_batch_size())?;
486 if checks.is_empty() {
487 return Ok(BatchDecision::Allowed(Vec::new()));
488 }
489
490 let input = BatchSqlInput::from_checks(checks);
494 let deadline = Instant::now() + self.config.operation_timeout();
495 let connection = acquire_check_connection(&self.pool, deadline).await?;
496 let mut guarded_connection = ConnectionCancellationGuard::new(connection);
497 let result =
498 run_check_transaction(guarded_connection.connection(), &input, checks, deadline).await;
499 if !result
500 .as_ref()
501 .is_err_and(CheckRunError::client_timeout_won)
502 {
503 guarded_connection.disarm();
504 }
505 result.map_err(CheckRunError::into_public)
506 }
507
508 pub async fn cleanup_expired(&self, maximum_rows: u32) -> Result<u64, MaintenanceError> {
522 if maximum_rows == 0 {
523 return Ok(0);
524 }
525
526 let deadline = Instant::now() + self.config.operation_timeout();
527 let connection = acquire_maintenance_connection(&self.pool, deadline).await?;
528 let mut guarded_connection = ConnectionCancellationGuard::new(connection);
529 let result =
530 run_cleanup_transaction(guarded_connection.connection(), maximum_rows, deadline).await;
531 if !result
532 .as_ref()
533 .is_err_and(MaintenanceRunError::client_timeout_won)
534 {
535 guarded_connection.disarm();
536 }
537 result.map_err(MaintenanceRunError::into_public)
538 }
539}
540
541#[derive(Debug, Error)]
543pub enum CheckError {
544 #[error(transparent)]
546 InvalidBatch(#[from] BatchError),
547
548 #[error("PostgreSQL rate-limit check failed before commit; quota was not consumed")]
550 DefinitelyNotConsumed(#[source] sqlx::Error),
551
552 #[error("PostgreSQL rate-limit commit outcome is unknown; quota may have been consumed")]
554 CommitOutcomeUnknown(#[source] sqlx::Error),
555
556 #[error("PostgreSQL rate-limit check timed out while {operation}; quota was not consumed")]
558 TimedOutBeforeCommit {
559 operation: &'static str,
561 },
562
563 #[error("PostgreSQL rate-limit commit timed out; quota may have been consumed")]
565 CommitTimedOut,
566
567 #[error("PostgreSQL rate-limit storage invariant failed: {0}")]
569 StorageInvariant(&'static str),
570
571 #[error(
573 "PostgreSQL rate-limit single-check response invariant failed after rollback; quota was not consumed"
574 )]
575 ResponseInvariant,
576
577 #[error(
579 "PostgreSQL rate-limit single-check response invariant failed after commit; quota was consumed"
580 )]
581 CommittedResponseInvariant,
582}
583
584impl CheckError {
585 pub const fn may_have_consumed_quota(&self) -> bool {
587 matches!(
588 self,
589 Self::CommitOutcomeUnknown(_) | Self::CommitTimedOut | Self::CommittedResponseInvariant
590 )
591 }
592}
593
594#[derive(Debug, Error)]
596pub enum MaintenanceError {
597 #[error("PostgreSQL expired-window cleanup failed before commit; no rows were removed")]
600 Database(#[source] sqlx::Error),
601 #[error("PostgreSQL expired-window cleanup timed out while {operation}; no rows were removed")]
603 TimedOutBeforeCommit {
604 operation: &'static str,
606 },
607 #[error(
609 "PostgreSQL expired-window cleanup commit outcome is unknown; rows may have been removed"
610 )]
611 CommitOutcomeUnknown(#[source] sqlx::Error),
612 #[error("PostgreSQL expired-window cleanup commit timed out; rows may have been removed")]
614 CommitTimedOut,
615}
616
617impl MaintenanceError {
618 pub const fn may_have_removed_rows(&self) -> bool {
620 matches!(self, Self::CommitOutcomeUnknown(_) | Self::CommitTimedOut)
621 }
622}
623
624fn single_decision_from_batch(batch: BatchDecision) -> Result<Decision, CheckError> {
625 batch
626 .try_into_single_decision()
627 .map_err(|invalid| match invalid {
628 BatchDecision::Allowed(_) => CheckError::CommittedResponseInvariant,
629 BatchDecision::Denied { .. } => CheckError::ResponseInvariant,
630 })
631}
632
633struct ConnectionCancellationGuard {
634 connection: PoolConnection<Postgres>,
635 armed: bool,
636}
637
638impl ConnectionCancellationGuard {
639 const fn new(connection: PoolConnection<Postgres>) -> Self {
640 Self {
641 connection,
642 armed: true,
643 }
644 }
645
646 const fn connection(&mut self) -> &mut PoolConnection<Postgres> {
647 &mut self.connection
648 }
649
650 const fn disarm(&mut self) {
651 self.armed = false;
652 }
653}
654
655impl Drop for ConnectionCancellationGuard {
656 fn drop(&mut self) {
657 if self.armed {
658 self.connection.close_on_drop();
663 }
664 }
665}
666
667#[derive(Debug)]
668enum CheckRunError {
669 Public(CheckError),
670 ClientTimedOutBeforeCommit { operation: &'static str },
671 ClientCommitTimedOut,
672}
673
674impl CheckRunError {
675 const fn client_timeout_won(&self) -> bool {
676 matches!(
677 self,
678 Self::ClientTimedOutBeforeCommit { .. } | Self::ClientCommitTimedOut
679 )
680 }
681
682 fn into_public(self) -> CheckError {
683 match self {
684 Self::Public(error) => error,
685 Self::ClientTimedOutBeforeCommit { operation } => {
686 CheckError::TimedOutBeforeCommit { operation }
687 }
688 Self::ClientCommitTimedOut => CheckError::CommitTimedOut,
689 }
690 }
691}
692
693impl From<CheckError> for CheckRunError {
694 fn from(error: CheckError) -> Self {
695 Self::Public(error)
696 }
697}
698
699#[derive(Debug)]
700enum MaintenanceRunError {
701 Public(MaintenanceError),
702 ClientTimedOutBeforeCommit { operation: &'static str },
703 ClientCommitTimedOut,
704}
705
706impl MaintenanceRunError {
707 const fn client_timeout_won(&self) -> bool {
708 matches!(
709 self,
710 Self::ClientTimedOutBeforeCommit { .. } | Self::ClientCommitTimedOut
711 )
712 }
713
714 fn into_public(self) -> MaintenanceError {
715 match self {
716 Self::Public(error) => error,
717 Self::ClientTimedOutBeforeCommit { operation } => {
718 MaintenanceError::TimedOutBeforeCommit { operation }
719 }
720 Self::ClientCommitTimedOut => MaintenanceError::CommitTimedOut,
721 }
722 }
723}
724
725#[derive(Debug)]
726enum PendingBatchOutcome {
727 Allowed(Vec<PendingAllowance>),
728 Denied { index: usize, denial: PendingDenial },
729}
730
731#[derive(Debug)]
732struct BatchSqlInput {
733 policy_ids: Vec<String>,
734 scope_ids: Vec<String>,
735 fingerprints: Vec<Vec<u8>>,
736 subjects: Vec<Vec<u8>>,
737 lock_input_positions: Vec<i64>,
738 advisory_lock_ids: Vec<i64>,
739 windows: Vec<PgInterval>,
740 costs: Vec<i64>,
741 limits: Vec<i64>,
742}
743
744impl BatchSqlInput {
745 fn from_checks(checks: &[Check<'_>]) -> Self {
746 let counter_keys = checks.iter().map(Check::counter_key).collect::<Vec<_>>();
747 let mut ordered_indices = (0..checks.len()).collect::<Vec<_>>();
748 ordered_indices.sort_unstable_by_key(|index| counter_keys[*index]);
749
750 let mut input = Self {
751 policy_ids: Vec::with_capacity(checks.len()),
752 scope_ids: Vec::with_capacity(checks.len()),
753 fingerprints: Vec::with_capacity(checks.len()),
754 subjects: Vec::with_capacity(checks.len()),
755 lock_input_positions: ordered_indices
756 .into_iter()
757 .map(|index| {
758 i64::try_from(index + 1)
759 .expect("bounded batch input positions fit PostgreSQL BIGINT")
760 })
761 .collect(),
762 advisory_lock_ids: counter_keys.iter().copied().map(advisory_lock_id).collect(),
763 windows: Vec::with_capacity(checks.len()),
764 costs: Vec::with_capacity(checks.len()),
765 limits: Vec::with_capacity(checks.len()),
766 };
767 input.advisory_lock_ids.sort_unstable();
768 input.advisory_lock_ids.dedup();
769
770 for (check, counter_key) in checks.iter().zip(counter_keys) {
771 let policy = check.policy();
772 input.policy_ids.push(policy.id().as_str().to_owned());
773 input.scope_ids.push(policy.scope().as_str().to_owned());
774 input
775 .fingerprints
776 .push(counter_key.fingerprint().as_bytes().to_vec());
777 input
778 .subjects
779 .push(counter_key.subject().as_bytes().to_vec());
780 input.windows.push(
781 PgInterval::try_from(policy.window())
782 .expect("core policy windows fit PostgreSQL INTERVAL exactly"),
783 );
784 input.costs.push(database_integer(check.cost()));
785 input.limits.push(database_integer(policy.limit()));
786 }
787 input
788 }
789}
790
791#[derive(Debug)]
792struct BatchPreflight {
793 database_now: DateTime<Utc>,
794 response_now: DateTime<Utc>,
795 denial: Option<(usize, PendingDenial)>,
796}
797
798#[derive(Debug)]
799struct PendingAllowance {
800 limit: u64,
801 remaining: u64,
802 reset_from_sample: Duration,
803}
804
805impl PendingAllowance {
806 fn finish(self, authoritative_elapsed: Duration) -> Decision {
807 Decision::allowed(
808 self.limit,
809 self.remaining,
810 self.reset_from_sample.saturating_sub(authoritative_elapsed),
811 )
812 }
813}
814
815#[derive(Debug)]
816struct PendingDenial {
817 limit: u64,
818 retry_from_sample: Duration,
819}
820
821impl PendingDenial {
822 fn finish(self, authoritative_elapsed: Duration) -> Denial {
823 Denial::QuotaExceeded {
824 limit: self.limit,
825 retry_after: self.retry_from_sample.saturating_sub(authoritative_elapsed),
826 }
827 }
828}
829
830async fn acquire_check_connection(
831 pool: &PgPool,
832 deadline: Instant,
833) -> Result<PoolConnection<Postgres>, CheckError> {
834 timeout_at(deadline, pool.acquire())
835 .await
836 .map_err(|_| CheckError::TimedOutBeforeCommit {
837 operation: "acquiring database connection",
838 })?
839 .map_err(CheckError::DefinitelyNotConsumed)
840}
841
842async fn run_check_transaction(
843 connection: &mut PoolConnection<Postgres>,
844 input: &BatchSqlInput,
845 checks: &[Check<'_>],
846 deadline: Instant,
847) -> Result<BatchDecision, CheckRunError> {
848 let mut transaction =
849 check_before_commit(deadline, "beginning transaction", connection.begin()).await?;
850
851 acquire_advisory_locks(&mut transaction, &input.advisory_lock_ids, deadline).await?;
855
856 acquire_existing_row_locks(&mut transaction, input, deadline).await?;
860
861 let (pending, authoritative_elapsed) =
862 execute_batch(&mut transaction, input, checks, deadline).await?;
863
864 match pending {
865 PendingBatchOutcome::Denied { index, denial } => {
866 set_check_server_timeouts(&mut transaction, deadline, "rolling back denied batch")
867 .await?;
868 check_before_commit(
869 deadline,
870 "rolling back denied batch",
871 transaction.rollback(),
872 )
873 .await?;
874 Ok(BatchDecision::Denied {
875 index,
876 denial: denial.finish(authoritative_elapsed),
877 })
878 }
879 PendingBatchOutcome::Allowed(allowances) => {
880 set_check_server_timeouts(&mut transaction, deadline, "committing counter batch")
881 .await?;
882 commit_check(deadline, transaction).await?;
883 Ok(BatchDecision::Allowed(
884 allowances
885 .into_iter()
886 .map(|allowance| allowance.finish(authoritative_elapsed))
887 .collect(),
888 ))
889 }
890 }
891}
892
893async fn check_before_commit<T, F>(
894 deadline: Instant,
895 operation: &'static str,
896 future: F,
897) -> Result<T, CheckRunError>
898where
899 F: Future<Output = Result<T, sqlx::Error>>,
900{
901 timeout_at(deadline, future)
902 .await
903 .map_err(|_| CheckRunError::ClientTimedOutBeforeCommit { operation })?
904 .map_err(|error| CheckRunError::Public(map_check_database_error(error, operation)))
905}
906
907async fn commit_check(
908 deadline: Instant,
909 transaction: Transaction<'_, Postgres>,
910) -> Result<(), CheckRunError> {
911 if Instant::now() >= deadline {
912 return Err(CheckRunError::Public(CheckError::TimedOutBeforeCommit {
913 operation: "starting commit",
914 }));
915 }
916
917 timeout_at(deadline, transaction.commit())
918 .await
919 .map_err(|_| CheckRunError::ClientCommitTimedOut)?
920 .map_err(|error| CheckRunError::Public(CheckError::CommitOutcomeUnknown(error)))
921}
922
923fn map_check_database_error(error: sqlx::Error, operation: &'static str) -> CheckError {
924 if is_server_timeout(&error) {
925 CheckError::TimedOutBeforeCommit { operation }
926 } else {
927 CheckError::DefinitelyNotConsumed(error)
928 }
929}
930
931fn is_server_timeout(error: &sqlx::Error) -> bool {
932 let sqlx::Error::Database(database_error) = error else {
933 return false;
934 };
935 matches!(database_error.code().as_deref(), Some("57014" | "55P03"))
936}
937
938fn remaining_server_timeout_settings(deadline: Instant) -> Option<(String, String)> {
939 let remaining = deadline.checked_duration_since(Instant::now())?;
940 let remaining_millis = remaining.as_millis();
941 let statement_millis = remaining_millis
942 .saturating_sub(SERVER_TIMEOUT_GRACE.as_millis())
943 .max(1);
944 let lock_millis = statement_millis.saturating_sub(1).max(1);
945 Some((format!("{statement_millis}ms"), format!("{lock_millis}ms")))
946}
947
948async fn set_check_server_timeouts(
949 transaction: &mut Transaction<'_, Postgres>,
950 deadline: Instant,
951 operation: &'static str,
952) -> Result<(), CheckRunError> {
953 let (statement_timeout, lock_timeout) = remaining_server_timeout_settings(deadline).ok_or(
954 CheckRunError::Public(CheckError::TimedOutBeforeCommit { operation }),
955 )?;
956 check_before_commit(
957 deadline,
958 operation,
959 sqlx::query(SET_LOCAL_TIMEOUTS_SQL)
960 .bind(statement_timeout)
961 .bind(lock_timeout)
962 .execute(&mut **transaction),
963 )
964 .await
965 .map(|_| ())
966}
967
968async fn acquire_advisory_locks(
969 transaction: &mut Transaction<'_, Postgres>,
970 advisory_lock_ids: &[i64],
971 deadline: Instant,
972) -> Result<(), CheckRunError> {
973 set_check_server_timeouts(transaction, deadline, "acquiring logical key lock").await?;
974 check_before_commit(deadline, "acquiring logical key lock", async {
975 sqlx::query(BATCH_ADVISORY_LOCK_SQL)
976 .bind(advisory_lock_ids)
977 .execute(&mut **transaction)
978 .await
979 .map(|_| ())
980 })
981 .await
982}
983
984fn advisory_lock_id(counter_key: CounterKey) -> i64 {
985 let mut digest = Sha256::new();
986 digest.update(ADVISORY_LOCK_DOMAIN);
987 digest.update(counter_key.to_bytes());
988 let digest = digest.finalize();
989 let mut id_bytes = [0_u8; 8];
990 let id_length = id_bytes.len();
991 id_bytes.copy_from_slice(&digest[..id_length]);
992 i64::from_be_bytes(id_bytes)
993}
994
995async fn acquire_existing_row_locks(
996 transaction: &mut Transaction<'_, Postgres>,
997 input: &BatchSqlInput,
998 deadline: Instant,
999) -> Result<(), CheckRunError> {
1000 set_check_server_timeouts(transaction, deadline, "acquiring counter row lock").await?;
1001 check_before_commit(deadline, "acquiring counter row lock", async {
1002 sqlx::query(BATCH_ROW_LOCK_SQL)
1003 .bind(input.fingerprints.as_slice())
1004 .bind(input.subjects.as_slice())
1005 .bind(input.lock_input_positions.as_slice())
1006 .execute(&mut **transaction)
1007 .await
1008 .map(|_| ())
1009 })
1010 .await
1011}
1012
1013async fn acquire_maintenance_connection(
1014 pool: &PgPool,
1015 deadline: Instant,
1016) -> Result<PoolConnection<Postgres>, MaintenanceError> {
1017 timeout_at(deadline, pool.acquire())
1018 .await
1019 .map_err(|_| MaintenanceError::TimedOutBeforeCommit {
1020 operation: "acquiring database connection",
1021 })?
1022 .map_err(MaintenanceError::Database)
1023}
1024
1025async fn run_cleanup_transaction(
1026 connection: &mut PoolConnection<Postgres>,
1027 maximum_rows: u32,
1028 deadline: Instant,
1029) -> Result<u64, MaintenanceRunError> {
1030 let mut transaction = maintenance_before_commit(
1031 deadline,
1032 "beginning cleanup transaction",
1033 connection.begin(),
1034 )
1035 .await?;
1036 set_maintenance_server_timeouts(&mut transaction, deadline, "deleting expired windows").await?;
1037 let result = maintenance_before_commit(
1038 deadline,
1039 "deleting expired windows",
1040 sqlx::query(CLEANUP_SQL)
1041 .bind(i64::from(maximum_rows))
1042 .execute(&mut *transaction),
1043 )
1044 .await?;
1045 let rows_affected = result.rows_affected();
1046
1047 set_maintenance_server_timeouts(
1048 &mut transaction,
1049 deadline,
1050 "committing expired-window cleanup",
1051 )
1052 .await?;
1053 commit_maintenance(deadline, transaction).await?;
1054 Ok(rows_affected)
1055}
1056
1057async fn maintenance_before_commit<T, F>(
1058 deadline: Instant,
1059 operation: &'static str,
1060 future: F,
1061) -> Result<T, MaintenanceRunError>
1062where
1063 F: Future<Output = Result<T, sqlx::Error>>,
1064{
1065 timeout_at(deadline, future)
1066 .await
1067 .map_err(|_| MaintenanceRunError::ClientTimedOutBeforeCommit { operation })?
1068 .map_err(|error| {
1069 let public = if is_server_timeout(&error) {
1070 MaintenanceError::TimedOutBeforeCommit { operation }
1071 } else {
1072 MaintenanceError::Database(error)
1073 };
1074 MaintenanceRunError::Public(public)
1075 })
1076}
1077
1078async fn set_maintenance_server_timeouts(
1079 transaction: &mut Transaction<'_, Postgres>,
1080 deadline: Instant,
1081 operation: &'static str,
1082) -> Result<(), MaintenanceRunError> {
1083 let (statement_timeout, lock_timeout) = remaining_server_timeout_settings(deadline).ok_or(
1084 MaintenanceRunError::Public(MaintenanceError::TimedOutBeforeCommit { operation }),
1085 )?;
1086 maintenance_before_commit(
1087 deadline,
1088 operation,
1089 sqlx::query(SET_LOCAL_TIMEOUTS_SQL)
1090 .bind(statement_timeout)
1091 .bind(lock_timeout)
1092 .execute(&mut **transaction),
1093 )
1094 .await
1095 .map(|_| ())
1096}
1097
1098async fn commit_maintenance(
1099 deadline: Instant,
1100 transaction: Transaction<'_, Postgres>,
1101) -> Result<(), MaintenanceRunError> {
1102 if Instant::now() >= deadline {
1103 return Err(MaintenanceRunError::Public(
1104 MaintenanceError::TimedOutBeforeCommit {
1105 operation: "starting expired-window cleanup commit",
1106 },
1107 ));
1108 }
1109
1110 timeout_at(deadline, transaction.commit())
1111 .await
1112 .map_err(|_| MaintenanceRunError::ClientCommitTimedOut)?
1113 .map_err(|error| MaintenanceRunError::Public(MaintenanceError::CommitOutcomeUnknown(error)))
1114}
1115
1116fn authoritative_elapsed(start: DateTime<Utc>, end: DateTime<Utc>) -> Duration {
1117 end.signed_duration_since(start)
1118 .to_std()
1119 .unwrap_or(Duration::ZERO)
1120}
1121
1122async fn execute_batch(
1123 transaction: &mut Transaction<'_, Postgres>,
1124 input: &BatchSqlInput,
1125 checks: &[Check<'_>],
1126 deadline: Instant,
1127) -> Result<(PendingBatchOutcome, Duration), CheckRunError> {
1128 let preflight = preflight_batch(transaction, input, checks, deadline).await?;
1129 if let Some((index, denial)) = preflight.denial {
1130 return Ok((
1131 PendingBatchOutcome::Denied { index, denial },
1132 authoritative_elapsed(preflight.database_now, preflight.response_now),
1133 ));
1134 }
1135 let (allowances, response_now) =
1136 upsert_batch(transaction, input, checks, preflight.database_now, deadline).await?;
1137 Ok((
1138 PendingBatchOutcome::Allowed(allowances),
1139 authoritative_elapsed(preflight.database_now, response_now),
1140 ))
1141}
1142
1143async fn preflight_batch(
1144 transaction: &mut Transaction<'_, Postgres>,
1145 input: &BatchSqlInput,
1146 checks: &[Check<'_>],
1147 deadline: Instant,
1148) -> Result<BatchPreflight, CheckRunError> {
1149 set_check_server_timeouts(transaction, deadline, "preflighting counter batch").await?;
1150 let preflight_row = check_before_commit(
1151 deadline,
1152 "preflighting counter batch",
1153 sqlx::query(BATCH_PREFLIGHT_SQL)
1154 .bind(input.fingerprints.as_slice())
1155 .bind(input.subjects.as_slice())
1156 .bind(input.costs.as_slice())
1157 .bind(input.limits.as_slice())
1158 .fetch_one(&mut **transaction),
1159 )
1160 .await?;
1161
1162 let database_now: DateTime<Utc> = preflight_row
1163 .try_get("database_now")
1164 .map_err(CheckError::DefinitelyNotConsumed)?;
1165 let preflight_response_now: DateTime<Utc> = preflight_row
1166 .try_get("response_now")
1167 .map_err(CheckError::DefinitelyNotConsumed)?;
1168 let denied_index: Option<i64> = preflight_row
1169 .try_get("input_index")
1170 .map_err(CheckError::DefinitelyNotConsumed)?;
1171 let denied_expiry: Option<DateTime<Utc>> = preflight_row
1172 .try_get("window_expires_at")
1173 .map_err(CheckError::DefinitelyNotConsumed)?;
1174
1175 let denial = match (denied_index, denied_expiry) {
1176 (Some(input_index), Some(expires_at)) => {
1177 let input_index = usize::try_from(input_index).map_err(|_| {
1178 CheckError::StorageInvariant("batch preflight returned an invalid input index")
1179 })?;
1180 let check = checks.get(input_index).ok_or(CheckError::StorageInvariant(
1181 "batch preflight returned an out-of-range input index",
1182 ))?;
1183 Some((
1184 input_index,
1185 PendingDenial {
1186 limit: check.policy().limit(),
1187 retry_from_sample: duration_until(expires_at, database_now)?,
1188 },
1189 ))
1190 }
1191 (None, None) => None,
1192 _ => {
1193 return Err(CheckError::StorageInvariant(
1194 "batch preflight returned an incomplete denial",
1195 )
1196 .into());
1197 }
1198 };
1199
1200 Ok(BatchPreflight {
1201 database_now,
1202 response_now: preflight_response_now,
1203 denial,
1204 })
1205}
1206
1207async fn upsert_batch(
1208 transaction: &mut Transaction<'_, Postgres>,
1209 input: &BatchSqlInput,
1210 checks: &[Check<'_>],
1211 database_now: DateTime<Utc>,
1212 deadline: Instant,
1213) -> Result<(Vec<PendingAllowance>, DateTime<Utc>), CheckRunError> {
1214 set_check_server_timeouts(transaction, deadline, "updating counter batch").await?;
1215 let rows = check_before_commit(
1216 deadline,
1217 "updating counter batch",
1218 sqlx::query(BATCH_UPSERT_SQL)
1219 .bind(input.policy_ids.as_slice())
1220 .bind(input.scope_ids.as_slice())
1221 .bind(input.fingerprints.as_slice())
1222 .bind(input.subjects.as_slice())
1223 .bind(input.windows.as_slice())
1224 .bind(input.costs.as_slice())
1225 .bind(input.limits.as_slice())
1226 .bind(database_now)
1227 .fetch_all(&mut **transaction),
1228 )
1229 .await?;
1230
1231 if rows.is_empty() {
1232 return Err(
1233 CheckError::StorageInvariant("allowed batch update returned no decisions").into(),
1234 );
1235 }
1236 let response_now: DateTime<Utc> = rows[0]
1237 .try_get("response_now")
1238 .map_err(CheckError::DefinitelyNotConsumed)?;
1239
1240 let mut allowances = Vec::with_capacity(checks.len());
1241 for (output_position, row) in rows.iter().enumerate() {
1242 let input_index: i64 = row
1243 .try_get("input_index")
1244 .map_err(CheckError::DefinitelyNotConsumed)?;
1245 let input_index = usize::try_from(input_index).map_err(|_| {
1246 CheckError::StorageInvariant("batch evaluation returned an invalid input index")
1247 })?;
1248 let Some(check) = checks.get(input_index) else {
1249 return Err(CheckError::StorageInvariant(
1250 "batch evaluation returned an out-of-range input index",
1251 )
1252 .into());
1253 };
1254 let expires_at = read_expiry(row)?;
1255 let remaining_from_sample = duration_until(expires_at, database_now)?;
1256
1257 if input_index != output_position {
1258 return Err(CheckError::StorageInvariant(
1259 "allowed batch decisions were not returned in caller order",
1260 )
1261 .into());
1262 }
1263 let used = read_used(row)?;
1264 let remaining =
1265 check
1266 .policy()
1267 .limit()
1268 .checked_sub(used)
1269 .ok_or(CheckError::StorageInvariant(
1270 "stored usage exceeds its policy limit",
1271 ))?;
1272 allowances.push(PendingAllowance {
1273 limit: check.policy().limit(),
1274 remaining,
1275 reset_from_sample: remaining_from_sample,
1276 });
1277 }
1278
1279 if allowances.len() != checks.len() {
1280 return Err(CheckError::StorageInvariant(
1281 "allowed batch returned an incomplete decision set",
1282 )
1283 .into());
1284 }
1285 Ok((allowances, response_now))
1286}
1287
1288fn database_integer(value: u64) -> i64 {
1289 i64::try_from(value)
1290 .expect("core policy and check validation keep database integers within i64")
1291}
1292
1293fn read_used(row: &PgRow) -> Result<u64, CheckError> {
1294 let used: i64 = row
1295 .try_get("used")
1296 .map_err(CheckError::DefinitelyNotConsumed)?;
1297 u64::try_from(used).map_err(|_| CheckError::StorageInvariant("stored usage is negative"))
1298}
1299
1300fn read_expiry(row: &PgRow) -> Result<DateTime<Utc>, CheckError> {
1301 row.try_get("window_expires_at")
1302 .map_err(CheckError::DefinitelyNotConsumed)
1303}
1304
1305fn duration_until(
1306 expires_at: DateTime<Utc>,
1307 database_now: DateTime<Utc>,
1308) -> Result<Duration, CheckError> {
1309 expires_at
1310 .signed_duration_since(database_now)
1311 .to_std()
1312 .map_err(|_| CheckError::StorageInvariant("stored window is already expired"))
1313}
1314
1315#[cfg(test)]
1316mod tests {
1317 use std::time::Duration;
1318
1319 use super::*;
1320 use runlimit_core::{
1321 FixedWindowPolicy, MAX_LIMIT, MAX_WINDOW, MAX_WINDOW_MILLIS, PolicyId, ScopeId, SubjectKey,
1322 };
1323
1324 fn policy(id: &str, scope: &str) -> FixedWindowPolicy {
1325 FixedWindowPolicy::new(
1326 PolicyId::new(id).expect("valid policy identifier"),
1327 ScopeId::new(scope).expect("valid scope identifier"),
1328 10,
1329 Duration::from_secs(60),
1330 )
1331 .expect("valid policy")
1332 }
1333
1334 fn key(byte: u8) -> SubjectKey {
1335 SubjectKey::from_digest([byte; 32])
1336 }
1337
1338 #[test]
1339 fn portable_core_boundaries_fit_database_values_exactly() {
1340 let policy = FixedWindowPolicy::new(
1341 PolicyId::new("portable.maximum").unwrap(),
1342 ScopeId::new("subject").unwrap(),
1343 MAX_LIMIT,
1344 MAX_WINDOW,
1345 )
1346 .unwrap();
1347 let check = Check::with_cost(&policy, key(1), MAX_LIMIT).unwrap();
1348
1349 assert_eq!(database_integer(policy.limit()), i64::MAX);
1350 assert_eq!(database_integer(check.cost()), i64::MAX);
1351 let interval = PgInterval::try_from(policy.window()).unwrap();
1352 assert_eq!(interval.months, 0);
1353 assert_eq!(interval.days, 0);
1354 assert_eq!(
1355 interval.microseconds,
1356 i64::try_from(MAX_WINDOW_MILLIS * 1_000).unwrap()
1357 );
1358 }
1359
1360 #[test]
1361 fn row_lock_permutation_orders_complete_counter_keys() {
1362 let alpha_client = policy("alpha", "client");
1363 let alpha_identity = policy("alpha", "identity");
1364 let beta_client = policy("beta", "client");
1365 let checks = [
1366 Check::new(&beta_client, key(0)),
1367 Check::new(&alpha_identity, key(0)),
1368 Check::new(&alpha_client, key(1)),
1369 Check::new(&alpha_client, key(0)),
1370 ];
1371
1372 let input = BatchSqlInput::from_checks(&checks);
1373 let locked_keys = input
1374 .lock_input_positions
1375 .iter()
1376 .map(|position| checks[usize::try_from(*position - 1).unwrap()].counter_key())
1377 .collect::<Vec<_>>();
1378
1379 assert!(locked_keys.windows(2).all(|pair| pair[0] < pair[1]));
1380 }
1381
1382 #[test]
1383 fn advisory_lock_protocol_has_a_golden_vector() {
1384 let policy = policy("login", "client");
1385 let check = Check::new(&policy, key(7));
1386
1387 assert_eq!(
1388 advisory_lock_id(check.counter_key()),
1389 4_549_358_434_381_602_087
1390 );
1391 }
1392
1393 #[test]
1394 fn response_metadata_subtracts_only_authoritative_elapsed_time() {
1395 let allowed = PendingAllowance {
1396 limit: 10,
1397 remaining: 4,
1398 reset_from_sample: Duration::from_millis(250),
1399 }
1400 .finish(Duration::from_millis(80));
1401 let denied = PendingDenial {
1402 limit: 10,
1403 retry_from_sample: Duration::from_millis(150),
1404 }
1405 .finish(Duration::from_millis(80));
1406
1407 assert_eq!(allowed.reset_after(), Some(Duration::from_millis(170)));
1408 assert_eq!(denied.retry_after(), Some(Duration::from_millis(70)));
1409 assert_eq!(denied.retry_after_seconds(), Some(1));
1410 }
1411
1412 #[test]
1413 fn configuration_bounds_batches_and_deadlines() {
1414 let defaults = PostgresConfig::new();
1415 assert_eq!(defaults.max_batch_size(), 32);
1416 assert_eq!(defaults.operation_timeout(), Duration::from_secs(3));
1417 assert_eq!(
1418 defaults.with_max_batch_size(0),
1419 Err(PostgresConfigError::ZeroBatchSize)
1420 );
1421 assert_eq!(
1422 defaults.with_operation_timeout(Duration::ZERO),
1423 Err(PostgresConfigError::ZeroOperationTimeout)
1424 );
1425 assert_eq!(
1426 defaults.with_operation_timeout(Duration::from_secs(61)),
1427 Err(PostgresConfigError::OperationTimeoutTooLong {
1428 actual: Duration::from_secs(61),
1429 maximum: Duration::from_secs(60),
1430 })
1431 );
1432 }
1433
1434 #[tokio::test]
1435 async fn duplicate_storage_keys_fail_before_connecting() {
1436 let alpha = policy("alpha", "client");
1437 let beta = policy("beta", "client");
1438 let checks = [
1439 Check::new(&beta, key(7)),
1440 Check::new(&beta, key(7)),
1441 Check::new(&alpha, key(8)),
1442 Check::new(&alpha, key(8)),
1443 ];
1444 let pool = sqlx::postgres::PgPoolOptions::new()
1445 .connect_lazy("postgresql://runlimit:runlimit@127.0.0.1:1/runlimit")
1446 .expect("syntactically valid database URL");
1447 let limiter = PostgresLimiter::new(pool);
1448
1449 let error = limiter
1450 .check_all(&checks)
1451 .await
1452 .expect_err("duplicate keys must be rejected");
1453
1454 assert!(matches!(
1455 error,
1456 CheckError::InvalidBatch(BatchError::DuplicateKey {
1457 first_index: 0,
1458 duplicate_index: 1,
1459 })
1460 ));
1461 }
1462
1463 #[tokio::test]
1464 async fn oversized_batch_fails_before_connecting() {
1465 let first_policy = policy("login", "client");
1466 let second_policy = policy("signup", "client");
1467 let checks = [
1468 Check::new(&first_policy, key(1)),
1469 Check::new(&second_policy, key(2)),
1470 ];
1471 let pool = sqlx::postgres::PgPoolOptions::new()
1472 .connect_lazy("postgresql://runlimit:runlimit@127.0.0.1:1/runlimit")
1473 .expect("syntactically valid database URL");
1474 let config = PostgresConfig::new()
1475 .with_max_batch_size(1)
1476 .expect("one is a valid batch limit");
1477 let limiter = PostgresLimiter::with_config(pool, config);
1478
1479 let error = limiter
1480 .check_all(&checks)
1481 .await
1482 .expect_err("oversized batch must be rejected");
1483
1484 assert!(matches!(
1485 error,
1486 CheckError::InvalidBatch(BatchError::BatchTooLarge {
1487 actual: 2,
1488 maximum: 1,
1489 })
1490 ));
1491 }
1492
1493 #[test]
1494 fn error_reports_consumption_uncertainty() {
1495 let definite = CheckError::DefinitelyNotConsumed(sqlx::Error::RowNotFound);
1496 let unknown = CheckError::CommitOutcomeUnknown(sqlx::Error::RowNotFound);
1497 let rolled_back_invariant = CheckError::ResponseInvariant;
1498 let committed_invariant = CheckError::CommittedResponseInvariant;
1499 let cleanup_definite = MaintenanceError::Database(sqlx::Error::RowNotFound);
1500 let cleanup_unknown = MaintenanceError::CommitOutcomeUnknown(sqlx::Error::RowNotFound);
1501
1502 assert!(!definite.may_have_consumed_quota());
1503 assert!(unknown.may_have_consumed_quota());
1504 assert!(CheckError::CommitTimedOut.may_have_consumed_quota());
1505 assert!(!rolled_back_invariant.may_have_consumed_quota());
1506 assert!(committed_invariant.may_have_consumed_quota());
1507 assert!(!cleanup_definite.may_have_removed_rows());
1508 assert!(cleanup_unknown.may_have_removed_rows());
1509 assert!(MaintenanceError::CommitTimedOut.may_have_removed_rows());
1510 }
1511
1512 #[test]
1513 fn malformed_single_responses_preserve_consumption_status() {
1514 let denial = Denial::QuotaExceeded {
1515 limit: 1,
1516 retry_after: Duration::from_secs(1),
1517 };
1518
1519 let committed = single_decision_from_batch(BatchDecision::Allowed(Vec::new()))
1520 .expect_err("malformed allowed response follows a confirmed commit");
1521 let rolled_back = single_decision_from_batch(BatchDecision::Denied { index: 1, denial })
1522 .expect_err("malformed denied response follows a rollback");
1523
1524 assert!(matches!(committed, CheckError::CommittedResponseInvariant));
1525 assert!(committed.may_have_consumed_quota());
1526 assert!(matches!(rolled_back, CheckError::ResponseInvariant));
1527 assert!(!rolled_back.may_have_consumed_quota());
1528 }
1529
1530 #[test]
1531 fn cleanup_query_is_bounded_and_nonblocking() {
1532 assert_eq!(CLEANUP_SQL.matches("clock_timestamp()").count(), 1);
1533 assert!(CLEANUP_SQL.contains("AS MATERIALIZED"));
1534 assert!(CLEANUP_SQL.contains("SELECT sampled_at\n FROM authoritative_time"));
1535 assert!(!CLEANUP_SQL.contains("window_expires_at <= clock_timestamp()"));
1536 assert!(CLEANUP_SQL.contains("LIMIT $1"));
1537 assert!(CLEANUP_SQL.contains("FOR UPDATE SKIP LOCKED"));
1538 }
1539}