1use std::{
88 fmt,
89 future::Future,
90 sync::Arc,
91 time::{Duration, Instant as WallClockInstant},
92};
93
94use runlimit_core::{
95 AdmissionObservation, AdmissionOperation, AdmissionOutcome, BatchDecision, BatchError, Check,
96 CleanupObservation, ConsumptionStatus, CounterKey, Decision, Denial, FixedWindowPolicy,
97 Limiter, Observation, Observer, QuotaMode, observe_safely, validate_batch,
98};
99use sha2::{Digest, Sha256};
100use sqlx::{
101 Acquire, PgPool, Postgres, Row, Transaction,
102 migrate::{MigrateError, Migrator},
103 pool::PoolConnection,
104 postgres::{PgRow, types::PgInterval},
105 types::chrono::{DateTime, Utc},
106};
107use thiserror::Error;
108use tokio::time::{Instant, timeout, timeout_at};
109
110pub const CREATE_RUNLIMIT_FIXED_WINDOWS_SQL: &str =
118 include_str!("../migrations/20260723000000_create_runlimit_fixed_windows.sql");
119
120pub const SET_RUNLIMIT_FIXED_WINDOWS_FILLFACTOR_SQL: &str =
128 include_str!("../migrations/20260725000000_set_runlimit_fixed_windows_fillfactor.sql");
129
130pub const BOUND_RUNLIMIT_FIXED_WINDOW_CARDINALITY_SQL: &str =
137 include_str!("../migrations/20260726000000_bound_runlimit_fixed_window_cardinality.sql");
138
139pub static MIGRATOR: Migrator = sqlx::migrate!("./migrations");
147
148const ADVISORY_LOCK_DOMAIN: &[u8] = b"runlimit/postgres-advisory-lock/v1\0";
149const SERVER_TIMEOUT_GRACE: Duration = Duration::from_millis(25);
150pub const CAPACITY_SHARD_COUNT: usize = 256;
155
156pub const HARD_MAX_ROWS_PER_SHARD: u32 = 65_536;
162
163const SET_LOCAL_TIMEOUTS_SQL: &str = r"
164SELECT
165 set_config('statement_timeout', $1, true),
166 set_config('lock_timeout', $2, true)
167";
168
169const BATCH_ADVISORY_LOCK_SQL: &str = r"
170WITH RECURSIVE acquired(position, locked) AS (
171 SELECT 1, pg_advisory_xact_lock($1[1])
172 WHERE cardinality($1::BIGINT[]) > 0
173
174 UNION ALL
175
176 SELECT
177 acquired.position + 1,
178 pg_advisory_xact_lock($1[acquired.position + 1])
179 FROM acquired
180 WHERE acquired.position < cardinality($1::BIGINT[])
181)
182SELECT count(*)
183FROM acquired
184";
185
186const BATCH_ROW_LOCK_SQL: &str = r"
187WITH input_keys AS (
188 SELECT *
189 FROM unnest(
190 $1::BYTEA[],
191 $2::BYTEA[]
192 ) WITH ORDINALITY AS keys(
193 config_fingerprint,
194 subject_key,
195 input_position
196 )
197),
198lock_order AS (
199 SELECT *
200 FROM unnest($3::BIGINT[])
201 WITH ORDINALITY AS positions(input_position, lock_position)
202),
203ordered_keys AS (
204 SELECT
205 input_keys.config_fingerprint,
206 input_keys.subject_key,
207 lock_order.lock_position
208 FROM input_keys
209 INNER JOIN lock_order
210 ON lock_order.input_position = input_keys.input_position
211 ORDER BY lock_order.lock_position
212)
213SELECT 1
214FROM ordered_keys
215INNER JOIN runlimit_fixed_windows AS windows
216 ON windows.config_fingerprint = ordered_keys.config_fingerprint
217 AND windows.subject_key = ordered_keys.subject_key
218ORDER BY ordered_keys.lock_position
219FOR UPDATE OF windows
220";
221
222const BATCH_CAPACITY_LOCK_SQL: &str = r"
223WITH input_keys AS (
224 SELECT *
225 FROM unnest(
226 $1::BYTEA[],
227 $2::BYTEA[],
228 $3::SMALLINT[]
229 ) WITH ORDINALITY AS keys(
230 config_fingerprint,
231 subject_key,
232 capacity_shard,
233 input_position
234 )
235),
236missing_keys AS MATERIALIZED (
237 SELECT
238 input_keys.capacity_shard,
239 input_keys.input_position
240 FROM input_keys
241 LEFT JOIN runlimit_fixed_windows AS windows
242 ON windows.config_fingerprint = input_keys.config_fingerprint
243 AND windows.subject_key = input_keys.subject_key
244 WHERE windows.config_fingerprint IS NULL
245),
246target_shards AS (
247 SELECT DISTINCT capacity_shard
248 FROM missing_keys
249),
250locked_shards AS MATERIALIZED (
251 SELECT
252 capacity.capacity_shard,
253 capacity.row_count
254 FROM runlimit_capacity_shards AS capacity
255 INNER JOIN target_shards
256 ON target_shards.capacity_shard = capacity.capacity_shard
257 ORDER BY capacity.capacity_shard
258 FOR UPDATE OF capacity
259),
260lock_barrier AS MATERIALIZED (
261 SELECT count(*) AS locked_count
262 FROM locked_shards
263)
264SELECT
265 missing_keys.input_position - 1 AS input_index,
266 missing_keys.capacity_shard,
267 locked_shards.row_count
268FROM missing_keys
269LEFT JOIN locked_shards
270 ON locked_shards.capacity_shard = missing_keys.capacity_shard
271CROSS JOIN lock_barrier
272ORDER BY missing_keys.input_position
273";
274
275const BATCH_PREFLIGHT_SQL: &str = r"
276WITH input AS (
277 SELECT *
278 FROM unnest(
279 $1::BYTEA[],
280 $2::BYTEA[],
281 $3::BIGINT[],
282 $4::BIGINT[]
283 ) WITH ORDINALITY AS checks(
284 config_fingerprint,
285 subject_key,
286 cost,
287 quota_limit,
288 input_position
289 )
290),
291sample AS (
292 SELECT pg_catalog.clock_timestamp() AS database_now
293),
294first_denial AS (
295 SELECT
296 input.input_position - 1 AS input_index,
297 windows.window_expires_at
298 FROM input
299 CROSS JOIN sample
300 INNER JOIN runlimit_fixed_windows AS windows
301 ON windows.config_fingerprint = input.config_fingerprint
302 AND windows.subject_key = input.subject_key
303 WHERE
304 windows.window_expires_at > sample.database_now
305 AND windows.used > input.quota_limit - input.cost
306 ORDER BY input.input_position
307 LIMIT 1
308)
309SELECT
310 sample.database_now,
311 pg_catalog.clock_timestamp() AS response_now,
312 first_denial.input_index,
313 first_denial.window_expires_at
314FROM sample
315LEFT JOIN first_denial ON TRUE
316";
317
318const BATCH_UPSERT_SQL: &str = r"
319WITH input AS (
320 SELECT *
321 FROM unnest(
322 $1::TEXT[],
323 $2::TEXT[],
324 $3::BYTEA[],
325 $4::BYTEA[],
326 $5::INTERVAL[],
327 $6::BIGINT[],
328 $7::BIGINT[]
329 ) WITH ORDINALITY AS checks(
330 policy_id,
331 scope_id,
332 config_fingerprint,
333 subject_key,
334 window_size,
335 cost,
336 quota_limit,
337 input_position
338 )
339),
340upserted AS (
341 INSERT INTO runlimit_fixed_windows (
342 policy_id,
343 scope_id,
344 config_fingerprint,
345 subject_key,
346 window_started_at,
347 window_expires_at,
348 used
349 )
350 SELECT
351 policy_id,
352 scope_id,
353 config_fingerprint,
354 subject_key,
355 $8,
356 $8 + window_size,
357 cost
358 FROM input
359 ON CONFLICT (config_fingerprint, subject_key)
360 DO UPDATE SET
361 policy_id = EXCLUDED.policy_id,
362 scope_id = EXCLUDED.scope_id,
363 window_started_at = CASE
364 WHEN runlimit_fixed_windows.window_expires_at <= $8 THEN $8
365 ELSE runlimit_fixed_windows.window_started_at
366 END,
367 window_expires_at = CASE
368 WHEN runlimit_fixed_windows.window_expires_at <= $8
369 THEN $8 + (EXCLUDED.window_expires_at - EXCLUDED.window_started_at)
370 ELSE runlimit_fixed_windows.window_expires_at
371 END,
372 used = CASE
373 WHEN runlimit_fixed_windows.window_expires_at <= $8 THEN EXCLUDED.used
374 ELSE runlimit_fixed_windows.used + EXCLUDED.used
375 END
376 WHERE
377 runlimit_fixed_windows.window_expires_at <= $8
378 OR runlimit_fixed_windows.used
379 <= (
380 SELECT quota_limit - cost
381 FROM input
382 WHERE
383 input.config_fingerprint = EXCLUDED.config_fingerprint
384 AND input.subject_key = EXCLUDED.subject_key
385 )
386 RETURNING
387 config_fingerprint,
388 subject_key,
389 used,
390 window_expires_at
391),
392response AS (
393 SELECT pg_catalog.clock_timestamp() AS response_now
394 FROM upserted
395 HAVING count(*) >= 0
396)
397SELECT
398 input.input_position - 1 AS input_index,
399 upserted.used,
400 upserted.window_expires_at,
401 response.response_now
402FROM input
403INNER JOIN upserted
404 ON upserted.config_fingerprint = input.config_fingerprint
405 AND upserted.subject_key = input.subject_key
406CROSS JOIN response
407ORDER BY input.input_position
408";
409
410const CLEANUP_SQL: &str = include_str!("cleanup_expired.sql");
411
412#[derive(Clone, Copy, Debug, Eq, PartialEq)]
420pub struct PostgresConfig {
421 maximum_rows_per_shard: u32,
422 max_batch_size: usize,
423 pool_acquire_timeout: Duration,
424 operation_timeout: Duration,
425}
426
427impl PostgresConfig {
428 pub const DEFAULT_MAXIMUM_ROWS_PER_SHARD: u32 = 4_096;
430
431 pub const DEFAULT_MAX_BATCH_SIZE: usize = 32;
433
434 pub const DEFAULT_POOL_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(3);
436
437 pub const DEFAULT_OPERATION_TIMEOUT: Duration = Duration::from_secs(3);
439
440 pub const MAXIMUM_POOL_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(60);
442
443 pub const MAXIMUM_OPERATION_TIMEOUT: Duration = Duration::from_secs(60);
445
446 pub const fn new() -> Self {
449 Self {
450 maximum_rows_per_shard: Self::DEFAULT_MAXIMUM_ROWS_PER_SHARD,
451 max_batch_size: Self::DEFAULT_MAX_BATCH_SIZE,
452 pool_acquire_timeout: Self::DEFAULT_POOL_ACQUIRE_TIMEOUT,
453 operation_timeout: Self::DEFAULT_OPERATION_TIMEOUT,
454 }
455 }
456
457 pub fn with_maximum_rows_per_shard(
470 mut self,
471 maximum: u32,
472 ) -> Result<Self, PostgresConfigError> {
473 if maximum == 0 {
474 return Err(PostgresConfigError::ZeroMaximumRowsPerShard);
475 }
476 if maximum > HARD_MAX_ROWS_PER_SHARD {
477 return Err(PostgresConfigError::MaximumRowsPerShardTooLarge {
478 actual: maximum,
479 maximum: HARD_MAX_ROWS_PER_SHARD,
480 });
481 }
482 self.maximum_rows_per_shard = maximum;
483 Ok(self)
484 }
485
486 pub fn with_max_batch_size(mut self, maximum: usize) -> Result<Self, PostgresConfigError> {
492 if maximum == 0 {
493 return Err(PostgresConfigError::ZeroBatchSize);
494 }
495 self.max_batch_size = maximum;
496 Ok(self)
497 }
498
499 pub fn with_pool_acquire_timeout(
509 mut self,
510 timeout: Duration,
511 ) -> Result<Self, PostgresConfigError> {
512 if timeout.is_zero() {
513 return Err(PostgresConfigError::ZeroPoolAcquireTimeout);
514 }
515 if timeout > Self::MAXIMUM_POOL_ACQUIRE_TIMEOUT {
516 return Err(PostgresConfigError::PoolAcquireTimeoutTooLong {
517 actual: timeout,
518 maximum: Self::MAXIMUM_POOL_ACQUIRE_TIMEOUT,
519 });
520 }
521 self.pool_acquire_timeout = timeout;
522 Ok(self)
523 }
524
525 pub fn with_operation_timeout(
535 mut self,
536 timeout: Duration,
537 ) -> Result<Self, PostgresConfigError> {
538 if timeout.is_zero() {
539 return Err(PostgresConfigError::ZeroOperationTimeout);
540 }
541 if timeout > Self::MAXIMUM_OPERATION_TIMEOUT {
542 return Err(PostgresConfigError::OperationTimeoutTooLong {
543 actual: timeout,
544 maximum: Self::MAXIMUM_OPERATION_TIMEOUT,
545 });
546 }
547 self.operation_timeout = timeout;
548 Ok(self)
549 }
550
551 pub const fn max_batch_size(&self) -> usize {
553 self.max_batch_size
554 }
555
556 pub const fn maximum_rows_per_shard(self) -> u32 {
558 self.maximum_rows_per_shard
559 }
560
561 pub const fn pool_acquire_timeout(self) -> Duration {
563 self.pool_acquire_timeout
564 }
565
566 pub const fn operation_timeout(self) -> Duration {
568 self.operation_timeout
569 }
570}
571
572impl Default for PostgresConfig {
573 fn default() -> Self {
574 Self::new()
575 }
576}
577
578#[cfg(feature = "serde")]
579#[derive(serde::Serialize)]
580struct PostgresConfigRef {
581 maximum_rows_per_shard: u32,
582 max_batch_size: usize,
583 pool_acquire_timeout: Duration,
584 operation_timeout: Duration,
585}
586
587#[cfg(feature = "serde")]
588#[derive(serde::Deserialize)]
589#[serde(deny_unknown_fields)]
590struct PostgresConfigWire {
591 #[serde(default = "default_postgres_maximum_rows_per_shard")]
592 maximum_rows_per_shard: u32,
593 #[serde(default = "default_postgres_max_batch_size")]
594 max_batch_size: usize,
595 #[serde(default = "default_postgres_pool_acquire_timeout")]
596 pool_acquire_timeout: Duration,
597 #[serde(default = "default_postgres_operation_timeout")]
598 operation_timeout: Duration,
599}
600
601#[cfg(feature = "serde")]
602const fn default_postgres_maximum_rows_per_shard() -> u32 {
603 PostgresConfig::DEFAULT_MAXIMUM_ROWS_PER_SHARD
604}
605
606#[cfg(feature = "serde")]
607const fn default_postgres_max_batch_size() -> usize {
608 PostgresConfig::DEFAULT_MAX_BATCH_SIZE
609}
610
611#[cfg(feature = "serde")]
612const fn default_postgres_pool_acquire_timeout() -> Duration {
613 PostgresConfig::DEFAULT_POOL_ACQUIRE_TIMEOUT
614}
615
616#[cfg(feature = "serde")]
617const fn default_postgres_operation_timeout() -> Duration {
618 PostgresConfig::DEFAULT_OPERATION_TIMEOUT
619}
620
621#[cfg(feature = "serde")]
622impl serde::Serialize for PostgresConfig {
623 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
624 where
625 S: serde::Serializer,
626 {
627 serde::Serialize::serialize(
628 &PostgresConfigRef {
629 maximum_rows_per_shard: self.maximum_rows_per_shard(),
630 max_batch_size: self.max_batch_size(),
631 pool_acquire_timeout: self.pool_acquire_timeout(),
632 operation_timeout: self.operation_timeout(),
633 },
634 serializer,
635 )
636 }
637}
638
639#[cfg(feature = "serde")]
640impl<'de> serde::Deserialize<'de> for PostgresConfig {
641 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
642 where
643 D: serde::Deserializer<'de>,
644 {
645 let wire = <PostgresConfigWire as serde::Deserialize>::deserialize(deserializer)?;
646 Self::new()
647 .with_maximum_rows_per_shard(wire.maximum_rows_per_shard)
648 .and_then(|config| config.with_max_batch_size(wire.max_batch_size))
649 .and_then(|config| config.with_pool_acquire_timeout(wire.pool_acquire_timeout))
650 .and_then(|config| config.with_operation_timeout(wire.operation_timeout))
651 .map_err(serde::de::Error::custom)
652 }
653}
654
655#[derive(Clone, Copy, Debug, Error, Eq, PartialEq)]
657#[non_exhaustive]
658pub enum PostgresConfigError {
659 #[error("maximum_rows_per_shard must be greater than zero")]
661 ZeroMaximumRowsPerShard,
662 #[error("maximum_rows_per_shard ({actual}) exceeds database-enforced maximum ({maximum})")]
665 MaximumRowsPerShardTooLarge {
666 actual: u32,
668 maximum: u32,
670 },
671 #[error("max_batch_size must be greater than zero")]
673 ZeroBatchSize,
674 #[error("PostgreSQL pool acquisition timeout must be greater than zero")]
676 ZeroPoolAcquireTimeout,
677 #[error("PostgreSQL pool acquisition timeout {actual:?} exceeds maximum {maximum:?}")]
680 PoolAcquireTimeoutTooLong {
681 actual: Duration,
683 maximum: Duration,
685 },
686 #[error("PostgreSQL rate-limit operation timeout must be greater than zero")]
688 ZeroOperationTimeout,
689 #[error("PostgreSQL rate-limit operation timeout {actual:?} exceeds maximum {maximum:?}")]
692 OperationTimeoutTooLong {
693 actual: Duration,
695 maximum: Duration,
697 },
698}
699
700#[derive(Clone)]
705pub struct PostgresLimiter {
706 pool: PgPool,
707 config: PostgresConfig,
708 observer: Option<Arc<dyn Observer>>,
709}
710
711impl fmt::Debug for PostgresLimiter {
712 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
713 formatter
714 .debug_struct("PostgresLimiter")
715 .field("pool", &self.pool)
716 .field("config", &self.config)
717 .field("has_observer", &self.observer.is_some())
718 .finish_non_exhaustive()
719 }
720}
721
722impl PostgresLimiter {
723 pub const fn new(pool: PgPool) -> Self {
728 Self {
729 pool,
730 config: PostgresConfig::new(),
731 observer: None,
732 }
733 }
734
735 pub const fn with_config(pool: PgPool, config: PostgresConfig) -> Self {
737 Self {
738 pool,
739 config,
740 observer: None,
741 }
742 }
743
744 #[must_use]
750 pub fn with_observer(mut self, observer: Arc<dyn Observer>) -> Self {
751 self.observer = Some(observer);
752 self
753 }
754
755 pub const fn pool(&self) -> &PgPool {
757 &self.pool
758 }
759
760 pub const fn config(&self) -> PostgresConfig {
762 self.config
763 }
764
765 pub async fn migrate(&self) -> Result<(), MigrateError> {
783 let connection = self.pool.acquire().await?;
784 let mut guarded_connection = ConnectionCancellationGuard::new(connection);
785 let mut migrator = sqlx::migrate!("./migrations");
786 migrator.set_ignore_missing(true);
787
788 let result = migrator
789 .run_direct(&mut **guarded_connection.connection())
790 .await;
791 if result.is_ok() {
792 guarded_connection.disarm();
793 }
794 result
795 }
796
797 pub async fn check(&self, check: &Check<'_>) -> Result<Decision, CheckError> {
808 let started = WallClockInstant::now();
809 let result = self
810 .check_all_unobserved(std::slice::from_ref(check))
811 .await
812 .and_then(single_decision_from_batch);
813 self.observe_single_admission(check, &result, started.elapsed());
814 result
815 }
816
817 pub async fn check_all(&self, checks: &[Check<'_>]) -> Result<BatchDecision, CheckError> {
844 let started = WallClockInstant::now();
845 let result = self.check_all_unobserved(checks).await;
846 self.observe_batch_admission(checks, &result, started.elapsed());
847 result
848 }
849
850 async fn check_all_unobserved(
851 &self,
852 checks: &[Check<'_>],
853 ) -> Result<BatchDecision, CheckError> {
854 validate_batch(checks, self.config.max_batch_size())?;
855 if checks.is_empty() {
856 return Ok(BatchDecision::Allowed(Vec::new()));
857 }
858
859 let input = BatchSqlInput::from_checks(checks);
863 let connection =
864 acquire_check_connection(&self.pool, self.config.pool_acquire_timeout()).await?;
865 let deadline = Instant::now() + self.config.operation_timeout();
869 let mut guarded_connection = ConnectionCancellationGuard::new(connection);
870 let result = run_check_transaction(
871 guarded_connection.connection(),
872 &input,
873 checks,
874 self.config.maximum_rows_per_shard(),
875 deadline,
876 )
877 .await;
878 match result {
879 Ok(outcome) => {
880 if outcome.connection_reusable {
881 guarded_connection.disarm();
882 }
883 Ok(outcome.decision)
884 }
885 Err(error) => {
886 if !error.client_timeout_won() {
887 guarded_connection.disarm();
888 }
889 Err(error.into_public())
890 }
891 }
892 }
893
894 pub async fn cleanup_expired(&self, maximum_rows: u32) -> Result<u64, MaintenanceError> {
910 let started = WallClockInstant::now();
911 let result = self.cleanup_expired_unobserved(maximum_rows).await;
912 self.observe_cleanup(maximum_rows, &result, started.elapsed());
913 result
914 }
915
916 async fn cleanup_expired_unobserved(&self, maximum_rows: u32) -> Result<u64, MaintenanceError> {
917 if maximum_rows == 0 {
918 return Ok(0);
919 }
920
921 let connection =
922 acquire_maintenance_connection(&self.pool, self.config.pool_acquire_timeout()).await?;
923 let deadline = Instant::now() + self.config.operation_timeout();
924 let mut guarded_connection = ConnectionCancellationGuard::new(connection);
925 let result =
926 run_cleanup_transaction(guarded_connection.connection(), maximum_rows, deadline).await;
927 if !result
928 .as_ref()
929 .is_err_and(MaintenanceRunError::client_timeout_won)
930 {
931 guarded_connection.disarm();
932 }
933 result.map_err(MaintenanceRunError::into_public)
934 }
935
936 fn observe_single_admission(
937 &self,
938 check: &Check<'_>,
939 result: &Result<Decision, CheckError>,
940 elapsed: Duration,
941 ) {
942 let (outcome, consumption) = match result {
943 Ok(decision) => (
944 decision_admission_outcome(decision),
945 decision_consumption(decision),
946 ),
947 Err(error) => (AdmissionOutcome::Failed, check_error_consumption(error)),
948 };
949 self.observe_admission(
950 AdmissionOperation::Check,
951 1,
952 Some(check),
953 outcome,
954 consumption,
955 elapsed,
956 );
957 }
958
959 fn observe_batch_admission(
960 &self,
961 checks: &[Check<'_>],
962 result: &Result<BatchDecision, CheckError>,
963 elapsed: Duration,
964 ) {
965 let relevant_check = match result {
966 Ok(BatchDecision::Allowed(decisions)) if decisions.len() == 1 => checks.first(),
967 Ok(BatchDecision::Denied { index, .. } | BatchDecision::ShadowDenied { index, .. }) => {
968 checks.get(*index)
969 }
970 Err(_) if checks.len() == 1 => checks.first(),
971 _ => None,
972 };
973 let (outcome, consumption) = match result {
974 Ok(decision) => (
975 batch_admission_outcome(decision),
976 batch_consumption(decision, checks.is_empty()),
977 ),
978 Err(error) => (AdmissionOutcome::Failed, check_error_consumption(error)),
979 };
980 self.observe_admission(
981 AdmissionOperation::Batch,
982 checks.len(),
983 relevant_check,
984 outcome,
985 consumption,
986 elapsed,
987 );
988 }
989
990 #[allow(clippy::too_many_arguments)]
991 fn observe_admission(
992 &self,
993 operation: AdmissionOperation,
994 batch_size: usize,
995 relevant_check: Option<&Check<'_>>,
996 outcome: AdmissionOutcome,
997 consumption: ConsumptionStatus,
998 elapsed: Duration,
999 ) {
1000 let Some(observer) = &self.observer else {
1001 return;
1002 };
1003 let (policy_id, scope_id) = relevant_check.map_or((None, None), |check| {
1004 (Some(check.policy().id()), Some(check.policy().scope()))
1005 });
1006 observe_safely(
1007 observer.as_ref(),
1008 &Observation::Admission(AdmissionObservation::new(
1009 operation,
1010 batch_size,
1011 policy_id,
1012 scope_id,
1013 outcome,
1014 consumption,
1015 elapsed,
1016 )),
1017 );
1018 }
1019
1020 fn observe_cleanup(
1021 &self,
1022 requested: u32,
1023 result: &Result<u64, MaintenanceError>,
1024 elapsed: Duration,
1025 ) {
1026 let Some(observer) = &self.observer else {
1027 return;
1028 };
1029 let (removed, consumption) = match result {
1030 Ok(removed) => (Some(*removed), ConsumptionStatus::Consumed),
1031 Err(error) if error.may_have_removed_rows() => {
1032 (None, ConsumptionStatus::PossiblyConsumed)
1033 }
1034 Err(_) => (Some(0), ConsumptionStatus::NotConsumed),
1035 };
1036 observe_safely(
1037 observer.as_ref(),
1038 &Observation::Cleanup(CleanupObservation::new(
1039 usize::try_from(requested).unwrap_or(usize::MAX),
1040 removed,
1041 elapsed,
1042 consumption,
1043 )),
1044 );
1045 }
1046}
1047
1048impl Limiter for PostgresLimiter {
1049 type Policy = FixedWindowPolicy;
1050 type Error = CheckError;
1051
1052 fn check(
1053 &self,
1054 check: &Check<'_>,
1055 ) -> impl Future<Output = Result<Decision, Self::Error>> + Send {
1056 PostgresLimiter::check(self, check)
1057 }
1058
1059 fn check_all(
1060 &self,
1061 checks: &[Check<'_>],
1062 ) -> impl Future<Output = Result<BatchDecision, Self::Error>> + Send {
1063 PostgresLimiter::check_all(self, checks)
1064 }
1065}
1066
1067fn decision_admission_outcome(decision: &Decision) -> AdmissionOutcome {
1068 if !decision.would_deny() {
1069 return AdmissionOutcome::Allowed;
1070 }
1071 if decision.is_shadow_denied() {
1072 return AdmissionOutcome::ShadowDenied;
1073 }
1074 match decision.denial() {
1075 Some(Denial::QuotaExceeded { .. }) => AdmissionOutcome::QuotaDenied,
1076 Some(Denial::StorageCapacity { .. }) => AdmissionOutcome::CapacityDenied,
1077 _ => AdmissionOutcome::Failed,
1078 }
1079}
1080
1081fn decision_consumption(decision: &Decision) -> ConsumptionStatus {
1082 if decision.would_deny() {
1083 ConsumptionStatus::NotConsumed
1084 } else {
1085 ConsumptionStatus::Consumed
1086 }
1087}
1088
1089fn batch_admission_outcome(decision: &BatchDecision) -> AdmissionOutcome {
1090 match decision {
1091 BatchDecision::Allowed(_) => AdmissionOutcome::Allowed,
1092 BatchDecision::ShadowDenied { .. } => AdmissionOutcome::ShadowDenied,
1093 BatchDecision::Denied {
1094 denial: Denial::QuotaExceeded { .. },
1095 ..
1096 } => AdmissionOutcome::QuotaDenied,
1097 BatchDecision::Denied {
1098 denial: Denial::StorageCapacity { .. },
1099 ..
1100 } => AdmissionOutcome::CapacityDenied,
1101 _ => AdmissionOutcome::Failed,
1102 }
1103}
1104
1105fn batch_consumption(decision: &BatchDecision, empty: bool) -> ConsumptionStatus {
1106 if matches!(decision, BatchDecision::Allowed(_)) && !empty {
1107 ConsumptionStatus::Consumed
1108 } else {
1109 ConsumptionStatus::NotConsumed
1110 }
1111}
1112
1113const fn check_error_consumption(error: &CheckError) -> ConsumptionStatus {
1114 match error {
1115 CheckError::CommitOutcomeUnknown(_) | CheckError::CommitTimedOut => {
1116 ConsumptionStatus::PossiblyConsumed
1117 }
1118 CheckError::CommittedResponseInvariant => ConsumptionStatus::Consumed,
1119 _ => ConsumptionStatus::NotConsumed,
1120 }
1121}
1122
1123#[derive(Debug, Error)]
1125#[non_exhaustive]
1126pub enum CheckError {
1127 #[error(transparent)]
1129 InvalidBatch(#[from] BatchError),
1130
1131 #[error("PostgreSQL rate-limit check failed before commit; quota was not consumed")]
1133 DefinitelyNotConsumed(#[source] sqlx::Error),
1134
1135 #[error("PostgreSQL rate-limit commit outcome is unknown; quota may have been consumed")]
1137 CommitOutcomeUnknown(#[source] sqlx::Error),
1138
1139 #[error("PostgreSQL rate-limit check timed out while {operation}; quota was not consumed")]
1142 TimedOutBeforeCommit {
1143 operation: &'static str,
1145 },
1146
1147 #[error("PostgreSQL rate-limit commit timed out; quota may have been consumed")]
1149 CommitTimedOut,
1150
1151 #[error("PostgreSQL rate-limit storage invariant failed: {0}")]
1153 StorageInvariant(&'static str),
1154
1155 #[error(
1157 "PostgreSQL rate-limit single-check response invariant failed after rollback; quota was not consumed"
1158 )]
1159 ResponseInvariant,
1160
1161 #[error(
1163 "PostgreSQL rate-limit single-check response invariant failed after commit; quota was consumed"
1164 )]
1165 CommittedResponseInvariant,
1166}
1167
1168impl CheckError {
1169 pub const fn may_have_consumed_quota(&self) -> bool {
1171 matches!(
1172 self,
1173 Self::CommitOutcomeUnknown(_) | Self::CommitTimedOut | Self::CommittedResponseInvariant
1174 )
1175 }
1176}
1177
1178#[derive(Debug, Error)]
1180#[non_exhaustive]
1181pub enum MaintenanceError {
1182 #[error("PostgreSQL expired-window cleanup failed before commit; no rows were removed")]
1185 Database(#[source] sqlx::Error),
1186 #[error("PostgreSQL expired-window cleanup timed out while {operation}; no rows were removed")]
1189 TimedOutBeforeCommit {
1190 operation: &'static str,
1192 },
1193 #[error(
1195 "PostgreSQL expired-window cleanup commit outcome is unknown; rows may have been removed"
1196 )]
1197 CommitOutcomeUnknown(#[source] sqlx::Error),
1198 #[error("PostgreSQL expired-window cleanup commit timed out; rows may have been removed")]
1200 CommitTimedOut,
1201}
1202
1203impl MaintenanceError {
1204 pub const fn may_have_removed_rows(&self) -> bool {
1206 matches!(self, Self::CommitOutcomeUnknown(_) | Self::CommitTimedOut)
1207 }
1208}
1209
1210fn single_decision_from_batch(batch: BatchDecision) -> Result<Decision, CheckError> {
1211 batch
1212 .try_into_single_decision()
1213 .map_err(|invalid| match invalid {
1214 BatchDecision::Denied { .. } | BatchDecision::ShadowDenied { .. } => {
1215 CheckError::ResponseInvariant
1216 }
1217 _ => CheckError::CommittedResponseInvariant,
1220 })
1221}
1222
1223struct ConnectionCancellationGuard {
1224 connection: PoolConnection<Postgres>,
1225 armed: bool,
1226}
1227
1228impl ConnectionCancellationGuard {
1229 const fn new(connection: PoolConnection<Postgres>) -> Self {
1230 Self {
1231 connection,
1232 armed: true,
1233 }
1234 }
1235
1236 const fn connection(&mut self) -> &mut PoolConnection<Postgres> {
1237 &mut self.connection
1238 }
1239
1240 const fn disarm(&mut self) {
1241 self.armed = false;
1242 }
1243}
1244
1245impl Drop for ConnectionCancellationGuard {
1246 fn drop(&mut self) {
1247 if self.armed {
1248 self.connection.close_on_drop();
1253 }
1254 }
1255}
1256
1257#[derive(Debug)]
1258enum CheckRunError {
1259 Public(CheckError),
1260 ClientTimedOutBeforeCommit { operation: &'static str },
1261 ClientCommitTimedOut,
1262}
1263
1264#[derive(Debug)]
1265struct CheckRunOutcome {
1266 decision: BatchDecision,
1267 connection_reusable: bool,
1268}
1269
1270impl CheckRunError {
1271 const fn client_timeout_won(&self) -> bool {
1272 matches!(
1273 self,
1274 Self::ClientTimedOutBeforeCommit { .. } | Self::ClientCommitTimedOut
1275 )
1276 }
1277
1278 fn into_public(self) -> CheckError {
1279 match self {
1280 Self::Public(error) => error,
1281 Self::ClientTimedOutBeforeCommit { operation } => {
1282 CheckError::TimedOutBeforeCommit { operation }
1283 }
1284 Self::ClientCommitTimedOut => CheckError::CommitTimedOut,
1285 }
1286 }
1287}
1288
1289impl From<CheckError> for CheckRunError {
1290 fn from(error: CheckError) -> Self {
1291 Self::Public(error)
1292 }
1293}
1294
1295#[derive(Debug)]
1296enum MaintenanceRunError {
1297 Public(MaintenanceError),
1298 ClientTimedOutBeforeCommit { operation: &'static str },
1299 ClientCommitTimedOut,
1300}
1301
1302impl MaintenanceRunError {
1303 const fn client_timeout_won(&self) -> bool {
1304 matches!(
1305 self,
1306 Self::ClientTimedOutBeforeCommit { .. } | Self::ClientCommitTimedOut
1307 )
1308 }
1309
1310 fn into_public(self) -> MaintenanceError {
1311 match self {
1312 Self::Public(error) => error,
1313 Self::ClientTimedOutBeforeCommit { operation } => {
1314 MaintenanceError::TimedOutBeforeCommit { operation }
1315 }
1316 Self::ClientCommitTimedOut => MaintenanceError::CommitTimedOut,
1317 }
1318 }
1319}
1320
1321#[derive(Debug)]
1322enum PendingBatchOutcome {
1323 Allowed(Vec<PendingAllowance>),
1324 Denied { index: usize, denial: PendingDenial },
1325}
1326
1327#[derive(Debug)]
1328struct BatchSqlInput {
1329 policy_ids: Vec<String>,
1330 scope_ids: Vec<String>,
1331 fingerprints: Vec<Vec<u8>>,
1332 subjects: Vec<Vec<u8>>,
1333 capacity_shards: Vec<i16>,
1334 lock_input_positions: Vec<i64>,
1335 advisory_lock_ids: Vec<i64>,
1336 windows: Vec<PgInterval>,
1337 costs: Vec<i64>,
1338 limits: Vec<i64>,
1339}
1340
1341impl BatchSqlInput {
1342 fn from_checks(checks: &[Check<'_>]) -> Self {
1343 let counter_keys = checks.iter().map(Check::counter_key).collect::<Vec<_>>();
1344 let mut ordered_indices = (0..checks.len()).collect::<Vec<_>>();
1345 ordered_indices.sort_unstable_by_key(|index| counter_keys[*index]);
1346
1347 let mut input = Self {
1348 policy_ids: Vec::with_capacity(checks.len()),
1349 scope_ids: Vec::with_capacity(checks.len()),
1350 fingerprints: Vec::with_capacity(checks.len()),
1351 subjects: Vec::with_capacity(checks.len()),
1352 capacity_shards: Vec::with_capacity(checks.len()),
1353 lock_input_positions: ordered_indices
1354 .into_iter()
1355 .map(|index| {
1356 i64::try_from(index + 1)
1357 .expect("bounded batch input positions fit PostgreSQL BIGINT")
1358 })
1359 .collect(),
1360 advisory_lock_ids: counter_keys.iter().copied().map(advisory_lock_id).collect(),
1361 windows: Vec::with_capacity(checks.len()),
1362 costs: Vec::with_capacity(checks.len()),
1363 limits: Vec::with_capacity(checks.len()),
1364 };
1365 input.advisory_lock_ids.sort_unstable();
1366 input.advisory_lock_ids.dedup();
1367
1368 for (check, counter_key) in checks.iter().zip(counter_keys) {
1369 let policy = check.policy();
1370 input.policy_ids.push(policy.id().as_str().to_owned());
1371 input.scope_ids.push(policy.scope().as_str().to_owned());
1372 input
1373 .fingerprints
1374 .push(counter_key.fingerprint().as_bytes().to_vec());
1375 input
1376 .subjects
1377 .push(counter_key.subject().as_bytes().to_vec());
1378 input.capacity_shards.push(capacity_shard(counter_key));
1379 input.windows.push(
1380 PgInterval::try_from(policy.window())
1381 .expect("core policy windows fit PostgreSQL INTERVAL exactly"),
1382 );
1383 input.costs.push(database_integer(check.cost()));
1384 input.limits.push(database_integer(policy.limit()));
1385 }
1386 input
1387 }
1388}
1389
1390#[derive(Debug)]
1391struct BatchPreflight {
1392 database_now: DateTime<Utc>,
1393 response_now: DateTime<Utc>,
1394 denial: Option<(usize, PendingDenial)>,
1395}
1396
1397#[derive(Debug)]
1398struct PendingAllowance {
1399 limit: u64,
1400 remaining: u64,
1401 reset_from_sample: Duration,
1402}
1403
1404impl PendingAllowance {
1405 fn finish(self, authoritative_elapsed: Duration) -> Decision {
1406 Decision::allowed(
1407 self.limit,
1408 self.remaining,
1409 self.reset_from_sample.saturating_sub(authoritative_elapsed),
1410 )
1411 }
1412}
1413
1414#[derive(Debug)]
1415enum PendingDenial {
1416 QuotaExceeded {
1417 limit: u64,
1418 retry_from_sample: Duration,
1419 },
1420 StorageCapacity,
1421}
1422
1423impl PendingDenial {
1424 const fn is_quota_exceeded(&self) -> bool {
1425 matches!(self, Self::QuotaExceeded { .. })
1426 }
1427
1428 fn finish(self, authoritative_elapsed: Duration) -> Denial {
1429 match self {
1430 Self::QuotaExceeded {
1431 limit,
1432 retry_from_sample,
1433 } => Denial::QuotaExceeded {
1434 capacity: limit,
1435 retry_after: retry_from_sample.saturating_sub(authoritative_elapsed),
1436 },
1437 Self::StorageCapacity => Denial::StorageCapacity { retry_after: None },
1438 }
1439 }
1440}
1441
1442async fn acquire_check_connection(
1443 pool: &PgPool,
1444 acquire_timeout: Duration,
1445) -> Result<PoolConnection<Postgres>, CheckError> {
1446 timeout(acquire_timeout, pool.acquire())
1447 .await
1448 .map_err(|_| CheckError::TimedOutBeforeCommit {
1449 operation: "acquiring database connection",
1450 })?
1451 .map_err(CheckError::DefinitelyNotConsumed)
1452}
1453
1454async fn run_check_transaction(
1455 connection: &mut PoolConnection<Postgres>,
1456 input: &BatchSqlInput,
1457 checks: &[Check<'_>],
1458 maximum_rows_per_shard: u32,
1459 deadline: Instant,
1460) -> Result<CheckRunOutcome, CheckRunError> {
1461 let mut transaction =
1462 check_before_commit(deadline, "beginning transaction", connection.begin()).await?;
1463 set_check_server_timeouts(
1464 &mut transaction,
1465 deadline,
1466 "configuring transaction timeouts",
1467 )
1468 .await?;
1469
1470 acquire_advisory_locks(&mut transaction, &input.advisory_lock_ids, deadline).await?;
1477
1478 acquire_existing_row_locks(&mut transaction, input, deadline).await?;
1482
1483 let (pending, authoritative_elapsed) = execute_batch(
1484 &mut transaction,
1485 input,
1486 checks,
1487 maximum_rows_per_shard,
1488 deadline,
1489 )
1490 .await?;
1491
1492 match pending {
1493 PendingBatchOutcome::Denied { index, denial } => {
1494 let shadow =
1495 denial.is_quota_exceeded() && checks[0].policy().quota_mode() == QuotaMode::Shadow;
1496 Ok(finish_denied_transaction(
1497 deadline,
1498 index,
1499 denial,
1500 shadow,
1501 authoritative_elapsed,
1502 transaction.rollback(),
1503 )
1504 .await)
1505 }
1506 PendingBatchOutcome::Allowed(allowances) => {
1507 commit_check(deadline, transaction).await?;
1508 Ok(CheckRunOutcome {
1509 decision: BatchDecision::Allowed(
1510 allowances
1511 .into_iter()
1512 .map(|allowance| allowance.finish(authoritative_elapsed))
1513 .collect(),
1514 ),
1515 connection_reusable: true,
1516 })
1517 }
1518 }
1519}
1520
1521async fn finish_denied_transaction<F>(
1522 deadline: Instant,
1523 index: usize,
1524 denial: PendingDenial,
1525 shadow: bool,
1526 authoritative_elapsed: Duration,
1527 rollback: F,
1528) -> CheckRunOutcome
1529where
1530 F: Future<Output = Result<(), sqlx::Error>>,
1531{
1532 let denial = denial.finish(authoritative_elapsed);
1533 let decision = if shadow {
1534 BatchDecision::ShadowDenied { index, denial }
1535 } else {
1536 BatchDecision::Denied { index, denial }
1537 };
1538 let connection_reusable = denied_rollback_succeeded(deadline, rollback).await;
1539 CheckRunOutcome {
1540 decision,
1541 connection_reusable,
1542 }
1543}
1544
1545async fn denied_rollback_succeeded<F>(deadline: Instant, rollback: F) -> bool
1546where
1547 F: Future<Output = Result<(), sqlx::Error>>,
1548{
1549 matches!(timeout_at(deadline, rollback).await, Ok(Ok(())))
1550}
1551
1552async fn check_before_commit<T, F>(
1553 deadline: Instant,
1554 operation: &'static str,
1555 future: F,
1556) -> Result<T, CheckRunError>
1557where
1558 F: Future<Output = Result<T, sqlx::Error>>,
1559{
1560 timeout_at(deadline, future)
1561 .await
1562 .map_err(|_| CheckRunError::ClientTimedOutBeforeCommit { operation })?
1563 .map_err(|error| CheckRunError::Public(map_check_database_error(error, operation)))
1564}
1565
1566async fn commit_check(
1567 deadline: Instant,
1568 transaction: Transaction<'_, Postgres>,
1569) -> Result<(), CheckRunError> {
1570 if Instant::now() >= deadline {
1571 return Err(CheckRunError::Public(CheckError::TimedOutBeforeCommit {
1572 operation: "starting commit",
1573 }));
1574 }
1575
1576 timeout_at(deadline, transaction.commit())
1577 .await
1578 .map_err(|_| CheckRunError::ClientCommitTimedOut)?
1579 .map_err(|error| CheckRunError::Public(CheckError::CommitOutcomeUnknown(error)))
1580}
1581
1582fn map_check_database_error(error: sqlx::Error, operation: &'static str) -> CheckError {
1583 if is_server_timeout(&error) {
1584 CheckError::TimedOutBeforeCommit { operation }
1585 } else {
1586 CheckError::DefinitelyNotConsumed(error)
1587 }
1588}
1589
1590fn is_server_timeout(error: &sqlx::Error) -> bool {
1591 let sqlx::Error::Database(database_error) = error else {
1592 return false;
1593 };
1594 matches!(database_error.code().as_deref(), Some("57014" | "55P03"))
1595}
1596
1597fn remaining_server_timeout_settings(deadline: Instant) -> Option<(String, String)> {
1598 let remaining = deadline.checked_duration_since(Instant::now())?;
1599 let remaining_millis = remaining.as_millis();
1600 let statement_millis = remaining_millis
1601 .saturating_sub(SERVER_TIMEOUT_GRACE.as_millis())
1602 .max(1);
1603 let lock_millis = statement_millis.saturating_sub(1).max(1);
1604 Some((format!("{statement_millis}ms"), format!("{lock_millis}ms")))
1605}
1606
1607async fn set_check_server_timeouts(
1608 transaction: &mut Transaction<'_, Postgres>,
1609 deadline: Instant,
1610 operation: &'static str,
1611) -> Result<(), CheckRunError> {
1612 let (statement_timeout, lock_timeout) = remaining_server_timeout_settings(deadline).ok_or(
1613 CheckRunError::Public(CheckError::TimedOutBeforeCommit { operation }),
1614 )?;
1615 check_before_commit(
1616 deadline,
1617 operation,
1618 sqlx::query(SET_LOCAL_TIMEOUTS_SQL)
1619 .bind(statement_timeout)
1620 .bind(lock_timeout)
1621 .execute(&mut **transaction),
1622 )
1623 .await
1624 .map(|_| ())
1625}
1626
1627async fn acquire_advisory_locks(
1628 transaction: &mut Transaction<'_, Postgres>,
1629 advisory_lock_ids: &[i64],
1630 deadline: Instant,
1631) -> Result<(), CheckRunError> {
1632 check_before_commit(deadline, "acquiring logical key lock", async {
1633 sqlx::query(BATCH_ADVISORY_LOCK_SQL)
1634 .bind(advisory_lock_ids)
1635 .execute(&mut **transaction)
1636 .await
1637 .map(|_| ())
1638 })
1639 .await
1640}
1641
1642fn advisory_lock_id(counter_key: CounterKey) -> i64 {
1643 let mut digest = Sha256::new();
1644 digest.update(ADVISORY_LOCK_DOMAIN);
1645 digest.update(counter_key.to_bytes());
1646 let digest = digest.finalize();
1647 let mut id_bytes = [0_u8; 8];
1648 let id_length = id_bytes.len();
1649 id_bytes.copy_from_slice(&digest[..id_length]);
1650 i64::from_be_bytes(id_bytes)
1651}
1652
1653fn capacity_shard(counter_key: CounterKey) -> i16 {
1654 i16::from(counter_key.fingerprint().as_bytes()[0] ^ counter_key.subject().as_bytes()[0])
1655}
1656
1657async fn acquire_existing_row_locks(
1658 transaction: &mut Transaction<'_, Postgres>,
1659 input: &BatchSqlInput,
1660 deadline: Instant,
1661) -> Result<(), CheckRunError> {
1662 check_before_commit(deadline, "acquiring counter row lock", async {
1663 sqlx::query(BATCH_ROW_LOCK_SQL)
1664 .bind(input.fingerprints.as_slice())
1665 .bind(input.subjects.as_slice())
1666 .bind(input.lock_input_positions.as_slice())
1667 .execute(&mut **transaction)
1668 .await
1669 .map(|_| ())
1670 })
1671 .await
1672}
1673
1674async fn first_capacity_denial(
1675 transaction: &mut Transaction<'_, Postgres>,
1676 input: &BatchSqlInput,
1677 maximum_rows_per_shard: u32,
1678 deadline: Instant,
1679) -> Result<Option<(usize, PendingDenial)>, CheckRunError> {
1680 let rows = check_before_commit(
1681 deadline,
1682 "acquiring capacity shard lock",
1683 sqlx::query(BATCH_CAPACITY_LOCK_SQL)
1684 .bind(input.fingerprints.as_slice())
1685 .bind(input.subjects.as_slice())
1686 .bind(input.capacity_shards.as_slice())
1687 .fetch_all(&mut **transaction),
1688 )
1689 .await?;
1690
1691 let mut pending_insertions = [0_u32; CAPACITY_SHARD_COUNT];
1692 for row in rows {
1693 let input_index: i64 = row
1694 .try_get("input_index")
1695 .map_err(CheckError::DefinitelyNotConsumed)?;
1696 let input_index = usize::try_from(input_index).map_err(|_| {
1697 CheckError::StorageInvariant("capacity preflight returned an invalid input index")
1698 })?;
1699 let expected_shard =
1700 input
1701 .capacity_shards
1702 .get(input_index)
1703 .ok_or(CheckError::StorageInvariant(
1704 "capacity preflight returned an out-of-range input index",
1705 ))?;
1706 let returned_shard: i16 = row
1707 .try_get("capacity_shard")
1708 .map_err(CheckError::DefinitelyNotConsumed)?;
1709 if &returned_shard != expected_shard {
1710 return Err(CheckError::StorageInvariant(
1711 "capacity preflight returned a mismatched shard",
1712 )
1713 .into());
1714 }
1715 let shard_index = usize::try_from(returned_shard).map_err(|_| {
1716 CheckError::StorageInvariant("capacity preflight returned a negative shard")
1717 })?;
1718 let pending =
1719 pending_insertions
1720 .get_mut(shard_index)
1721 .ok_or(CheckError::StorageInvariant(
1722 "capacity preflight returned an out-of-range shard",
1723 ))?;
1724 let stored_rows: Option<i64> = row
1725 .try_get("row_count")
1726 .map_err(CheckError::DefinitelyNotConsumed)?;
1727 let stored_rows = stored_rows.ok_or(CheckError::StorageInvariant(
1728 "capacity shard ledger row is missing",
1729 ))?;
1730 let stored_rows = u64::try_from(stored_rows)
1731 .map_err(|_| CheckError::StorageInvariant("capacity shard ledger count is negative"))?;
1732 let projected_rows = stored_rows
1733 .checked_add(u64::from(*pending))
1734 .and_then(|rows| rows.checked_add(1))
1735 .ok_or(CheckError::StorageInvariant(
1736 "capacity shard ledger count overflowed",
1737 ))?;
1738 if projected_rows > u64::from(maximum_rows_per_shard) {
1739 return Ok(Some((input_index, PendingDenial::StorageCapacity)));
1740 }
1741 *pending += 1;
1742 }
1743
1744 Ok(None)
1745}
1746
1747async fn acquire_maintenance_connection(
1748 pool: &PgPool,
1749 acquire_timeout: Duration,
1750) -> Result<PoolConnection<Postgres>, MaintenanceError> {
1751 timeout(acquire_timeout, pool.acquire())
1752 .await
1753 .map_err(|_| MaintenanceError::TimedOutBeforeCommit {
1754 operation: "acquiring database connection",
1755 })?
1756 .map_err(MaintenanceError::Database)
1757}
1758
1759async fn run_cleanup_transaction(
1760 connection: &mut PoolConnection<Postgres>,
1761 maximum_rows: u32,
1762 deadline: Instant,
1763) -> Result<u64, MaintenanceRunError> {
1764 let mut transaction = maintenance_before_commit(
1765 deadline,
1766 "beginning cleanup transaction",
1767 connection.begin(),
1768 )
1769 .await?;
1770 set_maintenance_server_timeouts(
1771 &mut transaction,
1772 deadline,
1773 "configuring cleanup transaction timeouts",
1774 )
1775 .await?;
1776 let result = maintenance_before_commit(
1777 deadline,
1778 "deleting expired windows",
1779 sqlx::query(CLEANUP_SQL)
1780 .bind(i64::from(maximum_rows))
1781 .execute(&mut *transaction),
1782 )
1783 .await?;
1784 let rows_affected = result.rows_affected();
1785
1786 commit_maintenance(deadline, transaction).await?;
1787 Ok(rows_affected)
1788}
1789
1790async fn maintenance_before_commit<T, F>(
1791 deadline: Instant,
1792 operation: &'static str,
1793 future: F,
1794) -> Result<T, MaintenanceRunError>
1795where
1796 F: Future<Output = Result<T, sqlx::Error>>,
1797{
1798 timeout_at(deadline, future)
1799 .await
1800 .map_err(|_| MaintenanceRunError::ClientTimedOutBeforeCommit { operation })?
1801 .map_err(|error| {
1802 let public = if is_server_timeout(&error) {
1803 MaintenanceError::TimedOutBeforeCommit { operation }
1804 } else {
1805 MaintenanceError::Database(error)
1806 };
1807 MaintenanceRunError::Public(public)
1808 })
1809}
1810
1811async fn set_maintenance_server_timeouts(
1812 transaction: &mut Transaction<'_, Postgres>,
1813 deadline: Instant,
1814 operation: &'static str,
1815) -> Result<(), MaintenanceRunError> {
1816 let (statement_timeout, lock_timeout) = remaining_server_timeout_settings(deadline).ok_or(
1817 MaintenanceRunError::Public(MaintenanceError::TimedOutBeforeCommit { operation }),
1818 )?;
1819 maintenance_before_commit(
1820 deadline,
1821 operation,
1822 sqlx::query(SET_LOCAL_TIMEOUTS_SQL)
1823 .bind(statement_timeout)
1824 .bind(lock_timeout)
1825 .execute(&mut **transaction),
1826 )
1827 .await
1828 .map(|_| ())
1829}
1830
1831async fn commit_maintenance(
1832 deadline: Instant,
1833 transaction: Transaction<'_, Postgres>,
1834) -> Result<(), MaintenanceRunError> {
1835 if Instant::now() >= deadline {
1836 return Err(MaintenanceRunError::Public(
1837 MaintenanceError::TimedOutBeforeCommit {
1838 operation: "starting expired-window cleanup commit",
1839 },
1840 ));
1841 }
1842
1843 timeout_at(deadline, transaction.commit())
1844 .await
1845 .map_err(|_| MaintenanceRunError::ClientCommitTimedOut)?
1846 .map_err(|error| MaintenanceRunError::Public(MaintenanceError::CommitOutcomeUnknown(error)))
1847}
1848
1849fn authoritative_elapsed(start: DateTime<Utc>, end: DateTime<Utc>) -> Duration {
1850 end.signed_duration_since(start)
1851 .to_std()
1852 .unwrap_or(Duration::ZERO)
1853}
1854
1855async fn execute_batch(
1856 transaction: &mut Transaction<'_, Postgres>,
1857 input: &BatchSqlInput,
1858 checks: &[Check<'_>],
1859 maximum_rows_per_shard: u32,
1860 deadline: Instant,
1861) -> Result<(PendingBatchOutcome, Duration), CheckRunError> {
1862 let capacity_denial =
1863 first_capacity_denial(transaction, input, maximum_rows_per_shard, deadline).await?;
1864 let preflight = preflight_batch(transaction, input, checks, deadline).await?;
1865 let first_denial = match (capacity_denial, preflight.denial) {
1866 (Some(capacity), Some(quota)) => Some(if capacity.0 < quota.0 {
1867 capacity
1868 } else {
1869 quota
1870 }),
1871 (Some(capacity), None) => Some(capacity),
1872 (None, Some(quota)) => Some(quota),
1873 (None, None) => None,
1874 };
1875 if let Some((index, denial)) = first_denial {
1876 return Ok((
1877 PendingBatchOutcome::Denied { index, denial },
1878 authoritative_elapsed(preflight.database_now, preflight.response_now),
1879 ));
1880 }
1881 let (allowances, response_now) =
1882 upsert_batch(transaction, input, checks, preflight.database_now, deadline).await?;
1883 Ok((
1884 PendingBatchOutcome::Allowed(allowances),
1885 authoritative_elapsed(preflight.database_now, response_now),
1886 ))
1887}
1888
1889async fn preflight_batch(
1890 transaction: &mut Transaction<'_, Postgres>,
1891 input: &BatchSqlInput,
1892 checks: &[Check<'_>],
1893 deadline: Instant,
1894) -> Result<BatchPreflight, CheckRunError> {
1895 let preflight_row = check_before_commit(
1896 deadline,
1897 "preflighting counter batch",
1898 sqlx::query(BATCH_PREFLIGHT_SQL)
1899 .bind(input.fingerprints.as_slice())
1900 .bind(input.subjects.as_slice())
1901 .bind(input.costs.as_slice())
1902 .bind(input.limits.as_slice())
1903 .fetch_one(&mut **transaction),
1904 )
1905 .await?;
1906
1907 let database_now: DateTime<Utc> = preflight_row
1908 .try_get("database_now")
1909 .map_err(CheckError::DefinitelyNotConsumed)?;
1910 let preflight_response_now: DateTime<Utc> = preflight_row
1911 .try_get("response_now")
1912 .map_err(CheckError::DefinitelyNotConsumed)?;
1913 let denied_index: Option<i64> = preflight_row
1914 .try_get("input_index")
1915 .map_err(CheckError::DefinitelyNotConsumed)?;
1916 let denied_expiry: Option<DateTime<Utc>> = preflight_row
1917 .try_get("window_expires_at")
1918 .map_err(CheckError::DefinitelyNotConsumed)?;
1919
1920 let denial = match (denied_index, denied_expiry) {
1921 (Some(input_index), Some(expires_at)) => {
1922 let input_index = usize::try_from(input_index).map_err(|_| {
1923 CheckError::StorageInvariant("batch preflight returned an invalid input index")
1924 })?;
1925 let check = checks.get(input_index).ok_or(CheckError::StorageInvariant(
1926 "batch preflight returned an out-of-range input index",
1927 ))?;
1928 Some((
1929 input_index,
1930 PendingDenial::QuotaExceeded {
1931 limit: check.policy().limit(),
1932 retry_from_sample: duration_until(expires_at, database_now)?,
1933 },
1934 ))
1935 }
1936 (None, None) => None,
1937 _ => {
1938 return Err(CheckError::StorageInvariant(
1939 "batch preflight returned an incomplete denial",
1940 )
1941 .into());
1942 }
1943 };
1944
1945 Ok(BatchPreflight {
1946 database_now,
1947 response_now: preflight_response_now,
1948 denial,
1949 })
1950}
1951
1952async fn upsert_batch(
1953 transaction: &mut Transaction<'_, Postgres>,
1954 input: &BatchSqlInput,
1955 checks: &[Check<'_>],
1956 database_now: DateTime<Utc>,
1957 deadline: Instant,
1958) -> Result<(Vec<PendingAllowance>, DateTime<Utc>), CheckRunError> {
1959 let rows = check_before_commit(
1960 deadline,
1961 "updating counter batch",
1962 sqlx::query(BATCH_UPSERT_SQL)
1963 .bind(input.policy_ids.as_slice())
1964 .bind(input.scope_ids.as_slice())
1965 .bind(input.fingerprints.as_slice())
1966 .bind(input.subjects.as_slice())
1967 .bind(input.windows.as_slice())
1968 .bind(input.costs.as_slice())
1969 .bind(input.limits.as_slice())
1970 .bind(database_now)
1971 .fetch_all(&mut **transaction),
1972 )
1973 .await?;
1974
1975 if rows.is_empty() {
1976 return Err(
1977 CheckError::StorageInvariant("allowed batch update returned no decisions").into(),
1978 );
1979 }
1980 let response_now: DateTime<Utc> = rows[0]
1981 .try_get("response_now")
1982 .map_err(CheckError::DefinitelyNotConsumed)?;
1983
1984 let mut allowances = Vec::with_capacity(checks.len());
1985 for (output_position, row) in rows.iter().enumerate() {
1986 let input_index: i64 = row
1987 .try_get("input_index")
1988 .map_err(CheckError::DefinitelyNotConsumed)?;
1989 let input_index = usize::try_from(input_index).map_err(|_| {
1990 CheckError::StorageInvariant("batch evaluation returned an invalid input index")
1991 })?;
1992 let Some(check) = checks.get(input_index) else {
1993 return Err(CheckError::StorageInvariant(
1994 "batch evaluation returned an out-of-range input index",
1995 )
1996 .into());
1997 };
1998 let expires_at = read_expiry(row)?;
1999 let remaining_from_sample = duration_until(expires_at, database_now)?;
2000
2001 if input_index != output_position {
2002 return Err(CheckError::StorageInvariant(
2003 "allowed batch decisions were not returned in caller order",
2004 )
2005 .into());
2006 }
2007 let used = read_used(row)?;
2008 let remaining =
2009 check
2010 .policy()
2011 .limit()
2012 .checked_sub(used)
2013 .ok_or(CheckError::StorageInvariant(
2014 "stored usage exceeds its policy limit",
2015 ))?;
2016 allowances.push(PendingAllowance {
2017 limit: check.policy().limit(),
2018 remaining,
2019 reset_from_sample: remaining_from_sample,
2020 });
2021 }
2022
2023 if allowances.len() != checks.len() {
2024 return Err(CheckError::StorageInvariant(
2025 "allowed batch returned an incomplete decision set",
2026 )
2027 .into());
2028 }
2029 Ok((allowances, response_now))
2030}
2031
2032fn database_integer(value: u64) -> i64 {
2033 i64::try_from(value)
2034 .expect("core policy and check validation keep database integers within i64")
2035}
2036
2037fn read_used(row: &PgRow) -> Result<u64, CheckError> {
2038 let used: i64 = row
2039 .try_get("used")
2040 .map_err(CheckError::DefinitelyNotConsumed)?;
2041 u64::try_from(used).map_err(|_| CheckError::StorageInvariant("stored usage is negative"))
2042}
2043
2044fn read_expiry(row: &PgRow) -> Result<DateTime<Utc>, CheckError> {
2045 row.try_get("window_expires_at")
2046 .map_err(CheckError::DefinitelyNotConsumed)
2047}
2048
2049fn duration_until(
2050 expires_at: DateTime<Utc>,
2051 database_now: DateTime<Utc>,
2052) -> Result<Duration, CheckError> {
2053 expires_at
2054 .signed_duration_since(database_now)
2055 .to_std()
2056 .map_err(|_| CheckError::StorageInvariant("stored window is already expired"))
2057}
2058
2059#[cfg(test)]
2060mod tests {
2061 use std::{
2062 sync::{Arc, Mutex},
2063 time::Duration,
2064 };
2065
2066 use super::*;
2067 use runlimit_core::{
2068 FixedWindowPolicy, MAX_LIMIT, MAX_WINDOW, MAX_WINDOW_MILLIS, PolicyId, ScopeId, SubjectKey,
2069 };
2070
2071 fn policy(id: &str, scope: &str) -> FixedWindowPolicy {
2072 FixedWindowPolicy::new(
2073 PolicyId::new(id).expect("valid policy identifier"),
2074 ScopeId::new(scope).expect("valid scope identifier"),
2075 10,
2076 Duration::from_secs(60),
2077 )
2078 .expect("valid policy")
2079 }
2080
2081 fn key(byte: u8) -> SubjectKey {
2082 SubjectKey::from_digest([byte; 32])
2083 }
2084
2085 #[derive(Default)]
2086 struct RecordingObserver {
2087 events: Mutex<Vec<(AdmissionOperation, AdmissionOutcome, ConsumptionStatus)>>,
2088 cleanups: Mutex<Vec<(usize, Option<u64>, ConsumptionStatus)>>,
2089 }
2090
2091 impl Observer for RecordingObserver {
2092 fn observe(&self, observation: &Observation<'_>) {
2093 match observation {
2094 Observation::Admission(admission) => self.events.lock().unwrap().push((
2095 admission.operation(),
2096 admission.outcome(),
2097 admission.consumption(),
2098 )),
2099 Observation::Cleanup(cleanup) => self.cleanups.lock().unwrap().push((
2100 cleanup.requested(),
2101 cleanup.removed(),
2102 cleanup.consumption(),
2103 )),
2104 _ => {}
2105 }
2106 }
2107 }
2108
2109 struct PanickingObserver;
2110
2111 impl Observer for PanickingObserver {
2112 fn observe(&self, _: &Observation<'_>) {
2113 panic!("injected observer panic");
2114 }
2115 }
2116
2117 #[test]
2118 fn portable_core_boundaries_fit_database_values_exactly() {
2119 let policy = FixedWindowPolicy::new(
2120 PolicyId::new("portable.maximum").unwrap(),
2121 ScopeId::new("subject").unwrap(),
2122 MAX_LIMIT,
2123 MAX_WINDOW,
2124 )
2125 .unwrap();
2126 let check = Check::with_cost(&policy, key(1), MAX_LIMIT).unwrap();
2127
2128 assert_eq!(database_integer(policy.limit()), i64::MAX);
2129 assert_eq!(database_integer(check.cost()), i64::MAX);
2130 let interval = PgInterval::try_from(policy.window()).unwrap();
2131 assert_eq!(interval.months, 0);
2132 assert_eq!(interval.days, 0);
2133 assert_eq!(
2134 interval.microseconds,
2135 i64::try_from(MAX_WINDOW_MILLIS * 1_000).unwrap()
2136 );
2137 }
2138
2139 #[test]
2140 fn row_lock_permutation_orders_complete_counter_keys() {
2141 let alpha_client = policy("alpha", "client");
2142 let alpha_identity = policy("alpha", "identity");
2143 let beta_client = policy("beta", "client");
2144 let checks = [
2145 Check::new(&beta_client, key(0)),
2146 Check::new(&alpha_identity, key(0)),
2147 Check::new(&alpha_client, key(1)),
2148 Check::new(&alpha_client, key(0)),
2149 ];
2150
2151 let input = BatchSqlInput::from_checks(&checks);
2152 let locked_keys = input
2153 .lock_input_positions
2154 .iter()
2155 .map(|position| checks[usize::try_from(*position - 1).unwrap()].counter_key())
2156 .collect::<Vec<_>>();
2157
2158 assert!(locked_keys.windows(2).all(|pair| pair[0] < pair[1]));
2159 }
2160
2161 #[test]
2162 fn advisory_lock_protocol_has_a_golden_vector() {
2163 let policy = policy("login", "client");
2164 let check = Check::new(&policy, key(7));
2165
2166 assert_eq!(
2167 advisory_lock_id(check.counter_key()),
2168 4_549_358_434_381_602_087
2169 );
2170 }
2171
2172 #[test]
2173 fn capacity_shard_protocol_has_a_golden_vector() {
2174 let policy = policy("login", "client");
2175 let check = Check::new(&policy, key(7));
2176
2177 assert_eq!(capacity_shard(check.counter_key()), 141);
2178 }
2179
2180 #[tokio::test]
2181 async fn observer_reports_connection_free_operations_and_isolates_panics() {
2182 let pool = sqlx::postgres::PgPoolOptions::new()
2183 .connect_lazy("postgresql://runlimit:runlimit@127.0.0.1:1/runlimit")
2184 .expect("syntactically valid database URL");
2185 let observer = Arc::new(RecordingObserver::default());
2186 let limiter = PostgresLimiter::new(pool.clone()).with_observer(observer.clone());
2187
2188 assert_eq!(
2189 limiter.check_all(&[]).await.unwrap(),
2190 BatchDecision::Allowed(Vec::new())
2191 );
2192 assert_eq!(limiter.cleanup_expired(0).await.unwrap(), 0);
2193 assert_eq!(
2194 observer.events.lock().unwrap().as_slice(),
2195 [(
2196 AdmissionOperation::Batch,
2197 AdmissionOutcome::Allowed,
2198 ConsumptionStatus::NotConsumed,
2199 )]
2200 );
2201 assert_eq!(
2202 observer.cleanups.lock().unwrap().as_slice(),
2203 [(0, Some(0), ConsumptionStatus::Consumed)]
2204 );
2205
2206 let panicking = PostgresLimiter::new(pool).with_observer(Arc::new(PanickingObserver));
2207 assert_eq!(
2208 panicking.check_all(&[]).await.unwrap(),
2209 BatchDecision::Allowed(Vec::new())
2210 );
2211 assert_eq!(panicking.cleanup_expired(0).await.unwrap(), 0);
2212 }
2213
2214 #[test]
2215 fn response_metadata_subtracts_only_authoritative_elapsed_time() {
2216 let allowed = PendingAllowance {
2217 limit: 10,
2218 remaining: 4,
2219 reset_from_sample: Duration::from_millis(250),
2220 }
2221 .finish(Duration::from_millis(80));
2222 let denied = PendingDenial::QuotaExceeded {
2223 limit: 10,
2224 retry_from_sample: Duration::from_millis(150),
2225 }
2226 .finish(Duration::from_millis(80));
2227
2228 assert_eq!(
2229 allowed.replenishes_after(),
2230 Some(Duration::from_millis(170))
2231 );
2232 assert_eq!(denied.retry_after(), Some(Duration::from_millis(70)));
2233 assert_eq!(denied.retry_after_seconds(), Some(1));
2234 }
2235
2236 #[tokio::test]
2237 async fn denied_decision_survives_rollback_failure_and_discards_connection() {
2238 let outcome = finish_denied_transaction(
2239 Instant::now() + Duration::from_secs(1),
2240 2,
2241 PendingDenial::QuotaExceeded {
2242 limit: 10,
2243 retry_from_sample: Duration::from_millis(150),
2244 },
2245 false,
2246 Duration::from_millis(20),
2247 std::future::ready(Err(sqlx::Error::Protocol(
2248 "injected rollback failure".to_owned(),
2249 ))),
2250 )
2251 .await;
2252
2253 assert!(matches!(
2254 outcome.decision,
2255 BatchDecision::Denied {
2256 index: 2,
2257 denial: Denial::QuotaExceeded {
2258 capacity: 10,
2259 retry_after
2260 }
2261 } if retry_after == Duration::from_millis(130)
2262 ));
2263 assert!(!outcome.connection_reusable);
2264 }
2265
2266 #[tokio::test]
2267 async fn denied_decision_survives_rollback_deadline_and_discards_connection() {
2268 let outcome = finish_denied_transaction(
2269 Instant::now(),
2270 0,
2271 PendingDenial::QuotaExceeded {
2272 limit: 1,
2273 retry_from_sample: Duration::from_millis(50),
2274 },
2275 false,
2276 Duration::ZERO,
2277 std::future::pending(),
2278 )
2279 .await;
2280
2281 assert!(matches!(
2282 outcome.decision,
2283 BatchDecision::Denied {
2284 index: 0,
2285 denial: Denial::QuotaExceeded { capacity: 1, .. }
2286 }
2287 ));
2288 assert!(!outcome.connection_reusable);
2289 }
2290
2291 #[tokio::test]
2292 async fn shadow_quota_denial_is_reported_after_rollback() {
2293 let outcome = finish_denied_transaction(
2294 Instant::now() + Duration::from_secs(1),
2295 0,
2296 PendingDenial::QuotaExceeded {
2297 limit: 1,
2298 retry_from_sample: Duration::from_secs(1),
2299 },
2300 true,
2301 Duration::ZERO,
2302 std::future::ready(Ok(())),
2303 )
2304 .await;
2305
2306 assert!(matches!(
2307 outcome.decision,
2308 BatchDecision::ShadowDenied {
2309 index: 0,
2310 denial: Denial::QuotaExceeded { capacity: 1, .. }
2311 }
2312 ));
2313 assert!(outcome.connection_reusable);
2314 }
2315
2316 #[test]
2317 fn configuration_bounds_batches_and_deadlines() {
2318 let defaults = PostgresConfig::new();
2319 assert_eq!(defaults.maximum_rows_per_shard(), 4_096);
2320 assert_eq!(defaults.max_batch_size(), 32);
2321 assert_eq!(defaults.pool_acquire_timeout(), Duration::from_secs(3));
2322 assert_eq!(defaults.operation_timeout(), Duration::from_secs(3));
2323 assert_eq!(
2324 defaults.with_maximum_rows_per_shard(0),
2325 Err(PostgresConfigError::ZeroMaximumRowsPerShard)
2326 );
2327 assert_eq!(
2328 defaults.with_maximum_rows_per_shard(HARD_MAX_ROWS_PER_SHARD + 1),
2329 Err(PostgresConfigError::MaximumRowsPerShardTooLarge {
2330 actual: HARD_MAX_ROWS_PER_SHARD + 1,
2331 maximum: HARD_MAX_ROWS_PER_SHARD,
2332 })
2333 );
2334 assert_eq!(
2335 defaults.with_max_batch_size(0),
2336 Err(PostgresConfigError::ZeroBatchSize)
2337 );
2338 assert_eq!(
2339 defaults.with_pool_acquire_timeout(Duration::ZERO),
2340 Err(PostgresConfigError::ZeroPoolAcquireTimeout)
2341 );
2342 assert_eq!(
2343 defaults.with_pool_acquire_timeout(Duration::from_secs(61)),
2344 Err(PostgresConfigError::PoolAcquireTimeoutTooLong {
2345 actual: Duration::from_secs(61),
2346 maximum: Duration::from_secs(60),
2347 })
2348 );
2349 assert_eq!(
2350 defaults.with_operation_timeout(Duration::ZERO),
2351 Err(PostgresConfigError::ZeroOperationTimeout)
2352 );
2353 assert_eq!(
2354 defaults.with_operation_timeout(Duration::from_secs(61)),
2355 Err(PostgresConfigError::OperationTimeoutTooLong {
2356 actual: Duration::from_secs(61),
2357 maximum: Duration::from_secs(60),
2358 })
2359 );
2360 }
2361
2362 #[tokio::test]
2363 async fn duplicate_storage_keys_fail_before_connecting() {
2364 let alpha = policy("alpha", "client");
2365 let beta = policy("beta", "client");
2366 let checks = [
2367 Check::new(&beta, key(7)),
2368 Check::new(&beta, key(7)),
2369 Check::new(&alpha, key(8)),
2370 Check::new(&alpha, key(8)),
2371 ];
2372 let pool = sqlx::postgres::PgPoolOptions::new()
2373 .connect_lazy("postgresql://runlimit:runlimit@127.0.0.1:1/runlimit")
2374 .expect("syntactically valid database URL");
2375 let limiter = PostgresLimiter::new(pool);
2376
2377 let error = limiter
2378 .check_all(&checks)
2379 .await
2380 .expect_err("duplicate keys must be rejected");
2381
2382 assert!(matches!(
2383 error,
2384 CheckError::InvalidBatch(BatchError::DuplicateKey {
2385 first_index: 0,
2386 duplicate_index: 1,
2387 })
2388 ));
2389 }
2390
2391 #[tokio::test]
2392 async fn oversized_batch_fails_before_connecting() {
2393 let first_policy = policy("login", "client");
2394 let second_policy = policy("signup", "client");
2395 let checks = [
2396 Check::new(&first_policy, key(1)),
2397 Check::new(&second_policy, key(2)),
2398 ];
2399 let pool = sqlx::postgres::PgPoolOptions::new()
2400 .connect_lazy("postgresql://runlimit:runlimit@127.0.0.1:1/runlimit")
2401 .expect("syntactically valid database URL");
2402 let config = PostgresConfig::new()
2403 .with_max_batch_size(1)
2404 .expect("one is a valid batch limit");
2405 let limiter = PostgresLimiter::with_config(pool, config);
2406
2407 let error = limiter
2408 .check_all(&checks)
2409 .await
2410 .expect_err("oversized batch must be rejected");
2411
2412 assert!(matches!(
2413 error,
2414 CheckError::InvalidBatch(BatchError::BatchTooLarge {
2415 actual: 2,
2416 maximum: 1,
2417 })
2418 ));
2419 }
2420
2421 #[test]
2422 fn error_reports_consumption_uncertainty() {
2423 let definite = CheckError::DefinitelyNotConsumed(sqlx::Error::RowNotFound);
2424 let unknown = CheckError::CommitOutcomeUnknown(sqlx::Error::RowNotFound);
2425 let rolled_back_invariant = CheckError::ResponseInvariant;
2426 let committed_invariant = CheckError::CommittedResponseInvariant;
2427 let cleanup_definite = MaintenanceError::Database(sqlx::Error::RowNotFound);
2428 let cleanup_unknown = MaintenanceError::CommitOutcomeUnknown(sqlx::Error::RowNotFound);
2429
2430 assert!(!definite.may_have_consumed_quota());
2431 assert!(unknown.may_have_consumed_quota());
2432 assert!(CheckError::CommitTimedOut.may_have_consumed_quota());
2433 assert!(!rolled_back_invariant.may_have_consumed_quota());
2434 assert!(committed_invariant.may_have_consumed_quota());
2435 assert_eq!(
2436 check_error_consumption(&definite),
2437 ConsumptionStatus::NotConsumed
2438 );
2439 assert_eq!(
2440 check_error_consumption(&unknown),
2441 ConsumptionStatus::PossiblyConsumed
2442 );
2443 assert_eq!(
2444 check_error_consumption(&CheckError::CommitTimedOut),
2445 ConsumptionStatus::PossiblyConsumed
2446 );
2447 assert_eq!(
2448 check_error_consumption(&rolled_back_invariant),
2449 ConsumptionStatus::NotConsumed
2450 );
2451 assert_eq!(
2452 check_error_consumption(&committed_invariant),
2453 ConsumptionStatus::Consumed
2454 );
2455 assert!(!cleanup_definite.may_have_removed_rows());
2456 assert!(cleanup_unknown.may_have_removed_rows());
2457 assert!(MaintenanceError::CommitTimedOut.may_have_removed_rows());
2458 }
2459
2460 #[test]
2461 fn malformed_single_responses_preserve_consumption_status() {
2462 let denial = Denial::QuotaExceeded {
2463 capacity: 1,
2464 retry_after: Duration::from_secs(1),
2465 };
2466
2467 let committed = single_decision_from_batch(BatchDecision::Allowed(Vec::new()))
2468 .expect_err("malformed allowed response follows a confirmed commit");
2469 let rolled_back = single_decision_from_batch(BatchDecision::Denied { index: 1, denial })
2470 .expect_err("malformed denied response follows a rollback");
2471
2472 assert!(matches!(committed, CheckError::CommittedResponseInvariant));
2473 assert!(committed.may_have_consumed_quota());
2474 assert!(matches!(rolled_back, CheckError::ResponseInvariant));
2475 assert!(!rolled_back.may_have_consumed_quota());
2476 }
2477
2478 #[test]
2479 fn cleanup_query_is_bounded_and_nonblocking() {
2480 assert_eq!(
2481 CLEANUP_SQL.matches("pg_catalog.clock_timestamp()").count(),
2482 1
2483 );
2484 assert!(CLEANUP_SQL.contains("AS MATERIALIZED"));
2485 assert!(CLEANUP_SQL.contains("SELECT sampled_at\n FROM authoritative_time"));
2486 assert!(!CLEANUP_SQL.contains("window_expires_at <= pg_catalog.clock_timestamp()"));
2487 assert!(CLEANUP_SQL.contains("LIMIT $1"));
2488 assert!(CLEANUP_SQL.contains("FOR UPDATE SKIP LOCKED"));
2489 assert!(CLEANUP_SQL.contains("ORDER BY capacity.capacity_shard"));
2490 assert!(CLEANUP_SQL.contains("FOR UPDATE OF capacity"));
2491 }
2492
2493 #[test]
2494 fn every_production_clock_call_is_catalog_qualified() {
2495 for (name, sql) in [
2496 ("batch preflight", BATCH_PREFLIGHT_SQL),
2497 ("batch upsert", BATCH_UPSERT_SQL),
2498 ("cleanup", CLEANUP_SQL),
2499 ] {
2500 let without_qualified_calls = sql.replace("pg_catalog.clock_timestamp()", "");
2501 assert!(
2502 !without_qualified_calls.contains("clock_timestamp()"),
2503 "{name} contains a search_path-resolved clock_timestamp() call"
2504 );
2505 }
2506 }
2507}