use std::{
fmt,
sync::Arc,
time::{Duration, Instant},
};
use runlimit_core::{
AdmissionObservation, BatchDecision, BatchError, CapacityObservation, Check,
CleanupObservation, ConsumptionStatus, CounterKey, Decision, Denial, Limiter, Observation,
Observer, QuotaDenial, observe_safely, validate_batch,
};
use thiserror::Error;
use crate::{
Clock, MemoryStoreConfig, SystemClock,
shards::{
BatchTopology, BoundedShards, CleanupEffect, Shard, ShardEffect, collect_shard_effects,
usize_to_u64,
},
};
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct Entry {
used: u64,
}
impl Shard<Entry> {
fn quota_denial(&self, check: &PreparedCheck, now_millis: u128) -> Option<QuotaDenial> {
let (entry, expires_at_millis) = self.active_entry(check.counter_key, now_millis)?;
let remaining = remaining_quota(check.limit, entry.used);
(check.cost > remaining).then(|| {
QuotaDenial::try_new(
check.limit,
duration_from_millis(expires_at_millis.saturating_sub(now_millis)),
)
.expect("prepared policy limits are validated")
})
}
fn consume(&mut self, check: &PreparedCheck, now_millis: u128) -> Decision {
let expires_at_millis = now_millis + u128::from(check.window_millis);
if let Some((entry, active_expires_at_millis)) =
self.active_entry(check.counter_key, now_millis)
{
let remaining = remaining_quota(check.limit, entry.used);
debug_assert!(
check.cost <= remaining,
"quota denial must be checked before consumption"
);
self.update_value(check.counter_key, |entry| {
entry.used = entry.used.saturating_add(check.cost);
})
.expect("the active entry was present");
return Decision::allowed(
check.limit,
remaining.saturating_sub(check.cost),
duration_from_millis(active_expires_at_millis.saturating_sub(now_millis)),
);
}
self.replace(
check.counter_key,
Entry { used: check.cost },
expires_at_millis,
);
Decision::allowed(
check.limit,
remaining_quota(check.limit, check.cost),
duration_from_millis(expires_at_millis.saturating_sub(now_millis)),
)
}
}
#[derive(Debug)]
struct PreparedCheck {
input_index: usize,
counter_key: CounterKey,
shard_position: usize,
limit: u64,
window_millis: u64,
cost: u64,
}
impl PreparedCheck {
fn new(input_index: usize, check: &Check<'_>, shard_position: usize) -> Self {
Self {
input_index,
counter_key: check.counter_key(),
shard_position,
limit: check.policy().limit(),
window_millis: check.policy().window_millis(),
cost: check.cost(),
}
}
}
#[derive(Debug)]
struct Evaluation<T, E> {
value: T,
effect: E,
}
pub struct MemoryStore<C = SystemClock> {
config: MemoryStoreConfig,
clock: C,
shards: BoundedShards<Entry>,
observer: Option<Arc<dyn Observer>>,
}
impl<C> fmt::Debug for MemoryStore<C> {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
let poisoned_shards = self.shards.poisoned_count();
formatter
.debug_struct("MemoryStore")
.field("config", &self.config)
.field("has_observer", &self.observer.is_some())
.field("poisoned_shards", &poisoned_shards)
.field("shard_state", &Redacted)
.finish_non_exhaustive()
}
}
struct Redacted;
impl fmt::Debug for Redacted {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("[REDACTED]")
}
}
impl MemoryStore<SystemClock> {
pub fn new(config: MemoryStoreConfig) -> Self {
Self::with_clock(config, SystemClock::new())
}
}
impl<C: Clock> MemoryStore<C> {
pub fn with_clock(config: MemoryStoreConfig, clock: C) -> Self {
let shards = BoundedShards::new(&config);
Self {
config,
clock,
shards,
observer: None,
}
}
#[must_use]
pub fn with_observer(mut self, observer: Arc<dyn Observer>) -> Self {
self.observer = Some(observer);
self
}
fn observe_shard_effects(&self, effects: &[ShardEffect]) {
for effect in effects {
self.observe_shard_effect(effect);
}
}
fn observe_shard_effect(&self, effect: &ShardEffect) {
let Some(observer) = &self.observer else {
return;
};
observe_safely(
observer.as_ref(),
&Observation::Cleanup(CleanupObservation::confirmed(
effect.cleanup.requested,
usize_to_u64(effect.cleanup.removed),
effect.cleanup.elapsed,
)),
);
observe_safely(
observer.as_ref(),
&Observation::Capacity(CapacityObservation::new(
usize_to_u64(effect.used),
usize_to_u64(effect.capacity),
Some(effect.shard_index),
)),
);
}
fn observe_admission<'a>(&self, build_observation: impl FnOnce() -> AdmissionObservation<'a>) {
let Some(observer) = &self.observer else {
return;
};
let admission = build_observation();
observe_safely(observer.as_ref(), &Observation::Admission(admission));
}
pub fn check(&self, check: &Check<'_>) -> Result<Decision, MemoryStoreError> {
let started = Instant::now();
let result = self.check_inner(check);
let elapsed = started.elapsed();
match &result {
Ok(evaluation) => {
self.observe_shard_effect(&evaluation.effect);
self.observe_admission(|| {
AdmissionObservation::from_check(check, &evaluation.value, elapsed)
});
}
Err(_) => self.observe_admission(|| {
AdmissionObservation::failed_check(check, ConsumptionStatus::NotConsumed, elapsed)
}),
}
result.map(|evaluation| evaluation.value)
}
fn check_inner(
&self,
check: &Check<'_>,
) -> Result<Evaluation<Decision, ShardEffect>, MemoryStoreError> {
let counter_key = check.counter_key();
let shard_index = self.shards.shard_index(&counter_key);
let prepared = PreparedCheck::new(0, check, 0);
let mut shard = self.shards.lock_shard(shard_index)?;
let observed_millis = self.clock.now().as_millis();
let now_millis = observed_millis.max(shard.latest_observed_millis());
shard.record_observed_millis(now_millis);
let cleanup_requested = self.config.max_expired_removals_per_check();
let cleanup_started = Instant::now();
let cleanup_removed = shard.prune_expired(now_millis, cleanup_requested);
let cleanup_elapsed = cleanup_started.elapsed();
let decision = if let Some(denial) = shard.quota_denial(&prepared, now_millis) {
if check.policy().quota_mode() == runlimit_core::QuotaMode::Shadow {
Decision::shadow_denied(denial)
} else {
Decision::denied(denial)
}
} else if !shard.contains_key(counter_key) && shard.used() >= shard.capacity() {
Decision::denied(Denial::storage_capacity(
shard
.capacity_retry_after_millis(now_millis)
.map(duration_from_millis),
))
} else {
shard.consume(&prepared, now_millis)
};
Ok(Evaluation {
value: decision,
effect: ShardEffect {
shard_index,
cleanup: CleanupEffect {
requested: cleanup_requested,
removed: cleanup_removed,
elapsed: cleanup_elapsed,
},
used: shard.used(),
capacity: shard.capacity(),
},
})
}
#[allow(clippy::too_many_lines)]
pub fn check_all(&self, checks: &[Check<'_>]) -> Result<BatchDecision, MemoryStoreError> {
let started = Instant::now();
let result = self.check_all_inner(checks);
let elapsed = started.elapsed();
match &result {
Ok(evaluation) => {
self.observe_shard_effects(&evaluation.effect);
self.observe_admission(|| {
AdmissionObservation::from_batch(checks, &evaluation.value, elapsed)
});
}
Err(_) => self.observe_admission(|| {
AdmissionObservation::failed_batch(
checks.len(),
ConsumptionStatus::NotConsumed,
elapsed,
)
}),
}
result.map(|evaluation| evaluation.value)
}
#[allow(clippy::too_many_lines)]
fn check_all_inner(
&self,
checks: &[Check<'_>],
) -> Result<Evaluation<BatchDecision, Vec<ShardEffect>>, MemoryStoreError> {
validate_batch(checks, self.config.max_batch_size())?;
if checks.is_empty() {
return Ok(Evaluation {
value: BatchDecision::allowed(Vec::new()),
effect: Vec::new(),
});
}
let topology = BatchTopology::new(
checks
.iter()
.map(|check| self.shards.shard_index(&check.counter_key())),
&self.config,
)?;
let prepared = checks
.iter()
.enumerate()
.map(|(input_index, check)| {
PreparedCheck::new(input_index, check, topology.shard_position(input_index))
})
.collect::<Vec<_>>();
let mut locked_shards = self.shards.lock_topology(&topology)?;
let observed_millis = self.clock.now().as_millis();
let now_millis = locked_shards
.iter()
.map(|(_, shard)| shard.latest_observed_millis())
.fold(observed_millis, u128::max);
let mut cleanup_effects = Vec::with_capacity(locked_shards.len());
for ((_, shard), check_count) in locked_shards.iter_mut().zip(topology.check_counts()) {
shard.record_observed_millis(now_millis);
let cleanup_limit = self
.config
.max_expired_removals_per_check()
.saturating_mul(check_count);
let cleanup_started = Instant::now();
let removed = shard.prune_expired(now_millis, cleanup_limit);
cleanup_effects.push(CleanupEffect {
requested: cleanup_limit,
removed,
elapsed: cleanup_started.elapsed(),
});
}
let mut pending_insertions = vec![0_usize; locked_shards.len()];
for check in &prepared {
let shard = &locked_shards[check.shard_position].1;
if let Some(denial) = shard.quota_denial(check, now_millis) {
let value = if checks[check.input_index].policy().quota_mode()
== runlimit_core::QuotaMode::Shadow
{
BatchDecision::shadow_denied(check.input_index, denial)
} else {
BatchDecision::denied(check.input_index, denial)
};
return Ok(Evaluation {
value,
effect: collect_shard_effects(&locked_shards, &cleanup_effects),
});
}
if !shard.contains_key(check.counter_key) {
let projected_len = shard.used() + pending_insertions[check.shard_position];
if projected_len >= shard.capacity() {
return Ok(Evaluation {
value: BatchDecision::denied(
check.input_index,
Denial::storage_capacity(
shard
.capacity_retry_after_millis(now_millis)
.map(duration_from_millis),
),
),
effect: collect_shard_effects(&locked_shards, &cleanup_effects),
});
}
pending_insertions[check.shard_position] += 1;
}
}
let mut decisions = Vec::with_capacity(prepared.len());
for check in &prepared {
let shard = &mut locked_shards[check.shard_position].1;
decisions.push(shard.consume(check, now_millis));
}
Ok(Evaluation {
value: BatchDecision::allowed(decisions),
effect: collect_shard_effects(&locked_shards, &cleanup_effects),
})
}
pub fn stats(&self) -> Result<MemoryStoreStats, MemoryStoreError> {
self.shards.stats(&self.config)
}
pub fn clear(&self) -> Result<(), MemoryStoreError> {
self.shards.clear()
}
pub fn recover_poisoned(&self) -> usize {
self.shards.recover_poisoned(&self.config)
}
}
fn remaining_quota(limit: u64, used: u64) -> u64 {
debug_assert!(
used <= limit,
"stored quota usage ({used}) exceeded its policy limit ({limit})"
);
limit.saturating_sub(used)
}
impl<C: Clock> Limiter for MemoryStore<C> {
type Policy = runlimit_core::FixedWindowPolicy;
type Error = MemoryStoreError;
fn check(
&self,
check: &Check<'_>,
) -> impl Future<Output = Result<Decision, Self::Error>> + Send {
std::future::ready(MemoryStore::check(self, check))
}
fn check_all(
&self,
checks: &[Check<'_>],
) -> impl Future<Output = Result<BatchDecision, Self::Error>> + Send {
std::future::ready(MemoryStore::check_all(self, checks))
}
}
fn duration_from_millis(millis: u128) -> Duration {
Duration::from_millis(
u64::try_from(millis)
.expect("a counter's remaining duration cannot exceed its portable policy window"),
)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct MemoryStoreStats {
entries: usize,
capacity: usize,
shard_count: usize,
}
impl MemoryStoreStats {
pub(crate) const fn from_parts(entries: usize, capacity: usize, shard_count: usize) -> Self {
Self {
entries,
capacity,
shard_count,
}
}
pub const fn entries(self) -> usize {
self.entries
}
pub const fn capacity(self) -> usize {
self.capacity
}
pub const fn shard_count(self) -> usize {
self.shard_count
}
}
#[cfg(feature = "serde")]
#[derive(serde::Serialize)]
struct MemoryStoreStatsWire {
entries: usize,
capacity: usize,
shard_count: usize,
}
#[cfg(feature = "serde")]
impl serde::Serialize for MemoryStoreStats {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serde::Serialize::serialize(
&MemoryStoreStatsWire {
entries: self.entries(),
capacity: self.capacity(),
shard_count: self.shard_count(),
},
serializer,
)
}
}
#[derive(Clone, Debug, Error, Eq, PartialEq)]
pub enum MemoryStoreError {
#[error(transparent)]
InvalidBatch(#[from] BatchError),
#[error(
"batch targets {key_count} distinct keys at memory-store shard {shard_index}, \
but that shard can store at most {capacity}"
)]
BatchExceedsShardCapacity {
shard_index: usize,
key_count: usize,
capacity: usize,
},
#[error("memory-store shard {shard_index} is poisoned and unavailable")]
PoisonedShard {
shard_index: usize,
},
}
#[cfg(test)]
mod tests {
use std::{
sync::{
Arc, Barrier, Mutex, Weak,
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
mpsc,
},
thread,
time::Duration,
};
use runlimit_core::{
AdmissionOperation, AdmissionOutcome, BatchDecision, BatchError, Check, ConsumptionStatus,
Decision, Denial, DenialKind, FixedWindowPolicy, KeyHasher, MAX_LIMIT, MAX_WINDOW,
MAX_WINDOW_MILLIS, Observation, Observer, PolicyId, QuotaDenial, QuotaMode, ScopeId,
SubjectKey,
};
use super::{Entry, MemoryStore, MemoryStoreError, remaining_quota};
use crate::{Clock, MemoryStoreConfig, shards::counter_key_hash};
fn quota(capacity: u64, retry_after: Duration) -> QuotaDenial {
QuotaDenial::try_new(capacity, retry_after).unwrap()
}
#[derive(Clone, Default)]
struct ManualClock {
now_millis: Arc<AtomicU64>,
}
impl ManualClock {
fn advance(&self, duration: Duration) {
let millis = u64::try_from(duration.as_millis()).unwrap();
self.now_millis.fetch_add(millis, Ordering::Relaxed);
}
fn set(&self, duration: Duration) {
let millis = u64::try_from(duration.as_millis()).unwrap();
self.now_millis.store(millis, Ordering::Relaxed);
}
}
impl Clock for ManualClock {
fn now(&self) -> Duration {
Duration::from_millis(self.now_millis.load(Ordering::Relaxed))
}
}
#[derive(Clone, Default)]
struct PanicOnceClock {
panic_next: Arc<AtomicBool>,
}
impl PanicOnceClock {
fn panic_once(&self) {
self.panic_next.store(true, Ordering::Relaxed);
}
}
impl Clock for PanicOnceClock {
fn now(&self) -> Duration {
assert!(
!self.panic_next.swap(false, Ordering::Relaxed),
"panic from test clock"
);
Duration::ZERO
}
}
#[derive(Clone)]
struct WideManualClock {
now_millis: Arc<Mutex<u128>>,
}
impl WideManualClock {
fn new(now_millis: u128) -> Self {
Self {
now_millis: Arc::new(Mutex::new(now_millis)),
}
}
fn advance(&self, millis: u128) {
let mut now_millis = self.now_millis.lock().unwrap();
*now_millis += millis;
}
}
impl Clock for WideManualClock {
fn now(&self) -> Duration {
let millis = *self.now_millis.lock().unwrap();
let seconds = u64::try_from(millis / 1_000).unwrap();
let subsec_millis = u32::try_from(millis % 1_000).unwrap();
Duration::new(seconds, subsec_millis * 1_000_000)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RecordedObservation {
Admission(AdmissionOutcome, ConsumptionStatus, usize),
Cleanup(usize, Option<u64>),
Capacity(u64, u64, u64),
}
#[derive(Default)]
struct RecordingObserver {
observations: Mutex<Vec<RecordedObservation>>,
}
impl Observer for RecordingObserver {
fn observe(&self, observation: &Observation<'_>) {
let owned = match observation {
Observation::Admission(admission) => RecordedObservation::Admission(
admission.outcome(),
admission.consumption(),
admission.batch_size(),
),
Observation::Cleanup(cleanup) => {
RecordedObservation::Cleanup(cleanup.requested(), cleanup.removed())
}
Observation::Capacity(capacity) => RecordedObservation::Capacity(
capacity.used(),
capacity.capacity(),
capacity.headroom(),
),
_ => return,
};
self.observations.lock().unwrap().push(owned);
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RecordedAdmissionMetadata {
operation: AdmissionOperation,
batch_size: usize,
has_policy_id: bool,
has_scope_id: bool,
has_policy_fingerprint: bool,
outcome: AdmissionOutcome,
consumption: ConsumptionStatus,
}
#[derive(Default)]
struct AdmissionMetadataObserver {
admissions: Mutex<Vec<RecordedAdmissionMetadata>>,
}
impl Observer for AdmissionMetadataObserver {
fn observe(&self, observation: &Observation<'_>) {
let Observation::Admission(admission) = observation else {
return;
};
self.admissions
.lock()
.unwrap()
.push(RecordedAdmissionMetadata {
operation: admission.operation(),
batch_size: admission.batch_size(),
has_policy_id: admission.policy_id().is_some(),
has_scope_id: admission.scope_id().is_some(),
has_policy_fingerprint: admission.policy_fingerprint().is_some(),
outcome: admission.outcome(),
consumption: admission.consumption(),
});
}
}
struct ReentrantObserver {
store: Mutex<Option<Weak<MemoryStore<ManualClock>>>>,
calls: AtomicUsize,
}
impl Observer for ReentrantObserver {
fn observe(&self, _observation: &Observation<'_>) {
self.calls.fetch_add(1, Ordering::Relaxed);
if let Some(store) = self.store.lock().unwrap().as_ref().and_then(Weak::upgrade) {
store.stats().unwrap();
}
}
}
struct PanickingObserver;
impl Observer for PanickingObserver {
fn observe(&self, _observation: &Observation<'_>) {
panic!("observer failure");
}
}
fn policy(policy_id: &str, scope_id: &str, limit: u64, window: Duration) -> FixedWindowPolicy {
FixedWindowPolicy::new(
PolicyId::new(policy_id).unwrap(),
ScopeId::new(scope_id).unwrap(),
limit,
window,
)
.unwrap()
}
fn subject(byte: u8) -> SubjectKey {
SubjectKey::from_digest([byte; 32])
}
fn subjects_for_shard<C: Clock>(
store: &MemoryStore<C>,
policy: &FixedWindowPolicy,
shard_index: usize,
count: usize,
) -> Vec<SubjectKey> {
assert!(shard_index < store.shards.len());
let mut subjects = Vec::with_capacity(count);
for value in 0_u64..10_000 {
let mut digest = [0_u8; 32];
digest[..8].copy_from_slice(&value.to_le_bytes());
let candidate = SubjectKey::from_digest(digest);
let counter_key = Check::new(policy, candidate).counter_key();
if store.shards.shard_index(&counter_key) == shard_index {
subjects.push(candidate);
if subjects.len() == count {
break;
}
}
}
assert_eq!(
subjects.len(),
count,
"could not find enough subjects for shard {shard_index}"
);
subjects
}
#[test]
fn hmac_subjects_are_evenly_and_deterministically_sharded() {
const SHARD_COUNT: usize = 16;
const SUBJECT_COUNT: u64 = 4_096;
let config = MemoryStoreConfig::new(SHARD_COUNT)
.unwrap()
.with_shard_count(SHARD_COUNT)
.unwrap();
let first = MemoryStore::with_clock(config.clone(), ManualClock::default());
let second = MemoryStore::with_clock(config, ManualClock::default());
let policy = policy("auth.login", "client", 2, Duration::from_secs(60));
let key_hasher = KeyHasher::new([0x42; 32]).unwrap();
let mut counts = [0_usize; SHARD_COUNT];
for value in 0_u64..SUBJECT_COUNT {
let subject = key_hasher.hash_for(&policy, value.to_le_bytes());
let key = Check::new(&policy, subject).counter_key();
let first_index = first.shards.shard_index(&key);
assert_eq!(first_index, second.shards.shard_index(&key));
counts[first_index] += 1;
}
assert!(
counts.iter().all(|&count| (176..=336).contains(&count)),
"uniform HMAC output should spread across every shard: {counts:?}"
);
}
#[test]
fn the_entries_map_distinguishes_deliberate_prehash_collisions() {
let first_policy = policy("auth.login", "client", 1, Duration::from_secs(60));
let second_policy = policy("auth.reset", "client", 2, Duration::from_secs(60));
let first_subject = SubjectKey::from_digest([0_u8; 32]);
let first_key = Check::new(&first_policy, first_subject).counter_key();
let zero_second_key =
Check::new(&second_policy, SubjectKey::from_digest([0_u8; 32])).counter_key();
let colliding_subject_word =
counter_key_hash(first_key) ^ counter_key_hash(zero_second_key);
let mut second_digest = [0_u8; 32];
second_digest[..8].copy_from_slice(&colliding_subject_word.to_le_bytes());
let second_subject = SubjectKey::from_digest(second_digest);
let second_key = Check::new(&second_policy, second_subject).counter_key();
assert_ne!(first_key, second_key);
assert_eq!(counter_key_hash(first_key), counter_key_hash(second_key));
let store =
MemoryStore::with_clock(MemoryStoreConfig::new(4).unwrap(), ManualClock::default());
let first_check = Check::new(&first_policy, first_subject);
let second_check = Check::new(&second_policy, second_subject);
assert!(store.check(&first_check).unwrap().is_allowed());
assert!(store.check(&second_check).unwrap().is_allowed());
assert_eq!(store.stats().unwrap().entries(), 2);
assert!(store.check(&first_check).unwrap().is_denied());
assert!(store.check(&second_check).unwrap().is_allowed());
}
#[test]
fn remaining_quota_reaches_zero_without_wrapping() {
assert_eq!(remaining_quota(3, 0), 3);
assert_eq!(remaining_quota(3, 3), 0);
}
#[cfg(debug_assertions)]
#[test]
#[should_panic(expected = "stored quota usage (4) exceeded its policy limit (3)")]
fn corrupt_quota_state_trips_the_debug_invariant() {
let _ = remaining_quota(3, 4);
}
#[cfg(not(debug_assertions))]
#[test]
fn corrupt_quota_state_fails_closed_in_release_builds() {
let policy = policy("auth.login", "client", 3, Duration::from_secs(60));
let check = Check::new(&policy, subject(1));
let key = check.counter_key();
let store =
MemoryStore::with_clock(MemoryStoreConfig::new(1).unwrap(), ManualClock::default());
{
let mut shard = store.shards.shard(0).lock().unwrap();
assert!(shard.replace(key, Entry { used: 4 }, 60_000).is_none());
}
assert_eq!(
store.check(&check),
Ok(Decision::denied(quota(3, Duration::from_secs(60))))
);
}
#[test]
fn debug_output_is_useful_without_exposing_counter_state() {
let store =
MemoryStore::with_clock(MemoryStoreConfig::new(8).unwrap(), ManualClock::default());
let policy = policy("auth.login", "client", 2, Duration::from_secs(60));
store
.check(&Check::new(&policy, subject(0xab)))
.expect("store is available");
let output = format!("{store:?}");
assert!(output.starts_with("MemoryStore { config: MemoryStoreConfig"));
assert!(output.contains("max_keys: 8"));
assert!(output.contains("poisoned_shards: 0"));
assert!(output.contains("shard_state: [REDACTED]"));
assert!(!output.contains("CounterKey"));
assert!(!output.contains("expires_at_millis"));
assert!(!output.contains("used:"));
}
#[test]
fn enforces_quota_and_reports_exact_retry_duration() {
let clock = ManualClock::default();
let store = MemoryStore::with_clock(MemoryStoreConfig::new(8).unwrap(), clock.clone());
let policy = policy("auth.login", "client", 2, Duration::from_secs(60));
let check = Check::new(&policy, subject(1));
let first = store.check(&check).unwrap();
assert!(first.is_allowed());
assert_eq!(first.available(), Some(1));
assert_eq!(first.retry_after(), None);
let second = store.check(&check).unwrap();
assert!(second.is_allowed());
assert_eq!(second.available(), Some(0));
let denied = store.check(&check).unwrap();
assert!(!denied.is_allowed());
assert_eq!(denied.retry_after(), Some(Duration::from_secs(60)));
clock.advance(Duration::from_secs(10));
assert_eq!(
store.check(&check).unwrap().retry_after(),
Some(Duration::from_secs(50))
);
clock.advance(Duration::from_secs(50));
let reset = store.check(&check).unwrap();
assert!(reset.is_allowed());
assert_eq!(reset.available(), Some(1));
assert_eq!(reset.replenishes_after(), Some(Duration::from_secs(60)));
}
#[test]
fn shadow_mode_reports_quota_without_consuming_or_rejecting() {
let clock = ManualClock::default();
let store = MemoryStore::with_clock(MemoryStoreConfig::new(4).unwrap(), clock.clone());
let enforced = policy("auth.login", "client", 1, Duration::from_secs(60));
let shadow = enforced.clone().with_quota_mode(QuotaMode::Shadow);
let subject = subject(7);
assert!(
store
.check(&Check::new(&shadow, subject))
.unwrap()
.is_allowed()
);
let shadow_denial = store.check(&Check::new(&shadow, subject)).unwrap();
assert!(shadow_denial.permits_request());
assert!(shadow_denial.is_shadow_denied());
assert_eq!(
shadow_denial.denial().map(Denial::kind),
Some(DenialKind::QuotaExceeded)
);
let enforced_denial = store.check(&Check::new(&enforced, subject)).unwrap();
assert!(enforced_denial.is_enforced_denial());
assert_eq!(
enforced_denial.denial().map(Denial::kind),
Some(DenialKind::QuotaExceeded)
);
clock.advance(Duration::from_secs(60));
assert!(
store
.check(&Check::new(&enforced, subject))
.unwrap()
.is_allowed()
);
}
#[test]
fn shadow_batches_roll_back_and_capacity_denials_remain_enforced() {
let store =
MemoryStore::with_clock(MemoryStoreConfig::new(3).unwrap(), ManualClock::default());
let shadow = policy("auth.login", "client", 1, Duration::from_secs(60))
.with_quota_mode(QuotaMode::Shadow);
let first = Check::new(&shadow, subject(1));
let second = Check::new(&shadow, subject(2));
assert!(store.check(&first).unwrap().is_allowed());
let result = store.check_all(&[first, second]).unwrap();
assert!(result.is_shadow_denied());
assert_eq!(result.denied_index(), Some(0));
assert!(
store.check(&second).unwrap().is_allowed(),
"a shadow-denied atomic batch must not consume another check"
);
let full_store =
MemoryStore::with_clock(MemoryStoreConfig::new(1).unwrap(), ManualClock::default());
assert!(full_store.check(&first).unwrap().is_allowed());
let capacity = full_store.check(&second).unwrap();
assert!(capacity.is_enforced_denial());
assert_eq!(
capacity.denial().map(Denial::kind),
Some(DenialKind::StorageCapacity)
);
}
#[test]
fn mixed_shadow_and_enforced_batches_fail_before_consumption() {
let store =
MemoryStore::with_clock(MemoryStoreConfig::new(4).unwrap(), ManualClock::default());
let enforced = policy("auth.login", "client", 1, Duration::from_secs(60));
let shadow = policy("auth.reset", "client", 1, Duration::from_secs(60))
.with_quota_mode(QuotaMode::Shadow);
let result = store.check_all(&[
Check::new(&enforced, subject(1)),
Check::new(&shadow, subject(2)),
]);
assert!(matches!(
result,
Err(MemoryStoreError::InvalidBatch(
BatchError::MixedQuotaModes { index: 1, .. }
))
));
assert_eq!(store.stats().unwrap().entries(), 0);
}
#[test]
fn observer_reports_outcomes_cleanup_and_capacity_headroom() {
let observer = Arc::new(RecordingObserver::default());
let store =
MemoryStore::with_clock(MemoryStoreConfig::new(1).unwrap(), ManualClock::default())
.with_observer(observer.clone());
let limited = policy("auth.login", "client", 1, Duration::from_secs(60));
assert!(
store
.check(&Check::new(&limited, subject(1)))
.unwrap()
.is_allowed()
);
assert!(
store
.check(&Check::new(&limited, subject(1)))
.unwrap()
.is_denied()
);
assert!(
store
.check(&Check::new(&limited, subject(2)))
.unwrap()
.is_denied()
);
let observations = observer.observations.lock().unwrap().clone();
assert_eq!(observations.len(), 9);
assert!(observations.contains(&RecordedObservation::Admission(
AdmissionOutcome::Allowed,
ConsumptionStatus::Consumed,
1,
)));
assert!(observations.contains(&RecordedObservation::Admission(
AdmissionOutcome::QuotaDenied,
ConsumptionStatus::NotConsumed,
1,
)));
assert!(observations.contains(&RecordedObservation::Admission(
AdmissionOutcome::CapacityDenied,
ConsumptionStatus::NotConsumed,
1,
)));
assert_eq!(
observations
.iter()
.filter(|event| matches!(event, RecordedObservation::Cleanup(8, Some(0))))
.count(),
3
);
assert_eq!(
observations
.iter()
.filter(|event| matches!(event, RecordedObservation::Capacity(1, 1, 0)))
.count(),
3
);
}
#[test]
fn observers_run_after_unlock_and_panics_are_isolated() {
let observer = Arc::new(ReentrantObserver {
store: Mutex::new(None),
calls: AtomicUsize::new(0),
});
let store = Arc::new(
MemoryStore::with_clock(MemoryStoreConfig::new(1).unwrap(), ManualClock::default())
.with_observer(observer.clone()),
);
*observer.store.lock().unwrap() = Some(Arc::downgrade(&store));
let limited = policy("auth.login", "client", 1, Duration::from_secs(60));
let check = Check::new(&limited, subject(1));
assert!(store.check(&check).unwrap().is_allowed());
assert_eq!(observer.calls.load(Ordering::Relaxed), 3);
let panicking_store =
MemoryStore::with_clock(MemoryStoreConfig::new(1).unwrap(), ManualClock::default())
.with_observer(Arc::new(PanickingObserver));
assert!(panicking_store.check(&check).unwrap().is_allowed());
assert!(panicking_store.check(&check).unwrap().is_denied());
}
#[test]
fn single_check_matches_a_one_element_batch_across_state_transitions() {
#[derive(Clone, Copy)]
struct Step {
now: Duration,
subject: u8,
cost: u64,
expected: Decision,
}
let config = MemoryStoreConfig::new(2)
.unwrap()
.with_shard_count(1)
.unwrap()
.with_max_expired_removals_per_check(1)
.unwrap();
let single_clock = ManualClock::default();
let batch_clock = ManualClock::default();
let single_store = MemoryStore::with_clock(config.clone(), single_clock.clone());
let batch_store = MemoryStore::with_clock(config, batch_clock.clone());
let policy = policy("auth.login", "client", 5, Duration::from_millis(10));
let steps = [
Step {
now: Duration::ZERO,
subject: 1,
cost: 2,
expected: Decision::allowed(5, 3, Duration::from_millis(10)),
},
Step {
now: Duration::from_millis(3),
subject: 1,
cost: 2,
expected: Decision::allowed(5, 1, Duration::from_millis(7)),
},
Step {
now: Duration::from_millis(3),
subject: 1,
cost: 2,
expected: Decision::denied(quota(5, Duration::from_millis(7))),
},
Step {
now: Duration::from_millis(9),
subject: 1,
cost: 1,
expected: Decision::allowed(5, 0, Duration::from_millis(1)),
},
Step {
now: Duration::from_millis(10),
subject: 1,
cost: 3,
expected: Decision::allowed(5, 2, Duration::from_millis(10)),
},
Step {
now: Duration::from_millis(8),
subject: 1,
cost: 1,
expected: Decision::allowed(5, 1, Duration::from_millis(10)),
},
Step {
now: Duration::from_millis(10),
subject: 2,
cost: 1,
expected: Decision::allowed(5, 4, Duration::from_millis(10)),
},
Step {
now: Duration::from_millis(10),
subject: 3,
cost: 1,
expected: Decision::denied(Denial::storage_capacity(Some(Duration::from_millis(
10,
)))),
},
Step {
now: Duration::from_millis(20),
subject: 3,
cost: 1,
expected: Decision::allowed(5, 4, Duration::from_millis(10)),
},
];
for (index, step) in steps.into_iter().enumerate() {
single_clock.set(step.now);
batch_clock.set(step.now);
let check = Check::with_cost(&policy, subject(step.subject), step.cost).unwrap();
let single = single_store.check(&check).unwrap();
let batch = batch_store
.check_all(std::slice::from_ref(&check))
.unwrap()
.try_into_single_decision()
.expect("a one-element batch returns one decision");
assert_eq!(single, batch, "step {index}");
assert_eq!(single, step.expected, "step {index}");
}
assert_eq!(single_store.stats().unwrap(), batch_store.stats().unwrap());
}
#[test]
fn windows_remain_exact_across_the_u64_millisecond_boundary() {
let clock = WideManualClock::new(u128::from(u64::MAX));
let store = MemoryStore::with_clock(MemoryStoreConfig::new(1).unwrap(), clock.clone());
let policy = policy("auth.login", "client", MAX_LIMIT, MAX_WINDOW);
let check = Check::with_cost(&policy, subject(1), MAX_LIMIT).unwrap();
let first = store.check(&check).unwrap();
assert!(first.is_allowed());
assert_eq!(first.available(), Some(0));
assert_eq!(first.replenishes_after(), Some(MAX_WINDOW));
let denied = store.check(&check).unwrap();
assert!(denied.is_denied());
assert_eq!(denied.retry_after(), Some(MAX_WINDOW));
clock.advance(u128::from(MAX_WINDOW_MILLIS - 1));
let nearly_reset = store.check(&check).unwrap();
assert!(nearly_reset.is_denied());
assert_eq!(nearly_reset.retry_after(), Some(Duration::from_millis(1)));
clock.advance(1);
let reset = store.check(&check).unwrap();
assert!(reset.is_allowed());
assert_eq!(reset.replenishes_after(), Some(MAX_WINDOW));
}
#[test]
fn policy_configuration_changes_use_independent_counters() {
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(2)
.unwrap()
.with_shard_count(1)
.unwrap(),
ManualClock::default(),
);
let strict = policy("auth.login", "client", 1, Duration::from_secs(60));
let relaxed = policy("auth.login", "client", 2, Duration::from_secs(60));
let key = subject(1);
assert!(store.check(&Check::new(&strict, key)).unwrap().is_allowed());
assert!(
store
.check(&Check::new(&relaxed, key))
.unwrap()
.is_allowed()
);
assert_eq!(store.stats().unwrap().entries(), 2);
}
#[test]
fn capacity_exhaustion_does_not_evict_an_active_entry() {
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(1)
.unwrap()
.with_shard_count(1)
.unwrap(),
ManualClock::default(),
);
let policy = policy("auth.login", "client", 10, Duration::from_secs(60));
assert!(
store
.check(&Check::new(&policy, subject(1)))
.unwrap()
.is_allowed()
);
let denied = store.check(&Check::new(&policy, subject(2))).unwrap();
assert!(!denied.is_allowed());
assert_eq!(denied.retry_after(), Some(Duration::from_secs(60)));
assert_eq!(store.stats().unwrap().entries(), 1);
}
#[test]
fn default_configuration_can_use_its_entire_capacity() {
let store =
MemoryStore::with_clock(MemoryStoreConfig::new(64).unwrap(), ManualClock::default());
let policy = policy("auth.login", "client", 10, Duration::from_secs(60));
for byte in 0..64 {
assert!(
store
.check(&Check::new(&policy, subject(byte)))
.unwrap()
.is_allowed()
);
}
let denied = store.check(&Check::new(&policy, subject(64))).unwrap();
assert!(denied.is_denied());
assert_eq!(store.stats().unwrap().entries(), 64);
}
#[test]
fn a_cross_shard_batch_preserves_input_order_and_updates_each_shard() {
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(4)
.unwrap()
.with_shard_count(2)
.unwrap(),
ManualClock::default(),
);
let policy = policy("auth.login", "client", 10, Duration::from_secs(60));
let shard_zero = subjects_for_shard(&store, &policy, 0, 1)[0];
let shard_one = subjects_for_shard(&store, &policy, 1, 2);
let checks = [
Check::with_cost(&policy, shard_one[0], 2).unwrap(),
Check::with_cost(&policy, shard_zero, 3).unwrap(),
Check::with_cost(&policy, shard_one[1], 4).unwrap(),
];
assert_eq!(
store.check_all(&checks),
Ok(BatchDecision::allowed(vec![
Decision::allowed(10, 8, Duration::from_secs(60)),
Decision::allowed(10, 7, Duration::from_secs(60)),
Decision::allowed(10, 6, Duration::from_secs(60)),
]))
);
assert_eq!(store.shards.shard(0).lock().unwrap().used(), 1);
assert_eq!(store.shards.shard(1).lock().unwrap().used(), 2);
}
#[test]
fn a_full_shard_denies_a_batch_while_another_shard_has_room() {
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(6)
.unwrap()
.with_shard_count(2)
.unwrap(),
ManualClock::default(),
);
let policy = policy("auth.login", "client", 10, Duration::from_secs(60));
let crowded = subjects_for_shard(&store, &policy, 0, 4);
let other = subjects_for_shard(&store, &policy, 1, 1)[0];
for key in &crowded[..2] {
assert!(
store
.check(&Check::new(&policy, *key))
.unwrap()
.is_allowed()
);
}
let checks = [
Check::new(&policy, other),
Check::new(&policy, crowded[2]),
Check::new(&policy, crowded[3]),
];
assert_eq!(
store.check_all(&checks),
Ok(BatchDecision::denied(
2,
Denial::storage_capacity(Some(Duration::from_secs(60))),
))
);
assert_eq!(store.shards.shard(0).lock().unwrap().used(), 2);
assert_eq!(store.shards.shard(1).lock().unwrap().used(), 0);
assert!(
store
.check(&Check::new(&policy, crowded[2]))
.unwrap()
.is_allowed()
);
assert!(
store
.check(&Check::new(&policy, other))
.unwrap()
.is_allowed()
);
}
#[test]
fn an_unsatisfiable_per_shard_batch_returns_a_structural_error() {
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(4)
.unwrap()
.with_shard_count(2)
.unwrap(),
ManualClock::default(),
);
let policy = policy("auth.login", "client", 2, Duration::from_secs(60));
let keys = subjects_for_shard(&store, &policy, 0, 3);
let other = subjects_for_shard(&store, &policy, 1, 1)[0];
let other_check = Check::new(&policy, other);
assert_eq!(
store.check(&other_check),
Ok(Decision::allowed(2, 1, Duration::from_secs(60)))
);
let checks = [
other_check,
Check::new(&policy, keys[0]),
Check::new(&policy, keys[1]),
Check::new(&policy, keys[2]),
];
assert_eq!(
store.check_all(&checks),
Err(MemoryStoreError::BatchExceedsShardCapacity {
shard_index: 0,
key_count: 3,
capacity: 2,
})
);
assert_eq!(store.stats().unwrap().entries(), 1);
assert_eq!(
store.check(&other_check),
Ok(Decision::allowed(2, 0, Duration::from_secs(60)))
);
}
#[test]
fn expiry_cleanup_is_bounded_and_capacity_remains_hard_bounded() {
let clock = ManualClock::default();
let config = MemoryStoreConfig::new(3)
.unwrap()
.with_shard_count(1)
.unwrap()
.with_max_expired_removals_per_check(1)
.unwrap();
let store = MemoryStore::with_clock(config, clock.clone());
let policy = policy("auth.login", "client", 10, Duration::from_secs(1));
for byte in 1..=3 {
assert!(
store
.check(&Check::new(&policy, subject(byte)))
.unwrap()
.is_allowed()
);
}
clock.advance(Duration::from_secs(1));
assert!(
store
.check(&Check::new(&policy, subject(4)))
.unwrap()
.is_allowed()
);
assert_eq!(store.stats().unwrap().entries(), 3);
assert!(
store
.check(&Check::new(&policy, subject(5)))
.unwrap()
.is_allowed()
);
assert_eq!(store.stats().unwrap().entries(), 3);
}
#[test]
fn a_cross_shard_batch_attributes_cleanup_to_each_checks_target() {
let clock = ManualClock::default();
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(8)
.unwrap()
.with_shard_count(2)
.unwrap()
.with_max_expired_removals_per_check(1)
.unwrap(),
clock.clone(),
);
let policy = policy("auth.login", "client", 10, Duration::from_secs(1));
let shard_zero = subjects_for_shard(&store, &policy, 0, 6);
let shard_one = subjects_for_shard(&store, &policy, 1, 5);
for key in shard_zero[..4].iter().chain(&shard_one[..4]) {
assert!(
store
.check(&Check::new(&policy, *key))
.unwrap()
.is_allowed()
);
}
clock.advance(Duration::from_secs(1));
let checks = [
Check::new(&policy, shard_zero[4]),
Check::new(&policy, shard_one[4]),
Check::new(&policy, shard_zero[5]),
];
assert_eq!(
store
.check_all(&checks)
.unwrap()
.allowed_decisions()
.map(<[Decision]>::len),
Some(3),
"the two checks targeting shard 0 need two cleanup slots"
);
let shard = store.shards.shard(0).lock().unwrap();
assert_eq!(shard.used(), 4);
assert_eq!(shard.expiration_count_at_or_before(1_000), 2);
drop(shard);
let shard = store.shards.shard(1).lock().unwrap();
assert_eq!(shard.used(), 4);
assert_eq!(shard.expiration_count_at_or_before(1_000), 3);
}
#[test]
fn an_atomic_batch_receives_the_cleanup_budget_for_each_check() {
let clock = ManualClock::default();
let config = MemoryStoreConfig::new(3)
.unwrap()
.with_shard_count(1)
.unwrap()
.with_max_expired_removals_per_check(1)
.unwrap();
let store = MemoryStore::with_clock(config, clock.clone());
let policy = policy("auth.login", "client", 10, Duration::from_secs(1));
for byte in 1..=3 {
assert!(
store
.check(&Check::new(&policy, subject(byte)))
.unwrap()
.is_allowed()
);
}
clock.advance(Duration::from_secs(1));
let checks = [4, 5, 6].map(|byte| Check::new(&policy, subject(byte)));
let result = store.check_all(&checks).unwrap();
assert_eq!(result.allowed_decisions().map(<[Decision]>::len), Some(3));
assert_eq!(store.stats().unwrap().entries(), 3);
}
#[test]
fn auxiliary_expiry_index_remains_bounded_across_many_windows() {
let clock = ManualClock::default();
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(1)
.unwrap()
.with_shard_count(1)
.unwrap(),
clock.clone(),
);
let policy = policy("auth.login", "client", 1, Duration::from_millis(1));
let check = Check::new(&policy, subject(1));
for _ in 0..1_000 {
assert!(store.check(&check).unwrap().is_allowed());
clock.advance(Duration::from_millis(1));
}
let shard = store.shards.shard(0).lock().unwrap();
assert_eq!(shard.used(), 1);
assert_eq!(shard.expiration_count(), 1);
}
#[test]
fn targeted_expired_refresh_keeps_indexes_aligned_for_single_and_batch() {
for use_batch in [false, true] {
let clock = ManualClock::default();
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(2)
.unwrap()
.with_shard_count(1)
.unwrap()
.with_max_expired_removals_per_check(1)
.unwrap(),
clock.clone(),
);
let policy = policy("auth.login", "client", 10, Duration::from_millis(2));
let earlier = Check::new(&policy, subject(1));
let target = Check::new(&policy, subject(2));
assert!(store.check(&earlier).unwrap().is_allowed());
clock.advance(Duration::from_millis(1));
assert!(store.check(&target).unwrap().is_allowed());
clock.advance(Duration::from_millis(2));
let decision = if use_batch {
store
.check_all(std::slice::from_ref(&target))
.unwrap()
.try_into_single_decision()
.expect("a one-element batch returns one decision")
} else {
store.check(&target).unwrap()
};
assert_eq!(decision, Decision::allowed(10, 9, Duration::from_millis(2)));
let key = target.counter_key();
let shard = store.shards.shard(0).lock().unwrap();
assert_eq!(shard.used(), 1);
assert_eq!(shard.expiration_count(), 1);
assert_eq!(shard.entry(key), Some((&Entry { used: 1 }, 5)));
assert!(shard.contains_expiration(key, 5));
}
}
#[test]
fn a_denied_batch_does_not_consume_other_checks() {
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(8)
.unwrap()
.with_shard_count(1)
.unwrap(),
ManualClock::default(),
);
let exhausted = policy("auth.login", "client", 1, Duration::from_secs(60));
let untouched = policy("auth.login", "identity", 1, Duration::from_secs(60));
let exhausted_check = Check::new(&exhausted, subject(1));
let untouched_check = Check::new(&untouched, subject(2));
assert!(store.check(&exhausted_check).unwrap().is_allowed());
let result = store
.check_all(&[exhausted_check, untouched_check])
.unwrap();
assert!(result.is_enforced_denial());
assert_eq!(result.denied_index(), Some(0));
assert!(
store
.check(&Check::new(&untouched, subject(2)))
.unwrap()
.is_allowed()
);
}
#[test]
fn duplicate_keys_are_rejected_before_consumption() {
let store =
MemoryStore::with_clock(MemoryStoreConfig::new(8).unwrap(), ManualClock::default());
let alpha = policy("auth.alpha", "client", 2, Duration::from_secs(60));
let beta = policy("auth.beta", "client", 2, Duration::from_secs(60));
let checks = [
Check::new(&beta, subject(1)),
Check::new(&beta, subject(1)),
Check::new(&alpha, subject(2)),
Check::new(&alpha, subject(2)),
];
assert_eq!(
store.check_all(&checks),
Err(MemoryStoreError::InvalidBatch(BatchError::DuplicateKey {
first_index: 0,
duplicate_index: 1,
}))
);
assert_eq!(store.stats().unwrap().entries(), 0);
}
#[test]
fn concurrent_checks_never_over_admit() {
let store = Arc::new(MemoryStore::with_clock(
MemoryStoreConfig::new(8).unwrap(),
ManualClock::default(),
));
let policy = Arc::new(policy("auth.login", "client", 25, Duration::from_secs(60)));
let allowed = (0..100)
.map(|_| {
let store = Arc::clone(&store);
let policy = Arc::clone(&policy);
thread::spawn(move || {
store
.check(&Check::new(&policy, subject(1)))
.unwrap()
.is_allowed()
})
})
.map(|thread| usize::from(thread.join().unwrap()))
.sum::<usize>();
assert_eq!(allowed, 25);
}
#[test]
fn concurrent_cross_shard_batches_lock_consistently_and_never_over_admit() {
const THREAD_COUNT: usize = 8;
const CALLS_PER_THREAD: usize = 64;
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(2)
.unwrap()
.with_shard_count(2)
.unwrap(),
ManualClock::default(),
);
let limit = u64::try_from(THREAD_COUNT * CALLS_PER_THREAD).unwrap();
let policy = policy("auth.login", "client", limit, Duration::from_secs(60));
let first = subjects_for_shard(&store, &policy, 0, 1)[0];
let second = subjects_for_shard(&store, &policy, 1, 1)[0];
let store = Arc::new(store);
let policy = Arc::new(policy);
let barrier = Arc::new(Barrier::new(THREAD_COUNT));
let (finished_sender, finished_receiver) = mpsc::channel();
let threads = (0..THREAD_COUNT)
.map(|thread_index| {
let store = Arc::clone(&store);
let policy = Arc::clone(&policy);
let barrier = Arc::clone(&barrier);
let finished_sender = finished_sender.clone();
thread::spawn(move || {
for call_index in 0..CALLS_PER_THREAD {
barrier.wait();
let subjects = if (thread_index + call_index) % 2 == 0 {
[first, second]
} else {
[second, first]
};
let checks = subjects.map(|subject| Check::new(policy.as_ref(), subject));
let result = store.check_all(&checks).unwrap();
assert!(
result
.allowed_decisions()
.is_some_and(|decisions| decisions.len() == checks.len()),
"every call through the configured limit must be allowed: {result:?}"
);
}
finished_sender.send(()).unwrap();
})
})
.collect::<Vec<_>>();
drop(finished_sender);
for _ in 0..THREAD_COUNT {
finished_receiver
.recv_timeout(Duration::from_secs(5))
.expect("cross-shard batches must not deadlock");
}
for thread in threads {
thread.join().unwrap();
}
assert_eq!(store.stats().unwrap().entries(), 2);
for key in [first, second] {
assert_eq!(
store.check(&Check::new(policy.as_ref(), key)),
Ok(Decision::denied(quota(limit, Duration::from_secs(60))))
);
}
}
#[test]
fn a_poisoned_default_shard_remains_fail_closed() {
let clock = PanicOnceClock::default();
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(1)
.unwrap()
.with_shard_count(1)
.unwrap(),
clock.clone(),
);
let policy = policy("auth.login", "client", 1, Duration::from_secs(60));
let exhausted = Check::new(&policy, subject(1));
let new_key = Check::new(&policy, subject(2));
assert!(store.check(&exhausted).unwrap().is_allowed());
clock.panic_once();
let panic_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _ = store.check(&new_key);
}));
assert!(panic_result.is_err());
let poisoned = Err(MemoryStoreError::PoisonedShard { shard_index: 0 });
assert_eq!(store.check(&exhausted), poisoned);
assert_eq!(store.check(&new_key), poisoned);
assert_eq!(store.check(&exhausted), poisoned);
assert_eq!(
store.stats(),
Err(MemoryStoreError::PoisonedShard { shard_index: 0 })
);
assert_eq!(
store.clear(),
Err(MemoryStoreError::PoisonedShard { shard_index: 0 })
);
}
#[test]
fn batch_preflight_preserves_validation_capacity_and_lock_precedence() {
let clock = PanicOnceClock::default();
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(1)
.unwrap()
.with_shard_count(1)
.unwrap()
.with_max_batch_size(2)
.unwrap(),
clock.clone(),
);
let policy = policy("auth.preflight", "client", 1, Duration::from_secs(60));
let checks = [
Check::new(&policy, subject(1)),
Check::new(&policy, subject(2)),
Check::new(&policy, subject(3)),
];
clock.panic_once();
let panic_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _ = store.check(&checks[0]);
}));
assert!(panic_result.is_err());
assert_eq!(store.check_all(&[]), Ok(BatchDecision::allowed(Vec::new())));
assert_eq!(
store.check_all(&[checks[0], checks[0]]),
Err(MemoryStoreError::InvalidBatch(BatchError::DuplicateKey {
first_index: 0,
duplicate_index: 1,
}))
);
assert_eq!(
store.check_all(&checks),
Err(MemoryStoreError::InvalidBatch(BatchError::BatchTooLarge {
actual: 3,
maximum: 2,
}))
);
assert_eq!(
store.check_all(&checks[..2]),
Err(MemoryStoreError::BatchExceedsShardCapacity {
shard_index: 0,
key_count: 2,
capacity: 1,
})
);
assert_eq!(
store.check_all(&checks[..1]),
Err(MemoryStoreError::PoisonedShard { shard_index: 0 })
);
}
#[test]
fn failed_checks_keep_policy_metadata_but_failed_batches_remain_anonymous() {
let clock = PanicOnceClock::default();
let observer = Arc::new(AdmissionMetadataObserver::default());
let store = MemoryStore::with_clock(
MemoryStoreConfig::new(1)
.unwrap()
.with_shard_count(1)
.unwrap(),
clock.clone(),
)
.with_observer(observer.clone());
let policy = policy(
"auth.observed-failure",
"client",
1,
Duration::from_secs(60),
);
let check = Check::new(&policy, subject(1));
clock.panic_once();
let panic_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _ = store.check(&check);
}));
assert!(panic_result.is_err());
assert_eq!(
store.check(&check),
Err(MemoryStoreError::PoisonedShard { shard_index: 0 })
);
assert_eq!(
store.check_all(&[check]),
Err(MemoryStoreError::PoisonedShard { shard_index: 0 })
);
assert_eq!(
observer.admissions.lock().unwrap().as_slice(),
[
RecordedAdmissionMetadata {
operation: AdmissionOperation::Check,
batch_size: 1,
has_policy_id: true,
has_scope_id: true,
has_policy_fingerprint: true,
outcome: AdmissionOutcome::Failed,
consumption: ConsumptionStatus::NotConsumed,
},
RecordedAdmissionMetadata {
operation: AdmissionOperation::Batch,
batch_size: 1,
has_policy_id: false,
has_scope_id: false,
has_policy_fingerprint: false,
outcome: AdmissionOutcome::Failed,
consumption: ConsumptionStatus::NotConsumed,
},
]
);
}
#[test]
fn explicit_recovery_resets_only_poisoned_shards_behind_an_arc() {
let clock = PanicOnceClock::default();
let store = Arc::new(MemoryStore::with_clock(
MemoryStoreConfig::new(4)
.unwrap()
.with_shard_count(2)
.unwrap(),
clock.clone(),
));
let policy = policy("auth.login", "client", 1, Duration::from_secs(60));
let reset_subject = subjects_for_shard(&store, &policy, 0, 1)[0];
let retained_subject = subjects_for_shard(&store, &policy, 1, 1)[0];
let reset_check = Check::new(&policy, reset_subject);
let retained_check = Check::new(&policy, retained_subject);
assert!(store.check(&reset_check).unwrap().is_allowed());
assert!(store.check(&retained_check).unwrap().is_allowed());
clock.panic_once();
let panicked_store = Arc::clone(&store);
let panic_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || {
let _ = panicked_store.check(&reset_check);
}));
assert!(panic_result.is_err());
assert_eq!(
store.check(&reset_check),
Err(MemoryStoreError::PoisonedShard { shard_index: 0 })
);
assert!(store.check(&retained_check).unwrap().is_denied());
assert!(format!("{store:?}").contains("poisoned_shards: 1"));
assert_eq!(store.recover_poisoned(), 1);
assert_eq!(
store.recover_poisoned(),
0,
"recovery must be idempotent when no new panic occurs"
);
assert_eq!(store.stats().unwrap().entries(), 1);
assert!(
store.check(&reset_check).unwrap().is_allowed(),
"the recovered shard starts with empty quota state"
);
assert!(
store.check(&retained_check).unwrap().is_denied(),
"healthy-shard quota state must survive recovery"
);
assert!(format!("{store:?}").contains("poisoned_shards: 0"));
}
#[test]
fn recovery_reopens_every_shard_poisoned_by_an_unwinding_batch() {
let clock = PanicOnceClock::default();
let store = Arc::new(MemoryStore::with_clock(
MemoryStoreConfig::new(2)
.unwrap()
.with_shard_count(2)
.unwrap(),
clock.clone(),
));
let policy = policy("auth.login", "client", 1, Duration::from_secs(60));
let shard_zero = subjects_for_shard(&store, &policy, 0, 1)[0];
let shard_one = subjects_for_shard(&store, &policy, 1, 1)[0];
let checks = [
Check::new(&policy, shard_one),
Check::new(&policy, shard_zero),
];
clock.panic_once();
let panicked_store = Arc::clone(&store);
let panic_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || {
let _ = panicked_store.check_all(&checks);
}));
assert!(panic_result.is_err());
assert_eq!(
store.check(&checks[0]),
Err(MemoryStoreError::PoisonedShard { shard_index: 1 })
);
assert_eq!(
store.check(&checks[1]),
Err(MemoryStoreError::PoisonedShard { shard_index: 0 })
);
assert_eq!(store.recover_poisoned(), 2);
assert_eq!(store.stats().unwrap().entries(), 0);
assert!(
store
.check_all(&checks)
.unwrap()
.allowed_decisions()
.is_some_and(|decisions| decisions.len() == 2),
"all shards must be usable after one atomic recovery"
);
}
}