use std::collections::HashMap;
use std::fmt;
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::{Duration, Instant};
use crate::limits::{SealedAdmissionKeyError, opaque_admission_digest, require_admission_field};
pub const DEFAULT_REVALIDATIONS_PER_PARTITION: usize = 2;
pub const HARD_REVALIDATIONS_PER_PARTITION: usize = 16;
pub const DEFAULT_REVALIDATIONS_PER_PROVIDER: usize = 32;
pub const HARD_REVALIDATIONS_PER_PROVIDER: usize = 256;
pub const DEFAULT_REVALIDATIONS_PER_DEPLOYMENT: usize = 256;
pub const HARD_REVALIDATIONS_PER_DEPLOYMENT: usize = 4_096;
pub const DEFAULT_ATTEMPTS_PER_MINUTE_PER_PARTITION: u32 = 120;
pub const HARD_ATTEMPTS_PER_MINUTE_PER_PARTITION: u32 = 6_000;
pub const DEFAULT_ATTEMPTS_PER_MINUTE_PER_PROVIDER: u32 = 10_000;
pub const HARD_ATTEMPTS_PER_MINUTE_PER_PROVIDER: u32 = 100_000;
pub const ATTEMPT_RATE_WINDOW: Duration = Duration::from_mins(1);
pub const AUDIENCE_BINDING_MAX_BYTES: usize = 8 * 1024;
fn require_audience_binding<'a>(
binding: &'a [&'a [u8]],
) -> Result<&'a [&'a [u8]], SealedAdmissionKeyError> {
if binding.is_empty() {
return Err(SealedAdmissionKeyError::EmptyField);
}
let mut total = 0_usize;
for part in binding {
if part.is_empty() {
return Err(SealedAdmissionKeyError::EmptyField);
}
total = total
.checked_add(part.len())
.ok_or(SealedAdmissionKeyError::FieldTooLong)?;
if total > AUDIENCE_BINDING_MAX_BYTES {
return Err(SealedAdmissionKeyError::FieldTooLong);
}
}
Ok(binding)
}
#[derive(Clone, PartialEq, Eq)]
pub struct PartitionDescriptor {
provider: String,
configuration_generation: u64,
issuer: String,
canonical_resource: String,
tenant: String,
subject: String,
client: String,
trust_generation: u64,
auth_policy_revision: u64,
identity: [u8; 32],
}
impl fmt::Debug for PartitionDescriptor {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("PartitionDescriptor")
.finish_non_exhaustive()
}
}
impl PartitionDescriptor {
#[allow(clippy::too_many_arguments)]
pub fn from_verified_facts(
provider: &str,
configuration_generation: u64,
issuer: &str,
canonical_resource: &str,
tenant: &str,
subject: &str,
client: &str,
trust_generation: u64,
auth_policy_revision: u64,
audience_binding: &[&[u8]],
) -> Result<Self, SealedAdmissionKeyError> {
let binding = require_audience_binding(audience_binding)?;
let configuration_generation_bytes = configuration_generation.to_be_bytes();
let trust_generation_bytes = trust_generation.to_be_bytes();
let auth_policy_revision_bytes = auth_policy_revision.to_be_bytes();
let binding_count_bytes = u64::try_from(binding.len())
.map_err(|_| SealedAdmissionKeyError::FieldTooLong)?
.to_be_bytes();
let mut parts: Vec<&[u8]> = vec![
b"auth-00-partition-descriptor-v2",
require_admission_field(provider)?,
&configuration_generation_bytes,
require_admission_field(issuer)?,
require_admission_field(canonical_resource)?,
require_admission_field(tenant)?,
require_admission_field(subject)?,
require_admission_field(client)?,
&trust_generation_bytes,
&auth_policy_revision_bytes,
&binding_count_bytes,
];
parts.extend_from_slice(binding);
Ok(Self {
provider: provider.to_owned(),
configuration_generation,
issuer: issuer.to_owned(),
canonical_resource: canonical_resource.to_owned(),
tenant: tenant.to_owned(),
subject: subject.to_owned(),
client: client.to_owned(),
trust_generation,
auth_policy_revision,
identity: opaque_admission_digest(&parts),
})
}
#[must_use]
pub fn provider(&self) -> &str {
&self.provider
}
#[must_use]
pub const fn configuration_generation(&self) -> u64 {
self.configuration_generation
}
#[must_use]
pub fn issuer(&self) -> &str {
&self.issuer
}
#[must_use]
pub fn canonical_resource(&self) -> &str {
&self.canonical_resource
}
#[must_use]
pub fn tenant(&self) -> &str {
&self.tenant
}
#[must_use]
pub fn subject(&self) -> &str {
&self.subject
}
#[must_use]
pub fn client(&self) -> &str {
&self.client
}
#[must_use]
pub const fn trust_generation(&self) -> u64 {
self.trust_generation
}
#[must_use]
pub const fn auth_policy_revision(&self) -> u64 {
self.auth_policy_revision
}
#[must_use]
pub const fn identity(&self) -> &[u8; 32] {
&self.identity
}
}
macro_rules! opaque_partition_key {
($(#[$meta:meta])* $name:ident) => {
$(#[$meta])*
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct $name {
digest: [u8; 32],
}
impl fmt::Debug for $name {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.debug_struct(stringify!($name)).finish_non_exhaustive()
}
}
impl $name {
#[must_use]
pub const fn as_bytes(&self) -> &[u8; 32] {
&self.digest
}
}
};
}
opaque_partition_key! {
CachePartitionKey
}
opaque_partition_key! {
ContinuationPartitionKey
}
opaque_partition_key! {
DurableOwnerKey
}
opaque_partition_key! {
SubscriptionPartitionKey
}
opaque_partition_key! {
CredentialStoreKey
}
opaque_partition_key! {
QuotaPartitionKey
}
opaque_partition_key! {
RevalidationFlightKey
}
opaque_partition_key! {
ReplayReservationKey
}
fn encode_grants(grants: &[&str]) -> Result<Vec<u8>, SealedAdmissionKeyError> {
let mut encoded = Vec::new();
encoded.extend_from_slice(&(grants.len() as u64).to_be_bytes());
for grant in grants {
let field = require_admission_field(grant)?;
encoded.extend_from_slice(&(field.len() as u64).to_be_bytes());
encoded.extend_from_slice(field);
}
Ok(encoded)
}
impl CachePartitionKey {
pub fn derive(
descriptor: &PartitionDescriptor,
effective_grants: &[&str],
token_instance: &str,
representation_policy: &str,
cache_domain: &str,
) -> Result<Self, SealedAdmissionKeyError> {
let grants = encode_grants(effective_grants)?;
Ok(Self {
digest: opaque_admission_digest(&[
b"auth-00-cache-partition-v1",
descriptor.identity(),
&grants,
require_admission_field(token_instance)?,
require_admission_field(representation_policy)?,
require_admission_field(cache_domain)?,
]),
})
}
}
impl ContinuationPartitionKey {
pub fn derive(
descriptor: &PartitionDescriptor,
grant_snapshot: &[&str],
method_parameter_binding: &str,
capability_fingerprint: &str,
continuation_policy: &str,
domain: &str,
) -> Result<Self, SealedAdmissionKeyError> {
let grants = encode_grants(grant_snapshot)?;
Ok(Self {
digest: opaque_admission_digest(&[
b"auth-00-continuation-partition-v1",
descriptor.identity(),
&grants,
require_admission_field(method_parameter_binding)?,
require_admission_field(capability_fingerprint)?,
require_admission_field(continuation_policy)?,
require_admission_field(domain)?,
]),
})
}
}
impl DurableOwnerKey {
pub fn derive(
descriptor: &PartitionDescriptor,
ownership_epoch: u64,
) -> Result<Self, SealedAdmissionKeyError> {
Ok(Self {
digest: opaque_admission_digest(&[
b"auth-00-durable-owner-v1",
require_admission_field(descriptor.issuer())?,
require_admission_field(descriptor.tenant())?,
require_admission_field(descriptor.subject())?,
require_admission_field(descriptor.client())?,
require_admission_field(descriptor.canonical_resource())?,
require_admission_field(descriptor.provider())?,
&ownership_epoch.to_be_bytes(),
]),
})
}
}
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct PartitionAuthorization {
binding: [u8; 32],
}
impl fmt::Debug for PartitionAuthorization {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("PartitionAuthorization")
.finish_non_exhaustive()
}
}
impl PartitionAuthorization {
#[must_use]
pub fn current(descriptor: &PartitionDescriptor, owner: &DurableOwnerKey) -> Self {
Self {
binding: opaque_admission_digest(&[
b"auth-00-partition-authorization-v1",
descriptor.identity(),
owner.as_bytes(),
]),
}
}
#[must_use]
pub const fn as_bytes(&self) -> &[u8; 32] {
&self.binding
}
}
impl SubscriptionPartitionKey {
pub fn derive(
descriptor: &PartitionDescriptor,
effective_grants: &[&str],
token_instance: &str,
subscription_topic: &str,
delivery_policy: &str,
) -> Result<Self, SealedAdmissionKeyError> {
let grants = encode_grants(effective_grants)?;
Ok(Self {
digest: opaque_admission_digest(&[
b"auth-00-subscription-partition-v1",
descriptor.identity(),
&grants,
require_admission_field(token_instance)?,
require_admission_field(subscription_topic)?,
require_admission_field(delivery_policy)?,
]),
})
}
}
impl CredentialStoreKey {
pub fn derive(
descriptor: &PartitionDescriptor,
store_domain: &str,
credential_class: &str,
token_instance: &str,
) -> Result<Self, SealedAdmissionKeyError> {
Ok(Self {
digest: opaque_admission_digest(&[
b"auth-00-credential-store-v1",
descriptor.identity(),
require_admission_field(store_domain)?,
require_admission_field(credential_class)?,
require_admission_field(token_instance)?,
]),
})
}
}
impl QuotaPartitionKey {
pub fn derive(
descriptor: &PartitionDescriptor,
quota_epoch: u64,
) -> Result<Self, SealedAdmissionKeyError> {
Ok(Self {
digest: opaque_admission_digest(&[
b"auth-00-quota-partition-v1",
require_admission_field(descriptor.provider())?,
require_admission_field(descriptor.canonical_resource())?,
require_admission_field(descriptor.tenant())?,
require_admission_field(descriptor.subject())?,
require_admission_field(descriptor.client())?,
"a_epoch.to_be_bytes(),
]),
})
}
}
impl RevalidationFlightKey {
pub fn derive(
descriptor: &PartitionDescriptor,
purpose: &str,
) -> Result<Self, SealedAdmissionKeyError> {
Ok(Self {
digest: opaque_admission_digest(&[
b"auth-00-revalidation-flight-v1",
descriptor.identity(),
require_admission_field(purpose)?,
]),
})
}
}
impl ReplayReservationKey {
pub fn derive(
descriptor: &PartitionDescriptor,
replay_alias: &str,
purpose: &str,
) -> Result<Self, SealedAdmissionKeyError> {
Ok(Self {
digest: opaque_admission_digest(&[
b"auth-00-replay-reservation-v1",
descriptor.identity(),
require_admission_field(replay_alias)?,
require_admission_field(purpose)?,
]),
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum LookupPurpose {
Cache,
Continuation,
Subscription,
CredentialStore,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PartitionSlot {
Cache(CachePartitionKey),
Continuation(ContinuationPartitionKey),
Subscription(SubscriptionPartitionKey),
CredentialStore(CredentialStoreKey),
}
impl PartitionSlot {
#[must_use]
pub const fn purpose(&self) -> LookupPurpose {
match self {
Self::Cache(_) => LookupPurpose::Cache,
Self::Continuation(_) => LookupPurpose::Continuation,
Self::Subscription(_) => LookupPurpose::Subscription,
Self::CredentialStore(_) => LookupPurpose::CredentialStore,
}
}
const fn key_bytes(&self) -> &[u8; 32] {
match self {
Self::Cache(key) => key.as_bytes(),
Self::Continuation(key) => key.as_bytes(),
Self::Subscription(key) => key.as_bytes(),
Self::CredentialStore(key) => key.as_bytes(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum LookupOutcome {
Present(Vec<u8>),
Absent,
}
impl LookupOutcome {
#[must_use]
pub const fn is_present(&self) -> bool {
matches!(self, Self::Present(_))
}
#[must_use]
pub const fn is_absent(&self) -> bool {
matches!(self, Self::Absent)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RevalidationLimitsError {
ZeroConcurrency,
ZeroAttemptRate,
ExceedsHardCeiling,
}
impl fmt::Display for RevalidationLimitsError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::ZeroConcurrency => {
formatter.write_str("revalidation concurrency must be positive")
}
Self::ZeroAttemptRate => {
formatter.write_str("revalidation attempt rate must be positive")
}
Self::ExceedsHardCeiling => {
formatter.write_str("revalidation limit exceeds its hard ceiling")
}
}
}
}
impl std::error::Error for RevalidationLimitsError {}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RevalidationLimits {
per_partition: usize,
per_provider: usize,
per_deployment: usize,
partition_attempts_per_minute: u32,
provider_attempts_per_minute: u32,
}
impl Default for RevalidationLimits {
fn default() -> Self {
Self {
per_partition: DEFAULT_REVALIDATIONS_PER_PARTITION,
per_provider: DEFAULT_REVALIDATIONS_PER_PROVIDER,
per_deployment: DEFAULT_REVALIDATIONS_PER_DEPLOYMENT,
partition_attempts_per_minute: DEFAULT_ATTEMPTS_PER_MINUTE_PER_PARTITION,
provider_attempts_per_minute: DEFAULT_ATTEMPTS_PER_MINUTE_PER_PROVIDER,
}
}
}
impl RevalidationLimits {
pub fn new(
per_partition: usize,
per_provider: usize,
per_deployment: usize,
partition_attempts_per_minute: u32,
provider_attempts_per_minute: u32,
) -> Result<Self, RevalidationLimitsError> {
if per_partition == 0 || per_provider == 0 || per_deployment == 0 {
return Err(RevalidationLimitsError::ZeroConcurrency);
}
if partition_attempts_per_minute == 0 || provider_attempts_per_minute == 0 {
return Err(RevalidationLimitsError::ZeroAttemptRate);
}
if per_partition > HARD_REVALIDATIONS_PER_PARTITION
|| per_provider > HARD_REVALIDATIONS_PER_PROVIDER
|| per_deployment > HARD_REVALIDATIONS_PER_DEPLOYMENT
|| partition_attempts_per_minute > HARD_ATTEMPTS_PER_MINUTE_PER_PARTITION
|| provider_attempts_per_minute > HARD_ATTEMPTS_PER_MINUTE_PER_PROVIDER
{
return Err(RevalidationLimitsError::ExceedsHardCeiling);
}
Ok(Self {
per_partition,
per_provider,
per_deployment,
partition_attempts_per_minute,
provider_attempts_per_minute,
})
}
#[must_use]
pub const fn per_partition(&self) -> usize {
self.per_partition
}
#[must_use]
pub const fn per_provider(&self) -> usize {
self.per_provider
}
#[must_use]
pub const fn per_deployment(&self) -> usize {
self.per_deployment
}
#[must_use]
pub const fn partition_attempts_per_minute(&self) -> u32 {
self.partition_attempts_per_minute
}
#[must_use]
pub const fn provider_attempts_per_minute(&self) -> u32 {
self.provider_attempts_per_minute
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PartitionAdmissionError {
ZeroUnits,
ZeroCapacity,
QuotaExhausted {
requested: usize,
in_use: usize,
limit: usize,
},
QuotaAlreadySettled,
RevalidationAlreadyInFlight,
PartitionRevalidationLimitExceeded {
in_flight: usize,
limit: usize,
},
ProviderRevalidationLimitExceeded {
in_flight: usize,
limit: usize,
},
DeploymentRevalidationLimitExceeded {
in_flight: usize,
limit: usize,
},
PartitionAttemptRateExceeded {
attempts: u32,
limit: u32,
},
ProviderAttemptRateExceeded {
attempts: u32,
limit: u32,
},
RevalidationAlreadyFinished,
ReplayAliasAlreadyReserved,
}
impl fmt::Display for PartitionAdmissionError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::ZeroUnits => formatter.write_str("quota reserve requires a positive unit count"),
Self::ZeroCapacity => formatter.write_str("quota capacity must be positive"),
Self::QuotaExhausted {
requested,
in_use,
limit,
} => write!(
formatter,
"quota partition capacity {limit} exceeded (in_use {in_use}, requested {requested})"
),
Self::QuotaAlreadySettled => formatter.write_str("quota reservation already settled"),
Self::RevalidationAlreadyInFlight => {
formatter.write_str("revalidation is already in flight for this partition purpose")
}
Self::PartitionRevalidationLimitExceeded { in_flight, limit } => write!(
formatter,
"partition revalidation concurrency {limit} exceeded (in_flight {in_flight})"
),
Self::ProviderRevalidationLimitExceeded { in_flight, limit } => write!(
formatter,
"provider revalidation concurrency {limit} exceeded (in_flight {in_flight})"
),
Self::DeploymentRevalidationLimitExceeded { in_flight, limit } => write!(
formatter,
"deployment revalidation concurrency {limit} exceeded (in_flight {in_flight})"
),
Self::PartitionAttemptRateExceeded { attempts, limit } => write!(
formatter,
"partition revalidation attempt rate {limit}/min exceeded (attempts {attempts})"
),
Self::ProviderAttemptRateExceeded { attempts, limit } => write!(
formatter,
"provider revalidation attempt rate {limit}/min exceeded (attempts {attempts})"
),
Self::RevalidationAlreadyFinished => {
formatter.write_str("revalidation flight is already finished")
}
Self::ReplayAliasAlreadyReserved => {
formatter.write_str("replay alias is already reserved in this partition")
}
}
}
}
impl std::error::Error for PartitionAdmissionError {}
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
struct RecordSlot {
purpose: LookupPurpose,
key: [u8; 32],
authorization: [u8; 32],
}
#[derive(Clone, Copy)]
struct AttemptWindow {
started: Instant,
attempts: u32,
}
#[derive(Default)]
struct PartitionState {
records: HashMap<RecordSlot, Vec<u8>>,
quota_in_use: HashMap<[u8; 32], usize>,
quota_live: HashMap<u64, ([u8; 32], usize)>,
next_quota_id: u64,
open_flights: HashMap<[u8; 32], u64>,
flight_owner: HashMap<u64, ([u8; 32], [u8; 32], [u8; 32])>,
next_flight_id: u64,
partition_flights: HashMap<[u8; 32], usize>,
provider_flights: HashMap<[u8; 32], usize>,
deployment_flights: usize,
partition_attempts: HashMap<[u8; 32], AttemptWindow>,
provider_attempts: HashMap<[u8; 32], AttemptWindow>,
replay_reservations: HashMap<[u8; 32], [u8; 32]>,
denial_count: usize,
}
fn lock_state(state: &Mutex<PartitionState>) -> MutexGuard<'_, PartitionState> {
state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn provider_bucket(descriptor: &PartitionDescriptor) -> [u8; 32] {
opaque_admission_digest(&[
b"auth-00-provider-bucket-v1",
descriptor.provider().as_bytes(),
])
}
#[derive(Clone)]
pub struct PartitionAdmissionController {
limits: RevalidationLimits,
quota_capacity: usize,
inner: Arc<Mutex<PartitionState>>,
}
impl fmt::Debug for PartitionAdmissionController {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("PartitionAdmissionController")
.field("limits", &self.limits)
.field("quota_capacity", &self.quota_capacity)
.finish_non_exhaustive()
}
}
impl PartitionAdmissionController {
pub fn new(
limits: RevalidationLimits,
quota_capacity: usize,
) -> Result<Self, PartitionAdmissionError> {
if quota_capacity == 0 {
return Err(PartitionAdmissionError::ZeroCapacity);
}
Ok(Self {
limits,
quota_capacity,
inner: Arc::new(Mutex::new(PartitionState::default())),
})
}
#[must_use]
pub const fn limits(&self) -> &RevalidationLimits {
&self.limits
}
#[must_use]
pub const fn quota_capacity(&self) -> usize {
self.quota_capacity
}
pub fn store(
&self,
authorization: &PartitionAuthorization,
slot: &PartitionSlot,
value: Vec<u8>,
) -> Option<Vec<u8>> {
let record = RecordSlot {
purpose: slot.purpose(),
key: *slot.key_bytes(),
authorization: *authorization.as_bytes(),
};
lock_state(&self.inner).records.insert(record, value)
}
#[must_use]
pub fn lookup(
&self,
authorization: &PartitionAuthorization,
slot: &PartitionSlot,
) -> LookupOutcome {
let record = RecordSlot {
purpose: slot.purpose(),
key: *slot.key_bytes(),
authorization: *authorization.as_bytes(),
};
lock_state(&self.inner)
.records
.get(&record)
.map_or(LookupOutcome::Absent, |value| {
LookupOutcome::Present(value.clone())
})
}
#[must_use]
pub fn record_count(&self) -> usize {
lock_state(&self.inner).records.len()
}
pub fn reserve_quota(
&self,
key: &QuotaPartitionKey,
units: usize,
) -> Result<QuotaReservation, PartitionAdmissionError> {
if units == 0 {
return Err(PartitionAdmissionError::ZeroUnits);
}
let bucket = *key.as_bytes();
let mut state = lock_state(&self.inner);
let in_use = state.quota_in_use.get(&bucket).copied().unwrap_or(0);
let next = in_use
.checked_add(units)
.ok_or(PartitionAdmissionError::QuotaExhausted {
requested: units,
in_use,
limit: self.quota_capacity,
})?;
if next > self.quota_capacity {
state.denial_count = state.denial_count.saturating_add(1);
return Err(PartitionAdmissionError::QuotaExhausted {
requested: units,
in_use,
limit: self.quota_capacity,
});
}
let id = state.next_quota_id;
state.next_quota_id = state.next_quota_id.saturating_add(1);
state.quota_in_use.insert(bucket, next);
state.quota_live.insert(id, (bucket, units));
drop(state);
Ok(QuotaReservation {
inner: Arc::clone(&self.inner),
id,
live: true,
})
}
#[must_use]
pub fn quota_in_use(&self, key: &QuotaPartitionKey) -> usize {
lock_state(&self.inner)
.quota_in_use
.get(key.as_bytes())
.copied()
.unwrap_or(0)
}
pub fn begin_revalidation(
&self,
descriptor: &PartitionDescriptor,
flight: &RevalidationFlightKey,
) -> Result<RevalidationFlight, PartitionAdmissionError> {
let flight_key = *flight.as_bytes();
let partition = *descriptor.identity();
let provider = provider_bucket(descriptor);
let now = Instant::now();
let mut state = lock_state(&self.inner);
if state.open_flights.contains_key(&flight_key) {
state.denial_count = state.denial_count.saturating_add(1);
return Err(PartitionAdmissionError::RevalidationAlreadyInFlight);
}
let partition_attempts = window_attempts(&state.partition_attempts, &partition, now);
if partition_attempts >= self.limits.partition_attempts_per_minute {
state.denial_count = state.denial_count.saturating_add(1);
return Err(PartitionAdmissionError::PartitionAttemptRateExceeded {
attempts: partition_attempts,
limit: self.limits.partition_attempts_per_minute,
});
}
let provider_attempts = window_attempts(&state.provider_attempts, &provider, now);
if provider_attempts >= self.limits.provider_attempts_per_minute {
state.denial_count = state.denial_count.saturating_add(1);
return Err(PartitionAdmissionError::ProviderAttemptRateExceeded {
attempts: provider_attempts,
limit: self.limits.provider_attempts_per_minute,
});
}
let partition_in_flight = state
.partition_flights
.get(&partition)
.copied()
.unwrap_or(0);
if partition_in_flight >= self.limits.per_partition {
state.denial_count = state.denial_count.saturating_add(1);
return Err(
PartitionAdmissionError::PartitionRevalidationLimitExceeded {
in_flight: partition_in_flight,
limit: self.limits.per_partition,
},
);
}
let provider_in_flight = state.provider_flights.get(&provider).copied().unwrap_or(0);
if provider_in_flight >= self.limits.per_provider {
state.denial_count = state.denial_count.saturating_add(1);
return Err(PartitionAdmissionError::ProviderRevalidationLimitExceeded {
in_flight: provider_in_flight,
limit: self.limits.per_provider,
});
}
if state.deployment_flights >= self.limits.per_deployment {
state.denial_count = state.denial_count.saturating_add(1);
return Err(
PartitionAdmissionError::DeploymentRevalidationLimitExceeded {
in_flight: state.deployment_flights,
limit: self.limits.per_deployment,
},
);
}
charge_attempt(&mut state.partition_attempts, partition, now);
charge_attempt(&mut state.provider_attempts, provider, now);
let id = state.next_flight_id;
state.next_flight_id = state.next_flight_id.saturating_add(1);
state.open_flights.insert(flight_key, id);
state
.flight_owner
.insert(id, (flight_key, partition, provider));
state
.partition_flights
.insert(partition, partition_in_flight.saturating_add(1));
state
.provider_flights
.insert(provider, provider_in_flight.saturating_add(1));
state.deployment_flights = state.deployment_flights.saturating_add(1);
drop(state);
Ok(RevalidationFlight {
inner: Arc::clone(&self.inner),
id,
live: true,
})
}
#[must_use]
pub fn partition_flights(&self, descriptor: &PartitionDescriptor) -> usize {
lock_state(&self.inner)
.partition_flights
.get(descriptor.identity())
.copied()
.unwrap_or(0)
}
#[must_use]
pub fn provider_flights(&self, descriptor: &PartitionDescriptor) -> usize {
lock_state(&self.inner)
.provider_flights
.get(&provider_bucket(descriptor))
.copied()
.unwrap_or(0)
}
#[must_use]
pub fn deployment_flights(&self) -> usize {
lock_state(&self.inner).deployment_flights
}
#[must_use]
pub fn partition_attempts(&self, descriptor: &PartitionDescriptor) -> u32 {
let state = lock_state(&self.inner);
window_attempts(
&state.partition_attempts,
descriptor.identity(),
Instant::now(),
)
}
#[must_use]
pub fn provider_attempts(&self, descriptor: &PartitionDescriptor) -> u32 {
let state = lock_state(&self.inner);
window_attempts(
&state.provider_attempts,
&provider_bucket(descriptor),
Instant::now(),
)
}
pub fn reserve_replay(
&self,
descriptor: &PartitionDescriptor,
replay: &ReplayReservationKey,
) -> Result<(), PartitionAdmissionError> {
let alias = *replay.as_bytes();
let partition = *descriptor.identity();
let mut state = lock_state(&self.inner);
if state.replay_reservations.contains_key(&alias) {
state.denial_count = state.denial_count.saturating_add(1);
return Err(PartitionAdmissionError::ReplayAliasAlreadyReserved);
}
state.replay_reservations.insert(alias, partition);
Ok(())
}
#[must_use]
pub fn replay_reserved(&self, replay: &ReplayReservationKey) -> bool {
lock_state(&self.inner)
.replay_reservations
.contains_key(replay.as_bytes())
}
#[must_use]
pub fn replay_reservation_count(&self) -> usize {
lock_state(&self.inner).replay_reservations.len()
}
#[must_use]
pub fn denial_count(&self) -> usize {
lock_state(&self.inner).denial_count
}
}
fn window_attempts(
windows: &HashMap<[u8; 32], AttemptWindow>,
bucket: &[u8; 32],
now: Instant,
) -> u32 {
windows.get(bucket).map_or(0, |window| {
if now.duration_since(window.started) >= ATTEMPT_RATE_WINDOW {
0
} else {
window.attempts
}
})
}
fn charge_attempt(windows: &mut HashMap<[u8; 32], AttemptWindow>, bucket: [u8; 32], now: Instant) {
let entry = windows.entry(bucket).or_insert(AttemptWindow {
started: now,
attempts: 0,
});
if now.duration_since(entry.started) >= ATTEMPT_RATE_WINDOW {
entry.started = now;
entry.attempts = 0;
}
entry.attempts = entry.attempts.saturating_add(1);
}
pub struct QuotaReservation {
inner: Arc<Mutex<PartitionState>>,
id: u64,
live: bool,
}
impl fmt::Debug for QuotaReservation {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("QuotaReservation")
.finish_non_exhaustive()
}
}
impl QuotaReservation {
pub fn release(&mut self) -> Result<(), PartitionAdmissionError> {
if !self.live {
return Err(PartitionAdmissionError::QuotaAlreadySettled);
}
self.live = false;
let mut state = lock_state(&self.inner);
let Some((bucket, units)) = state.quota_live.remove(&self.id) else {
return Err(PartitionAdmissionError::QuotaAlreadySettled);
};
let remaining = state
.quota_in_use
.get(&bucket)
.copied()
.unwrap_or(0)
.saturating_sub(units);
if remaining == 0 {
state.quota_in_use.remove(&bucket);
} else {
state.quota_in_use.insert(bucket, remaining);
}
Ok(())
}
}
impl Drop for QuotaReservation {
fn drop(&mut self) {
if self.live {
let _ = self.release();
}
}
}
pub struct RevalidationFlight {
inner: Arc<Mutex<PartitionState>>,
id: u64,
live: bool,
}
impl fmt::Debug for RevalidationFlight {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("RevalidationFlight")
.finish_non_exhaustive()
}
}
impl RevalidationFlight {
pub fn finish(&mut self) -> Result<(), PartitionAdmissionError> {
if !self.live {
return Err(PartitionAdmissionError::RevalidationAlreadyFinished);
}
self.live = false;
let mut state = lock_state(&self.inner);
let Some((flight_key, partition, provider)) = state.flight_owner.remove(&self.id) else {
return Ok(());
};
state.open_flights.remove(&flight_key);
let partition_remaining = state
.partition_flights
.get(&partition)
.copied()
.unwrap_or(0)
.saturating_sub(1);
if partition_remaining == 0 {
state.partition_flights.remove(&partition);
} else {
state
.partition_flights
.insert(partition, partition_remaining);
}
let provider_remaining = state
.provider_flights
.get(&provider)
.copied()
.unwrap_or(0)
.saturating_sub(1);
if provider_remaining == 0 {
state.provider_flights.remove(&provider);
} else {
state.provider_flights.insert(provider, provider_remaining);
}
state.deployment_flights = state.deployment_flights.saturating_sub(1);
Ok(())
}
}
impl Drop for RevalidationFlight {
fn drop(&mut self) {
if self.live {
let _ = self.finish();
}
}
}