Skip to main content

tollgate_store/
traits.rs

1//! The storage traits and their shared vocabulary.
2
3use std::num::NonZeroUsize;
4
5use async_trait::async_trait;
6use jiff::{SignedDuration, Timestamp};
7use tokio::sync::broadcast;
8
9use tollgate_core::{
10    AccountId, AccountStatus, BudgetSchedule, CapacityClass, CostUnits, FencingToken, Generation,
11    KeyId, LeaseGrant, LeaseId, Principal, PublishableSnapshot, UsageEvent,
12};
13
14/// Backend failure unrelated to domain rules (connection lost, transaction
15/// aborted). Callers treat it as retryable-with-backoff; it must never be
16/// conflated with a domain refusal.
17#[derive(Debug, Clone, PartialEq, Eq)]
18pub struct StoreError(pub String);
19
20impl std::fmt::Display for StoreError {
21    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
22        write!(f, "store error: {}", self.0)
23    }
24}
25
26impl std::error::Error for StoreError {}
27
28/// Domain refusals from the allocator.
29#[derive(Debug, Clone, PartialEq, Eq)]
30pub enum AllocateError {
31    /// No such account.
32    UnknownAccount,
33    /// The account exists but is not in a state that may spend.
34    AccountInactive,
35    /// Nothing left to lease. Distinct from `AccountInactive`: the client
36    /// should keep polling, because usage settlement or a top-up can restore
37    /// balance.
38    InsufficientBalance,
39    /// The ledger confirms no funding remains, including outstanding leases.
40    BalanceExhausted(tollgate_core::BalanceExhaustion),
41    /// No grant is possible, and the ledger confirms how much funding remains
42    /// outside this instance's reach — all of it held in other leases.
43    /// `remaining` is never zero; zero is [`Self::BalanceExhausted`].
44    /// `InsufficientBalance` stays the refusal that carries no attestation.
45    BalanceInsufficient(tollgate_core::BalanceShortfall),
46    /// A lease must specify one unambiguous, strictly positive lifetime.
47    InvalidTtl,
48    /// No lease record has this `lease_id`.
49    UnknownLease,
50    /// The fencing token does not match the lease record named by `lease_id`.
51    /// Token ordering across different active leases is irrelevant
52    /// (INVARIANTS.md GL-4).
53    Fenced,
54    /// The lease exists but is no longer active (already released, expired,
55    /// or reclaimed).
56    LeaseNotActive,
57    /// A release claimed more unspent units than the lease can still hold
58    /// (`unspent + recorded usage > granted`) — a client accounting bug,
59    /// surfaced rather than absorbed.
60    InvalidRelease,
61    /// A backend failure unrelated to domain rules; see [`StoreError`].
62    Storage(StoreError),
63}
64
65impl AllocateError {
66    /// Stable metric labels, in [`index`](AllocateError::index) order.
67    ///
68    /// Variant names, not [`Display`](std::fmt::Display) output: `Storage`
69    /// wraps a backend message that varies per failure, so labelling by
70    /// rendered text would mint a fresh time series per connection error.
71    pub const NAMES: [&'static str; Self::COUNT] = [
72        "unknown_account",
73        "account_inactive",
74        "insufficient_balance",
75        "invalid_ttl",
76        "unknown_lease",
77        "fenced",
78        "lease_not_active",
79        "invalid_release",
80        "storage",
81        "balance_exhausted",
82        "balance_insufficient",
83    ];
84
85    /// How many distinct refusals exist — the width of a per-reason tally.
86    pub const COUNT: usize = 11;
87
88    /// This refusal's dense slot, for direct-indexed per-reason counters.
89    ///
90    /// `Storage` carries data, so there is no discriminant to cast; the
91    /// mapping is written out and the match is exhaustive, so a new variant
92    /// fails to compile until it is given a slot rather than silently landing
93    /// in another's bucket. Mirrors `DenyReason::index`.
94    #[must_use]
95    pub const fn index(&self) -> usize {
96        match self {
97            AllocateError::UnknownAccount => 0,
98            AllocateError::AccountInactive => 1,
99            AllocateError::InsufficientBalance => 2,
100            AllocateError::InvalidTtl => 3,
101            AllocateError::UnknownLease => 4,
102            AllocateError::Fenced => 5,
103            AllocateError::LeaseNotActive => 6,
104            AllocateError::InvalidRelease => 7,
105            AllocateError::Storage(_) => 8,
106            AllocateError::BalanceExhausted(_) => 9,
107            AllocateError::BalanceInsufficient(_) => 10,
108        }
109    }
110
111    /// This refusal's stable metric label.
112    #[must_use]
113    pub const fn name(&self) -> &'static str {
114        Self::NAMES[self.index()]
115    }
116}
117
118impl std::fmt::Display for AllocateError {
119    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
120        match self {
121            AllocateError::UnknownAccount => f.write_str("unknown account"),
122            AllocateError::AccountInactive => f.write_str("account inactive"),
123            AllocateError::InsufficientBalance => f.write_str("insufficient balance"),
124            AllocateError::BalanceExhausted(_) => f.write_str("account balance exhausted"),
125            AllocateError::BalanceInsufficient(evidence) => write!(
126                f,
127                "insufficient balance ({} units remain, all held in leases)",
128                evidence.remaining
129            ),
130            AllocateError::InvalidTtl => {
131                f.write_str("lease TTL must specify one positive duration")
132            }
133            AllocateError::UnknownLease => f.write_str("unknown lease"),
134            AllocateError::Fenced => f.write_str("fencing token mismatch"),
135            AllocateError::LeaseNotActive => f.write_str("lease not active"),
136            AllocateError::InvalidRelease => f.write_str("invalid release"),
137            AllocateError::Storage(e) => write!(f, "{e}"),
138        }
139    }
140}
141
142impl std::error::Error for AllocateError {}
143
144impl From<StoreError> for AllocateError {
145    fn from(error: StoreError) -> Self {
146        AllocateError::Storage(error)
147    }
148}
149
150/// Admin-side inputs when creating an account.
151#[derive(Debug, Clone, Copy)]
152pub struct AccountConfig {
153    /// The new account's identifier.
154    pub account_id: AccountId,
155    /// Opening balance. Deposited as a top-up, so it counts toward
156    /// [`Conservation::deposited`] and never expires at a period boundary.
157    pub initial_balance: CostUnits,
158    /// The account's administrative status at birth. Anything but
159    /// [`AccountStatus::Active`] refuses leases while keeping the ledger, and
160    /// `Closed` is terminal from creation onwards (INVARIANTS.md GL-22).
161    ///
162    /// An [`AccountStatus`] rather than a bool so creation and
163    /// [`AdminStore::set_account_status`] speak one vocabulary about one
164    /// column; the bool could not express `Closed`, which is what let
165    /// terminality be a convention instead of a check (GL-51).
166    pub status: AccountStatus,
167    /// The account's execution-capacity class at birth (GL-99).
168    ///
169    /// Present at creation for the reason `status` is: creation and
170    /// [`AdminStore::set_capacity_class`] speak one vocabulary about one
171    /// column, so there is no window in which an account exists without a
172    /// class and no second place that decides the default.
173    pub capacity_class: CapacityClass,
174}
175
176/// Per-account conservation view for reconciliation.
177///
178/// Read the equation as a funding statement: the left side is everything the
179/// account was ever funded with, the right side is where those units now sit.
180/// The two sources of funding are money in ([`deposited`](Self::deposited))
181/// and credit extended ([`overage_recorded`](Self::overage_recorded)); the
182/// three resting places are unspent balance, capacity currently out on lease,
183/// and units already consumed or written off.
184#[derive(Debug, Clone, Copy, PartialEq, Eq)]
185pub struct Conservation {
186    /// Every unit ever deposited: the opening balance, top-ups, and periodic
187    /// allowances. Monotonic.
188    pub deposited: CostUnits,
189    /// Unfunded units billed under [`EnforcementMode::Elastic`]: spend no
190    /// deposit paid for and no lease debited.
191    ///
192    /// A *funding* term, on the left of the equation beside `deposited`, not
193    /// a bucket on the right. Overage usage also lands in `settled_usage`, so
194    /// without a matching term on the left the equation would fail by exactly
195    /// the overage — which is the whole reason this field exists rather than
196    /// the ledger simply recording the usage and saying nothing else.
197    ///
198    /// [`EnforcementMode::Elastic`]: tollgate_core::EnforcementMode::Elastic
199    pub overage_recorded: CostUnits,
200    /// Spendable units not out on lease: allowance plus top-up.
201    pub balance: CostUnits,
202    /// Units granted to leases that have not settled, including usage already
203    /// recorded against them.
204    pub active_lease_grants: CostUnits,
205    /// Usage billed against leases that have settled (released or expired),
206    /// plus all overage usage — which belongs to no lease and is therefore
207    /// settled the moment it is recorded. Usage on active leases is inside
208    /// `active_lease_grants`.
209    pub settled_usage: CostUnits,
210    /// Units granted to leases that settled without usage accounting for them:
211    /// unclaimed at release or forfeited at expiry reclaim. Usage for such a lease
212    /// that arrives later moves units from here into `settled_usage`.
213    pub settlement_loss: CostUnits,
214    /// Units that were funded but will never be spent, because the period
215    /// that funded them ended (GL-97).
216    ///
217    /// A resting place on the right of the equation, beside `settlement_loss`
218    /// and for the same reason: both are units the account was funded with
219    /// that no longer sit in a balance, on a lease, or in billed usage.
220    /// Without it, an allowance that resets each month would make the equation
221    /// fail by exactly the unspent remainder — the ledger reporting corruption
222    /// every time a budget did the one thing it exists to do.
223    ///
224    /// Monotonic, like `deposited` and `overage_recorded`: expiry is a fact
225    /// about a period that has closed, and closing a period is not reversible.
226    pub expired: CostUnits,
227}
228
229impl Conservation {
230    /// `deposited + overage == balance + active grants + settled usage + loss
231    /// + expired`, exactly.
232    ///
233    /// Both sides accumulate with checked arithmetic and an overflow answers
234    /// `false`, never a wrap or a panic: this function exists to *detect*
235    /// corrupt ledger state, so arithmetic that could not represent the state
236    /// must report a violation rather than quietly produce a total that
237    /// happens to match (INVARIANTS.md GL-11).
238    #[must_use]
239    pub fn holds(&self) -> bool {
240        let Some(funded) = self.deposited.checked_add(self.overage_recorded) else {
241            return false;
242        };
243        let mut sum = self.balance;
244        for part in [
245            self.active_lease_grants,
246            self.settled_usage,
247            self.settlement_loss,
248            self.expired,
249        ] {
250            match sum.checked_add(part) {
251                Some(next) => sum = next,
252                None => return false,
253            }
254        }
255        sum == funded
256    }
257}
258
259/// How an allocator sizes grants as an account's balance shrinks.
260///
261/// Deep balances grant the full request; near exhaustion the grant is capped
262/// at `balance / shrink_divisor` (floored at `min_grant`, and never above the
263/// remaining balance). This is the design-review answer to the quota-edge
264/// problem: N instances can no longer strand a small balance behind one
265/// holder's oversized lease. A consolidation may exceed the cap only to fund
266/// a quote the holder already refused ([`Self::consolidation_grant`]), which
267/// is demand rather than hoarding.
268#[derive(Debug, Clone, Copy)]
269pub struct GrantPolicy {
270    /// Divisor applied to the balance to cap a grant: the cap is
271    /// `balance / shrink_divisor`, floored at `min_grant`. `1` caps at the whole
272    /// balance. Must be positive.
273    pub shrink_divisor: u64,
274    /// Floor for the shrink cap, so a shrinking balance still grants leases of a
275    /// useful size; a grant never exceeds the balance. Must be positive.
276    pub min_grant: CostUnits,
277    /// Hard cap on any single lease's TTL; requests beyond it are clamped.
278    pub max_ttl: SignedDuration,
279    /// How long past a lease's `expires_at` the allocator waits before
280    /// reclaiming its unspent units. Holders stop spending at
281    /// `expires_at - safety margin` (their side of the protocol), so work
282    /// committed inside the usability window has `margin + grace` to be
283    /// flushed and billed before settlement could reject it. Releases are
284    /// also accepted through the grace window (review finding GL-1).
285    pub reclaim_grace: SignedDuration,
286}
287
288/// Invalid allocator policy. Duration signs are part of the lease safety
289/// protocol, so invalid values are rejected at backend construction rather
290/// than normalized into a potentially unsafe policy.
291#[derive(Debug, Clone, Copy, PartialEq, Eq)]
292pub struct GrantPolicyError(pub &'static str);
293
294impl std::fmt::Display for GrantPolicyError {
295    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
296        f.write_str(self.0)
297    }
298}
299
300impl std::error::Error for GrantPolicyError {}
301
302impl Default for GrantPolicy {
303    fn default() -> Self {
304        GrantPolicy {
305            shrink_divisor: 2,
306            min_grant: CostUnits(1),
307            max_ttl: SignedDuration::from_secs(300),
308            reclaim_grace: SignedDuration::from_secs(30),
309        }
310    }
311}
312
313impl GrantPolicy {
314    /// Greatest expiry whose full grace window has elapsed at `now`.
315    /// A validated policy has nonnegative grace; subtraction underflow means
316    /// no representable expiry is due. In particular, a deadline beyond
317    /// Timestamp::MAX is never shortened to that last representable instant.
318    pub fn reclaim_cutoff(&self, now: Timestamp) -> Option<Timestamp> {
319        now.checked_sub(self.reclaim_grace).ok()
320    }
321
322    /// Check that the policy is safe to allocate with: `shrink_divisor`,
323    /// `min_grant` and `max_ttl` positive, and `reclaim_grace` nonnegative. Backends
324    /// call this at construction.
325    ///
326    /// # Errors
327    ///
328    /// A [`GrantPolicyError`] naming the first invalid field.
329    pub fn validate(&self) -> Result<(), GrantPolicyError> {
330        if self.shrink_divisor == 0 {
331            return Err(GrantPolicyError("shrink_divisor must be positive"));
332        }
333        if self.min_grant.is_zero() {
334            return Err(GrantPolicyError("min_grant must be positive"));
335        }
336        if self.max_ttl <= SignedDuration::ZERO {
337            return Err(GrantPolicyError("max_ttl must be positive"));
338        }
339        if self.reclaim_grace < SignedDuration::ZERO {
340            return Err(GrantPolicyError("reclaim_grace must not be negative"));
341        }
342        Ok(())
343    }
344
345    /// The units a request for `requested` receives from `balance`, or `None`
346    /// when the balance cannot fund any grant.
347    #[must_use]
348    pub fn grant(&self, requested: CostUnits, balance: CostUnits) -> Option<CostUnits> {
349        // Constructors reject these policy/request states, but `grant` is a
350        // public pure helper too. Keep direct use fail-closed instead of
351        // panicking on a zero divisor or manufacturing a unit for a zero
352        // request.
353        if requested.is_zero()
354            || balance.is_zero()
355            || self.shrink_divisor == 0
356            || self.min_grant.is_zero()
357        {
358            return None;
359        }
360        let cap = (balance.get() / self.shrink_divisor).max(self.min_grant.get());
361        Some(CostUnits(
362            requested.get().min(cap).min(balance.get()).max(1),
363        ))
364    }
365
366    /// The units a consolidation re-grants, or `None` exactly when
367    /// [`Self::grant`] refuses.
368    ///
369    /// `balance` already includes the credit the exchange restores, and
370    /// `floor` is that credit: the ordinary answer may not shrink the holding
371    /// (GL-109). `needed` is the largest quote the returned lease refused, and
372    /// the answer grows to it when `balance` can fund it (GL-131). The shrink
373    /// cap stops one holder hoarding a small balance ahead of demand; a quote
374    /// the holder already failed to fund is demand, and the request that
375    /// proved it spends it. A quote `balance` cannot fund grows nothing,
376    /// because no grant would serve it. A plain acquire is `floor` and
377    /// `needed` both zero, which is [`Self::grant`] unchanged.
378    #[must_use]
379    pub fn consolidation_grant(
380        &self,
381        requested: CostUnits,
382        balance: CostUnits,
383        floor: CostUnits,
384        needed: CostUnits,
385    ) -> Option<CostUnits> {
386        let sized = self.grant(requested, balance)?.max(floor.min(balance));
387        Some(if needed <= balance {
388            sized.max(needed)
389        } else {
390            sized
391        })
392    }
393}
394
395/// One lease settled by an expiry sweep.
396#[derive(Debug, Clone, Copy, PartialEq, Eq)]
397#[cfg_attr(feature = "wire", derive(serde::Serialize, serde::Deserialize))]
398pub struct ReclaimedLease {
399    /// The lease the sweep settled.
400    pub lease_id: LeaseId,
401    /// The account the lease was drawn from.
402    pub account_id: AccountId,
403    /// Units recorded as provisional settlement loss: `granted - recorded
404    /// usage` at the sweep. Nothing is credited back, because a holder that
405    /// never released cannot prove any unit unspent (GL-136); usage for the
406    /// lease that arrives later converts loss into billed usage.
407    pub forfeited: CostUnits,
408}
409
410/// The production-sized upper bound for one expiry-reclaim transaction.
411///
412/// The limit bounds locks, row materialization, and SQL parameters per
413/// transaction; it does not cap a legitimate backlog because the maintenance
414/// task drains saturated batches until it reaches a partial one.
415pub const DEFAULT_RECLAIM_BATCH_LIMIT: NonZeroUsize =
416    NonZeroUsize::new(256).expect("the reclaim batch limit is nonzero");
417
418/// Verified evidence returned by one bounded expiry-reclaim transaction.
419///
420/// The fields are private so `saturated` cannot disagree with the requested
421/// limit. Callers may therefore use it to decide whether another batch is
422/// required without re-deriving the backend's result.
423#[derive(Debug, Clone, PartialEq, Eq)]
424pub struct ReclaimBatch {
425    reclaimed: Vec<ReclaimedLease>,
426    saturated: bool,
427}
428
429impl ReclaimBatch {
430    /// Build a batch and derive its saturation evidence from `limit`.
431    pub fn try_new(
432        reclaimed: Vec<ReclaimedLease>,
433        limit: NonZeroUsize,
434    ) -> Result<Self, StoreError> {
435        if reclaimed.len() > limit.get() {
436            return Err(StoreError(format!(
437                "reclaim backend returned {} leases for a batch limit of {}",
438                reclaimed.len(),
439                limit
440            )));
441        }
442        Ok(ReclaimBatch {
443            saturated: reclaimed.len() == limit.get(),
444            reclaimed,
445        })
446    }
447
448    /// The leases this batch settled.
449    #[must_use]
450    pub fn reclaimed(&self) -> &[ReclaimedLease] {
451        &self.reclaimed
452    }
453
454    /// How many leases this batch settled.
455    #[must_use]
456    pub fn len(&self) -> usize {
457        self.reclaimed.len()
458    }
459
460    /// Whether this batch settled nothing.
461    #[must_use]
462    pub fn is_empty(&self) -> bool {
463        self.reclaimed.is_empty()
464    }
465
466    /// Whether the batch reached its limit, so more expired leases may remain and
467    /// the caller should run another batch.
468    #[must_use]
469    pub fn is_saturated(&self) -> bool {
470        self.saturated
471    }
472
473    /// Consume the batch, returning the leases it settled.
474    #[must_use]
475    pub fn into_reclaimed(self) -> Vec<ReclaimedLease> {
476        self.reclaimed
477    }
478}
479
480/// A grant, and the ledger's remaining funding as of the transaction that
481/// made it.
482///
483/// `funding` is read from the committed ledger after the grant and any
484/// consolidation settlement, so it counts the new lease's units. It is an
485/// upper bound on what the account can still spend (see
486/// [`tollgate_core::BalanceShortfall`]), carried apart from the grant because
487/// the grant is a capability and this is evidence about the account. `None`
488/// means the answering allocator attested nothing, as an older server does.
489#[derive(Debug, Clone, Copy, PartialEq, Eq)]
490#[cfg_attr(feature = "wire", derive(serde::Serialize, serde::Deserialize))]
491pub struct Allocation {
492    /// The lease capability. Its fields are flattened into the wire object.
493    #[cfg_attr(feature = "wire", serde(flatten))]
494    pub grant: LeaseGrant,
495    /// The account's remaining funding after this grant, or `None` when the
496    /// allocator attested nothing. Omitted from the wire when `None`.
497    #[cfg_attr(
498        feature = "wire",
499        serde(default, skip_serializing_if = "Option::is_none")
500    )]
501    pub funding: Option<tollgate_core::BalanceShortfall>,
502}
503
504/// Atomic lease allocation against the account balance — the amortization
505/// point: one `acquire` funds thousands of local reservations.
506#[async_trait]
507pub trait LeaseAllocator: Send + Sync {
508    /// Atomically debit a grant from the account. The granted size follows
509    /// the backend's [`GrantPolicy`] and may be smaller than `requested`;
510    /// the fencing token comes from a strictly increasing per-account
511    /// sequence. It remains a capability for this lease only; allocating a
512    /// newer token does not invalidate another active lease.
513    async fn acquire(
514        &self,
515        account: AccountId,
516        requested: CostUnits,
517        ttl: SignedDuration,
518        now: Timestamp,
519    ) -> Result<Allocation, AllocateError>;
520
521    /// Graceful return: require the stored `(lease_id, fencing_token)` pair,
522    /// credit `unspent` back, and close the lease. Callers should flush usage
523    /// first when possible; events arriving after release are accepted only
524    /// when they fit its provisional settlement loss.
525    async fn release(
526        &self,
527        lease_id: LeaseId,
528        fencing_token: FencingToken,
529        unspent: CostUnits,
530        now: Timestamp,
531    ) -> Result<(), AllocateError>;
532
533    /// Atomically return an active lease's `unspent` units and re-grant
534    /// against the restored balance: [`release`](Self::release) followed by
535    /// [`acquire`](Self::acquire), in one transaction, for the same account
536    /// the lease names.
537    ///
538    /// This exists because the holder cannot compose it from the two calls.
539    /// A holder whose grant is too small for the work it is being offered is
540    /// holding exactly the units the next grant needs, and separating the
541    /// return from the request loses them twice over: the
542    /// [`GrantPolicy`] re-sizes against a balance the returned units have
543    /// already rejoined, so a `shrink_divisor` above one can hand back
544    /// *less* than was returned (49 units returned into a balance of 58
545    /// re-grants 29 under the default policy), and in the gap between the two
546    /// calls another instance can take them. Neither is recoverable by the
547    /// holder, which is why the exchange belongs to the component that owns
548    /// both the policy and the transaction (INVARIANTS.md GL-1, GL-6).
549    ///
550    /// **The grant is never smaller than the credit actually restored.**
551    /// Allowance funded by a closed period expires at settlement; only its
552    /// surviving top-up credit supplies a floor. Otherwise all `unspent`
553    /// units return. The result is the larger of this floor and the ordinary
554    /// policy grant, so it can exceed `requested` when preserving a larger
555    /// holding. A zero request or a balance unable to fund any grant refuses
556    /// the whole exchange.
557    ///
558    /// **The grant grows to `needed` when the restored balance can fund it.**
559    /// `needed` is the largest quote the returned lease refused, zero when
560    /// none. The shrink cap exists so one holder cannot hoard a small balance
561    /// ahead of demand; a refused quote is demand already proven, so under a
562    /// `shrink_divisor` above one it is what lets a single holder reach a
563    /// quote above `balance / shrink_divisor` at all. A `needed` the balance
564    /// cannot fund changes nothing. Sizing is
565    /// [`GrantPolicy::consolidation_grant`] in every backend.
566    ///
567    /// The transaction applies both halves or neither. A domain refusal
568    /// (`InsufficientBalance`, `BalanceExhausted`, `BalanceInsufficient`,
569    /// `UnknownAccount`, `AccountInactive`, or `InvalidTtl`) leaves the
570    /// original lease unchanged.
571    /// `InvalidRelease` also leaves it unchanged but reports an
572    /// accounting-integrity fault.
573    /// `UnknownLease`, `Fenced`, and `LeaseNotActive` provide no authority to
574    /// resume spending from the old lease.
575    ///
576    /// **`Storage`, timeouts, and cancellation have an ambiguous outcome.**
577    /// The transaction may have committed before its reply was lost, including
578    /// during an HTTP response or transaction-commit failure. A holder must
579    /// keep the old lease out of service and attempt its release; reinstating
580    /// it could spend credited units twice. The unanswered replacement grant
581    /// cannot be recovered through the old capability, must be reported as
582    /// uncertain, and remains bounded by TTL reclaim.
583    #[allow(
584        clippy::too_many_arguments,
585        reason = "one transactional exchange: the release half's capability and credit, the \
586                  grant half's size and demand, and the shared lifetime and clock"
587    )]
588    async fn consolidate(
589        &self,
590        lease_id: LeaseId,
591        fencing_token: FencingToken,
592        unspent: CostUnits,
593        requested: CostUnits,
594        needed: CostUnits,
595        ttl: SignedDuration,
596        now: Timestamp,
597    ) -> Result<Allocation, AllocateError>;
598
599    /// Settle at most `limit` active leases whose TTL (plus the policy's
600    /// reclaim grace) has lapsed and whose holder never released them.
601    ///
602    /// Nothing is credited back. Each lease's `granted - recorded usage` is
603    /// recorded as provisional settlement loss, exactly as a release claiming
604    /// nothing unspent would record it, because a holder that never released
605    /// cannot prove any unit unspent: it may have committed work it never
606    /// flushed (INVARIANTS.md GL-9, GL-136). Usage for the lease that arrives
607    /// later fits in that loss and converts it into billed usage.
608    ///
609    /// One call is one bounded atomic
610    /// transaction; [`ReclaimBatch::is_saturated`] is verified evidence that
611    /// the caller should immediately run another batch. It must be safe to
612    /// run concurrently with everything else.
613    async fn reclaim_expired_batch(
614        &self,
615        now: Timestamp,
616        limit: NonZeroUsize,
617    ) -> Result<ReclaimBatch, StoreError>;
618
619    /// Settle every currently expired lease through bounded transactions.
620    ///
621    /// This preserves the original full-drain caller API. If a later batch
622    /// fails, earlier batches are already committed, so the returned error
623    /// explicitly reports that partial progress rather than presenting the
624    /// operation as all-or-nothing.
625    async fn reclaim_expired(&self, now: Timestamp) -> Result<Vec<ReclaimedLease>, StoreError> {
626        drain_reclaim_expired(self, now).await
627    }
628}
629
630/// The drain loop [`LeaseAllocator::reclaim_expired`] performs, as a free
631/// function so that an override can reuse it instead of re-deriving it.
632///
633/// Rust has no `super` for a trait default, so a wrapper that overrides
634/// `reclaim_expired` cannot call the body it overrides. Without this it must
635/// choose between forwarding to an inner allocator — which silently discards
636/// the wrapper's own `reclaim_expired_batch` override, and so discards any
637/// failure that override injects — and copying this loop, which is how two
638/// copies drift apart. Calling this keeps one body and re-dispatches every
639/// batch through `allocator`, whatever `allocator` is (GL-83).
640///
641/// `#[doc(hidden)]` marks it cross-crate-visible for that purpose rather than
642/// part of the documented surface, as [`Reservation::reserve_at_locality`] is
643/// in `tollgate-core`.
644///
645/// [`Reservation::reserve_at_locality`]: https://docs.rs/tollgate-core
646#[doc(hidden)]
647pub async fn drain_reclaim_expired<A>(
648    allocator: &A,
649    now: Timestamp,
650) -> Result<Vec<ReclaimedLease>, StoreError>
651where
652    A: LeaseAllocator + ?Sized,
653{
654    let mut reclaimed: Vec<ReclaimedLease> = Vec::new();
655    loop {
656        let batch = match allocator
657            .reclaim_expired_batch(now, DEFAULT_RECLAIM_BATCH_LIMIT)
658            .await
659        {
660            Ok(batch) => batch,
661            Err(error) if reclaimed.is_empty() => return Err(error),
662            Err(error) => {
663                let units: u128 = reclaimed
664                    .iter()
665                    .map(|lease| u128::from(lease.forfeited.get()))
666                    .sum();
667                return Err(StoreError(format!(
668                    "reclaim drain failed after {} leases forfeiting {units} units were committed: {error}",
669                    reclaimed.len()
670                )));
671            }
672        };
673        let saturated = batch.is_saturated();
674        reclaimed.extend(batch.into_reclaimed());
675        if !saturated {
676            return Ok(reclaimed);
677        }
678        tokio::task::yield_now().await;
679    }
680}
681
682/// Authoritative state returned by a snapshot pull or push.
683#[derive(Debug, Clone)]
684pub enum SnapshotResolution {
685    /// A compiled snapshot is currently authoritative.
686    Present(PublishableSnapshot),
687    /// The principal existed but was revoked at this generation. Sources must
688    /// retain this watermark so a delayed older positive cannot resurrect it.
689    Revoked {
690        /// The generation the tombstone was published at. A positive snapshot
691        /// at or below it cannot resurrect the principal (INVARIANTS.md 15).
692        generation: Generation,
693    },
694    /// The source has never observed this principal.
695    Unknown,
696}
697
698/// Slots in a backend's snapshot push channel.
699///
700/// Shared so the two backends cannot drift, and named rather than inlined
701/// because two things must agree on it: the channel, and the warning that
702/// fires when one operation would out-run it. Each slot retains an
703/// `Arc<AccountSnapshot>`, so this is also a bound on how much snapshot memory
704/// one slow subscriber can pin.
705pub const PUSH_CHANNEL_CAPACITY: usize = 256;
706
707/// Whether pushing `principals` updates at once will out-run the push channel.
708///
709/// A status change republishes every live snapshot of an account (GL-51), so a
710/// wide account can exceed the channel in one operation. Past this point every
711/// subscriber lags and resyncs its whole tracked set — correct, and bounded by
712/// the client's `max_concurrent_fetches`, but expensive enough that an
713/// operator should not have to infer it from a latency graph.
714///
715/// Strictly greater: a batch that exactly fills the channel is delivered, so
716/// warning at equality would cry wolf on the largest successful case. Pure, so
717/// the boundary is pinned by a test rather than by whichever backend is being
718/// read.
719#[must_use]
720pub fn pushes_exceed_capacity(principals: usize) -> bool {
721    principals > PUSH_CHANNEL_CAPACITY
722}
723
724/// One pushed snapshot update.
725#[derive(Debug, Clone)]
726pub struct SnapshotPush {
727    /// The principal whose authoritative state changed.
728    pub principal: Principal,
729    /// Its new authoritative state.
730    pub resolution: SnapshotResolution,
731}
732
733/// Where compiled snapshots come from.
734#[async_trait]
735pub trait SnapshotSource: Send + Sync {
736    /// Fetch the authoritative state for a principal. Revocation is distinct
737    /// from never-known so pull, lag recovery, and restart preserve the
738    /// generation watermark required for anti-resurrection semantics.
739    /// Reads used to reconstruct reclaimed local history must be linearizable
740    /// against durable publications/tombstones. Start a new source operation;
741    /// an earlier cached response or lagging replica cannot establish that
742    /// principal's forgotten generation floor. Return an error if this
743    /// authority is unavailable. MemoryStore and primary PostgresStore reads
744    /// supply this ordering; HTTP deployments must preserve it end to end.
745    async fn snapshot(&self, principal: Principal) -> Result<SnapshotResolution, StoreError>;
746
747    /// Subscribe to pushes. A lagging receiver may miss updates; the
748    /// contract is that a fresh `snapshot()` fetch after a lag error
749    /// observes at least the newest generation.
750    fn subscribe(&self) -> broadcast::Receiver<SnapshotPush>;
751
752    /// Every principal this source knows, including revoked ones — a
753    /// tombstone is still a principal an instance must track, so that it
754    /// knows the revocation (INVARIANTS.md GL-15).
755    ///
756    /// For instances that serve any customer rather than a configured slice
757    /// (GL-48). Pushes alone cannot answer this: they carry deltas from the
758    /// moment of subscribing, so a cold instance has no way to learn the set
759    /// that already exists.
760    ///
761    /// `Ok(None)` means this source cannot enumerate, and the manager stays
762    /// on its configured set — exactly today's behaviour. `Err` means
763    /// enumeration *failed* and is retried. The two are deliberately
764    /// distinct: collapsing them would let a broken source look like a
765    /// limited one, and an instance would quietly serve a stale set forever.
766    ///
767    /// Defaulted so a source that has no catalogue — a test double, an
768    /// embedder's own adapter — is unaffected.
769    async fn principals(&self) -> Result<Option<Vec<Principal>>, StoreError> {
770        Ok(None)
771    }
772}
773
774/// Liveness of the backing store, for readiness probes: a server must not
775/// report ready while its source of truth is unreachable (review finding
776/// GL-11).
777#[async_trait]
778pub trait StoreHealth: Send + Sync {
779    /// Succeed only when the backing store can currently answer.
780    /// `MemoryStore` always succeeds; `PostgresStore` runs a trivial query.
781    ///
782    /// # Errors
783    ///
784    /// A [`StoreError`] when the store cannot be reached.
785    async fn ping(&self) -> Result<(), StoreError>;
786}
787
788/// Refusals from account creation (review finding GL-7): creation is never
789/// destructive and never silently idempotent — recreating an existing
790/// account is a surfaced error in every backend, because an overwrite would
791/// reset balances/fencing under live leases and a silent no-op would hide
792/// operator mistakes. Resetting an account is a deliberate, separate
793/// workflow, not a create.
794#[derive(Debug, Clone, PartialEq, Eq)]
795pub enum CreateAccountError {
796    /// An account with this id already exists. It is left untouched.
797    AlreadyExists,
798    /// A backend failure unrelated to domain rules; see [`StoreError`].
799    Storage(StoreError),
800}
801
802impl std::fmt::Display for CreateAccountError {
803    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
804        match self {
805            CreateAccountError::AlreadyExists => f.write_str("account already exists"),
806            CreateAccountError::Storage(e) => write!(f, "{e}"),
807        }
808    }
809}
810
811impl std::error::Error for CreateAccountError {}
812
813/// What a status transition actually did.
814///
815/// The blast radius of the operation, returned rather than logged, because an
816/// operator suspending an account has no other way to learn it: the ledger
817/// half is one row, but the snapshot half is however many credentials that
818/// account has, and 204 says nothing.
819///
820/// `republished == 0` is the interesting value. It means the account had no
821/// live snapshots to change — either it has no credentials yet, or the ones it
822/// has are all revoked, or a status change was repeated and everything was
823/// already at the target. All three are worth knowing at the moment of the
824/// call rather than from a later denial.
825#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
826pub struct StatusChange {
827    /// Live snapshots republished with the new status, at `generation + 1`.
828    /// Excludes tombstones, which are never republished, and snapshots already
829    /// carrying the target status, which are not rewritten.
830    pub republished: usize,
831    /// Rows that changed durably but could not be decoded well enough to push.
832    ///
833    /// Always zero in `MemoryStore`, which holds validated snapshots rather
834    /// than encoded ones. In a stored backend a row can be undecodable — it
835    /// already was before the transition touched it, and the request path
836    /// already refuses it — and the transition deliberately does not fail
837    /// whole over one corrupt credential. But it is not silently absorbed
838    /// either: those principals did not get a push, so they will not converge
839    /// until their next refresh.
840    pub unreadable: usize,
841}
842
843/// One account moved across a period boundary by a rollover pass.
844#[derive(Debug, Clone, Copy, PartialEq, Eq)]
845pub struct RolledAccount {
846    /// The account whose period boundary was crossed.
847    pub account_id: AccountId,
848    /// The new period's allowance, deposited by this pass.
849    pub deposited: CostUnits,
850    /// Unspent allowance from the period that just closed. Manual top-ups are
851    /// never included: they persist across a boundary (GL-97).
852    pub expired: CostUnits,
853}
854
855/// The production-sized upper bound for one rollover transaction.
856///
857/// Every scheduled account comes due at the same instant — that is what a
858/// calendar boundary means — so this is not a cap on a rare backlog but the
859/// normal shape of the first pass after midnight on the 1st. It bounds locks,
860/// row materialization, and transaction size per statement; the sweep drains
861/// saturated batches until it reaches a partial one, exactly as the expiry
862/// reclaim does.
863pub const DEFAULT_ROLLOVER_BATCH_LIMIT: NonZeroUsize =
864    NonZeroUsize::new(256).expect("the rollover batch limit is nonzero");
865
866/// Verified evidence returned by one bounded rollover transaction.
867///
868/// Private fields, so `saturated` cannot disagree with the requested limit —
869/// the same contract [`ReclaimBatch`] carries, and for the same reason: the
870/// caller decides whether to ask for another batch from this, without
871/// re-deriving the backend's result.
872#[derive(Debug, Clone, PartialEq, Eq)]
873pub struct RolloverBatch {
874    rolled: Vec<RolledAccount>,
875    saturated: bool,
876}
877
878impl RolloverBatch {
879    /// Build a batch and derive its saturation evidence from `limit`.
880    pub fn try_new(rolled: Vec<RolledAccount>, limit: NonZeroUsize) -> Result<Self, StoreError> {
881        if rolled.len() > limit.get() {
882            return Err(StoreError(format!(
883                "rollover backend returned {} accounts for a batch limit of {}",
884                rolled.len(),
885                limit
886            )));
887        }
888        Ok(RolloverBatch {
889            saturated: rolled.len() == limit.get(),
890            rolled,
891        })
892    }
893
894    /// The accounts this batch rolled.
895    #[must_use]
896    pub fn rolled(&self) -> &[RolledAccount] {
897        &self.rolled
898    }
899
900    /// How many accounts this batch rolled.
901    #[must_use]
902    pub fn len(&self) -> usize {
903        self.rolled.len()
904    }
905
906    /// Whether this batch rolled nothing.
907    #[must_use]
908    pub fn is_empty(&self) -> bool {
909        self.rolled.is_empty()
910    }
911
912    /// Whether the batch reached its limit, so more due accounts may remain and
913    /// the caller should run another batch.
914    #[must_use]
915    pub fn is_saturated(&self) -> bool {
916        self.saturated
917    }
918}
919
920/// Refusals from setting or rolling a budget schedule (GL-97).
921#[derive(Debug, Clone, PartialEq, Eq)]
922pub enum BudgetError {
923    /// No such account. Never a silent no-op: an operator setting a schedule
924    /// on a mistyped id has to learn it now rather than at the next boundary.
925    UnknownAccount,
926    /// A backend failure unrelated to domain rules; see [`StoreError`].
927    Storage(StoreError),
928}
929
930impl std::fmt::Display for BudgetError {
931    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
932        match self {
933            BudgetError::UnknownAccount => f.write_str("no such account"),
934            BudgetError::Storage(e) => write!(f, "{e}"),
935        }
936    }
937}
938
939impl std::error::Error for BudgetError {}
940
941impl From<StoreError> for BudgetError {
942    fn from(error: StoreError) -> Self {
943        BudgetError::Storage(error)
944    }
945}
946
947/// Refusals from an account-status transition (GL-51).
948///
949/// Deliberately not an [`AllocateError`]: that enum's `NAMES`/`COUNT`/`index`
950/// are the width of `LeaseCounters`' per-reason tally, and a status refusal
951/// can never come out of `acquire`, so widening it would export a slot that
952/// is permanently zero in every deployment. [`CreateAccountError`] is the
953/// existing precedent for this shape — domain refusals plus `Storage`.
954#[derive(Debug, Clone, PartialEq, Eq)]
955pub enum SetStatusError {
956    /// No such account. Never a silent no-op, and the same answer whichever
957    /// status was asked for.
958    UnknownAccount,
959    /// [`AccountStatus::Closed`] is terminal: an account enters it from any
960    /// status and leaves it never. The refusal changes nothing — not the
961    /// ledger, not one snapshot, not one generation.
962    AccountClosed,
963    /// A backend failure unrelated to domain rules; see [`StoreError`].
964    Storage(StoreError),
965}
966
967impl std::fmt::Display for SetStatusError {
968    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
969        match self {
970            SetStatusError::UnknownAccount => f.write_str("unknown account"),
971            SetStatusError::AccountClosed => f.write_str("account is closed"),
972            SetStatusError::Storage(e) => write!(f, "{e}"),
973        }
974    }
975}
976
977impl std::error::Error for SetStatusError {}
978
979impl From<StoreError> for SetStatusError {
980    fn from(error: StoreError) -> Self {
981        SetStatusError::Storage(error)
982    }
983}
984
985/// Refusals from publishing a snapshot (GL-51).
986///
987/// `publish_snapshot` used to return a bare [`StoreError`], which left it free
988/// to write a status contradicting the ledger and recreate the divergence
989/// [`AdminStore::set_account_status`] exists to abolish.
990#[derive(Debug, Clone, PartialEq, Eq)]
991pub enum PublishSnapshotError {
992    /// The stated credential is missing or belongs to another principal/account.
993    CredentialMismatch {
994        /// The credential the snapshot states.
995        key_id: KeyId,
996    },
997    /// The snapshot's status disagrees with the account ledger. An account's
998    /// status is changed through [`AdminStore::set_account_status`], which
999    /// republishes; a publish may carry the current status but may not change
1000    /// it.
1001    StatusMismatch {
1002        /// The status the account ledger holds.
1003        ledger: AccountStatus,
1004        /// The status the snapshot carried.
1005        submitted: AccountStatus,
1006    },
1007    /// The snapshot's execution-capacity class disagrees with the account
1008    /// ledger (GL-99). The class is an account-owned fact changed through
1009    /// [`AdminStore::set_capacity_class`], which republishes; a publish may
1010    /// carry the current class but may not change it. Two writers for one
1011    /// fact is the divergence the status guard above already exists to
1012    /// abolish, and a second field must not reintroduce it.
1013    CapacityClassMismatch {
1014        /// The class the account ledger holds.
1015        ledger: CapacityClass,
1016        /// The class the snapshot carried.
1017        submitted: CapacityClass,
1018    },
1019    /// A backend failure unrelated to domain rules; see [`StoreError`].
1020    Storage(StoreError),
1021}
1022
1023impl std::fmt::Display for PublishSnapshotError {
1024    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1025        match self {
1026            PublishSnapshotError::CredentialMismatch { key_id } => {
1027                write!(
1028                    f,
1029                    "credential {key_id} does not bind the published principal and account"
1030                )
1031            }
1032            PublishSnapshotError::StatusMismatch { ledger, submitted } => write!(
1033                f,
1034                "snapshot status {} contradicts account status {}",
1035                submitted.as_str(),
1036                ledger.as_str()
1037            ),
1038            PublishSnapshotError::CapacityClassMismatch { ledger, submitted } => write!(
1039                f,
1040                "snapshot capacity class {} contradicts account capacity class {}",
1041                submitted.as_str(),
1042                ledger.as_str()
1043            ),
1044            PublishSnapshotError::Storage(e) => write!(f, "{e}"),
1045        }
1046    }
1047}
1048
1049impl std::error::Error for PublishSnapshotError {}
1050
1051impl From<StoreError> for PublishSnapshotError {
1052    fn from(error: StoreError) -> Self {
1053        PublishSnapshotError::Storage(error)
1054    }
1055}
1056
1057/// One account's administrative state, for an operator read (GL-121).
1058///
1059/// Assembled from types that already exist rather than a parallel vocabulary,
1060/// so the HTTP surface reports the same terms the ledger reasons in and a
1061/// reader can hold a dashboard next to a conservation check.
1062///
1063/// **Funding is not billing, and the shape says so.** A falling `balance` does
1064/// not mean units were billed: it also falls when they go out on a lease that
1065/// has not settled, and it falls when a budget period closes and takes its
1066/// unspent allowance with it. Those are three different facts, and
1067/// [`Conservation`] keeps them apart — `active_lease_grants` is capacity
1068/// currently out, `settled_usage` is what was actually consumed,
1069/// `settlement_loss` and `expired` are what will never be. A surface that
1070/// reported only a balance would let a customer read depletion as spend, which
1071/// is exactly what GL-121 asks not to do.
1072#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1073pub struct AccountView {
1074    /// The account read.
1075    pub account_id: AccountId,
1076    /// Administrative status, from the ledger.
1077    pub status: AccountStatus,
1078    /// Execution-capacity class, from the ledger.
1079    pub capacity_class: CapacityClass,
1080    /// The periodic allowance, if this account has one. `None` is "no
1081    /// schedule, the balance does not expire" — not "unknown".
1082    pub schedule: Option<BudgetSchedule>,
1083    /// First instant of the period currently in force. Meaningful only
1084    /// alongside a `schedule`; it is the marker rollover is idempotent
1085    /// against.
1086    pub period_start: Timestamp,
1087    /// Every term of the funding equation, so a caller can distinguish
1088    /// remaining funding from outstanding grants from settled usage.
1089    pub conservation: Conservation,
1090}
1091
1092/// Administrative writes: the control plane's mutation surface. Kept apart
1093/// from the data-plane traits so a read-only replica can implement those
1094/// without this.
1095///
1096/// HTTP-facing mutations return [`crate::AdminReceipt`] captured at the same
1097/// serialization point as the write. Its `outcome` contains the operation's
1098/// result; before/after describe the fields owned by that operation. A separate
1099/// read before or after the transaction is not a valid receipt under concurrency.
1100/// No-op receipts have equal states; errors carry no confirmed transition.
1101/// Direct callers attach their own actor and audit delivery policy.
1102#[async_trait]
1103pub trait AdminStore: Send + Sync {
1104    /// Create an account from `config`, with its opening balance deposited as a
1105    /// top-up and its fencing sequence starting at one.
1106    ///
1107    /// Never destructive and never silently idempotent (INVARIANTS.md 14): an
1108    /// existing account is refused with [`CreateAccountError::AlreadyExists`] and
1109    /// left untouched, including its balance, ledger totals, fencing sequence and
1110    /// leases. The receipt's `before` is [`AdminState::Absent`](crate::AdminState::Absent).
1111    async fn create_account(
1112        &self,
1113        config: AccountConfig,
1114    ) -> Result<crate::AdminReceipt<()>, CreateAccountError>;
1115    /// Add `units` to an existing account as a top-up, which survives period
1116    /// boundaries, raising its balance and its `deposited` total together.
1117    ///
1118    /// A missing account is [`AllocateError::UnknownAccount`]. An overflow of
1119    /// either counter is [`AllocateError::Storage`] and moves neither. Account
1120    /// status is not checked. The receipt carries
1121    /// [`AdminState::Funding`](crate::AdminState::Funding) before and after.
1122    async fn deposit(
1123        &self,
1124        account: AccountId,
1125        units: CostUnits,
1126    ) -> Result<crate::AdminReceipt<()>, AllocateError>;
1127    /// Set an existing account's administrative status, in one transaction:
1128    /// the ledger's status, and a republication of every *live* snapshot of
1129    /// that account carrying the new status at `generation + 1`.
1130    ///
1131    /// This is the whole operator action. Before GL-51 the ledger flag and the
1132    /// published `AccountStatus` were two records with two propagation paths
1133    /// and nothing checking them against each other, so "deactivate" returned
1134    /// success while the request path kept admitting.
1135    ///
1136    /// Rules, all enforced here rather than by caller discipline:
1137    /// - A missing account is [`SetStatusError::UnknownAccount`], never a
1138    ///   silent no-op, whichever status was asked for.
1139    /// - [`AccountStatus::Closed`] is terminal
1140    ///   ([`SetStatusError::AccountClosed`]); `Closed` → `Closed` is a no-op.
1141    /// - Revoked principals are never republished: resurrecting a tombstone
1142    ///   is what INVARIANTS.md GL-15 forbids, and revocation stays a separate
1143    ///   per-credential mechanism.
1144    /// - Snapshots already at the target status are not rewritten, so a
1145    ///   repeat converges and bumps no generation.
1146    /// - Outstanding leases are **not** reclaimed. Lease acquisition refuses
1147    ///   at once, but admission stops only when the new snapshot installs —
1148    ///   one `SnapshotManager` refresh interval, and already-debited units
1149    ///   settle at release or TTL reclaim (GL-9).
1150    async fn set_account_status(
1151        &self,
1152        account: AccountId,
1153        status: AccountStatus,
1154    ) -> Result<crate::AdminReceipt<StatusChange>, SetStatusError>;
1155
1156    /// Set an existing account's execution-capacity class, in one
1157    /// transaction: the ledger's class, and a republication of every *live*
1158    /// snapshot of that account carrying the new class at `generation + 1`
1159    /// (GL-99).
1160    ///
1161    /// The same operator action, and the same ownership argument, as
1162    /// [`set_account_status`](Self::set_account_status): the class is one
1163    /// fact with one writer. A control plane that published it per credential
1164    /// instead would recreate exactly the divergence GL-51 abolished — some of
1165    /// an account's principals assured and some best-effort, with nothing
1166    /// checking them against each other, and a request's treatment depending
1167    /// on which credential it arrived with.
1168    ///
1169    /// Rules, all enforced here rather than by caller discipline:
1170    /// - A missing account is [`SetStatusError::UnknownAccount`].
1171    /// - A closed account is [`SetStatusError::AccountClosed`]. Reclassifying
1172    ///   a terminally closed account is meaningless and the refusal changes
1173    ///   nothing, exactly as it does for a status change.
1174    /// - Revoked principals are never republished (INVARIANTS.md GL-15).
1175    /// - Snapshots already at the target class are not rewritten, so a repeat
1176    ///   converges and bumps no generation.
1177    /// - Nothing about funding changes. The class decides whether an instance
1178    ///   starts an already-funded request, so quota, leases, and outstanding
1179    ///   usage are untouched — an account reclassified mid-flight keeps every
1180    ///   charge it has already committed.
1181    ///
1182    /// Reuses [`SetStatusError`] rather than declaring a near-identical twin:
1183    /// the two refusals are the same two conditions about the same ledger row,
1184    /// and a second enum would be two vocabularies for one answer.
1185    async fn set_capacity_class(
1186        &self,
1187        account: AccountId,
1188        class: CapacityClass,
1189    ) -> Result<crate::AdminReceipt<StatusChange>, SetStatusError>;
1190
1191    /// Give an account a periodic allowance, or take it away.
1192    ///
1193    /// Setting a schedule does not deposit anything: the first allowance
1194    /// arrives at the first [`roll_due_periods`](Self::roll_due_periods) pass after the
1195    /// schedule exists. Depositing here would make "set a schedule" and "give
1196    /// this account units now" the same operation, and an operator correcting
1197    /// a mistyped allowance would fund the account twice.
1198    ///
1199    /// `None` removes the schedule and leaves the balance alone — including
1200    /// any unspent allowance, which simply stops expiring. Removing a schedule
1201    /// is not a way to claw units back.
1202    /// Set or clear an account's periodic allowance.
1203    ///
1204    /// Returns a receipt rather than `()` so the change can be audited like
1205    /// every other administrative mutation: an operator surface has to be able
1206    /// to report what a call actually committed, and a bare `Ok` cannot say
1207    /// whether a schedule was introduced, replaced, or was already what the
1208    /// caller asked for (GL-121). A repeat returns equal before/after states,
1209    /// which is how the convention expresses an idempotent no-op.
1210    async fn set_budget_schedule(
1211        &self,
1212        account: AccountId,
1213        schedule: Option<BudgetSchedule>,
1214    ) -> Result<crate::AdminReceipt<()>, BudgetError>;
1215
1216    /// Cross the period boundary for up to `limit` accounts that are past it:
1217    /// expire each closed period's unspent allowance and deposit the next one,
1218    /// one transaction per batch.
1219    ///
1220    /// **Idempotency is this method's job, not its caller's.** The pass runs
1221    /// on every control-plane replica, so two of them will race a boundary;
1222    /// the backend crosses it under a row lock, and the second caller then
1223    /// reads the period the first one wrote and skips the account. A caller
1224    /// that read each period first and then rolled would produce two deposits
1225    /// under exactly the race this exists to survive.
1226    ///
1227    /// Safe to call at any cadence: before a boundary it selects nothing, and
1228    /// after one the first caller wins. It never rolls an account more than
1229    /// one period at a time — an account left unrolled for two months lands in
1230    /// the current period with one allowance, because an allowance is what the
1231    /// account is entitled to now, not a backlog to be paid out.
1232    ///
1233    /// Bounded, and saturating means there is more: every scheduled account
1234    /// comes due at the same instant, so a caller must drain saturated batches
1235    /// until one comes back partial. Unscheduled accounts are never selected.
1236    async fn roll_due_periods(
1237        &self,
1238        now: Timestamp,
1239        limit: NonZeroUsize,
1240    ) -> Result<RolloverBatch, StoreError>;
1241    /// Publish a principal's compiled snapshot.
1242    ///
1243    /// Refused with [`PublishSnapshotError::StatusMismatch`] when the
1244    /// snapshot's status contradicts the account ledger, so the two records
1245    /// [`set_account_status`](AdminStore::set_account_status) unifies cannot
1246    /// be pulled apart again one principal at a time. A snapshot whose
1247    /// account the ledger does not hold publishes unchanged, as before.
1248    async fn publish_snapshot(
1249        &self,
1250        principal: Principal,
1251        snapshot: PublishableSnapshot,
1252    ) -> Result<crate::AdminReceipt<()>, PublishSnapshotError>;
1253    /// Withdraw a principal's snapshot by tombstoning it at its current
1254    /// generation, and push the revocation to subscribers. The tombstone is
1255    /// durable, so no positive snapshot at or below that generation can resurrect
1256    /// the principal (INVARIANTS.md 15).
1257    ///
1258    /// A principal with no snapshot, or one already tombstoned, is left unchanged
1259    /// and the receipt's states are equal.
1260    async fn remove_snapshot(
1261        &self,
1262        principal: Principal,
1263    ) -> Result<crate::AdminReceipt<()>, StoreError>;
1264
1265    /// One account's administrative state, or `None` if no such account (GL-121).
1266    ///
1267    /// The read an operator surface needs and the traits did not have. Both
1268    /// backends already expose `conservation` as an inherent method, but with
1269    /// different signatures — one synchronous returning an `Option`, one
1270    /// asynchronous returning a `Result` — so nothing generic over a backend
1271    /// could read an account at all.
1272    ///
1273    /// A single call rather than several, because the terms have to agree with
1274    /// each other: status, schedule and the funding equation read separately
1275    /// can straddle a rollover or a suspension and describe a state the account
1276    /// was never in. A backend answers this from one consistent read.
1277    async fn account_view(&self, account: AccountId) -> Result<Option<AccountView>, StoreError>;
1278}
1279
1280/// Outcome of one ingest batch.
1281#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1282#[cfg_attr(feature = "wire", derive(serde::Serialize, serde::Deserialize))]
1283pub struct IngestReport {
1284    /// Newly recorded events.
1285    pub accepted: u64,
1286    /// Events whose `request_id` was already recorded (idempotent replay —
1287    /// INVARIANTS.md GL-7).
1288    pub duplicate: u64,
1289    /// Events refused: unknown lease, lease-capability mismatch, or no
1290    /// remaining accounting capacity, or units outside the backend's storage
1291    /// domain. These are bounded billing loss,
1292    /// visible to reconciliation.
1293    pub rejected: u64,
1294    /// Newly accepted events with absent, unknown or different-account key
1295    /// attribution. Duplicates never supply fresh activity evidence. `None`
1296    /// means this sink does not report attribution (including older servers),
1297    /// not that every event was attributed.
1298    #[cfg_attr(feature = "wire", serde(default))]
1299    pub unattributed: Option<u64>,
1300}
1301
1302impl IngestReport {
1303    /// A complete acknowledgement partitions this batch exactly once. Validate
1304    /// before releasing queued evidence, including replies from custom sinks.
1305    pub fn validate(&self, submitted: usize) -> Result<(), StoreError> {
1306        let total = self
1307            .accepted
1308            .checked_add(self.duplicate)
1309            .and_then(|n| n.checked_add(self.rejected));
1310        if !total.is_some_and(|n| u64::try_from(submitted) == Ok(n))
1311            || self.unattributed.is_some_and(|n| n > self.accepted)
1312        {
1313            return Err(StoreError(
1314                "invalid usage acknowledgement cardinality".into(),
1315            ));
1316        }
1317        Ok(())
1318    }
1319}
1320
1321/// The billing ledger's write side.
1322#[async_trait]
1323pub trait UsageSink: Send + Sync {
1324    /// Record a batch. Idempotent on `request_id`; every event must match its
1325    /// stored `(lease_id, account_id, fencing_token)` capability before lease
1326    /// state and accounting capacity are checked. Partial acceptance is
1327    /// normal — the report says what happened.
1328    ///
1329    /// Duplicates are classified before inspecting their payload. A new event
1330    /// outside the backend's unit domain is rejected individually; it is not
1331    /// remembered as accepted and cannot poison otherwise valid neighbors.
1332    /// MemoryStore supports `u64` units; PostgreSQL supports nonnegative
1333    /// `BIGINT` units (`0..=i64::MAX`). A representable event that overflows an
1334    /// accumulated accounting total refuses the entire batch atomically.
1335    async fn ingest(
1336        &self,
1337        events: &[UsageEvent],
1338        now: Timestamp,
1339    ) -> Result<IngestReport, IngestError>;
1340}
1341
1342/// One credential's durable record.
1343///
1344/// The `digest` is opaque here on purpose. The HMAC secret that produced it
1345/// lives with the verifier (`tollgate-auth`) and never reaches a store, so a
1346/// backend holds material that verifies nothing on its own — the property
1347/// `HmacRegistry` is built around, preserved across the persistence boundary.
1348/// A store that could compute a digest would be a store whose compromise is
1349/// sufficient to mint credentials.
1350///
1351/// `principal` is the digest's own truncation, so it is derived rather than
1352/// assigned: the request path is keyed by it, and per-credential revocation
1353/// is `install_revoked` for exactly this value.
1354#[derive(Clone, PartialEq, Eq)]
1355pub struct KeyRecord {
1356    /// The credential's non-secret identifier, chosen by its issuer.
1357    pub key_id: KeyId,
1358    /// The account the credential authenticates for.
1359    pub account_id: AccountId,
1360    /// The principal this credential authenticates as: the leading 128 bits
1361    /// of `digest`.
1362    pub principal: Principal,
1363    /// HMAC-SHA256 of the secret under the verifier's server secret.
1364    pub digest: [u8; 32],
1365    /// When the credential stops being valid of its own accord, independent
1366    /// of revocation. Surfaced to the request path through
1367    /// `Verified::reusable_until`, so a session cache cannot outlive it.
1368    pub not_after: Option<Timestamp>,
1369}
1370
1371impl std::fmt::Debug for KeyRecord {
1372    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1373        f.debug_struct("KeyRecord")
1374            .field("key_id", &self.key_id)
1375            .field("account_id", &self.account_id)
1376            .field("principal", &self.principal)
1377            .field("not_after", &self.not_after)
1378            .finish_non_exhaustive()
1379    }
1380}
1381
1382/// One requested key's activity. No observation is not proof of non-use.
1383#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1384pub struct CredentialActivity {
1385    /// The requested key.
1386    pub key_id: KeyId,
1387    /// What the store has recorded for it.
1388    pub state: CredentialActivityState,
1389}
1390
1391/// A credential's recorded commitment activity (INVARIANTS.md 35).
1392/// It is never authentication or authorization evidence.
1393#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1394pub enum CredentialActivityState {
1395    /// The directory holds no credential with this id.
1396    Unknown,
1397    /// The credential exists, but no accepted, attributable usage has been
1398    /// recorded for it. Not proof that it was never used.
1399    Unobserved,
1400    /// Maximum accepted, attributable execution-start time, at microsecond
1401    /// precision. Never an authorization or independent server-clock fact.
1402    Committed {
1403        /// The latest recorded execution-start time.
1404        last_committed_at: Timestamp,
1405    },
1406}
1407
1408/// What a revocation actually did.
1409///
1410/// Returned rather than inferred, for the reason [`StatusChange`] is: an
1411/// operator retiring a suspicious credential needs to know whether they
1412/// retired anything. "Already revoked" and "no such key" are different
1413/// answers to the same request and only one of them is a mistake.
1414#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1415pub enum Revocation {
1416    /// The credential was live and is now retired.
1417    Retired,
1418    /// The credential was already retired; nothing changed.
1419    AlreadyRetired,
1420}
1421
1422/// The largest usage batch any sink accepts in one call, in events.
1423///
1424/// A batch cap has to admit the largest batch the system can legitimately
1425/// produce, not a comfortable one: the shipped example writes 256 per flush,
1426/// and this leaves a factor of sixteen for an embedder that batches harder to
1427/// cut round-trips. Beyond it the answer is more flushes, not a bigger body —
1428/// an ingest is idempotent, so splitting costs a round trip and risks nothing.
1429///
1430/// `UsageWriterConfig::validate` refuses a `max_batch` above this, so the
1431/// misconfiguration is a startup error rather than a permanently-rejected
1432/// batch discovered in production (GL-61).
1433pub const MAX_INGEST_BATCH: usize = 4_096;
1434
1435/// Why an ingest attempt failed, and whether replaying it unchanged could
1436/// ever succeed.
1437///
1438/// The distinction exists because the writer's correct response to the two is
1439/// opposite. An unreachable sink is a *duration*: retrying the same batch is
1440/// the designed behaviour, and giving up would lose billable events over a
1441/// blip. A refused batch is a *fact about the batch*: retrying it unchanged
1442/// gets the same answer forever, and every event queued behind it waits for a
1443/// recovery that cannot come — a permanent, deterministic error laundered
1444/// into an unbounded billing and availability outage (GL-61).
1445///
1446/// [`From<StoreError>`] yields [`Unavailable`](Self::Unavailable), so a
1447/// backend that does not classify keeps the retry-forever behaviour it had.
1448/// Terminality is asserted, never assumed.
1449#[derive(Debug, Clone, PartialEq, Eq)]
1450pub enum IngestError {
1451    /// The sink could not be reached, or could not answer in time. The batch
1452    /// is unchanged and will be retried.
1453    Unavailable(StoreError),
1454    /// The sink refused this batch and will refuse it again unchanged: a body
1455    /// over the endpoint's limit, an event it cannot decode, a contract it
1456    /// does not implement. Retrying cannot help.
1457    Refused(StoreError),
1458}
1459
1460impl IngestError {
1461    /// Whether replaying this batch unchanged could ever succeed.
1462    #[must_use]
1463    pub const fn is_retryable(&self) -> bool {
1464        matches!(self, IngestError::Unavailable(_))
1465    }
1466}
1467
1468impl From<StoreError> for IngestError {
1469    fn from(error: StoreError) -> Self {
1470        IngestError::Unavailable(error)
1471    }
1472}
1473
1474impl std::fmt::Display for IngestError {
1475    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1476        match self {
1477            IngestError::Unavailable(e) => write!(f, "{e}"),
1478            IngestError::Refused(e) => write!(f, "refused: {e}"),
1479        }
1480    }
1481}
1482
1483impl std::error::Error for IngestError {}
1484
1485/// Refusals from credential lifecycle operations.
1486#[derive(Debug, Clone, PartialEq, Eq)]
1487pub enum KeyError {
1488    /// No such credential. Never a silent no-op — an operator revoking a key
1489    /// that does not exist has either the wrong id or a false belief about
1490    /// what is live, and both are worth surfacing.
1491    UnknownKey,
1492    /// The credential's account does not exist, so nothing could authenticate
1493    /// as it. Refused at issuance rather than producing a key that verifies
1494    /// and is then denied by every admission.
1495    UnknownAccount,
1496    /// This `key_id` is already recorded. Issuance is never destructive, for
1497    /// the reason account creation is not ([`CreateAccountError`]): an
1498    /// overwrite would silently retire a live credential.
1499    ///
1500    /// This is also the retry answer. A caller that supplies the `key_id` and
1501    /// loses the response resends the same one and is told the credential
1502    /// exists — which is the truth, and which discloses no secret. That is why
1503    /// issuance must never become an upsert (GL-121).
1504    AlreadyExists,
1505    /// The account already holds `limit` live credentials, so issuing another
1506    /// would exceed the bound the caller supplied.
1507    ///
1508    /// "Live" excludes revoked keys and keys whose `not_after` has passed: a
1509    /// bound that counted expired credentials would strand an account behind
1510    /// keys nobody can authenticate with.
1511    ActiveKeyLimit {
1512        /// The bound the caller supplied.
1513        limit: NonZeroUsize,
1514    },
1515    /// A backend failure unrelated to domain rules; see [`StoreError`].
1516    Storage(StoreError),
1517}
1518
1519impl std::fmt::Display for KeyError {
1520    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1521        match self {
1522            KeyError::UnknownKey => f.write_str("no such credential"),
1523            KeyError::UnknownAccount => f.write_str("no such account"),
1524            KeyError::AlreadyExists => f.write_str("credential already exists"),
1525            KeyError::ActiveKeyLimit { limit } => {
1526                write!(f, "account already holds {limit} live credentials")
1527            }
1528            KeyError::Storage(e) => write!(f, "{e}"),
1529        }
1530    }
1531}
1532
1533impl std::error::Error for KeyError {}
1534
1535/// Refusals from binding or withdrawing a snapshot by the credential it was
1536/// issued as (GL-143).
1537#[derive(Debug, Clone, PartialEq, Eq)]
1538pub enum KeySnapshotError {
1539    /// No such credential, or it belongs to another account. One answer for
1540    /// both, as revocation gives: a foreign `key_id` discloses nothing.
1541    UnknownCredential,
1542    /// The credential was revoked. Revocation is terminal (INVARIANTS.md
1543    /// GL-27), so it is never granted positive authorization again; withdrawal
1544    /// remains allowed.
1545    Retired {
1546        /// The revoked credential.
1547        key_id: KeyId,
1548    },
1549    /// Publication refused as [`AdminStore::publish_snapshot`] would refuse it.
1550    Publish(PublishSnapshotError),
1551    /// A backend failure unrelated to domain rules; see [`StoreError`].
1552    Storage(StoreError),
1553}
1554
1555impl std::fmt::Display for KeySnapshotError {
1556    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1557        match self {
1558            KeySnapshotError::UnknownCredential => f.write_str("no such credential"),
1559            KeySnapshotError::Retired { key_id } => {
1560                write!(f, "credential {key_id} is revoked")
1561            }
1562            KeySnapshotError::Publish(e) => write!(f, "{e}"),
1563            KeySnapshotError::Storage(e) => write!(f, "{e}"),
1564        }
1565    }
1566}
1567
1568impl std::error::Error for KeySnapshotError {}
1569
1570impl From<PublishSnapshotError> for KeySnapshotError {
1571    fn from(error: PublishSnapshotError) -> Self {
1572        Self::Publish(error)
1573    }
1574}
1575
1576impl From<StoreError> for KeySnapshotError {
1577    fn from(error: StoreError) -> Self {
1578        Self::Storage(error)
1579    }
1580}
1581
1582impl From<StoreError> for KeyError {
1583    fn from(error: StoreError) -> Self {
1584        Self::Storage(error)
1585    }
1586}
1587
1588/// One credential as an *administrator* sees it (GL-121).
1589///
1590/// Deliberately not a [`KeyRecord`]. A record carries `digest` — the HMAC the
1591/// verifier compares against — and `principal`, documented there as "the
1592/// leading 128 bits of `digest`". Both are digest material, and an
1593/// account-scoped listing is reachable by an application backend and from
1594/// there a browser, so neither may appear in it. `key_id` is the non-secret
1595/// handle: what the caller chose, what revocation names, what an audit shows.
1596///
1597/// Expiry and revocation are surfaced separately. A credential that lapsed on
1598/// its own is a different operational fact from one an operator withdrew, and
1599/// collapsing both into "inactive" loses the distinction exactly where someone
1600/// is deciding whether to issue a replacement.
1601#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1602pub struct KeySummary {
1603    /// The credential's non-secret identifier.
1604    pub key_id: KeyId,
1605    /// When the credential stops being valid of its own accord, if ever.
1606    pub not_after: Option<Timestamp>,
1607    /// When an operator withdrew it, if they did. Terminal.
1608    pub revoked_at: Option<Timestamp>,
1609}
1610
1611impl KeySummary {
1612    /// Whether this credential can still authenticate at `now`.
1613    ///
1614    /// The predicate the issuance bound counts with, written once so a listing
1615    /// and a limit cannot disagree about what "live" means.
1616    #[must_use]
1617    pub fn is_live(&self, now: Timestamp) -> bool {
1618        self.revoked_at.is_none() && self.not_after.is_none_or(|until| now < until)
1619    }
1620}
1621
1622/// Durable credential lifecycle: the half of key management that outlives a
1623/// process and is shared by a fleet.
1624///
1625/// **Why this is a store trait rather than registry state.** Every other
1626/// control-plane fact here — accounts, balances, leases, statuses, snapshots —
1627/// is transactional in a backend with a local read projection, and credentials
1628/// are the same class of fact. An in-memory-only registry would lose issued
1629/// keys on restart, keep verifying a credential another instance revoked, and
1630/// make a fleet-wide active-key limit unenforceable, because no instance sees
1631/// the fleet.
1632///
1633/// **Durability before disclosure.** [`insert_key`](Self::insert_key) must
1634/// commit before its caller returns the secret to anyone. A crash between the
1635/// two hands out a credential the server has never heard of, which no later
1636/// reconciliation can repair: the digest is unrecoverable from the record,
1637/// which is the point of storing digests.
1638///
1639/// The verifier's projection is rebuilt from [`active_keys`](Self::active_keys),
1640/// so a backend decides what "active" means once, here, rather than in each
1641/// reader.
1642#[async_trait]
1643pub trait KeyDirectory: crate::KeySource {
1644    /// Inspect requested keys in input order, including repeated IDs and
1645    /// retired keys. Every input has one explicit result or the read fails.
1646    /// This operator read uses O(keys.len()) output memory; backends bound
1647    /// individual queries internally. Multiple chunks need not share an instant.
1648    async fn credential_activity(
1649        &self,
1650        keys: &[KeyId],
1651    ) -> Result<Vec<CredentialActivity>, StoreError>;
1652
1653    /// Record a minted credential. The caller has already generated the
1654    /// secret and computed its digest; this stores what remains.
1655    async fn insert_key(&self, record: KeyRecord) -> Result<(), KeyError>;
1656
1657    /// Retire one credential, reporting whether it was live.
1658    ///
1659    /// Revocation is durable and terminal: a retired credential is never
1660    /// resurrected, for the same reason a snapshot tombstone is not
1661    /// (INVARIANTS.md GL-15).
1662    async fn revoke_key(&self, key_id: KeyId, now: Timestamp) -> Result<Revocation, KeyError>;
1663
1664    /// Unbounded operator read of every credential valid at `now`. Serving
1665    /// instances use `KeySource` pages and an owned, bounded drain instead.
1666    /// Retained for existing direct-store lifecycle tooling; no hidden page cap.
1667    async fn active_keys(&self, now: Timestamp) -> Result<Vec<KeyRecord>, StoreError>;
1668
1669    /// One account's credentials, ordered by `key_id`, for an operator
1670    /// listing (GL-121).
1671    ///
1672    /// Distinct from [`active_keys`](Self::active_keys), which is the
1673    /// fleet-wide, digest-bearing projection an *instance* pulls: this is
1674    /// account-scoped, bounded, and carries no digest material, because it
1675    /// answers a different question for a different caller.
1676    ///
1677    /// Paginate with `after` — the greatest `key_id` already seen, exclusive.
1678    /// Revoked and expired credentials are included, because an administrator
1679    /// deciding whether to issue a replacement needs to see what became of the
1680    /// last one; [`KeySummary::is_live`] separates them.
1681    async fn account_keys(
1682        &self,
1683        account: AccountId,
1684        after: Option<KeyId>,
1685        limit: NonZeroUsize,
1686    ) -> Result<Vec<KeySummary>, StoreError>;
1687
1688    /// Record a credential only if the account holds fewer than `max_active`
1689    /// live ones, counting and inserting indivisibly (GL-121).
1690    ///
1691    /// The bound is supplied per call rather than stored: what counts as a
1692    /// reasonable number of credentials belongs to the application's plan, not
1693    /// to Tollgate, and a value in the request is one the caller can change
1694    /// without a migration.
1695    ///
1696    /// **Why this is not [`active_keys`](Self::active_keys) then
1697    /// [`insert_key`](Self::insert_key).** Two issuers racing that pair both
1698    /// read `max_active - 1`, both insert, and the account ends up over the
1699    /// bound with no error raised anywhere — the more replicas, the likelier.
1700    /// A backend must make the count and the insert one indivisible step:
1701    /// `MemoryStore` holds a single lock across both, and `PostgresStore`
1702    /// takes the account row `FOR UPDATE` first — the row
1703    /// `set_account_status` already serialises against.
1704    ///
1705    /// Live excludes revoked credentials, and those whose `not_after` has
1706    /// passed at `now`, so an account cannot be stranded behind keys that can
1707    /// no longer authenticate.
1708    async fn insert_key_within(
1709        &self,
1710        record: KeyRecord,
1711        max_active: NonZeroUsize,
1712        now: Timestamp,
1713    ) -> Result<(), KeyError>;
1714
1715    /// Bounded issuance with lifecycle evidence captured under the mutation lock.
1716    /// HTTP administrators must use this receipt rather than synthesize history.
1717    async fn insert_key_within_audited(
1718        &self,
1719        record: KeyRecord,
1720        max_active: NonZeroUsize,
1721        now: Timestamp,
1722    ) -> Result<crate::AdminReceipt<()>, KeyError>;
1723
1724    /// Retire a credential and capture its actual owner, key and predecessor
1725    /// under the mutation lock. A repeated revocation returns equal states.
1726    async fn revoke_key_audited(
1727        &self,
1728        key_id: KeyId,
1729        now: Timestamp,
1730    ) -> Result<crate::AdminReceipt<Revocation>, KeyError>;
1731
1732    /// Publish `snapshot` for the principal of `account`'s credential `key`,
1733    /// resolved inside the store (GL-143).
1734    ///
1735    /// An operator holds `(account, key)`; the principal is digest material
1736    /// and never leaves the server. Resolution, the retirement check and the
1737    /// publication are one indivisible step, so a concurrent revocation
1738    /// either precedes it — and the publish is refused as
1739    /// [`KeySnapshotError::Retired`] — or follows it.
1740    ///
1741    /// `snapshot` must state `key_id == Some(key)`; anything else is
1742    /// [`PublishSnapshotError::CredentialMismatch`]. Every other rule is
1743    /// [`AdminStore::publish_snapshot`]'s, including the generation no-op.
1744    async fn publish_key_snapshot(
1745        &self,
1746        account: AccountId,
1747        key: KeyId,
1748        snapshot: PublishableSnapshot,
1749    ) -> Result<crate::AdminReceipt<()>, KeySnapshotError>;
1750
1751    /// Withdraw the snapshot of `account`'s credential `key`, tombstoning it
1752    /// as [`AdminStore::remove_snapshot`] does. Allowed for a revoked
1753    /// credential: revocation does not withdraw its snapshot, and withdrawal
1754    /// is the safe direction.
1755    async fn remove_key_snapshot(
1756        &self,
1757        account: AccountId,
1758        key: KeyId,
1759    ) -> Result<crate::AdminReceipt<()>, KeySnapshotError>;
1760}
1761
1762#[cfg(test)]
1763mod tests {
1764    use std::sync::atomic::{AtomicUsize, Ordering};
1765
1766    use super::KeySummary;
1767    use jiff::Timestamp;
1768    use tollgate_core::KeyId;
1769
1770    fn at(seconds: i64) -> Timestamp {
1771        Timestamp::from_second(seconds).expect("a test instant")
1772    }
1773
1774    /// `not_after` is exclusive, and the instant itself is the whole question.
1775    ///
1776    /// Both backends filter with `now < not_after` — `memory.rs` in three
1777    /// places, and four SQL predicates written `not_after > $now`. `is_live`
1778    /// is the summary of exactly those queries, so `<=` here would not merely
1779    /// be off by an instant: a listing would report a credential live for the
1780    /// one instant at which every query that selects credentials has already
1781    /// dropped it, and the issuance bound counts with this predicate.
1782    #[test]
1783    fn a_credential_is_dead_at_its_expiry_instant_not_after_it() {
1784        let expiring = |not_after| KeySummary {
1785            key_id: KeyId(1),
1786            not_after: Some(not_after),
1787            revoked_at: None,
1788        };
1789        assert!(
1790            expiring(at(100)).is_live(at(99)),
1791            "live up to the instant before"
1792        );
1793        assert!(
1794            !expiring(at(100)).is_live(at(100)),
1795            "dead *at* the boundary: expiry is exclusive, as both backends filter it"
1796        );
1797        assert!(!expiring(at(100)).is_live(at(101)), "and dead after it");
1798    }
1799
1800    use super::*;
1801
1802    #[derive(Clone, Copy)]
1803    enum ReclaimScript {
1804        FailFirst,
1805        FullBatchThenFail,
1806    }
1807
1808    struct ScriptedReclaimer {
1809        script: ReclaimScript,
1810        calls: AtomicUsize,
1811    }
1812
1813    #[async_trait]
1814    impl LeaseAllocator for ScriptedReclaimer {
1815        async fn acquire(
1816            &self,
1817            _account: AccountId,
1818            _requested: CostUnits,
1819            _ttl: SignedDuration,
1820            _now: Timestamp,
1821        ) -> Result<Allocation, AllocateError> {
1822            unreachable!("the full-drain tests only reclaim")
1823        }
1824
1825        async fn release(
1826            &self,
1827            _lease_id: LeaseId,
1828            _fencing_token: FencingToken,
1829            _unspent: CostUnits,
1830            _now: Timestamp,
1831        ) -> Result<(), AllocateError> {
1832            unreachable!("the full-drain tests only reclaim")
1833        }
1834
1835        async fn consolidate(
1836            &self,
1837            _lease_id: LeaseId,
1838            _fencing_token: FencingToken,
1839            _unspent: CostUnits,
1840            _requested: CostUnits,
1841            _needed: CostUnits,
1842            _ttl: SignedDuration,
1843            _now: Timestamp,
1844        ) -> Result<Allocation, AllocateError> {
1845            unreachable!("the reclaim drain never consolidates")
1846        }
1847
1848        async fn reclaim_expired_batch(
1849            &self,
1850            _now: Timestamp,
1851            limit: NonZeroUsize,
1852        ) -> Result<ReclaimBatch, StoreError> {
1853            let call = self.calls.fetch_add(1, Ordering::AcqRel);
1854            if matches!(self.script, ReclaimScript::FailFirst) || call > 0 {
1855                return Err(StoreError("scripted reclaim failure".into()));
1856            }
1857            let reclaimed = (0..limit.get())
1858                .map(|id| ReclaimedLease {
1859                    lease_id: LeaseId(u128::try_from(id).unwrap()),
1860                    account_id: AccountId(1),
1861                    forfeited: CostUnits(1),
1862                })
1863                .collect();
1864            ReclaimBatch::try_new(reclaimed, limit)
1865        }
1866    }
1867
1868    /// GL-131: under the default divisor of 2, a 60-unit balance re-granted as
1869    /// 30 forever, however often a 51-unit quote was refused.
1870    #[test]
1871    fn consolidation_grows_only_to_a_fundable_needed_quote() {
1872        let policy = GrantPolicy::default();
1873        let size = |requested, balance, floor, needed| {
1874            policy.consolidation_grant(
1875                CostUnits(requested),
1876                CostUnits(balance),
1877                CostUnits(floor),
1878                CostUnits(needed),
1879            )
1880        };
1881        assert_eq!(
1882            size(1_000, 60, 30, 0),
1883            Some(CostUnits(30)),
1884            "the GL-109 floor"
1885        );
1886        assert_eq!(
1887            size(1_000, 60, 30, 51),
1888            Some(CostUnits(51)),
1889            "proven demand"
1890        );
1891        assert_eq!(
1892            size(1_000, 60, 30, 60),
1893            Some(CostUnits(60)),
1894            "all of it, inclusive"
1895        );
1896        assert_eq!(
1897            size(1_000, 60, 30, 61),
1898            Some(CostUnits(30)),
1899            "an unfundable quote grows nothing"
1900        );
1901        assert_eq!(
1902            size(1_000, 60, 40, 35),
1903            Some(CostUnits(40)),
1904            "demand never shrinks the floor"
1905        );
1906        assert_eq!(
1907            size(10, 60, 0, 51),
1908            Some(CostUnits(51)),
1909            "past a small target"
1910        );
1911        assert_eq!(
1912            size(1_000, 0, 0, 51),
1913            None,
1914            "an empty balance still refuses"
1915        );
1916        assert_eq!(size(0, 60, 0, 51), None, "a zero request still refuses");
1917        for (requested, balance) in [(1_000, 60), (7, 60), (1_000, 1)] {
1918            assert_eq!(
1919                size(requested, balance, 0, 0),
1920                policy.grant(CostUnits(requested), CostUnits(balance)),
1921                "a plain acquire is the ordinary policy"
1922            );
1923        }
1924    }
1925
1926    /// Every variant, once. Sized by `COUNT`, so adding a refusal without
1927    /// widening this array fails to compile.
1928    fn all() -> [AllocateError; AllocateError::COUNT] {
1929        [
1930            AllocateError::UnknownAccount,
1931            AllocateError::AccountInactive,
1932            AllocateError::InsufficientBalance,
1933            AllocateError::InvalidTtl,
1934            AllocateError::UnknownLease,
1935            AllocateError::Fenced,
1936            AllocateError::LeaseNotActive,
1937            AllocateError::InvalidRelease,
1938            AllocateError::Storage(StoreError("connection reset".into())),
1939            AllocateError::BalanceExhausted(tollgate_core::BalanceExhaustion { period_end: None }),
1940            AllocateError::BalanceInsufficient(tollgate_core::BalanceShortfall {
1941                remaining: CostUnits(1),
1942                period_end: None,
1943            }),
1944        ]
1945    }
1946
1947    /// The indices must be a permutation of `0..COUNT`: two refusals sharing
1948    /// a slot would silently merge their tallies, and a slot no refusal maps
1949    /// to would export a counter that can never move.
1950    #[test]
1951    fn indices_cover_every_slot_exactly_once() {
1952        let mut seen = [false; AllocateError::COUNT];
1953        for error in all() {
1954            let index = error.index();
1955            assert!(index < AllocateError::COUNT, "{error} indexes out of range");
1956            assert!(!seen[index], "{error} shares slot {index}");
1957            seen[index] = true;
1958        }
1959        assert!(seen.iter().all(|hit| *hit), "every slot must be claimed");
1960    }
1961
1962    /// Labels are read by whatever scrapes the counters, so they are a
1963    /// contract: distinct, and free of the backend text `Display` carries.
1964    /// `Storage` wraps an arbitrary message, so labelling by rendered text
1965    /// would mint a fresh time series per connection failure.
1966    #[test]
1967    fn labels_are_distinct_and_free_of_backend_text() {
1968        for (position, error) in all().iter().enumerate() {
1969            assert_eq!(error.name(), AllocateError::NAMES[position]);
1970        }
1971        let storage = AllocateError::Storage(StoreError("connection reset".into()));
1972        assert_eq!(storage.name(), "storage");
1973        assert!(
1974            !storage.name().contains("connection"),
1975            "the label must not carry the backend's message"
1976        );
1977        let mut names = AllocateError::NAMES;
1978        names.sort_unstable();
1979        names.iter().reduce(|previous, next| {
1980            assert_ne!(previous, next, "duplicate label {next}");
1981            next
1982        });
1983    }
1984
1985    /// The payload must not affect the slot, or one refusal would scatter
1986    /// across slots as its message varied.
1987    #[test]
1988    fn payload_does_not_affect_the_slot() {
1989        assert_eq!(
1990            AllocateError::Storage(StoreError("a".into())).index(),
1991            AllocateError::Storage(StoreError("b".into())).index()
1992        );
1993    }
1994
1995    #[test]
1996    fn conservation_requires_an_exact_equation_without_overflow() {
1997        let balanced = Conservation {
1998            deposited: CostUnits(10),
1999            overage_recorded: CostUnits::ZERO,
2000            balance: CostUnits(1),
2001            active_lease_grants: CostUnits(2),
2002            settled_usage: CostUnits(3),
2003            settlement_loss: CostUnits(4),
2004            expired: CostUnits::ZERO,
2005        };
2006        assert!(balanced.holds());
2007
2008        let drifted = Conservation {
2009            deposited: CostUnits(11),
2010            ..balanced
2011        };
2012        assert!(!drifted.holds());
2013
2014        let overflowing = Conservation {
2015            deposited: CostUnits(u64::MAX),
2016            overage_recorded: CostUnits::ZERO,
2017            balance: CostUnits(u64::MAX),
2018            active_lease_grants: CostUnits(1),
2019            settled_usage: CostUnits::ZERO,
2020            settlement_loss: CostUnits::ZERO,
2021            expired: CostUnits::ZERO,
2022        };
2023        assert!(!overflowing.holds());
2024    }
2025
2026    /// Overage funds the left side, so usage it paid for closes the equation
2027    /// rather than breaking it — and the same numbers without the funding term
2028    /// must *not* balance, or the field would be decorative.
2029    #[test]
2030    fn overage_funds_the_usage_it_bills() {
2031        let elastic = Conservation {
2032            deposited: CostUnits(10),
2033            overage_recorded: CostUnits(5),
2034            balance: CostUnits(1),
2035            active_lease_grants: CostUnits(2),
2036            settled_usage: CostUnits(8),
2037            settlement_loss: CostUnits(4),
2038            expired: CostUnits::ZERO,
2039        };
2040        assert!(elastic.holds());
2041        assert!(
2042            !Conservation {
2043                overage_recorded: CostUnits::ZERO,
2044                ..elastic
2045            }
2046            .holds(),
2047            "the same ledger without the funding term must fail by exactly the overage"
2048        );
2049    }
2050
2051    /// The left side is checked too. Overflowing the funding sum answers
2052    /// `false` rather than wrapping to a total that might coincidentally match
2053    /// the right side (INVARIANTS.md GL-11).
2054    #[test]
2055    fn overflowing_the_funding_sum_is_a_violation_not_a_wrap() {
2056        let overflowing = Conservation {
2057            deposited: CostUnits(u64::MAX),
2058            overage_recorded: CostUnits(1),
2059            balance: CostUnits::ZERO,
2060            active_lease_grants: CostUnits::ZERO,
2061            settled_usage: CostUnits::ZERO,
2062            settlement_loss: CostUnits::ZERO,
2063            expired: CostUnits::ZERO,
2064        };
2065        assert!(!overflowing.holds());
2066    }
2067
2068    /// An allowance that expired at a period boundary left the balance without
2069    /// being spent, so the equation only closes if `expired` is on the right
2070    /// side — and the same ledger without the term must fail by exactly the
2071    /// units that expired, or the field would be decorative (GL-97).
2072    #[test]
2073    fn expiry_accounts_for_an_allowance_that_was_never_spent() {
2074        let rolled = Conservation {
2075            deposited: CostUnits(10),
2076            overage_recorded: CostUnits::ZERO,
2077            balance: CostUnits(1),
2078            active_lease_grants: CostUnits(2),
2079            settled_usage: CostUnits(3),
2080            settlement_loss: CostUnits::ZERO,
2081            expired: CostUnits(4),
2082        };
2083        assert!(rolled.holds());
2084        assert!(
2085            !Conservation {
2086                expired: CostUnits::ZERO,
2087                ..rolled
2088            }
2089            .holds(),
2090            "the same ledger without the expiry term must fail by exactly the expired units"
2091        );
2092    }
2093
2094    /// A rollover pass reports what it did, and its caller decides whether to
2095    /// ask for another batch from `is_saturated` alone. Every accessor is
2096    /// asserted directly: a `len` that always answered 1, or an `is_empty`
2097    /// stuck either way, would send the server's drain loop into an endless
2098    /// round of empty batches or stop it one batch short of the boundary it
2099    /// was crossing.
2100    #[test]
2101    fn rollover_batch_reports_what_it_rolled() {
2102        let limit = NonZeroUsize::new(2).unwrap();
2103        let rolled = |account| RolledAccount {
2104            account_id: AccountId(account),
2105            deposited: CostUnits(100),
2106            expired: CostUnits::ZERO,
2107        };
2108
2109        let empty = RolloverBatch::try_new(Vec::new(), limit).unwrap();
2110        assert!(empty.is_empty());
2111        assert_eq!(empty.len(), 0);
2112        assert!(!empty.is_saturated());
2113        assert!(empty.rolled().is_empty());
2114
2115        let partial = RolloverBatch::try_new(vec![rolled(1)], limit).unwrap();
2116        assert!(!partial.is_empty());
2117        assert_eq!(partial.len(), 1);
2118        assert!(
2119            !partial.is_saturated(),
2120            "a partial batch is what ends the drain"
2121        );
2122        assert_eq!(partial.rolled(), &[rolled(1)]);
2123
2124        let full = RolloverBatch::try_new(vec![rolled(1), rolled(2)], limit).unwrap();
2125        assert_eq!(full.len(), 2);
2126        assert!(
2127            full.is_saturated(),
2128            "a batch at the limit means there may be more"
2129        );
2130    }
2131
2132    /// Saturation is derived from the limit the caller asked for, so a backend
2133    /// returning more than it was allowed is corruption to refuse rather than
2134    /// a batch to trust — the same contract `ReclaimBatch::try_new` carries.
2135    #[test]
2136    fn a_rollover_batch_beyond_its_limit_is_refused() {
2137        let rolled = |account| RolledAccount {
2138            account_id: AccountId(account),
2139            deposited: CostUnits(100),
2140            expired: CostUnits::ZERO,
2141        };
2142        assert!(
2143            RolloverBatch::try_new(vec![rolled(1), rolled(2)], NonZeroUsize::new(1).unwrap())
2144                .is_err()
2145        );
2146    }
2147
2148    #[test]
2149    fn reclaim_batch_reports_an_empty_result() {
2150        let batch = ReclaimBatch::try_new(Vec::new(), NonZeroUsize::new(2).unwrap()).unwrap();
2151        assert!(batch.is_empty());
2152        assert_eq!(batch.len(), 0);
2153        assert!(!batch.is_saturated());
2154        assert!(batch.reclaimed().is_empty());
2155    }
2156
2157    #[tokio::test]
2158    async fn full_drain_preserves_an_initial_batch_error() {
2159        let allocator = ScriptedReclaimer {
2160            script: ReclaimScript::FailFirst,
2161            calls: AtomicUsize::new(0),
2162        };
2163        let expected = StoreError("scripted reclaim failure".into());
2164        assert_eq!(
2165            allocator.reclaim_expired(Timestamp::MIN).await,
2166            Err(expected)
2167        );
2168        assert_eq!(allocator.calls.load(Ordering::Acquire), 1);
2169    }
2170
2171    #[tokio::test]
2172    async fn full_drain_reports_progress_before_a_later_batch_error() {
2173        let allocator = ScriptedReclaimer {
2174            script: ReclaimScript::FullBatchThenFail,
2175            calls: AtomicUsize::new(0),
2176        };
2177        let original = StoreError("scripted reclaim failure".into());
2178        let error = allocator.reclaim_expired(Timestamp::MIN).await.unwrap_err();
2179        assert_ne!(error, original);
2180        assert!(error.0.contains("256 leases"));
2181        assert!(error.0.contains("256 units"));
2182        assert_eq!(allocator.calls.load(Ordering::Acquire), 2);
2183    }
2184
2185    /// The push-capacity boundary, pinned where both backends read it.
2186    ///
2187    /// Strictly greater is the whole content of the rule: a batch that exactly
2188    /// fills the channel is delivered, so warning at equality would fire on the
2189    /// largest successful case and train an operator to ignore it. Mutation
2190    /// testing found this untested — the comparison could be flipped to `<`,
2191    /// `<=` or `>=` and every scenario stayed green, because nothing observed
2192    /// the warning at all (GL-51).
2193    #[test]
2194    fn the_push_capacity_warning_fires_only_above_the_channel() {
2195        assert!(
2196            !pushes_exceed_capacity(0),
2197            "an empty batch is not a capacity problem"
2198        );
2199        assert!(!pushes_exceed_capacity(PUSH_CHANNEL_CAPACITY - 1));
2200        assert!(
2201            !pushes_exceed_capacity(PUSH_CHANNEL_CAPACITY),
2202            "a batch that exactly fills the channel is still delivered"
2203        );
2204        assert!(
2205            pushes_exceed_capacity(PUSH_CHANNEL_CAPACITY + 1),
2206            "one more than the channel holds is what makes a subscriber lag"
2207        );
2208    }
2209}