Skip to main content

agent_core/core/auth/
quota_keeper.rs

1//! G6 — quota keeper: pure scheduler state machine, private persisted state
2//! and singleton/attempt locking.
3//!
4//! The keeper watches each kept account's **weekly** window (identified by its
5//! provider-reported duration — never by the name `primary`) and tracks the
6//! provider-reported reset instant as the authoritative *reset generation*.
7//! When a generation's reset has passed and fresh usage still shows no newer
8//! window, the account is *due for activation*. In read-only mode (the
9//! default) the keeper only records and alerts. Only for accounts the
10//! operator explicitly opted in, the runner performs **one** bounded,
11//! tool-free request per generation; the keeper never declares the window
12//! active until a *fresh* usage read shows a strictly later reset.
13//!
14//! ## Invariants
15//!
16//! * No I/O, no clock, no network in the state machine: every function takes
17//!   `now_ms`. Persistence and locking are separate helpers.
18//! * State is keyed by broker source + provider seat identity. For an
19//!   opted-in account the identity is a **strong** seat fingerprint (digest
20//!   of the full provider account id carried by the credential itself), so
21//!   the key is canonical across aliases: two labels for one seat share one
22//!   attempt ledger and a sequential alias switch cannot spend twice. A
23//!   read-only account may carry only a weak metadata preview (label-scoped
24//!   key). A metadata prefix is **not** an identity: activation is refused
25//!   without a strong fingerprint, and the runner re-verifies the seat from
26//!   the freshly vended token immediately before the request. Re-login of
27//!   the same alias with a different seat never inherits a ticket.
28//! * An attempt is recorded and must be **persisted before** the request is
29//!   sent ([`begin_attempt`]); ambiguous outcomes (timeout, crash between
30//!   begin and finish, 5xx) consume the per-generation budget of exactly
31//!   one. Only a definitive *not sent* (typed pre-flight stage) or a
32//!   whitelisted pre-inference 4xx is refunded, and those are bounded
33//!   separately with backoff. A second attempt for the same generation
34//!   requires an explicit operator [`rearm`].
35//! * A newly observed window is labelled **verified**; correlation with a
36//!   keeper attempt is recorded, but never claimed as causal.
37//! * Natural rollover is never assumed: with no evidence the phase is
38//!   `Unknown`/`AwaitingActivation`, never `Active`.
39//! * **Proven capacity before any inference.** An activation attempt is
40//!   authorized only when the *latest* poll succeeded, is fresh, carries no
41//!   overall `limit_reached` assertion, and every window applicable to the
42//!   activation model (5h, weekly, model-scoped) shows headroom. A passed
43//!   reset while the provider still asserts 100 % / `limit_reached` is
44//!   `Exhausted` (reset passed), never due.
45
46use std::collections::BTreeMap;
47use std::fs::{File, OpenOptions};
48use std::path::{Path, PathBuf};
49
50use serde::{Deserialize, Serialize};
51
52use super::account::CredentialRef;
53use super::OAuthProviderId;
54use super::quota_policy::{
55    self, weekly_window, AccountCapacity, ModelAvailability, QuotaObservation, SelectionRequest,
56    Strategy, WindowLimit, WindowVerdict, DEFAULT_EXHAUSTED_AT_PERCENT,
57    MAX_FUTURE_OBSERVATION_SKEW_MS,
58};
59
60pub const STATE_VERSION: u32 = 1;
61/// File name of the persisted state inside the state dir.
62pub const STATE_FILE_NAME: &str = "state.json";
63/// File name of the singleton lock inside the state dir.
64pub const SINGLETON_LOCK_NAME: &str = "keeper.lock";
65/// Sub-directory (of the canonical keeper dir) holding per-account locks.
66pub const ACCOUNT_LOCK_DIR: &str = "locks";
67
68const SEC: u64 = 1000;
69const MIN: u64 = 60 * SEC;
70const HOUR: u64 = 60 * MIN;
71const DAY: u64 = 24 * HOUR;
72
73// Bounds for operator-supplied intervals (untrusted input; always clamped).
74pub const DEFAULT_POLL_INTERVAL_MS: u64 = 5 * MIN;
75pub const MIN_POLL_INTERVAL_MS: u64 = MIN;
76pub const MAX_POLL_INTERVAL_MS: u64 = HOUR;
77pub const DEFAULT_MAX_BACKOFF_MS: u64 = 4 * HOUR;
78pub const MAX_MAX_BACKOFF_MS: u64 = 12 * HOUR;
79pub const DEFAULT_STALE_AFTER_MS: u64 = 30 * MIN;
80pub const MIN_STALE_AFTER_MS: u64 = MIN;
81pub const MAX_STALE_AFTER_MS: u64 = DAY;
82/// Poll shortly after the reported reset instead of waiting a full interval.
83pub const DEFAULT_RESET_GRACE_MS: u64 = 30 * SEC;
84pub const MAX_RESET_GRACE_MS: u64 = 10 * MIN;
85/// Re-poll cadence while an attempt awaits verification.
86pub const DEFAULT_VERIFY_POLL_MS: u64 = 20 * SEC;
87/// How long an attempt may stay pending before it is labelled unverified.
88pub const DEFAULT_VERIFY_WINDOW_MS: u64 = 10 * MIN;
89pub const MAX_VERIFY_WINDOW_MS: u64 = 2 * HOUR;
90/// Attempts per reset generation. Fixed at one: a second attempt for the
91/// same generation only ever happens through an explicit operator
92/// [`rearm`], never automatically.
93pub const DEFAULT_MAX_ATTEMPTS: u32 = 1;
94pub const MAX_MAX_ATTEMPTS: u32 = 1;
95/// Definitive not-sent/rejected outcomes tolerated per generation.
96pub const MAX_NOT_SENT_PER_GENERATION: u32 = 5;
97pub const DEFAULT_ATTEMPT_TIMEOUT_MS: u64 = 60 * SEC;
98pub const MIN_ATTEMPT_TIMEOUT_MS: u64 = 10 * SEC;
99pub const MAX_ATTEMPT_TIMEOUT_MS: u64 = 5 * MIN;
100pub const DEFAULT_USAGE_TIMEOUT_MS: u64 = 20 * SEC;
101pub const DEFAULT_BANKED_EXPIRY_ALERT_MS: u64 = 3 * DAY;
102/// Attempt history retained per account.
103pub const MAX_HISTORY: usize = 24;
104
105// ── Config ───────────────────────────────────────────────────────────────────
106
107#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
108pub struct KeeperConfig {
109    pub poll_interval_ms: u64,
110    pub max_backoff_ms: u64,
111    pub stale_after_ms: u64,
112    pub reset_grace_ms: u64,
113    pub verify_poll_ms: u64,
114    pub verify_window_ms: u64,
115    pub max_attempts_per_generation: u32,
116    pub attempt_timeout_ms: u64,
117    pub usage_timeout_ms: u64,
118    pub banked_expiry_alert_ms: u64,
119    pub exhausted_at_percent: f64,
120}
121
122impl Default for KeeperConfig {
123    fn default() -> Self {
124        Self {
125            poll_interval_ms: DEFAULT_POLL_INTERVAL_MS,
126            max_backoff_ms: DEFAULT_MAX_BACKOFF_MS,
127            stale_after_ms: DEFAULT_STALE_AFTER_MS,
128            reset_grace_ms: DEFAULT_RESET_GRACE_MS,
129            verify_poll_ms: DEFAULT_VERIFY_POLL_MS,
130            verify_window_ms: DEFAULT_VERIFY_WINDOW_MS,
131            max_attempts_per_generation: DEFAULT_MAX_ATTEMPTS,
132            attempt_timeout_ms: DEFAULT_ATTEMPT_TIMEOUT_MS,
133            usage_timeout_ms: DEFAULT_USAGE_TIMEOUT_MS,
134            banked_expiry_alert_ms: DEFAULT_BANKED_EXPIRY_ALERT_MS,
135            exhausted_at_percent: DEFAULT_EXHAUSTED_AT_PERCENT,
136        }
137    }
138}
139
140impl KeeperConfig {
141    /// Clamp every operator-controlled value into its safe range.
142    pub fn bounded(mut self) -> Self {
143        self.poll_interval_ms = self
144            .poll_interval_ms
145            .clamp(MIN_POLL_INTERVAL_MS, MAX_POLL_INTERVAL_MS);
146        self.max_backoff_ms = self
147            .max_backoff_ms
148            .clamp(self.poll_interval_ms, MAX_MAX_BACKOFF_MS);
149        self.stale_after_ms = self
150            .stale_after_ms
151            .clamp(MIN_STALE_AFTER_MS, MAX_STALE_AFTER_MS);
152        self.reset_grace_ms = self.reset_grace_ms.min(MAX_RESET_GRACE_MS);
153        self.verify_poll_ms = self.verify_poll_ms.clamp(5 * SEC, self.poll_interval_ms);
154        self.verify_window_ms = self
155            .verify_window_ms
156            .clamp(self.verify_poll_ms, MAX_VERIFY_WINDOW_MS);
157        self.max_attempts_per_generation = self
158            .max_attempts_per_generation
159            .clamp(1, MAX_MAX_ATTEMPTS);
160        self.attempt_timeout_ms = self
161            .attempt_timeout_ms
162            .clamp(MIN_ATTEMPT_TIMEOUT_MS, MAX_ATTEMPT_TIMEOUT_MS);
163        self.usage_timeout_ms = self.usage_timeout_ms.clamp(5 * SEC, 2 * MIN);
164        if !self.exhausted_at_percent.is_finite() || !(1.0..=100.0).contains(&self.exhausted_at_percent) {
165            self.exhausted_at_percent = DEFAULT_EXHAUSTED_AT_PERCENT;
166        }
167        self
168    }
169}
170
171// ── Identity ─────────────────────────────────────────────────────────────────
172
173/// Prefix of a *strong* seat fingerprint (see [`AccountIdentity::seat_fingerprint`]).
174pub const SEAT_FP_PREFIX: &str = "seat:";
175
176/// Namespace of one kept account. Every field is non-secret display text.
177#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
178pub struct AccountIdentity {
179    /// Broker source: `local:<auth.json path>` or `remote:<endpoint>`.
180    pub source: String,
181    /// Storage key: `provider` or `provider@label`.
182    pub credential: String,
183    /// Provider-side identity fingerprint: a strong `seat:<digest>` (see
184    /// [`AccountIdentity::seat_fingerprint`]) or a weak preview from
185    /// [`AccountIdentity::fingerprint`]. Only a strong fingerprint is an
186    /// identity; a weak one is a discriminator for read-only tracking.
187    pub identity_fp: String,
188}
189
190impl AccountIdentity {
191    /// Map key. Changing any component yields a fresh state entry.
192    ///
193    /// With a **strong** seat fingerprint the key is canonical across
194    /// aliases (`source|provider|seat:…`, the label is not part of it), so
195    /// two labels pointing at one provider seat share ONE attempt ledger and
196    /// a sequential alias switch can never spend a second activation on the
197    /// same generation. With a weak fingerprint the key stays label-scoped.
198    pub fn key(&self) -> String {
199        format!("{}|{}|{}", self.source, self.ledger_scope(), self.identity_fp)
200    }
201
202    /// The credential component of the key: the provider alone for a strong
203    /// seat identity, the full `provider@label` otherwise.
204    pub fn ledger_scope(&self) -> String {
205        if self.is_strong() {
206            self.credential
207                .split_once('@')
208                .map(|(p, _)| p.to_string())
209                .unwrap_or_else(|| self.credential.clone())
210        } else {
211            self.credential.clone()
212        }
213    }
214
215    /// Whether this identity is a full provider seat identity (established
216    /// from the credential itself), as opposed to a metadata preview.
217    pub fn is_strong(&self) -> bool {
218        self.identity_fp.starts_with(SEAT_FP_PREFIX)
219    }
220
221    /// Strong seat fingerprint from the FULL provider account id carried by
222    /// the credential (e.g. the Codex `chatgpt_account_id` claim of the token
223    /// the broker vends for exactly this slot). Same recipe as the broker's
224    /// listed [`AccountSummary::seat_fingerprint`](super::AccountSummary)
225    /// ([`super::seat_fingerprint`]), so a listing value and a token-derived
226    /// value compare equal. Digest only — the state file never stores the
227    /// id itself. `None` for an empty id.
228    pub fn seat_fingerprint(provider: OAuthProviderId, account_id: &str) -> Option<String> {
229        super::account::seat_fingerprint(provider, account_id).map(Self::strong_from_listed)
230    }
231
232    /// Wrap a broker-listed seat fingerprint as a strong identity value.
233    pub fn strong_from_listed(listed: impl AsRef<str>) -> String {
234        format!("{SEAT_FP_PREFIX}{}", listed.as_ref().trim())
235    }
236
237    /// WEAK fingerprint from non-secret provider metadata. Prefers the
238    /// provider account-id prefix, then a hash of the identity string, then
239    /// the login time. Providers exposing none of these yield `unknown`,
240    /// which is reported so the operator knows re-login inheritance is
241    /// possible. A prefix is not an identity: activation requires
242    /// [`seat_fingerprint`](Self::seat_fingerprint).
243    pub fn fingerprint(
244        account_id_prefix: Option<&str>,
245        identity: Option<&str>,
246        added_at: Option<u64>,
247    ) -> String {
248        if let Some(p) = account_id_prefix.map(str::trim).filter(|p| !p.is_empty()) {
249            return format!("id:{p}");
250        }
251        if let Some(i) = identity.map(str::trim).filter(|i| !i.is_empty()) {
252            use sha2::{Digest, Sha256};
253            let digest = Sha256::digest(i.as_bytes());
254            let hex: String = digest.iter().take(6).map(|b| format!("{b:02x}")).collect();
255            return format!("who:{hex}");
256        }
257        if let Some(t) = added_at {
258            return format!("added:{t}");
259        }
260        "unknown".to_string()
261    }
262}
263
264// ── Observations (input) ─────────────────────────────────────────────────────
265
266/// Banked reset inventory (Codex `rate_limit_reset_credits`). Alert-only.
267#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
268pub struct BankedResets {
269    /// `None` = the provider did not report a count (unknown ≠ zero).
270    #[serde(default, skip_serializing_if = "Option::is_none")]
271    pub available_count: Option<u64>,
272    /// Earliest expiry among the inventory, if exposed.
273    #[serde(default, skip_serializing_if = "Option::is_none")]
274    pub earliest_expiry_ms: Option<u64>,
275    #[serde(default, skip_serializing_if = "Option::is_none")]
276    pub inventory_error: Option<String>,
277}
278
279/// What one usage poll returned. Produced by the runner's usage adapter.
280#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
281pub struct UsageObservation {
282    /// When the body was received (epoch ms).
283    pub observed_at_ms: u64,
284    pub outcome: ObservationOutcome,
285}
286
287#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
288#[serde(tag = "kind", rename_all = "snake_case")]
289pub enum ObservationOutcome {
290    Ok {
291        windows: Vec<WindowLimit>,
292        #[serde(default, skip_serializing_if = "Option::is_none")]
293        models: Option<Vec<ModelAvailability>>,
294        /// Provider-asserted overall flag (Codex `rate_limit.limit_reached`).
295        /// `Some(true)` is an assertion of no capacity regardless of windows.
296        #[serde(default, skip_serializing_if = "Option::is_none")]
297        limit_reached: Option<bool>,
298        #[serde(default, skip_serializing_if = "Option::is_none")]
299        banked: Option<BankedResets>,
300        /// Provider account-id prefix carried by the response, if any.
301        #[serde(default, skip_serializing_if = "Option::is_none")]
302        identity_prefix: Option<String>,
303    },
304    AuthError {
305        detail: String,
306    },
307    /// Retryable (network, timeout, 5xx, 429 on the usage endpoint).
308    Transport {
309        detail: String,
310    },
311    Malformed {
312        detail: String,
313    },
314    Unsupported {
315        detail: String,
316    },
317}
318
319/// Fold a provider-asserted overall `limit_reached: true` into the
320/// account-wide windows so every policy consumer honours it. With no
321/// account-wide window present a synthetic `overall` window is added.
322pub fn windows_with_overall_flag(
323    windows: &[WindowLimit],
324    limit_reached: Option<bool>,
325) -> Vec<WindowLimit> {
326    let mut out: Vec<WindowLimit> = windows.to_vec();
327    if limit_reached == Some(true) {
328        let mut any_account_wide = false;
329        for w in out.iter_mut().filter(|w| w.models.is_none()) {
330            any_account_wide = true;
331            w.limit_reached = Some(true);
332        }
333        if !any_account_wide {
334            out.push(WindowLimit {
335                id: "overall".into(),
336                duration_ms: None,
337                used_percent: None,
338                limit_reached: Some(true),
339                resets_at_ms: None,
340                models: None,
341            });
342        }
343    }
344    out
345}
346
347impl UsageObservation {
348    /// The policy view of this observation (for `quota_policy::select`).
349    pub fn to_quota_observation(&self) -> QuotaObservation {
350        match &self.outcome {
351            ObservationOutcome::Ok {
352                windows,
353                models,
354                limit_reached,
355                ..
356            } => QuotaObservation::Ok {
357                windows: windows_with_overall_flag(windows, *limit_reached),
358                models: models.clone(),
359            },
360            ObservationOutcome::AuthError { .. } => QuotaObservation::AuthError,
361            ObservationOutcome::Transport { .. } => QuotaObservation::Unknown,
362            ObservationOutcome::Malformed { .. } => QuotaObservation::Malformed,
363            ObservationOutcome::Unsupported { .. } => QuotaObservation::Unsupported,
364        }
365    }
366}
367
368// ── State ────────────────────────────────────────────────────────────────────
369
370/// How a verified window relates to keeper activity. Correlation is recorded;
371/// causation is never claimed (an external request or a provider-side reset
372/// could equally have anchored the window).
373#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
374#[serde(tag = "kind", rename_all = "snake_case")]
375pub enum Attribution {
376    /// A window was observed without any keeper attempt for the prior generation.
377    Observed,
378    /// A keeper attempt for the prior generation preceded the new window.
379    /// Verified window; **not proven causal**.
380    AttemptCorrelated { attempt: u32 },
381}
382
383/// Where a request definitively stopped before leaving the process or
384/// before reaching the provider. Only these stages may be reported as
385/// `NotSent`; anything after the request was written to the socket is
386/// `Ambiguous`.
387#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
388#[serde(rename_all = "snake_case")]
389pub enum NotSentStage {
390    /// The provider/account is not activatable by this runner.
391    Unsupported,
392    /// Broker refused to vend a token for the tracked credential.
393    TokenVend,
394    /// The token carries no provider account id to pair with the bearer.
395    AccountId,
396    /// The token vended immediately before the request belongs to a
397    /// DIFFERENT provider seat than the one the keeper tracks (re-login
398    /// under the same label). The stale usage evidence must never authorize
399    /// inference on the new seat; the runner disables activation for the
400    /// account until it is restarted and the identity re-established.
401    IdentityMismatch,
402    /// The request body could not be built.
403    RequestBuild,
404    /// TCP/TLS connect failed; no bytes were sent.
405    Connect,
406}
407
408/// HTTP statuses that prove the provider rejected the request before any
409/// inference (malformed request, auth, missing route, payload shape).
410/// Everything else — notably 429, 402, 408, 409 — is not refunded.
411pub const REFUNDABLE_REJECT_STATUSES: &[u16] = &[400, 401, 403, 404, 413, 415, 422];
412
413/// Terminal classification of one activation attempt, as reported by the
414/// runner. Only a typed `NotSent` and a whitelisted `Rejected` status are
415/// definitive "no inference" outcomes; everything else may have spent quota.
416#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
417#[serde(tag = "kind", rename_all = "snake_case")]
418pub enum AttemptOutcome {
419    /// Provider accepted the request (2xx) and the bounded body was consumed.
420    Sent { http_status: u16 },
421    /// Provider returned a 4xx. Refunded only for [`REFUNDABLE_REJECT_STATUSES`].
422    Rejected { http_status: u16 },
423    /// Nothing reached the provider; `stage` names the pre-flight step.
424    NotSent { stage: NotSentStage, reason: String },
425    /// Timeout, 5xx, stream cut, or crash between begin and finish.
426    Ambiguous { reason: String },
427}
428
429impl AttemptOutcome {
430    /// Whether the attempt is refunded (definitively no inference happened).
431    pub fn is_refundable(&self) -> bool {
432        match self {
433            Self::NotSent { .. } => true,
434            Self::Rejected { http_status } => REFUNDABLE_REJECT_STATUSES.contains(http_status),
435            Self::Sent { .. } | Self::Ambiguous { .. } => false,
436        }
437    }
438}
439
440#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
441#[serde(tag = "phase", rename_all = "snake_case")]
442pub enum Phase {
443    /// No usable weekly-window evidence.
444    Unknown { reason: String },
445    /// Credential rejected by the provider; needs re-login.
446    AuthError { detail: String },
447    /// No headroom. `reset_passed` = the provider still asserts the limit
448    /// although the reported reset instant has passed — never due.
449    Exhausted {
450        generation: u64,
451        #[serde(default)]
452        reset_passed: bool,
453    },
454    /// Live window with headroom. `verified_at_ms` = observation time.
455    Active {
456        generation: u64,
457        #[serde(default, skip_serializing_if = "Option::is_none")]
458        used_percent: Option<f64>,
459        verified_at_ms: u64,
460        attribution: Attribution,
461    },
462    /// Reset passed; fresh usage still shows no newer window.
463    AwaitingActivation { generation: u64, due_since_ms: u64 },
464    /// An attempt is recorded (persisted before send) and not yet verified.
465    ActivationPending {
466        generation: u64,
467        attempt: u32,
468        started_ms: u64,
469        #[serde(default, skip_serializing_if = "Option::is_none")]
470        outcome: Option<AttemptOutcome>,
471    },
472    /// Attempt budget spent for this generation without verification.
473    Unverified { generation: u64, attempts: u32 },
474}
475
476impl Phase {
477    pub fn label(&self) -> &'static str {
478        match self {
479            Self::Unknown { .. } => "unknown",
480            Self::AuthError { .. } => "auth_error",
481            Self::Exhausted { .. } => "exhausted",
482            Self::Active { .. } => "active",
483            Self::AwaitingActivation { .. } => "awaiting_activation",
484            Self::ActivationPending { .. } => "activation_pending",
485            Self::Unverified { .. } => "unverified",
486        }
487    }
488
489    /// Generation carried by the phase, if any.
490    pub fn generation(&self) -> Option<u64> {
491        match self {
492            Self::Unknown { .. } | Self::AuthError { .. } => None,
493            Self::Exhausted { generation, .. }
494            | Self::Active { generation, .. }
495            | Self::AwaitingActivation { generation, .. }
496            | Self::ActivationPending { generation, .. }
497            | Self::Unverified { generation, .. } => Some(*generation),
498        }
499    }
500}
501
502/// One activation attempt, for the operator record.
503#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
504pub struct AttemptRecord {
505    pub attempt: u32,
506    pub generation: u64,
507    pub started_ms: u64,
508    #[serde(default, skip_serializing_if = "Option::is_none")]
509    pub finished_ms: Option<u64>,
510    #[serde(default, skip_serializing_if = "Option::is_none")]
511    pub outcome: Option<AttemptOutcome>,
512    /// Set when a later window was verified after this attempt (correlation).
513    #[serde(default, skip_serializing_if = "Option::is_none")]
514    pub verified_at_ms: Option<u64>,
515    #[serde(default, skip_serializing_if = "Option::is_none")]
516    pub new_generation: Option<u64>,
517}
518
519#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
520pub struct AccountState {
521    pub identity: AccountIdentity,
522    pub phase: Phase,
523    /// Latest provider-reported weekly reset (epoch ms) we track.
524    #[serde(default, skip_serializing_if = "Option::is_none")]
525    pub generation: Option<u64>,
526    /// Attempts that may have spent quota in `generation`.
527    #[serde(default)]
528    pub attempts_in_generation: u32,
529    /// Refunded (definitively not-sent/rejected) attempts in `generation`.
530    #[serde(default)]
531    pub not_sent_in_generation: u32,
532    #[serde(default, skip_serializing_if = "Option::is_none")]
533    pub last_poll_ms: Option<u64>,
534    #[serde(default, skip_serializing_if = "Option::is_none")]
535    pub last_ok_observation_ms: Option<u64>,
536    /// Whether the most recent poll (of any kind) succeeded.
537    #[serde(default)]
538    pub last_poll_ok: bool,
539    /// Windows/models/flag from the last successful poll (capacity evidence).
540    #[serde(default)]
541    pub last_windows: Vec<WindowLimit>,
542    #[serde(default, skip_serializing_if = "Option::is_none")]
543    pub last_models: Option<Vec<ModelAvailability>>,
544    #[serde(default, skip_serializing_if = "Option::is_none")]
545    pub last_limit_reached: Option<bool>,
546    #[serde(default, skip_serializing_if = "Option::is_none")]
547    pub last_error: Option<String>,
548    #[serde(default)]
549    pub consecutive_errors: u32,
550    #[serde(default)]
551    pub backoff_ms: u64,
552    /// Earliest time the runner should act on this account again.
553    #[serde(default)]
554    pub next_action_at_ms: u64,
555    /// Not before this may another activation attempt start (after refunds).
556    #[serde(default)]
557    pub next_attempt_allowed_ms: u64,
558    /// Measured on the last verified new window: anchor − previous reset.
559    #[serde(default, skip_serializing_if = "Option::is_none")]
560    pub last_idle_delay_ms: Option<u64>,
561    /// `true` when the delay is an upper bound (anchor inferred from the
562    /// observation, not from a reported duration).
563    #[serde(default)]
564    pub last_idle_delay_is_upper_bound: bool,
565    #[serde(default, skip_serializing_if = "Option::is_none")]
566    pub weekly: Option<WindowLimit>,
567    #[serde(default, skip_serializing_if = "Option::is_none")]
568    pub banked: Option<BankedResets>,
569    #[serde(default)]
570    pub history: Vec<AttemptRecord>,
571}
572
573impl AccountState {
574    pub fn new(identity: AccountIdentity) -> Self {
575        Self {
576            identity,
577            phase: Phase::Unknown {
578                reason: "never polled".into(),
579            },
580            generation: None,
581            attempts_in_generation: 0,
582            not_sent_in_generation: 0,
583            last_poll_ms: None,
584            last_ok_observation_ms: None,
585            last_poll_ok: false,
586            last_windows: Vec::new(),
587            last_models: None,
588            last_limit_reached: None,
589            last_error: None,
590            consecutive_errors: 0,
591            backoff_ms: 0,
592            next_action_at_ms: 0,
593            next_attempt_allowed_ms: 0,
594            last_idle_delay_ms: None,
595            last_idle_delay_is_upper_bound: false,
596            weekly: None,
597            banked: None,
598            history: Vec::new(),
599        }
600    }
601
602    fn push_history(&mut self, rec: AttemptRecord) {
603        self.history.push(rec);
604        if self.history.len() > MAX_HISTORY {
605            let drop = self.history.len() - MAX_HISTORY;
606            self.history.drain(..drop);
607        }
608    }
609}
610
611#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
612pub struct KeeperState {
613    pub version: u32,
614    #[serde(default)]
615    pub updated_at_ms: u64,
616    #[serde(default)]
617    pub accounts: BTreeMap<String, AccountState>,
618}
619
620impl Default for KeeperState {
621    fn default() -> Self {
622        Self {
623            version: STATE_VERSION,
624            updated_at_ms: 0,
625            accounts: BTreeMap::new(),
626        }
627    }
628}
629
630impl KeeperState {
631    /// Fetch or create the entry for `identity`. An existing entry (same
632    /// key) adopts the caller's identity so that, for a seat-canonical key,
633    /// the recorded `credential` names the alias currently in use.
634    pub fn entry(&mut self, identity: &AccountIdentity) -> &mut AccountState {
635        let st = self
636            .accounts
637            .entry(identity.key())
638            .or_insert_with(|| AccountState::new(identity.clone()));
639        if st.identity != *identity {
640            st.identity = identity.clone();
641        }
642        st
643    }
644
645    /// Prune entries that are NOT in `keep`, carry no attempt history, are
646    /// not mid-attempt, and have been idle longer than `max_idle_ms`.
647    ///
648    /// Attempt ledgers are never deleted: an account temporarily excluded by
649    /// `--account`/`--provider` (or re-logged-in later with the same seat)
650    /// must find its spent generation intact, otherwise a filter switch could
651    /// burn a second activation. Returns the pruned keys.
652    pub fn prune_idle(&mut self, keep: &[AccountIdentity], now_ms: u64, max_idle_ms: u64) -> Vec<String> {
653        let keep: Vec<String> = keep.iter().map(AccountIdentity::key).collect();
654        let pruned: Vec<String> = self
655            .accounts
656            .iter()
657            .filter(|(k, a)| {
658                !keep.contains(k)
659                    && a.history.is_empty()
660                    && !matches!(
661                        a.phase,
662                        Phase::ActivationPending { .. } | Phase::Unverified { .. }
663                    )
664                    && now_ms.saturating_sub(a.last_poll_ms.unwrap_or(0)) > max_idle_ms
665            })
666            .map(|(k, _)| k.clone())
667            .collect();
668        for k in &pruned {
669            self.accounts.remove(k);
670        }
671        pruned
672    }
673
674    /// Accounts whose next action time has arrived.
675    pub fn due(&self, now_ms: u64) -> Vec<String> {
676        self.accounts
677            .iter()
678            .filter(|(_, a)| a.next_action_at_ms <= now_ms)
679            .map(|(k, _)| k.clone())
680            .collect()
681    }
682
683    /// [`due`](Self::due) restricted to `keys` (the accounts a runner keeps).
684    pub fn due_among(&self, now_ms: u64, keys: &[String]) -> Vec<String> {
685        self.due(now_ms)
686            .into_iter()
687            .filter(|k| keys.contains(k))
688            .collect()
689    }
690
691    /// Earliest next action across all accounts.
692    pub fn next_wake_ms(&self) -> Option<u64> {
693        self.accounts.values().map(|a| a.next_action_at_ms).min()
694    }
695
696    /// Earliest next action among `keys`.
697    pub fn next_wake_among(&self, keys: &[String]) -> Option<u64> {
698        self.accounts
699            .iter()
700            .filter(|(k, _)| keys.contains(k))
701            .map(|(_, a)| a.next_action_at_ms)
702            .min()
703    }
704}
705
706// ── Events ───────────────────────────────────────────────────────────────────
707
708/// Operator-visible outcome of a state-machine step. Secret-free.
709#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
710#[serde(tag = "event", rename_all = "snake_case")]
711pub enum KeeperEvent {
712    PhaseChanged {
713        key: String,
714        from: String,
715        to: String,
716    },
717    PollFailed {
718        key: String,
719        detail: String,
720        backoff_ms: u64,
721    },
722    /// A new weekly window was verified from fresh usage.
723    NewWindow {
724        key: String,
725        from_generation: Option<u64>,
726        to_generation: u64,
727        #[serde(default, skip_serializing_if = "Option::is_none")]
728        idle_delay_ms: Option<u64>,
729        idle_delay_is_upper_bound: bool,
730        attribution: Attribution,
731    },
732    /// The reset passed but the provider still asserts no capacity.
733    LimitAssertedAfterReset {
734        key: String,
735        generation: u64,
736    },
737    /// The reset passed and no newer window is reported.
738    ActivationDue {
739        key: String,
740        generation: u64,
741        due_since_ms: u64,
742    },
743    AttemptStarted {
744        key: String,
745        generation: u64,
746        attempt: u32,
747    },
748    AttemptFinished {
749        key: String,
750        generation: u64,
751        attempt: u32,
752        outcome: AttemptOutcome,
753        refunded: bool,
754    },
755    Unverified {
756        key: String,
757        generation: u64,
758        attempts: u32,
759    },
760    /// Operator explicitly re-armed one more attempt for a generation.
761    Rearmed {
762        key: String,
763        generation: u64,
764        previous_attempts: u32,
765    },
766    AuthError {
767        key: String,
768        detail: String,
769    },
770    BankedResets {
771        key: String,
772        available_count: u64,
773        #[serde(default, skip_serializing_if = "Option::is_none")]
774        earliest_expiry_ms: Option<u64>,
775        expiring_soon: bool,
776    },
777}
778
779// ── Errors ───────────────────────────────────────────────────────────────────
780
781#[derive(Debug)]
782pub enum KeeperError {
783    UnknownAccount(String),
784    /// `begin_attempt` called while the account is not awaiting activation.
785    NotDue { key: String, phase: String },
786    /// `begin_attempt` called for an account without a strong seat identity.
787    IdentityRequired { key: String },
788    /// Persisted state could not be read/parsed. Fail closed — never activate.
789    CorruptState { path: PathBuf, detail: String },
790    /// Persisted state has a newer schema than this binary understands.
791    UnsupportedVersion { path: PathBuf, version: u32 },
792    Io { path: PathBuf, detail: String },
793    /// Another process holds the lock.
794    Locked { path: PathBuf, holder: Option<String> },
795}
796
797impl std::fmt::Display for KeeperError {
798    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
799        match self {
800            Self::UnknownAccount(k) => write!(f, "unknown keeper account '{k}'"),
801            Self::NotDue { key, phase } => {
802                write!(f, "account '{key}' is not awaiting activation (phase {phase})")
803            }
804            Self::IdentityRequired { key } => write!(
805                f,
806                "account '{key}' has no established seat identity; activation refused"
807            ),
808            Self::CorruptState { path, detail } => write!(
809                f,
810                "keeper state at {} is unreadable ({detail}); refusing to run — move it aside to reset",
811                path.display()
812            ),
813            Self::UnsupportedVersion { path, version } => write!(
814                f,
815                "keeper state at {} has version {version} (this build supports {STATE_VERSION})",
816                path.display()
817            ),
818            Self::Io { path, detail } => write!(f, "{}: {detail}", path.display()),
819            Self::Locked { path, holder } => match holder {
820                Some(h) => write!(f, "{} is held by {h}", path.display()),
821                None => write!(f, "{} is held by another process", path.display()),
822            },
823        }
824    }
825}
826
827impl std::error::Error for KeeperError {}
828
829// ── State machine ────────────────────────────────────────────────────────────
830
831fn set_phase(st: &mut AccountState, events: &mut Vec<KeeperEvent>, to: Phase) {
832    if st.phase != to {
833        let from = st.phase.label().to_string();
834        let to_label = to.label().to_string();
835        st.phase = to;
836        if from != to_label {
837            events.push(KeeperEvent::PhaseChanged {
838                key: st.identity.key(),
839                from,
840                to: to_label,
841            });
842        }
843    }
844}
845
846fn grow_backoff(st: &mut AccountState, cfg: &KeeperConfig) -> u64 {
847    st.backoff_ms = if st.backoff_ms == 0 {
848        cfg.poll_interval_ms
849    } else {
850        st.backoff_ms.saturating_mul(2).min(cfg.max_backoff_ms)
851    };
852    st.backoff_ms
853}
854
855/// Schedule the next poll: the regular interval, or just after a known
856/// future reset — whichever is sooner — so a rollover is never missed by a
857/// whole interval.
858fn schedule_regular(st: &mut AccountState, cfg: &KeeperConfig, now_ms: u64) {
859    let mut next = now_ms + cfg.poll_interval_ms;
860    if let Some(g) = st.generation {
861        if g > now_ms {
862            next = next.min(g + cfg.reset_grace_ms);
863        }
864    }
865    st.next_action_at_ms = next;
866}
867
868/// Account-wide capacity evidence in a fresh reading: no overall limit
869/// assertion and headroom on every account-wide window (at least one).
870fn account_wide_headroom(windows: &[WindowLimit], limit_reached: Option<bool>, exhausted_at: f64) -> bool {
871    if limit_reached == Some(true) {
872        return false;
873    }
874    let mut seen = false;
875    for w in windows.iter().filter(|w| w.models.is_none()) {
876        seen = true;
877        if !matches!(w.verdict(exhausted_at), WindowVerdict::Headroom { .. }) {
878            return false;
879        }
880    }
881    seen
882}
883
884/// Feed one fresh usage observation for `key` into the state machine.
885pub fn observe(
886    state: &mut KeeperState,
887    cfg: &KeeperConfig,
888    now_ms: u64,
889    key: &str,
890    obs: &UsageObservation,
891) -> Result<Vec<KeeperEvent>, KeeperError> {
892    let st = state
893        .accounts
894        .get_mut(key)
895        .ok_or_else(|| KeeperError::UnknownAccount(key.to_string()))?;
896    let mut events = Vec::new();
897    st.last_poll_ms = Some(now_ms);
898    st.last_poll_ok = false;
899    state.updated_at_ms = now_ms;
900
901    // A reading stamped in the future is malformed — never evidence.
902    let future_stamped = obs.observed_at_ms > now_ms.saturating_add(MAX_FUTURE_OBSERVATION_SKEW_MS);
903
904    match &obs.outcome {
905        ObservationOutcome::Transport { detail } => {
906            st.last_error = Some(detail.clone());
907            st.consecutive_errors += 1;
908            let b = grow_backoff(st, cfg);
909            st.next_action_at_ms = now_ms + b;
910            events.push(KeeperEvent::PollFailed {
911                key: key.to_string(),
912                detail: detail.clone(),
913                backoff_ms: b,
914            });
915            // A pending attempt cannot be verified through a failed poll;
916            // the verification window keeps running.
917            expire_pending(st, cfg, now_ms, &mut events);
918            Ok(events)
919        }
920        ObservationOutcome::AuthError { detail } => {
921            st.last_error = Some(detail.clone());
922            st.consecutive_errors += 1;
923            st.backoff_ms = cfg.max_backoff_ms;
924            st.next_action_at_ms = now_ms + cfg.max_backoff_ms;
925            set_phase(
926                st,
927                &mut events,
928                Phase::AuthError {
929                    detail: detail.clone(),
930                },
931            );
932            events.push(KeeperEvent::AuthError {
933                key: key.to_string(),
934                detail: detail.clone(),
935            });
936            Ok(events)
937        }
938        ObservationOutcome::Malformed { detail } | ObservationOutcome::Unsupported { detail } => {
939            st.last_error = Some(detail.clone());
940            st.consecutive_errors += 1;
941            let b = grow_backoff(st, cfg);
942            st.next_action_at_ms = now_ms + b;
943            set_phase(
944                st,
945                &mut events,
946                Phase::Unknown {
947                    reason: detail.clone(),
948                },
949            );
950            events.push(KeeperEvent::PollFailed {
951                key: key.to_string(),
952                detail: detail.clone(),
953                backoff_ms: b,
954            });
955            expire_pending(st, cfg, now_ms, &mut events);
956            Ok(events)
957        }
958        ObservationOutcome::Ok { .. } if future_stamped => {
959            let detail = "observation timestamp is in the future".to_string();
960            st.last_error = Some(detail.clone());
961            st.consecutive_errors += 1;
962            let b = grow_backoff(st, cfg);
963            st.next_action_at_ms = now_ms + b;
964            set_phase(
965                st,
966                &mut events,
967                Phase::Unknown {
968                    reason: detail.clone(),
969                },
970            );
971            events.push(KeeperEvent::PollFailed {
972                key: key.to_string(),
973                detail,
974                backoff_ms: b,
975            });
976            expire_pending(st, cfg, now_ms, &mut events);
977            Ok(events)
978        }
979        ObservationOutcome::Ok {
980            windows,
981            models,
982            limit_reached,
983            banked,
984            ..
985        } => {
986            st.last_error = None;
987            st.consecutive_errors = 0;
988            st.backoff_ms = 0;
989            st.last_poll_ok = true;
990            st.last_ok_observation_ms = Some(obs.observed_at_ms);
991            st.last_windows = windows.clone();
992            st.last_models = models.clone();
993            st.last_limit_reached = *limit_reached;
994            if let Some(b) = banked {
995                if let Some(count) = b.available_count.filter(|c| *c > 0) {
996                    let expiring_soon = b
997                        .earliest_expiry_ms
998                        .is_some_and(|e| e <= now_ms.saturating_add(cfg.banked_expiry_alert_ms));
999                    events.push(KeeperEvent::BankedResets {
1000                        key: key.to_string(),
1001                        available_count: count,
1002                        earliest_expiry_ms: b.earliest_expiry_ms,
1003                        expiring_soon,
1004                    });
1005                }
1006            }
1007            st.banked = banked.clone();
1008            let weekly = weekly_window(windows).cloned();
1009            st.weekly = weekly.clone();
1010            let observed = obs.observed_at_ms;
1011            let headroom_now =
1012                account_wide_headroom(windows, *limit_reached, cfg.exhausted_at_percent);
1013
1014            match weekly.as_ref().and_then(|w| w.resets_at_ms.map(|r| (w, r))) {
1015                None => {
1016                    // No weekly reset evidence in the fresh reading.
1017                    let reason = if weekly.is_some() {
1018                        "weekly window reported without a reset time"
1019                    } else {
1020                        "no weekly window evidence"
1021                    };
1022                    match st.generation {
1023                        Some(g) if g <= now_ms && headroom_now => {
1024                            // Known window ended; nothing newer reported; the
1025                            // provider asserts capacity → due.
1026                            window_ended(st, cfg, now_ms, g, &mut events);
1027                        }
1028                        Some(g) if g <= now_ms => {
1029                            // Ended, but no headroom evidence → not due.
1030                            expire_pending(st, cfg, now_ms, &mut events);
1031                            if !matches!(st.phase, Phase::ActivationPending { .. }) {
1032                                set_phase(
1033                                    st,
1034                                    &mut events,
1035                                    Phase::Unknown {
1036                                        reason: format!("{reason}; no headroom evidence after reset"),
1037                                    },
1038                                );
1039                            }
1040                            st.generation = Some(g);
1041                            schedule_regular(st, cfg, now_ms);
1042                        }
1043                        _ => {
1044                            set_phase(
1045                                st,
1046                                &mut events,
1047                                Phase::Unknown {
1048                                    reason: reason.into(),
1049                                },
1050                            );
1051                            schedule_regular(st, cfg, now_ms);
1052                        }
1053                    }
1054                }
1055                Some((w, r)) if r > now_ms => {
1056                    // Live window with a future reset.
1057                    let prev = st.generation;
1058                    if let Some(p) = prev {
1059                        if r < p {
1060                            // Provider went backwards: contradictory evidence.
1061                            set_phase(
1062                                st,
1063                                &mut events,
1064                                Phase::Unknown {
1065                                    reason: "reported reset earlier than the tracked generation".into(),
1066                                },
1067                            );
1068                            schedule_regular(st, cfg, now_ms);
1069                            return Ok(events);
1070                        }
1071                    }
1072                    let is_new = prev != Some(r);
1073                    let mut attribution = match &st.phase {
1074                        Phase::Active { attribution, .. } if !is_new => *attribution,
1075                        _ => Attribution::Observed,
1076                    };
1077                    if is_new {
1078                        // Correlate with an attempt for the previous generation.
1079                        let attempt = match &st.phase {
1080                            Phase::ActivationPending { attempt, .. } => Some(*attempt),
1081                            Phase::Unverified { attempts, .. } if *attempts > 0 => Some(*attempts),
1082                            _ => None,
1083                        };
1084                        if let Some(a) = attempt {
1085                            attribution = Attribution::AttemptCorrelated { attempt: a };
1086                        }
1087                        // Idle delay: anchor − previous reset.
1088                        let (idle, upper) = match (prev, w.duration_ms) {
1089                            (Some(p), Some(d)) if r >= d => {
1090                                let anchor = r - d;
1091                                (Some(anchor.saturating_sub(p)), false)
1092                            }
1093                            (Some(p), _) => (Some(observed.saturating_sub(p)), true),
1094                            (None, _) => (None, false),
1095                        };
1096                        st.last_idle_delay_ms = idle;
1097                        st.last_idle_delay_is_upper_bound = upper;
1098                        for rec in st.history.iter_mut().rev() {
1099                            if rec.new_generation.is_none() && rec.generation != r {
1100                                rec.verified_at_ms = Some(observed);
1101                                rec.new_generation = Some(r);
1102                                break;
1103                            }
1104                        }
1105                        st.attempts_in_generation = 0;
1106                        st.not_sent_in_generation = 0;
1107                        st.next_attempt_allowed_ms = 0;
1108                        events.push(KeeperEvent::NewWindow {
1109                            key: key.to_string(),
1110                            from_generation: prev,
1111                            to_generation: r,
1112                            idle_delay_ms: idle,
1113                            idle_delay_is_upper_bound: upper,
1114                            attribution,
1115                        });
1116                    }
1117                    st.generation = Some(r);
1118                    let phase = if *limit_reached == Some(true) {
1119                        Phase::Exhausted {
1120                            generation: r,
1121                            reset_passed: false,
1122                        }
1123                    } else {
1124                        match w.verdict(cfg.exhausted_at_percent) {
1125                            WindowVerdict::Headroom { used_percent } => Phase::Active {
1126                                generation: r,
1127                                used_percent,
1128                                verified_at_ms: observed,
1129                                attribution,
1130                            },
1131                            WindowVerdict::Exhausted { .. } => Phase::Exhausted {
1132                                generation: r,
1133                                reset_passed: false,
1134                            },
1135                            WindowVerdict::Unknown => Phase::Unknown {
1136                                reason: "weekly window has no usage evidence".into(),
1137                            },
1138                        }
1139                    };
1140                    set_phase(st, &mut events, phase);
1141                    schedule_regular(st, cfg, now_ms);
1142                }
1143                Some((w, r)) => {
1144                    // Reset already passed and the provider still reports it.
1145                    if let Some(p) = st.generation {
1146                        if r < p {
1147                            set_phase(
1148                                st,
1149                                &mut events,
1150                                Phase::Unknown {
1151                                    reason: "reported reset earlier than the tracked generation".into(),
1152                                },
1153                            );
1154                            schedule_regular(st, cfg, now_ms);
1155                            return Ok(events);
1156                        }
1157                    }
1158                    if st.generation != Some(r) {
1159                        st.generation = Some(r);
1160                        st.attempts_in_generation = 0;
1161                        st.not_sent_in_generation = 0;
1162                        st.next_attempt_allowed_ms = 0;
1163                    }
1164                    let weekly_headroom =
1165                        matches!(w.verdict(cfg.exhausted_at_percent), WindowVerdict::Headroom { .. });
1166                    if headroom_now && weekly_headroom {
1167                        // Ended, nothing newer, capacity asserted → due.
1168                        window_ended(st, cfg, now_ms, r, &mut events);
1169                    } else {
1170                        // The provider still asserts no capacity after the
1171                        // reset: never due. Keep a pending attempt's
1172                        // verification clock running.
1173                        expire_pending(st, cfg, now_ms, &mut events);
1174                        if !matches!(st.phase, Phase::ActivationPending { .. }) {
1175                            set_phase(
1176                                st,
1177                                &mut events,
1178                                Phase::Exhausted {
1179                                    generation: r,
1180                                    reset_passed: true,
1181                                },
1182                            );
1183                        }
1184                        events.push(KeeperEvent::LimitAssertedAfterReset {
1185                            key: key.to_string(),
1186                            generation: r,
1187                        });
1188                        st.next_action_at_ms = now_ms + cfg.poll_interval_ms;
1189                    }
1190                }
1191            }
1192            Ok(events)
1193        }
1194    }
1195}
1196
1197/// The window for `g` has ended, fresh usage shows nothing newer and the
1198/// provider asserts capacity.
1199fn window_ended(
1200    st: &mut AccountState,
1201    cfg: &KeeperConfig,
1202    now_ms: u64,
1203    g: u64,
1204    events: &mut Vec<KeeperEvent>,
1205) {
1206    match st.phase.clone() {
1207        Phase::ActivationPending { generation, .. } if generation == g => {
1208            // Still awaiting verification.
1209            expire_pending(st, cfg, now_ms, events);
1210            if matches!(st.phase, Phase::ActivationPending { .. }) {
1211                st.next_action_at_ms = now_ms + cfg.verify_poll_ms;
1212            } else {
1213                st.next_action_at_ms = now_ms + cfg.poll_interval_ms;
1214            }
1215        }
1216        Phase::Unverified { generation, .. } if generation == g => {
1217            st.next_action_at_ms = now_ms + cfg.poll_interval_ms;
1218        }
1219        Phase::AwaitingActivation {
1220            generation,
1221            due_since_ms,
1222        } if generation == g => {
1223            events.push(KeeperEvent::ActivationDue {
1224                key: st.identity.key(),
1225                generation: g,
1226                due_since_ms,
1227            });
1228            st.next_action_at_ms = now_ms + cfg.poll_interval_ms;
1229        }
1230        _ => {
1231            if st.attempts_in_generation >= cfg.max_attempts_per_generation {
1232                let attempts = st.attempts_in_generation;
1233                set_phase(
1234                    st,
1235                    events,
1236                    Phase::Unverified {
1237                        generation: g,
1238                        attempts,
1239                    },
1240                );
1241            } else {
1242                // Due since the reset instant itself: idle delay is measured
1243                // against the generation, not against poll times.
1244                set_phase(
1245                    st,
1246                    events,
1247                    Phase::AwaitingActivation {
1248                        generation: g,
1249                        due_since_ms: g,
1250                    },
1251                );
1252                events.push(KeeperEvent::ActivationDue {
1253                    key: st.identity.key(),
1254                    generation: g,
1255                    due_since_ms: g,
1256                });
1257            }
1258            st.next_action_at_ms = now_ms + cfg.poll_interval_ms;
1259        }
1260    }
1261}
1262
1263/// Move a pending attempt whose verification window elapsed to `Unverified`
1264/// (or back to `AwaitingActivation` when budget remains).
1265fn expire_pending(
1266    st: &mut AccountState,
1267    cfg: &KeeperConfig,
1268    now_ms: u64,
1269    events: &mut Vec<KeeperEvent>,
1270) {
1271    if let Phase::ActivationPending {
1272        generation,
1273        started_ms,
1274        ..
1275    } = st.phase.clone()
1276    {
1277        if now_ms.saturating_sub(started_ms) >= cfg.verify_window_ms {
1278            let attempts = st.attempts_in_generation;
1279            if attempts >= cfg.max_attempts_per_generation {
1280                set_phase(
1281                    st,
1282                    events,
1283                    Phase::Unverified {
1284                        generation,
1285                        attempts,
1286                    },
1287                );
1288                events.push(KeeperEvent::Unverified {
1289                    key: st.identity.key(),
1290                    generation,
1291                    attempts,
1292                });
1293            } else {
1294                set_phase(
1295                    st,
1296                    events,
1297                    Phase::AwaitingActivation {
1298                        generation,
1299                        due_since_ms: generation,
1300                    },
1301                );
1302            }
1303        }
1304    }
1305}
1306
1307/// Runner-facing decision after a poll.
1308#[derive(Debug, Clone, PartialEq, Eq)]
1309pub enum ActivationDecision {
1310    Activate { generation: u64, attempt: u32 },
1311    Skip { reason: String },
1312}
1313
1314/// Capacity view of the last successful poll, for `quota_policy`.
1315pub fn capacity_view(st: &AccountState) -> Option<AccountCapacity> {
1316    let credential = CredentialRef::parse_storage_key(&st.identity.credential)?;
1317    Some(AccountCapacity {
1318        credential,
1319        observed_at_ms: st.last_ok_observation_ms,
1320        observation: QuotaObservation::Ok {
1321            windows: windows_with_overall_flag(&st.last_windows, st.last_limit_reached),
1322            models: st.last_models.clone(),
1323        },
1324        cooldown_until_ms: None,
1325        failure: None,
1326    })
1327}
1328
1329/// Whether the runner may start an activation attempt now.
1330///
1331/// Read-only unless `opted_in`. Requires: a **strong** seat identity (a
1332/// metadata prefix is not an identity), an awaiting phase with budget, the
1333/// **latest** poll successful and fresh, no overall limit assertion, and
1334/// headroom on every window applicable to `model` (5h, weekly and
1335/// model-scoped alike) with the model not listed exhausted/unknown — the
1336/// same fail-closed evaluation the selection policy uses.
1337pub fn activation_decision(
1338    state: &KeeperState,
1339    cfg: &KeeperConfig,
1340    now_ms: u64,
1341    key: &str,
1342    opted_in: bool,
1343    model: &str,
1344) -> ActivationDecision {
1345    let skip = |r: &str| ActivationDecision::Skip {
1346        reason: r.to_string(),
1347    };
1348    let Some(st) = state.accounts.get(key) else {
1349        return skip("unknown account");
1350    };
1351    if !opted_in {
1352        return skip("not opted in (read-only)");
1353    }
1354    if !st.identity.is_strong() {
1355        return skip("seat identity not established (a metadata prefix is not an identity)");
1356    }
1357    let generation = match &st.phase {
1358        Phase::AwaitingActivation { generation, .. } => *generation,
1359        Phase::ActivationPending { .. } => return skip("attempt pending verification"),
1360        Phase::Unverified { .. } => return skip("attempt budget spent for this generation"),
1361        other => return skip(&format!("not due (phase {})", other.label())),
1362    };
1363    if st.attempts_in_generation >= cfg.max_attempts_per_generation {
1364        return skip("attempt budget spent for this generation");
1365    }
1366    if st.not_sent_in_generation >= MAX_NOT_SENT_PER_GENERATION {
1367        return skip("too many unsent attempts for this generation");
1368    }
1369    if now_ms < st.next_attempt_allowed_ms {
1370        return skip("backoff after an unsent attempt");
1371    }
1372    if !st.last_poll_ok {
1373        return skip("latest poll failed; no fresh evidence");
1374    }
1375    if st.last_limit_reached == Some(true) {
1376        return skip("provider asserts limit_reached");
1377    }
1378    let Some(cap) = capacity_view(st) else {
1379        return skip("unparseable credential key");
1380    };
1381    let req = SelectionRequest {
1382        model: Some(model),
1383        strategy: Strategy::PreferenceOrder,
1384        exhausted_at_percent: cfg.exhausted_at_percent,
1385        ..SelectionRequest::new(cap.credential.provider, now_ms, cfg.stale_after_ms)
1386    };
1387    if let Err(reason) = quota_policy::evaluate(&req, &cap) {
1388        return skip(&format!("no proven capacity: {reason}"));
1389    }
1390    ActivationDecision::Activate {
1391        generation,
1392        attempt: st.attempts_in_generation + 1,
1393    }
1394}
1395
1396/// Ticket for one attempt. The caller MUST persist the state between
1397/// [`begin_attempt`] and the request, so a crash leaves a pending record.
1398#[derive(Debug, Clone, PartialEq, Eq)]
1399pub struct AttemptTicket {
1400    pub key: String,
1401    pub generation: u64,
1402    pub attempt: u32,
1403}
1404
1405/// Record an attempt as started. Consumes one unit of the generation budget.
1406pub fn begin_attempt(
1407    state: &mut KeeperState,
1408    now_ms: u64,
1409    key: &str,
1410) -> Result<(AttemptTicket, Vec<KeeperEvent>), KeeperError> {
1411    let st = state
1412        .accounts
1413        .get_mut(key)
1414        .ok_or_else(|| KeeperError::UnknownAccount(key.to_string()))?;
1415    if !st.identity.is_strong() {
1416        return Err(KeeperError::IdentityRequired {
1417            key: key.to_string(),
1418        });
1419    }
1420    let generation = match &st.phase {
1421        Phase::AwaitingActivation { generation, .. } => *generation,
1422        other => {
1423            return Err(KeeperError::NotDue {
1424                key: key.to_string(),
1425                phase: other.label().to_string(),
1426            })
1427        }
1428    };
1429    st.attempts_in_generation += 1;
1430    let attempt = st.attempts_in_generation;
1431    let mut events = Vec::new();
1432    set_phase(
1433        st,
1434        &mut events,
1435        Phase::ActivationPending {
1436            generation,
1437            attempt,
1438            started_ms: now_ms,
1439            outcome: None,
1440        },
1441    );
1442    st.push_history(AttemptRecord {
1443        attempt,
1444        generation,
1445        started_ms: now_ms,
1446        finished_ms: None,
1447        outcome: None,
1448        verified_at_ms: None,
1449        new_generation: None,
1450    });
1451    st.next_action_at_ms = now_ms;
1452    state.updated_at_ms = now_ms;
1453    events.push(KeeperEvent::AttemptStarted {
1454        key: key.to_string(),
1455        generation,
1456        attempt,
1457    });
1458    Ok((
1459        AttemptTicket {
1460            key: key.to_string(),
1461            generation,
1462            attempt,
1463        },
1464        events,
1465    ))
1466}
1467
1468/// Record the outcome of an attempt. Refundable outcomes return the budget
1469/// unit and apply backoff; all others stay pending until verified/expired.
1470pub fn finish_attempt(
1471    state: &mut KeeperState,
1472    cfg: &KeeperConfig,
1473    now_ms: u64,
1474    ticket: &AttemptTicket,
1475    outcome: AttemptOutcome,
1476) -> Result<Vec<KeeperEvent>, KeeperError> {
1477    let st = state
1478        .accounts
1479        .get_mut(&ticket.key)
1480        .ok_or_else(|| KeeperError::UnknownAccount(ticket.key.clone()))?;
1481    let mut events = Vec::new();
1482    if let Some(rec) = st
1483        .history
1484        .iter_mut()
1485        .rev()
1486        .find(|r| r.generation == ticket.generation && r.attempt == ticket.attempt)
1487    {
1488        rec.finished_ms = Some(now_ms);
1489        rec.outcome = Some(outcome.clone());
1490    }
1491    let refunded = outcome.is_refundable();
1492    match &st.phase {
1493        Phase::ActivationPending {
1494            generation,
1495            attempt,
1496            ..
1497        } if *generation == ticket.generation && *attempt == ticket.attempt => {}
1498        _ => {
1499            // Phase moved on (e.g. verified through a concurrent poll); keep
1500            // the record only.
1501            events.push(KeeperEvent::AttemptFinished {
1502                key: ticket.key.clone(),
1503                generation: ticket.generation,
1504                attempt: ticket.attempt,
1505                outcome,
1506                refunded: false,
1507            });
1508            state.updated_at_ms = now_ms;
1509            return Ok(events);
1510        }
1511    }
1512    if refunded {
1513        st.attempts_in_generation = st.attempts_in_generation.saturating_sub(1);
1514        st.not_sent_in_generation += 1;
1515        let b = grow_backoff(st, cfg);
1516        st.next_attempt_allowed_ms = now_ms + b;
1517        st.next_action_at_ms = now_ms + b;
1518        if let AttemptOutcome::Rejected { http_status } = &outcome {
1519            if matches!(http_status, 401 | 403) {
1520                set_phase(
1521                    st,
1522                    &mut events,
1523                    Phase::AuthError {
1524                        detail: format!("activation rejected with HTTP {http_status}"),
1525                    },
1526                );
1527                events.push(KeeperEvent::AuthError {
1528                    key: ticket.key.clone(),
1529                    detail: format!("activation rejected with HTTP {http_status}"),
1530                });
1531            }
1532        }
1533        if !matches!(st.phase, Phase::AuthError { .. }) {
1534            set_phase(
1535                st,
1536                &mut events,
1537                Phase::AwaitingActivation {
1538                    generation: ticket.generation,
1539                    due_since_ms: ticket.generation,
1540                },
1541            );
1542        }
1543        st.last_error = Some(match &outcome {
1544            AttemptOutcome::NotSent { stage, reason } => {
1545                format!("activation not sent ({stage:?}): {reason}")
1546            }
1547            AttemptOutcome::Rejected { http_status } => {
1548                format!("activation rejected: HTTP {http_status}")
1549            }
1550            _ => "activation refunded".into(),
1551        });
1552    } else {
1553        st.phase = Phase::ActivationPending {
1554            generation: ticket.generation,
1555            attempt: ticket.attempt,
1556            started_ms: match &st.phase {
1557                Phase::ActivationPending { started_ms, .. } => *started_ms,
1558                _ => now_ms,
1559            },
1560            outcome: Some(outcome.clone()),
1561        };
1562        // Verify from fresh usage as soon as possible.
1563        st.next_action_at_ms = now_ms;
1564        if let AttemptOutcome::Ambiguous { reason } = &outcome {
1565            st.last_error = Some(format!("activation ambiguous: {reason}"));
1566        }
1567    }
1568    state.updated_at_ms = now_ms;
1569    events.push(KeeperEvent::AttemptFinished {
1570        key: ticket.key.clone(),
1571        generation: ticket.generation,
1572        attempt: ticket.attempt,
1573        outcome,
1574        refunded,
1575    });
1576    Ok(events)
1577}
1578
1579/// Housekeeping on startup: a pending attempt with no recorded outcome means
1580/// the previous process died between begin and finish — mark it ambiguous.
1581pub fn recover_after_restart(state: &mut KeeperState, now_ms: u64) -> Vec<KeeperEvent> {
1582    let mut events = Vec::new();
1583    for (key, st) in state.accounts.iter_mut() {
1584        if let Phase::ActivationPending {
1585            generation,
1586            attempt,
1587            started_ms,
1588            outcome: None,
1589        } = st.phase.clone()
1590        {
1591            let outcome = AttemptOutcome::Ambiguous {
1592                reason: "process restarted before the attempt outcome was recorded".into(),
1593            };
1594            st.phase = Phase::ActivationPending {
1595                generation,
1596                attempt,
1597                started_ms,
1598                outcome: Some(outcome.clone()),
1599            };
1600            if let Some(rec) = st
1601                .history
1602                .iter_mut()
1603                .rev()
1604                .find(|r| r.generation == generation && r.attempt == attempt)
1605            {
1606                rec.finished_ms = Some(now_ms);
1607                rec.outcome = Some(outcome.clone());
1608            }
1609            st.next_action_at_ms = now_ms;
1610            events.push(KeeperEvent::AttemptFinished {
1611                key: key.clone(),
1612                generation,
1613                attempt,
1614                outcome,
1615                refunded: false,
1616            });
1617        }
1618    }
1619    events
1620}
1621
1622/// Explicit operator re-arm: allow exactly one more attempt for the
1623/// generation currently in `Unverified`. Never automatic. Returns `Ok(None)`
1624/// when the account is not unverified (nothing to re-arm).
1625pub fn rearm(
1626    state: &mut KeeperState,
1627    now_ms: u64,
1628    key: &str,
1629) -> Result<Option<KeeperEvent>, KeeperError> {
1630    let st = state
1631        .accounts
1632        .get_mut(key)
1633        .ok_or_else(|| KeeperError::UnknownAccount(key.to_string()))?;
1634    let Phase::Unverified {
1635        generation,
1636        attempts,
1637    } = st.phase.clone()
1638    else {
1639        return Ok(None);
1640    };
1641    st.attempts_in_generation = 0;
1642    st.not_sent_in_generation = 0;
1643    st.next_attempt_allowed_ms = 0;
1644    st.phase = Phase::AwaitingActivation {
1645        generation,
1646        due_since_ms: generation,
1647    };
1648    st.next_action_at_ms = now_ms;
1649    state.updated_at_ms = now_ms;
1650    Ok(Some(KeeperEvent::Rearmed {
1651        key: key.to_string(),
1652        generation,
1653        previous_attempts: attempts,
1654    }))
1655}
1656
1657// ── Persistence ──────────────────────────────────────────────────────────────
1658
1659/// Private, atomic JSON state (dir `0700`, file `0600`, tmp+rename).
1660#[derive(Debug, Clone)]
1661pub struct StateStore {
1662    dir: PathBuf,
1663}
1664
1665impl StateStore {
1666    pub fn new(dir: &Path) -> Self {
1667        Self {
1668            dir: dir.to_path_buf(),
1669        }
1670    }
1671
1672    pub fn dir(&self) -> &Path {
1673        &self.dir
1674    }
1675
1676    pub fn path(&self) -> PathBuf {
1677        self.dir.join(STATE_FILE_NAME)
1678    }
1679
1680    /// Missing file → default state. Unreadable/unparseable → error (fail closed).
1681    pub fn load(&self) -> Result<KeeperState, KeeperError> {
1682        let path = self.path();
1683        let raw = match std::fs::read(&path) {
1684            Ok(b) => b,
1685            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
1686                return Ok(KeeperState::default())
1687            }
1688            Err(e) => {
1689                return Err(KeeperError::Io {
1690                    path,
1691                    detail: e.to_string(),
1692                })
1693            }
1694        };
1695        let state: KeeperState =
1696            serde_json::from_slice(&raw).map_err(|e| KeeperError::CorruptState {
1697                path: path.clone(),
1698                detail: e.to_string(),
1699            })?;
1700        if state.version > STATE_VERSION {
1701            return Err(KeeperError::UnsupportedVersion {
1702                path,
1703                version: state.version,
1704            });
1705        }
1706        Ok(state)
1707    }
1708
1709    pub fn save(&self, state: &KeeperState) -> Result<(), KeeperError> {
1710        crate::private_fs::ensure_private_dir(&self.dir).map_err(|e| KeeperError::Io {
1711            path: self.dir.clone(),
1712            detail: e.to_string(),
1713        })?;
1714        let mut out = KeeperState {
1715            version: STATE_VERSION,
1716            ..state.clone()
1717        };
1718        if out.updated_at_ms == 0 {
1719            out.updated_at_ms = state.updated_at_ms;
1720        }
1721        let bytes = serde_json::to_vec_pretty(&out).map_err(|e| KeeperError::Io {
1722            path: self.path(),
1723            detail: e.to_string(),
1724        })?;
1725        crate::private_fs::write_atomic_private(&self.path(), &bytes).map_err(|e| KeeperError::Io {
1726            path: self.path(),
1727            detail: e.to_string(),
1728        })
1729    }
1730}
1731
1732// ── Locks ────────────────────────────────────────────────────────────────────
1733
1734/// Exclusive advisory `flock` held for the process lifetime (released on
1735/// drop or process death). Used for the state-dir singleton and for one lock
1736/// per kept account in the canonical keeper dir, so a second keeper with a
1737/// different `--state-dir` still cannot act on the same account.
1738#[derive(Debug)]
1739pub struct KeeperLock {
1740    _file: File,
1741    path: PathBuf,
1742}
1743
1744impl KeeperLock {
1745    pub fn acquire(dir: &Path, name: &str, holder: &str) -> Result<Self, KeeperError> {
1746        use fs4::fs_std::FileExt;
1747        crate::private_fs::ensure_private_dir(dir).map_err(|e| KeeperError::Io {
1748            path: dir.to_path_buf(),
1749            detail: e.to_string(),
1750        })?;
1751        let path = dir.join(name);
1752        let file = OpenOptions::new()
1753            .create(true)
1754            .truncate(false)
1755            .read(true)
1756            .write(true)
1757            .open(&path)
1758            .map_err(|e| KeeperError::Io {
1759                path: path.clone(),
1760                detail: e.to_string(),
1761            })?;
1762        #[cfg(unix)]
1763        {
1764            use std::os::unix::fs::PermissionsExt;
1765            let _ = file.set_permissions(std::fs::Permissions::from_mode(0o600));
1766        }
1767        let locked = FileExt::try_lock_exclusive(&file).map_err(|e| KeeperError::Io {
1768            path: path.clone(),
1769            detail: e.to_string(),
1770        })?;
1771        if !locked {
1772            let body = std::fs::read_to_string(&path).unwrap_or_default();
1773            let holder = body.lines().next().map(str::to_string).filter(|s| !s.is_empty());
1774            return Err(KeeperError::Locked { path, holder });
1775        }
1776        use std::io::Write;
1777        let _ = file.set_len(0);
1778        let _ = (&file).write_all(format!("{holder}\n").as_bytes());
1779        Ok(Self { _file: file, path })
1780    }
1781
1782    pub fn path(&self) -> &Path {
1783        &self.path
1784    }
1785}
1786
1787/// File-name-safe form of a storage key for per-account locks.
1788pub fn account_lock_name(storage_key: &str) -> String {
1789    let safe: String = storage_key
1790        .chars()
1791        .map(|c| {
1792            if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-' | '@') {
1793                c
1794            } else {
1795                '_'
1796            }
1797        })
1798        .collect();
1799    format!("{safe}.lock")
1800}
1801
1802#[cfg(test)]
1803mod tests {
1804    use super::*;
1805    use crate::auth::quota_policy::ModelState;
1806
1807    const NOW: u64 = 1_800_000_000_000;
1808    const WEEK: u64 = 7 * DAY;
1809    const MODEL: &str = "gpt-5.4-mini";
1810
1811    /// Strong seat identity (what the runner establishes for an opted-in
1812    /// account from the vended token). One seat per label in these tests.
1813    fn ident(label: &str) -> AccountIdentity {
1814        AccountIdentity {
1815            source: "local:/tmp/auth.json".into(),
1816            credential: format!("openai-codex@{label}"),
1817            identity_fp: AccountIdentity::seat_fingerprint(
1818                OAuthProviderId::OpenAiCodex,
1819                &format!("2b2f1234-seat-{label}"),
1820            )
1821            .unwrap(),
1822        }
1823    }
1824
1825    /// Weak metadata-only identity (read-only accounts).
1826    fn weak_ident(label: &str) -> AccountIdentity {
1827        AccountIdentity {
1828            source: "local:/tmp/auth.json".into(),
1829            credential: format!("openai-codex@{label}"),
1830            identity_fp: "id:2b2f1234".into(),
1831        }
1832    }
1833
1834    fn cfg() -> KeeperConfig {
1835        KeeperConfig::default().bounded()
1836    }
1837
1838    fn five_hour(observed: u64, used: f64) -> WindowLimit {
1839        WindowLimit {
1840            id: "primary".into(),
1841            duration_ms: Some(5 * HOUR),
1842            used_percent: Some(used),
1843            limit_reached: Some(used >= 100.0),
1844            resets_at_ms: Some(observed + HOUR),
1845            models: None,
1846        }
1847    }
1848
1849    fn weekly(used: f64, resets: Option<u64>) -> WindowLimit {
1850        WindowLimit {
1851            id: "secondary".into(),
1852            duration_ms: Some(WEEK),
1853            used_percent: Some(used),
1854            limit_reached: Some(used >= 100.0),
1855            resets_at_ms: resets,
1856            models: None,
1857        }
1858    }
1859
1860    /// Reading with a 5h window at 1 % and a weekly window at `used`.
1861    fn ok_obs(observed: u64, used: f64, resets: Option<u64>) -> UsageObservation {
1862        UsageObservation {
1863            observed_at_ms: observed,
1864            outcome: ObservationOutcome::Ok {
1865                windows: vec![five_hour(observed, 1.0), weekly(used, resets)],
1866                models: None,
1867                limit_reached: Some(used >= 100.0),
1868                banked: None,
1869                identity_prefix: Some("2b2f1234".into()),
1870            },
1871        }
1872    }
1873
1874    fn no_weekly_obs(observed: u64) -> UsageObservation {
1875        UsageObservation {
1876            observed_at_ms: observed,
1877            outcome: ObservationOutcome::Ok {
1878                windows: vec![five_hour(observed, 0.0)],
1879                models: None,
1880                limit_reached: Some(false),
1881                banked: None,
1882                identity_prefix: None,
1883            },
1884        }
1885    }
1886
1887    fn transport(t: u64) -> UsageObservation {
1888        UsageObservation {
1889            observed_at_ms: t,
1890            outcome: ObservationOutcome::Transport {
1891                detail: "connect".into(),
1892            },
1893        }
1894    }
1895
1896    fn fresh(label: &str) -> (KeeperState, String) {
1897        let mut s = KeeperState::default();
1898        let id = ident(label);
1899        s.entry(&id);
1900        (s, id.key())
1901    }
1902
1903    /// Drive an account to "due": exhausted live window, reset passes, fresh
1904    /// reading shows the old (past) reset with headroom (provider asserts
1905    /// capacity again but has not anchored a new window).
1906    fn make_due(label: &str) -> (KeeperState, String, u64, u64) {
1907        let c = cfg();
1908        let (mut s, k) = fresh(label);
1909        let g1 = NOW + HOUR;
1910        observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
1911        assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { reset_passed: false, .. }));
1912        let t = g1 + MIN;
1913        observe(&mut s, &c, t, &k, &ok_obs(t, 0.0, Some(g1))).unwrap();
1914        assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { generation, due_since_ms } if generation == g1 && due_since_ms == g1));
1915        (s, k, g1, t)
1916    }
1917
1918    fn decide(s: &KeeperState, t: u64, k: &str, opted_in: bool) -> ActivationDecision {
1919        activation_decision(s, &cfg(), t, k, opted_in, MODEL)
1920    }
1921
1922    #[test]
1923    fn config_bounds_untrusted_intervals() {
1924        let c = KeeperConfig {
1925            poll_interval_ms: 1,
1926            max_backoff_ms: 0,
1927            stale_after_ms: u64::MAX,
1928            reset_grace_ms: u64::MAX,
1929            verify_poll_ms: 0,
1930            verify_window_ms: u64::MAX,
1931            max_attempts_per_generation: 99,
1932            attempt_timeout_ms: 0,
1933            usage_timeout_ms: u64::MAX,
1934            banked_expiry_alert_ms: 0,
1935            exhausted_at_percent: f64::NAN,
1936        }
1937        .bounded();
1938        assert_eq!(c.poll_interval_ms, MIN_POLL_INTERVAL_MS);
1939        assert_eq!(c.max_backoff_ms, MIN_POLL_INTERVAL_MS);
1940        assert_eq!(c.stale_after_ms, MAX_STALE_AFTER_MS);
1941        assert_eq!(c.reset_grace_ms, MAX_RESET_GRACE_MS);
1942        assert_eq!(c.max_attempts_per_generation, 1);
1943        assert_eq!(c.attempt_timeout_ms, MIN_ATTEMPT_TIMEOUT_MS);
1944        assert_eq!(c.exhausted_at_percent, DEFAULT_EXHAUSTED_AT_PERCENT);
1945        assert!(c.verify_window_ms <= MAX_VERIFY_WINDOW_MS);
1946        assert!(c.verify_poll_ms >= 5 * SEC);
1947        assert_eq!(KeeperConfig::default().bounded(), KeeperConfig::default());
1948        assert_eq!(KeeperConfig::default().poll_interval_ms, 5 * MIN);
1949    }
1950
1951    #[test]
1952    fn identity_fingerprint_prefers_account_id_and_changes_on_relogin() {
1953        assert_eq!(
1954            AccountIdentity::fingerprint(Some("2b2f1234"), Some("a@b"), Some(1)),
1955            "id:2b2f1234"
1956        );
1957        let a = AccountIdentity::fingerprint(None, Some("a@b"), Some(1));
1958        let b = AccountIdentity::fingerprint(None, Some("c@d"), Some(1));
1959        assert!(a.starts_with("who:") && b.starts_with("who:") && a != b);
1960        assert_eq!(AccountIdentity::fingerprint(None, None, Some(7)), "added:7");
1961        assert_eq!(AccountIdentity::fingerprint(None, None, None), "unknown");
1962        let mut i1 = weak_ident("x");
1963        let mut i2 = weak_ident("x");
1964        i1.identity_fp = "id:aaaa".into();
1965        i2.identity_fp = "id:bbbb".into();
1966        assert_ne!(i1.key(), i2.key());
1967        // Re-login under the same alias with another seat → a due ticket
1968        // does not carry over.
1969        let (mut s, k, _g, _t) = make_due("x");
1970        let relogged = AccountIdentity {
1971            identity_fp: AccountIdentity::seat_fingerprint(OAuthProviderId::OpenAiCodex, "another-seat").unwrap(),
1972            ..ident("x")
1973        };
1974        s.entry(&relogged);
1975        assert!(matches!(s.accounts[&relogged.key()].phase, Phase::Unknown { .. }));
1976        assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { .. }));
1977        assert_eq!(s.accounts.len(), 2);
1978    }
1979
1980    #[test]
1981    fn strong_seat_identity_is_canonical_across_aliases_and_weak_is_label_scoped() {
1982        // Strong: digest of provider + FULL account id; label not in the key.
1983        let codex = OAuthProviderId::OpenAiCodex;
1984        let fp = AccountIdentity::seat_fingerprint(codex, "2b2f1234-full-id").unwrap();
1985        assert!(fp.starts_with(SEAT_FP_PREFIX));
1986        assert!(!fp.contains("2b2f1234"), "state never carries the id itself");
1987        assert_ne!(fp, AccountIdentity::seat_fingerprint(codex, "2b2f1234-other").unwrap());
1988        assert_ne!(fp, AccountIdentity::seat_fingerprint(OAuthProviderId::Anthropic, "2b2f1234-full-id").unwrap());
1989        assert_eq!(fp, AccountIdentity::seat_fingerprint(codex, "  2b2f1234-full-id ").unwrap());
1990        assert_eq!(AccountIdentity::seat_fingerprint(codex, "  "), None);
1991        // Byte-for-byte the broker's listed recipe, so listing ⇔ token compare.
1992        let listed = super::super::account::seat_fingerprint(codex, "2b2f1234-full-id").unwrap();
1993        assert_eq!(fp, AccountIdentity::strong_from_listed(&listed));
1994        assert_eq!(fp, format!("seat:{listed}"));
1995        let a = AccountIdentity {
1996            source: "local:/tmp/auth.json".into(),
1997            credential: "openai-codex@a".into(),
1998            identity_fp: fp.clone(),
1999        };
2000        let b = AccountIdentity {
2001            credential: "openai-codex@b".into(),
2002            ..a.clone()
2003        };
2004        let default = AccountIdentity {
2005            credential: "openai-codex".into(),
2006            ..a.clone()
2007        };
2008        assert!(a.is_strong());
2009        assert_eq!(a.key(), b.key(), "aliases of one seat share one ledger");
2010        assert_eq!(a.key(), default.key());
2011        assert_eq!(a.ledger_scope(), "openai-codex");
2012        assert_eq!(a.key(), format!("local:/tmp/auth.json|openai-codex|{fp}"));
2013        // Weak: label-scoped, never strong.
2014        let wa = weak_ident("a");
2015        let wb = weak_ident("b");
2016        assert!(!wa.is_strong());
2017        assert_ne!(wa.key(), wb.key());
2018        assert_eq!(wa.ledger_scope(), "openai-codex@a");
2019        // Sequential alias switch keeps the spent generation: run 1 spends
2020        // under alias `a`, run 2 comes back under alias `b` (same seat).
2021        let c = cfg();
2022        let mut s = KeeperState::default();
2023        s.entry(&a);
2024        let g1 = NOW + HOUR;
2025        observe(&mut s, &c, NOW, &a.key(), &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2026        let t = g1 + MIN;
2027        observe(&mut s, &c, t, &a.key(), &ok_obs(t, 0.0, Some(g1))).unwrap();
2028        let (ticket, _) = begin_attempt(&mut s, t, &a.key()).unwrap();
2029        finish_attempt(&mut s, &c, t, &ticket, AttemptOutcome::Ambiguous { reason: "timeout".into() }).unwrap();
2030        let st = s.entry(&b);
2031        assert_eq!(st.identity.credential, "openai-codex@b", "entry adopts the alias in use");
2032        assert_eq!(st.attempts_in_generation, 1);
2033        assert_eq!(s.accounts.len(), 1);
2034        assert!(matches!(decide(&s, t + SEC, &b.key(), true), ActivationDecision::Skip { .. }));
2035        assert!(begin_attempt(&mut s, t + SEC, &b.key()).is_err());
2036    }
2037
2038    #[test]
2039    fn activation_requires_a_strong_seat_identity() {
2040        // Same journey as make_due, but the account only has a metadata prefix.
2041        let c = cfg();
2042        let mut s = KeeperState::default();
2043        let w = weak_ident("w");
2044        s.entry(&w);
2045        let k = w.key();
2046        let g1 = NOW + HOUR;
2047        observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2048        let t = g1 + MIN;
2049        observe(&mut s, &c, t, &k, &ok_obs(t, 0.0, Some(g1))).unwrap();
2050        assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { .. }), "due, alerts");
2051        match decide(&s, t, &k, true) {
2052            ActivationDecision::Skip { reason } => assert!(reason.contains("identity"), "{reason}"),
2053            other => panic!("weak identity must never activate: {other:?}"),
2054        }
2055        assert!(matches!(
2056            begin_attempt(&mut s, t, &k),
2057            Err(KeeperError::IdentityRequired { .. })
2058        ));
2059        assert_eq!(s.accounts[&k].attempts_in_generation, 0);
2060        // The strong twin is allowed.
2061        let (s2, k2, _g, t2) = make_due("s");
2062        assert!(matches!(decide(&s2, t2, &k2, true), ActivationDecision::Activate { .. }));
2063    }
2064
2065    #[test]
2066    fn identity_mismatch_before_send_is_refunded_and_recorded() {
2067        let c = cfg();
2068        let (mut s, k, g1, t) = make_due("m");
2069        let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2070        let ev = finish_attempt(
2071            &mut s,
2072            &c,
2073            t + SEC,
2074            &ticket,
2075            AttemptOutcome::NotSent {
2076                stage: NotSentStage::IdentityMismatch,
2077                reason: "token belongs to another seat".into(),
2078            },
2079        )
2080        .unwrap();
2081        let st = &s.accounts[&k];
2082        assert_eq!(st.attempts_in_generation, 0, "nothing was sent → refunded");
2083        assert_eq!(st.not_sent_in_generation, 1);
2084        assert!(matches!(st.phase, Phase::AwaitingActivation { generation, .. } if generation == g1));
2085        assert!(st.last_error.as_deref().unwrap().contains("IdentityMismatch"));
2086        assert!(ev.iter().any(|e| matches!(
2087            e,
2088            KeeperEvent::AttemptFinished { refunded: true, outcome: AttemptOutcome::NotSent { stage: NotSentStage::IdentityMismatch, .. }, .. }
2089        )));
2090        assert!(st.history.last().unwrap().outcome.is_some());
2091    }
2092
2093    #[test]
2094    fn live_window_tracks_generation_and_schedules_reset_grace() {
2095        let (mut s, k) = fresh("a");
2096        let reset = NOW + 2 * MIN;
2097        let ev = observe(&mut s, &cfg(), NOW, &k, &ok_obs(NOW, 10.0, Some(reset))).unwrap();
2098        let st = &s.accounts[&k];
2099        assert!(matches!(st.phase, Phase::Active { generation, .. } if generation == reset));
2100        assert_eq!(st.generation, Some(reset));
2101        assert!(st.last_poll_ok);
2102        // reset sooner than the poll interval → poll at reset + grace
2103        assert_eq!(st.next_action_at_ms, reset + cfg().reset_grace_ms);
2104        assert!(ev.iter().any(|e| matches!(e, KeeperEvent::NewWindow { from_generation: None, .. })));
2105        // exhausted live window
2106        observe(&mut s, &cfg(), NOW + 1, &k, &ok_obs(NOW + 1, 100.0, Some(reset))).unwrap();
2107        assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { generation, reset_passed: false } if generation == reset));
2108        // overall limit flag alone makes it exhausted even with weekly headroom
2109        let mut obs = ok_obs(NOW + 1, 10.0, Some(reset));
2110        if let ObservationOutcome::Ok { limit_reached, .. } = &mut obs.outcome {
2111            *limit_reached = Some(true);
2112        }
2113        observe(&mut s, &cfg(), NOW + 1, &k, &obs).unwrap();
2114        assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { .. }));
2115        // far reset → regular interval
2116        let far = NOW + 3 * DAY;
2117        observe(&mut s, &cfg(), NOW + 2, &k, &ok_obs(NOW + 2, 50.0, Some(far))).unwrap();
2118        assert_eq!(s.accounts[&k].next_action_at_ms, NOW + 2 + cfg().poll_interval_ms);
2119    }
2120
2121    #[test]
2122    fn weekly_from_duration_not_primary_name() {
2123        // "primary" is weekly here, "secondary" is 5h.
2124        let obs = UsageObservation {
2125            observed_at_ms: NOW,
2126            outcome: ObservationOutcome::Ok {
2127                windows: vec![
2128                    WindowLimit {
2129                        id: "primary".into(),
2130                        duration_ms: Some(WEEK),
2131                        used_percent: Some(30.0),
2132                        limit_reached: Some(false),
2133                        resets_at_ms: Some(NOW + 3 * DAY),
2134                        models: None,
2135                    },
2136                    WindowLimit {
2137                        id: "secondary".into(),
2138                        duration_ms: Some(5 * HOUR),
2139                        used_percent: Some(100.0),
2140                        limit_reached: Some(true),
2141                        resets_at_ms: Some(NOW + HOUR),
2142                        models: None,
2143                    },
2144                ],
2145                models: None,
2146                limit_reached: Some(false),
2147                banked: None,
2148                identity_prefix: None,
2149            },
2150        };
2151        let (mut s, k) = fresh("a");
2152        observe(&mut s, &cfg(), NOW, &k, &obs).unwrap();
2153        assert!(matches!(s.accounts[&k].phase, Phase::Active { generation, .. } if generation == NOW + 3 * DAY));
2154        assert_eq!(s.accounts[&k].weekly.as_ref().unwrap().id, "primary");
2155    }
2156
2157    #[test]
2158    fn passed_reset_with_asserted_limit_is_exhausted_never_due() {
2159        let c = cfg();
2160        let (mut s, k) = fresh("a");
2161        let g1 = NOW + HOUR;
2162        observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2163        // Reset passed; provider still says weekly 100 % / limit_reached.
2164        let t = g1 + MIN;
2165        let ev = observe(&mut s, &c, t, &k, &ok_obs(t, 100.0, Some(g1))).unwrap();
2166        assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { generation, reset_passed: true } if generation == g1));
2167        assert!(ev.iter().any(|e| matches!(e, KeeperEvent::LimitAssertedAfterReset { .. })));
2168        assert!(!ev.iter().any(|e| matches!(e, KeeperEvent::ActivationDue { .. })));
2169        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2170        assert!(matches!(begin_attempt(&mut s, t, &k), Err(KeeperError::NotDue { .. })));
2171        // Weekly 0 % but overall limit_reached true (some other limit) → still not due.
2172        let mut obs = ok_obs(t, 0.0, Some(g1));
2173        if let ObservationOutcome::Ok { limit_reached, .. } = &mut obs.outcome {
2174            *limit_reached = Some(true);
2175        }
2176        observe(&mut s, &c, t, &k, &obs).unwrap();
2177        assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { reset_passed: true, .. }));
2178        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2179        // Weekly 0 % but the 5h window is exhausted → not due either.
2180        let mut obs = ok_obs(t, 0.0, Some(g1));
2181        if let ObservationOutcome::Ok { windows, .. } = &mut obs.outcome {
2182            windows[0] = five_hour(t, 100.0);
2183        }
2184        observe(&mut s, &c, t, &k, &obs).unwrap();
2185        assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { reset_passed: true, .. }));
2186        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2187    }
2188
2189    #[test]
2190    fn passed_reset_with_headroom_is_due_and_read_only_never_activates() {
2191        let (mut s, k, g1, t) = make_due("a");
2192        for _ in 0..5 {
2193            assert!(matches!(
2194                decide(&s, t, &k, false),
2195                ActivationDecision::Skip { reason } if reason.contains("read-only")
2196            ));
2197        }
2198        // due_since stays pinned to the generation across cycles
2199        let t2 = t + 3 * cfg().poll_interval_ms;
2200        let ev = observe(&mut s, &cfg(), t2, &k, &ok_obs(t2, 0.0, Some(g1))).unwrap();
2201        assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { due_since_ms, .. } if due_since_ms == g1));
2202        assert!(ev.iter().any(|e| matches!(e, KeeperEvent::ActivationDue { due_since_ms, .. } if *due_since_ms == g1)));
2203        // opted in → exactly one activate decision
2204        assert_eq!(
2205            decide(&s, t2, &k, true),
2206            ActivationDecision::Activate {
2207                generation: g1,
2208                attempt: 1
2209            }
2210        );
2211    }
2212
2213    #[test]
2214    fn missing_weekly_window_needs_headroom_evidence() {
2215        let c = cfg();
2216        let (mut s, k) = fresh("a");
2217        let g1 = NOW + HOUR;
2218        observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2219        let t = g1 + MIN;
2220        // Weekly dropped, 5h window headroom, overall false → due.
2221        observe(&mut s, &c, t, &k, &no_weekly_obs(t)).unwrap();
2222        assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { generation, .. } if generation == g1));
2223        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Activate { .. }));
2224        // Weekly dropped and NO windows at all → no evidence → not due.
2225        let (mut s, k) = fresh("b");
2226        observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2227        let empty = UsageObservation {
2228            observed_at_ms: t,
2229            outcome: ObservationOutcome::Ok {
2230                windows: vec![],
2231                models: None,
2232                limit_reached: Some(false),
2233                banked: None,
2234                identity_prefix: None,
2235            },
2236        };
2237        observe(&mut s, &c, t, &k, &empty).unwrap();
2238        assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2239        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2240        // Weekly dropped, 5h headroom but overall limit_reached true → not due.
2241        let (mut s, k) = fresh("c");
2242        observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2243        let mut obs = no_weekly_obs(t);
2244        if let ObservationOutcome::Ok { limit_reached, .. } = &mut obs.outcome {
2245            *limit_reached = Some(true);
2246        }
2247        observe(&mut s, &c, t, &k, &obs).unwrap();
2248        assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2249        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2250    }
2251
2252    #[test]
2253    fn never_observed_account_with_no_reset_evidence_stays_unknown() {
2254        let (mut s, k) = fresh("a");
2255        observe(&mut s, &cfg(), NOW, &k, &no_weekly_obs(NOW)).unwrap();
2256        assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2257        assert!(matches!(decide(&s, NOW, &k, true), ActivationDecision::Skip { .. }));
2258        // weekly window without reset time → unknown too
2259        observe(&mut s, &cfg(), NOW, &k, &ok_obs(NOW, 0.0, None)).unwrap();
2260        assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2261        assert!(matches!(decide(&s, NOW, &k, true), ActivationDecision::Skip { .. }));
2262    }
2263
2264    #[test]
2265    fn activation_requires_latest_poll_ok_fresh_and_all_windows_for_model() {
2266        let c = cfg();
2267        // Latest poll failed → blocked even though the last OK reading is fresh.
2268        let (mut s, k, g1, t) = make_due("a");
2269        observe(&mut s, &c, t + SEC, &k, &transport(t + SEC)).unwrap();
2270        assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { .. }));
2271        assert!(!s.accounts[&k].last_poll_ok);
2272        assert!(matches!(
2273            decide(&s, t + SEC, &k, true),
2274            ActivationDecision::Skip { reason } if reason.contains("latest poll failed")
2275        ));
2276        // A fresh OK poll restores eligibility.
2277        observe(&mut s, &c, t + 2 * SEC, &k, &ok_obs(t + 2 * SEC, 0.0, Some(g1))).unwrap();
2278        assert!(matches!(decide(&s, t + 2 * SEC, &k, true), ActivationDecision::Activate { .. }));
2279        // Stale: time passes without a poll.
2280        let late = t + 2 * SEC + c.stale_after_ms + SEC;
2281        assert!(matches!(
2282            decide(&s, late, &k, true),
2283            ActivationDecision::Skip { reason } if reason.contains("stale")
2284        ));
2285        // Future-stamped reading is malformed evidence, not capacity.
2286        let (mut s, k, g1, t) = make_due("b");
2287        let fut = ok_obs(t + HOUR, 0.0, Some(g1));
2288        observe(&mut s, &c, t, &k, &fut).unwrap();
2289        assert!(!s.accounts[&k].last_poll_ok);
2290        assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2291        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2292        // Model-scoped window for the activation model exhausted → blocked.
2293        let (mut s, k, g1, t) = make_due("c");
2294        let mut obs = ok_obs(t, 0.0, Some(g1));
2295        if let ObservationOutcome::Ok { windows, .. } = &mut obs.outcome {
2296            windows.push(WindowLimit {
2297                id: "model_weekly".into(),
2298                duration_ms: Some(WEEK),
2299                used_percent: Some(100.0),
2300                limit_reached: Some(true),
2301                resets_at_ms: Some(g1 + DAY),
2302                models: Some(vec![MODEL.into()]),
2303            });
2304        }
2305        observe(&mut s, &c, t, &k, &obs).unwrap();
2306        assert!(matches!(
2307            decide(&s, t, &k, true),
2308            ActivationDecision::Skip { reason } if reason.contains("exhausted")
2309        ));
2310        // Model listed exhausted / unknown → blocked; available → ok.
2311        let (mut s, k, g1, t) = make_due("d");
2312        for (state, ok) in [
2313            (ModelState::Exhausted, false),
2314            (ModelState::Unknown, false),
2315            (ModelState::Available, true),
2316        ] {
2317            let mut obs = ok_obs(t, 0.0, Some(g1));
2318            if let ObservationOutcome::Ok { models, .. } = &mut obs.outcome {
2319                *models = Some(vec![ModelAvailability {
2320                    model: MODEL.into(),
2321                    state,
2322                }]);
2323            }
2324            observe(&mut s, &c, t, &k, &obs).unwrap();
2325            assert_eq!(
2326                matches!(decide(&s, t, &k, true), ActivationDecision::Activate { .. }),
2327                ok,
2328                "{state:?}"
2329            );
2330        }
2331        // Short (5h) window with an unknown percentage → no evidence → blocked.
2332        let (mut s, k, g1, t) = make_due("e");
2333        let mut obs = ok_obs(t, 0.0, Some(g1));
2334        if let ObservationOutcome::Ok { windows, .. } = &mut obs.outcome {
2335            windows[0].used_percent = None;
2336            windows[0].limit_reached = None;
2337        }
2338        observe(&mut s, &c, t, &k, &obs).unwrap();
2339        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2340    }
2341
2342    #[test]
2343    fn one_activation_per_generation_verified_only_from_fresh_usage() {
2344        let c = cfg();
2345        let (mut s, k, g1, t) = make_due("a");
2346        assert_eq!(
2347            decide(&s, t, &k, true),
2348            ActivationDecision::Activate {
2349                generation: g1,
2350                attempt: 1
2351            }
2352        );
2353        let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2354        assert!(matches!(s.accounts[&k].phase, Phase::ActivationPending { attempt: 1, .. }));
2355        assert_eq!(s.accounts[&k].attempts_in_generation, 1);
2356        // while pending nothing else may start
2357        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2358        assert!(matches!(begin_attempt(&mut s, t, &k), Err(KeeperError::NotDue { .. })));
2359        finish_attempt(&mut s, &c, t + SEC, &ticket, AttemptOutcome::Sent { http_status: 200 }).unwrap();
2360        // NOT active yet: the request succeeded but no fresh evidence.
2361        assert!(matches!(s.accounts[&k].phase, Phase::ActivationPending { .. }));
2362        // Poll still shows the old reset → still pending, verify cadence.
2363        observe(&mut s, &c, t + 2 * SEC, &k, &ok_obs(t + 2 * SEC, 0.0, Some(g1))).unwrap();
2364        assert!(matches!(s.accounts[&k].phase, Phase::ActivationPending { .. }));
2365        assert_eq!(s.accounts[&k].next_action_at_ms, t + 2 * SEC + c.verify_poll_ms);
2366        // Provider now asserts a limit after the reset → still pending (clock runs), never due.
2367        observe(&mut s, &c, t + 3 * SEC, &k, &ok_obs(t + 3 * SEC, 100.0, Some(g1))).unwrap();
2368        assert!(matches!(s.accounts[&k].phase, Phase::ActivationPending { .. }));
2369        // fresh usage shows a strictly later reset → active, attempt-correlated (not causal)
2370        let anchor = t + 30 * SEC;
2371        let g2 = anchor + WEEK;
2372        let ev = observe(&mut s, &c, t + MIN, &k, &ok_obs(t + MIN, 0.0, Some(g2))).unwrap();
2373        let st = &s.accounts[&k];
2374        assert!(matches!(
2375            st.phase,
2376            Phase::Active {
2377                generation,
2378                attribution: Attribution::AttemptCorrelated { attempt: 1 },
2379                ..
2380            } if generation == g2
2381        ));
2382        assert_eq!(st.generation, Some(g2));
2383        assert_eq!(st.attempts_in_generation, 0);
2384        assert_eq!(st.last_idle_delay_ms, Some(anchor - g1));
2385        assert!(!st.last_idle_delay_is_upper_bound);
2386        assert!(ev.iter().any(|e| matches!(
2387            e,
2388            KeeperEvent::NewWindow {
2389                from_generation: Some(f),
2390                to_generation,
2391                attribution: Attribution::AttemptCorrelated { attempt: 1 },
2392                ..
2393            } if *f == g1 && *to_generation == g2
2394        )));
2395        let rec = st.history.last().unwrap();
2396        assert_eq!(rec.new_generation, Some(g2));
2397        assert!(rec.verified_at_ms.is_some());
2398        assert!(matches!(decide(&s, t + MIN, &k, true), ActivationDecision::Skip { .. }));
2399    }
2400
2401    #[test]
2402    fn ambiguous_attempt_and_crash_never_burn_twice() {
2403        let c = cfg();
2404        let (mut s, k, g1, t) = make_due("a");
2405        let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2406        // Simulate crash: persist, reload, recover.
2407        let json = serde_json::to_vec(&s).unwrap();
2408        let mut s2: KeeperState = serde_json::from_slice(&json).unwrap();
2409        let ev = recover_after_restart(&mut s2, t + 5 * MIN);
2410        assert!(ev.iter().any(|e| matches!(
2411            e,
2412            KeeperEvent::AttemptFinished {
2413                outcome: AttemptOutcome::Ambiguous { .. },
2414                refunded: false,
2415                ..
2416            }
2417        )));
2418        assert!(matches!(
2419            s2.accounts[&k].phase,
2420            Phase::ActivationPending {
2421                outcome: Some(AttemptOutcome::Ambiguous { .. }),
2422                ..
2423            }
2424        ));
2425        // Duplicate restart: recovery is idempotent.
2426        assert!(recover_after_restart(&mut s2, t + 6 * MIN).is_empty());
2427        // Polls keep showing the old generation with headroom past the verify window.
2428        let t2 = t + c.verify_window_ms + SEC;
2429        let ev = observe(&mut s2, &c, t2, &k, &ok_obs(t2, 0.0, Some(g1))).unwrap();
2430        assert!(matches!(s2.accounts[&k].phase, Phase::Unverified { attempts: 1, .. }));
2431        assert!(ev.iter().any(|e| matches!(e, KeeperEvent::Unverified { .. })));
2432        // No further activation for this generation, ever.
2433        for i in 0..10u64 {
2434            let tt = t2 + i * c.poll_interval_ms;
2435            observe(&mut s2, &c, tt, &k, &ok_obs(tt, 0.0, Some(g1))).unwrap();
2436            assert!(matches!(decide(&s2, tt, &k, true), ActivationDecision::Skip { .. }));
2437            assert!(begin_attempt(&mut s2, tt, &k).is_err());
2438        }
2439        // The late-finishing original process (ticket) cannot re-arm anything.
2440        let ev = finish_attempt(&mut s2, &c, t2 + 1, &ticket, AttemptOutcome::Sent { http_status: 200 }).unwrap();
2441        assert!(matches!(s2.accounts[&k].phase, Phase::Unverified { .. }));
2442        assert!(ev.iter().any(|e| matches!(e, KeeperEvent::AttemptFinished { refunded: false, .. })));
2443        // A later external window is still recognized (and correlated).
2444        let g2 = t2 + DAY + WEEK;
2445        observe(&mut s2, &c, t2 + DAY, &k, &ok_obs(t2 + DAY, 0.0, Some(g2))).unwrap();
2446        assert!(matches!(
2447            s2.accounts[&k].phase,
2448            Phase::Active {
2449                attribution: Attribution::AttemptCorrelated { attempt: 1 },
2450                ..
2451            }
2452        ));
2453    }
2454
2455    #[test]
2456    fn not_sent_is_refunded_with_backoff_but_bounded() {
2457        let c = cfg();
2458        let (mut s, k, g1, mut t) = make_due("a");
2459        for i in 0..MAX_NOT_SENT_PER_GENERATION {
2460            assert_eq!(
2461                decide(&s, t, &k, true),
2462                ActivationDecision::Activate {
2463                    generation: g1,
2464                    attempt: 1
2465                },
2466                "iteration {i}"
2467            );
2468            let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2469            finish_attempt(
2470                &mut s,
2471                &c,
2472                t,
2473                &ticket,
2474                AttemptOutcome::NotSent {
2475                    stage: NotSentStage::TokenVend,
2476                    reason: "token vend failed".into(),
2477                },
2478            )
2479            .unwrap();
2480            let st = &s.accounts[&k];
2481            assert_eq!(st.attempts_in_generation, 0, "refunded");
2482            assert!(matches!(st.phase, Phase::AwaitingActivation { due_since_ms, .. } if due_since_ms == g1));
2483            // backoff blocks an immediate retry
2484            assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2485            t = st.next_attempt_allowed_ms;
2486            observe(&mut s, &c, t, &k, &ok_obs(t, 0.0, Some(g1))).unwrap();
2487        }
2488        assert!(matches!(
2489            decide(&s, t, &k, true),
2490            ActivationDecision::Skip { reason } if reason.contains("unsent")
2491        ));
2492        assert!(s.accounts[&k].backoff_ms <= c.max_backoff_ms);
2493    }
2494
2495    #[test]
2496    fn refund_whitelist_is_explicit() {
2497        for s in REFUNDABLE_REJECT_STATUSES {
2498            assert!(AttemptOutcome::Rejected { http_status: *s }.is_refundable());
2499        }
2500        for s in [402u16, 408, 409, 418, 429, 500, 502] {
2501            assert!(!AttemptOutcome::Rejected { http_status: s }.is_refundable(), "{s}");
2502        }
2503        assert!(AttemptOutcome::NotSent {
2504            stage: NotSentStage::Connect,
2505            reason: "x".into()
2506        }
2507        .is_refundable());
2508        assert!(!AttemptOutcome::Ambiguous { reason: "x".into() }.is_refundable());
2509        assert!(!AttemptOutcome::Sent { http_status: 200 }.is_refundable());
2510    }
2511
2512    #[test]
2513    fn rearm_is_explicit_and_only_from_unverified() {
2514        let c = cfg();
2515        let (mut s, k, g1, t) = make_due("a");
2516        // nothing to re-arm while due
2517        assert_eq!(rearm(&mut s, t, &k).unwrap(), None);
2518        let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2519        finish_attempt(&mut s, &c, t, &ticket, AttemptOutcome::Ambiguous { reason: "timeout".into() }).unwrap();
2520        let t2 = t + c.verify_window_ms + SEC;
2521        observe(&mut s, &c, t2, &k, &ok_obs(t2, 0.0, Some(g1))).unwrap();
2522        assert!(matches!(s.accounts[&k].phase, Phase::Unverified { attempts: 1, .. }));
2523        assert!(matches!(decide(&s, t2, &k, true), ActivationDecision::Skip { .. }));
2524        let ev = rearm(&mut s, t2, &k).unwrap();
2525        assert!(matches!(ev, Some(KeeperEvent::Rearmed { previous_attempts: 1, .. })));
2526        assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { generation, due_since_ms } if generation == g1 && due_since_ms == g1));
2527        assert_eq!(decide(&s, t2, &k, true), ActivationDecision::Activate { generation: g1, attempt: 1 });
2528        // history retains the first attempt
2529        assert_eq!(s.accounts[&k].history.len(), 1);
2530        // a second rearm without a new unverified state is a no-op
2531        assert_eq!(rearm(&mut s, t2, &k).unwrap(), None);
2532    }
2533
2534    #[test]
2535    fn rejected_401_becomes_auth_error_and_429_is_not_refunded() {
2536        let c = cfg();
2537        let (mut s, k, _g1, t) = make_due("a");
2538        let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2539        finish_attempt(&mut s, &c, t, &ticket, AttemptOutcome::Rejected { http_status: 401 }).unwrap();
2540        assert!(matches!(s.accounts[&k].phase, Phase::AuthError { .. }));
2541        assert_eq!(s.accounts[&k].attempts_in_generation, 0);
2542
2543        let (mut s, k, _g1, t) = make_due("b");
2544        let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2545        finish_attempt(&mut s, &c, t, &ticket, AttemptOutcome::Rejected { http_status: 429 }).unwrap();
2546        assert!(matches!(s.accounts[&k].phase, Phase::ActivationPending { .. }));
2547        assert_eq!(s.accounts[&k].attempts_in_generation, 1);
2548    }
2549
2550    #[test]
2551    fn transport_errors_back_off_exponentially_and_recover() {
2552        let c = cfg();
2553        let (mut s, k) = fresh("a");
2554        let mut t = NOW;
2555        let mut last = 0;
2556        for _ in 0..8 {
2557            observe(&mut s, &c, t, &k, &transport(t)).unwrap();
2558            let b = s.accounts[&k].backoff_ms;
2559            assert!(b >= last && b <= c.max_backoff_ms);
2560            assert_eq!(s.accounts[&k].next_action_at_ms, t + b);
2561            last = b;
2562            t += b;
2563        }
2564        assert_eq!(last, c.max_backoff_ms);
2565        observe(&mut s, &c, t, &k, &ok_obs(t, 1.0, Some(t + DAY))).unwrap();
2566        assert_eq!(s.accounts[&k].backoff_ms, 0);
2567        assert_eq!(s.accounts[&k].consecutive_errors, 0);
2568    }
2569
2570    #[test]
2571    fn auth_error_unsupported_and_malformed_are_labelled_not_activated() {
2572        let c = cfg();
2573        let (mut s, k, _g1, t) = make_due("a");
2574        observe(
2575            &mut s,
2576            &c,
2577            t,
2578            &k,
2579            &UsageObservation {
2580                observed_at_ms: t,
2581                outcome: ObservationOutcome::AuthError {
2582                    detail: "401".into(),
2583                },
2584            },
2585        )
2586        .unwrap();
2587        assert!(matches!(s.accounts[&k].phase, Phase::AuthError { .. }));
2588        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2589        observe(
2590            &mut s,
2591            &c,
2592            t,
2593            &k,
2594            &UsageObservation {
2595                observed_at_ms: t,
2596                outcome: ObservationOutcome::Malformed {
2597                    detail: "rate_limit missing".into(),
2598                },
2599            },
2600        )
2601        .unwrap();
2602        assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2603        observe(
2604            &mut s,
2605            &c,
2606            t,
2607            &k,
2608            &UsageObservation {
2609                observed_at_ms: t,
2610                outcome: ObservationOutcome::Unsupported {
2611                    detail: "no adapter".into(),
2612                },
2613            },
2614        )
2615        .unwrap();
2616        assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2617        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2618    }
2619
2620    #[test]
2621    fn external_new_window_is_observed_not_attributed() {
2622        let c = cfg();
2623        let (mut s, k) = fresh("a");
2624        let g1 = NOW + HOUR;
2625        observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2626        let t = g1 + 2 * HOUR;
2627        let g2 = t + WEEK - 10 * MIN; // anchored 10 min ago by someone else
2628        let ev = observe(&mut s, &c, t, &k, &ok_obs(t, 1.0, Some(g2))).unwrap();
2629        assert!(matches!(
2630            s.accounts[&k].phase,
2631            Phase::Active {
2632                attribution: Attribution::Observed,
2633                ..
2634            }
2635        ));
2636        assert_eq!(s.accounts[&k].last_idle_delay_ms, Some(2 * HOUR - 10 * MIN));
2637        assert!(ev.iter().any(|e| matches!(e, KeeperEvent::NewWindow { attribution: Attribution::Observed, .. })));
2638        // Reset earlier than the tracked generation → contradictory → unknown.
2639        let (mut s, k) = fresh("b");
2640        observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2641        observe(&mut s, &c, t, &k, &ok_obs(t, 1.0, Some(t + WEEK))).unwrap();
2642        observe(&mut s, &c, t + 1, &k, &ok_obs(t + 1, 1.0, Some(t + DAY))).unwrap();
2643        assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2644    }
2645
2646    #[test]
2647    fn banked_resets_alert_only() {
2648        let c = cfg();
2649        let (mut s, k) = fresh("a");
2650        let mut obs = ok_obs(NOW, 10.0, Some(NOW + DAY));
2651        if let ObservationOutcome::Ok { banked, .. } = &mut obs.outcome {
2652            *banked = Some(BankedResets {
2653                available_count: Some(2),
2654                earliest_expiry_ms: Some(NOW + DAY),
2655                inventory_error: None,
2656            });
2657        }
2658        let ev = observe(&mut s, &c, NOW, &k, &obs).unwrap();
2659        assert!(ev.iter().any(|e| matches!(
2660            e,
2661            KeeperEvent::BankedResets {
2662                available_count: 2,
2663                expiring_soon: true,
2664                ..
2665            }
2666        )));
2667        assert_eq!(s.accounts[&k].banked.as_ref().unwrap().available_count, Some(2));
2668        // zero or unreported inventory → no alert
2669        for count in [Some(0), None] {
2670            if let ObservationOutcome::Ok { banked, .. } = &mut obs.outcome {
2671                *banked = Some(BankedResets {
2672                    available_count: count,
2673                    earliest_expiry_ms: None,
2674                    inventory_error: None,
2675                });
2676            }
2677            let ev = observe(&mut s, &c, NOW, &k, &obs).unwrap();
2678            assert!(!ev.iter().any(|e| matches!(e, KeeperEvent::BankedResets { .. })));
2679        }
2680    }
2681
2682    #[test]
2683    fn overall_limit_flag_folds_into_policy_view() {
2684        let w = vec![five_hour(NOW, 1.0)];
2685        let folded = windows_with_overall_flag(&w, Some(true));
2686        assert_eq!(folded[0].limit_reached, Some(true));
2687        let folded = windows_with_overall_flag(&[], Some(true));
2688        assert_eq!(folded.len(), 1);
2689        assert_eq!(folded[0].id, "overall");
2690        let folded = windows_with_overall_flag(&w, Some(false));
2691        assert_eq!(folded, w);
2692    }
2693
2694    #[test]
2695    fn prune_never_deletes_attempt_ledgers_and_scheduling_is_scoped() {
2696        let mut s = KeeperState::default();
2697        let a = ident("a");
2698        let mut b = ident("a");
2699        b.identity_fp = AccountIdentity::seat_fingerprint(OAuthProviderId::OpenAiCodex, "other-seat").unwrap();
2700        s.entry(&a);
2701        s.entry(&b);
2702        s.accounts.get_mut(&a.key()).unwrap().next_action_at_ms = NOW + 10;
2703        s.accounts.get_mut(&b.key()).unwrap().next_action_at_ms = NOW + 5;
2704        assert_eq!(s.next_wake_ms(), Some(NOW + 5));
2705        assert_eq!(s.due(NOW + 5), vec![b.key()]);
2706        assert_eq!(s.next_wake_among(&[a.key()]), Some(NOW + 10));
2707        assert!(s.due_among(NOW + 5, &[a.key()]).is_empty());
2708        // Not kept, no ledger → prunable only after the idle window.
2709        s.accounts.get_mut(&a.key()).unwrap().last_poll_ms = Some(NOW);
2710        assert!(s.prune_idle(&[b.clone()], NOW + DAY, 30 * DAY).is_empty());
2711        assert_eq!(s.prune_idle(&[b.clone()], NOW + 31 * DAY, 30 * DAY), vec![a.key()]);
2712        // An excluded account with a spent attempt is never pruned.
2713        let (mut s, k, _g, t) = make_due("c");
2714        let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2715        finish_attempt(&mut s, &cfg(), t, &ticket, AttemptOutcome::Sent { http_status: 200 }).unwrap();
2716        assert!(s.prune_idle(&[], t + 365 * DAY, 30 * DAY).is_empty());
2717        assert_eq!(s.accounts[&k].attempts_in_generation, 1);
2718        // Excluded then restored: the spent generation is still spent.
2719        s.entry(&ident("c"));
2720        assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2721    }
2722
2723    #[test]
2724    fn state_store_is_private_atomic_and_fails_closed_on_corruption() {
2725        let dir = tempfile::tempdir().unwrap();
2726        let store = StateStore::new(&dir.path().join("keeper"));
2727        assert_eq!(store.load().unwrap(), KeeperState::default());
2728        let (mut s, k) = fresh("a");
2729        observe(&mut s, &cfg(), NOW, &k, &ok_obs(NOW, 1.0, Some(NOW + DAY))).unwrap();
2730        store.save(&s).unwrap();
2731        let back = store.load().unwrap();
2732        assert_eq!(back.accounts[&k].phase, s.accounts[&k].phase);
2733        #[cfg(unix)]
2734        {
2735            use std::os::unix::fs::PermissionsExt;
2736            let fm = std::fs::metadata(store.path()).unwrap().permissions().mode() & 0o777;
2737            let dm = std::fs::metadata(store.dir()).unwrap().permissions().mode() & 0o777;
2738            assert_eq!(fm, 0o600);
2739            assert_eq!(dm, 0o700);
2740        }
2741        let raw = std::fs::read_to_string(store.path()).unwrap();
2742        assert!(!raw.contains("Bearer") && !raw.contains("refresh"));
2743        std::fs::write(store.path(), b"{ not json").unwrap();
2744        assert!(matches!(store.load(), Err(KeeperError::CorruptState { .. })));
2745        std::fs::write(store.path(), b"{\"version\": 99, \"accounts\": {}}").unwrap();
2746        assert!(matches!(store.load(), Err(KeeperError::UnsupportedVersion { .. })));
2747    }
2748
2749    #[test]
2750    fn locks_are_exclusive_per_name_and_released_on_drop() {
2751        let dir = tempfile::tempdir().unwrap();
2752        let l1 = KeeperLock::acquire(dir.path(), SINGLETON_LOCK_NAME, "pid 1").unwrap();
2753        match KeeperLock::acquire(dir.path(), SINGLETON_LOCK_NAME, "pid 2") {
2754            Err(KeeperError::Locked { holder, .. }) => assert_eq!(holder.as_deref(), Some("pid 1")),
2755            other => panic!("expected lock conflict, got {other:?}"),
2756        }
2757        let name = account_lock_name("openai-codex@astra2");
2758        assert_eq!(name, "openai-codex@astra2.lock");
2759        assert_eq!(account_lock_name("weird/../key"), "weird_.._key.lock");
2760        let _l2 = KeeperLock::acquire(dir.path(), &name, "pid 1").unwrap();
2761        drop(l1);
2762        let _l3 = KeeperLock::acquire(dir.path(), SINGLETON_LOCK_NAME, "pid 3").unwrap();
2763    }
2764}