1use std::collections::BTreeMap;
47use std::fs::{File, OpenOptions};
48use std::path::{Path, PathBuf};
49
50use serde::{Deserialize, Serialize};
51
52use super::account::CredentialRef;
53use super::OAuthProviderId;
54use super::quota_policy::{
55 self, weekly_window, AccountCapacity, ModelAvailability, QuotaObservation, SelectionRequest,
56 Strategy, WindowLimit, WindowVerdict, DEFAULT_EXHAUSTED_AT_PERCENT,
57 MAX_FUTURE_OBSERVATION_SKEW_MS,
58};
59
60pub const STATE_VERSION: u32 = 1;
61pub const STATE_FILE_NAME: &str = "state.json";
63pub const SINGLETON_LOCK_NAME: &str = "keeper.lock";
65pub const ACCOUNT_LOCK_DIR: &str = "locks";
67
68const SEC: u64 = 1000;
69const MIN: u64 = 60 * SEC;
70const HOUR: u64 = 60 * MIN;
71const DAY: u64 = 24 * HOUR;
72
73pub const DEFAULT_POLL_INTERVAL_MS: u64 = 5 * MIN;
75pub const MIN_POLL_INTERVAL_MS: u64 = MIN;
76pub const MAX_POLL_INTERVAL_MS: u64 = HOUR;
77pub const DEFAULT_MAX_BACKOFF_MS: u64 = 4 * HOUR;
78pub const MAX_MAX_BACKOFF_MS: u64 = 12 * HOUR;
79pub const DEFAULT_STALE_AFTER_MS: u64 = 30 * MIN;
80pub const MIN_STALE_AFTER_MS: u64 = MIN;
81pub const MAX_STALE_AFTER_MS: u64 = DAY;
82pub const DEFAULT_RESET_GRACE_MS: u64 = 30 * SEC;
84pub const MAX_RESET_GRACE_MS: u64 = 10 * MIN;
85pub const DEFAULT_VERIFY_POLL_MS: u64 = 20 * SEC;
87pub const DEFAULT_VERIFY_WINDOW_MS: u64 = 10 * MIN;
89pub const MAX_VERIFY_WINDOW_MS: u64 = 2 * HOUR;
90pub const DEFAULT_MAX_ATTEMPTS: u32 = 1;
94pub const MAX_MAX_ATTEMPTS: u32 = 1;
95pub const MAX_NOT_SENT_PER_GENERATION: u32 = 5;
97pub const DEFAULT_ATTEMPT_TIMEOUT_MS: u64 = 60 * SEC;
98pub const MIN_ATTEMPT_TIMEOUT_MS: u64 = 10 * SEC;
99pub const MAX_ATTEMPT_TIMEOUT_MS: u64 = 5 * MIN;
100pub const DEFAULT_USAGE_TIMEOUT_MS: u64 = 20 * SEC;
101pub const DEFAULT_BANKED_EXPIRY_ALERT_MS: u64 = 3 * DAY;
102pub const MAX_HISTORY: usize = 24;
104
105#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
108pub struct KeeperConfig {
109 pub poll_interval_ms: u64,
110 pub max_backoff_ms: u64,
111 pub stale_after_ms: u64,
112 pub reset_grace_ms: u64,
113 pub verify_poll_ms: u64,
114 pub verify_window_ms: u64,
115 pub max_attempts_per_generation: u32,
116 pub attempt_timeout_ms: u64,
117 pub usage_timeout_ms: u64,
118 pub banked_expiry_alert_ms: u64,
119 pub exhausted_at_percent: f64,
120}
121
122impl Default for KeeperConfig {
123 fn default() -> Self {
124 Self {
125 poll_interval_ms: DEFAULT_POLL_INTERVAL_MS,
126 max_backoff_ms: DEFAULT_MAX_BACKOFF_MS,
127 stale_after_ms: DEFAULT_STALE_AFTER_MS,
128 reset_grace_ms: DEFAULT_RESET_GRACE_MS,
129 verify_poll_ms: DEFAULT_VERIFY_POLL_MS,
130 verify_window_ms: DEFAULT_VERIFY_WINDOW_MS,
131 max_attempts_per_generation: DEFAULT_MAX_ATTEMPTS,
132 attempt_timeout_ms: DEFAULT_ATTEMPT_TIMEOUT_MS,
133 usage_timeout_ms: DEFAULT_USAGE_TIMEOUT_MS,
134 banked_expiry_alert_ms: DEFAULT_BANKED_EXPIRY_ALERT_MS,
135 exhausted_at_percent: DEFAULT_EXHAUSTED_AT_PERCENT,
136 }
137 }
138}
139
140impl KeeperConfig {
141 pub fn bounded(mut self) -> Self {
143 self.poll_interval_ms = self
144 .poll_interval_ms
145 .clamp(MIN_POLL_INTERVAL_MS, MAX_POLL_INTERVAL_MS);
146 self.max_backoff_ms = self
147 .max_backoff_ms
148 .clamp(self.poll_interval_ms, MAX_MAX_BACKOFF_MS);
149 self.stale_after_ms = self
150 .stale_after_ms
151 .clamp(MIN_STALE_AFTER_MS, MAX_STALE_AFTER_MS);
152 self.reset_grace_ms = self.reset_grace_ms.min(MAX_RESET_GRACE_MS);
153 self.verify_poll_ms = self.verify_poll_ms.clamp(5 * SEC, self.poll_interval_ms);
154 self.verify_window_ms = self
155 .verify_window_ms
156 .clamp(self.verify_poll_ms, MAX_VERIFY_WINDOW_MS);
157 self.max_attempts_per_generation = self
158 .max_attempts_per_generation
159 .clamp(1, MAX_MAX_ATTEMPTS);
160 self.attempt_timeout_ms = self
161 .attempt_timeout_ms
162 .clamp(MIN_ATTEMPT_TIMEOUT_MS, MAX_ATTEMPT_TIMEOUT_MS);
163 self.usage_timeout_ms = self.usage_timeout_ms.clamp(5 * SEC, 2 * MIN);
164 if !self.exhausted_at_percent.is_finite() || !(1.0..=100.0).contains(&self.exhausted_at_percent) {
165 self.exhausted_at_percent = DEFAULT_EXHAUSTED_AT_PERCENT;
166 }
167 self
168 }
169}
170
171pub const SEAT_FP_PREFIX: &str = "seat:";
175
176#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
178pub struct AccountIdentity {
179 pub source: String,
181 pub credential: String,
183 pub identity_fp: String,
188}
189
190impl AccountIdentity {
191 pub fn key(&self) -> String {
199 format!("{}|{}|{}", self.source, self.ledger_scope(), self.identity_fp)
200 }
201
202 pub fn ledger_scope(&self) -> String {
205 if self.is_strong() {
206 self.credential
207 .split_once('@')
208 .map(|(p, _)| p.to_string())
209 .unwrap_or_else(|| self.credential.clone())
210 } else {
211 self.credential.clone()
212 }
213 }
214
215 pub fn is_strong(&self) -> bool {
218 self.identity_fp.starts_with(SEAT_FP_PREFIX)
219 }
220
221 pub fn seat_fingerprint(provider: OAuthProviderId, account_id: &str) -> Option<String> {
229 super::account::seat_fingerprint(provider, account_id).map(Self::strong_from_listed)
230 }
231
232 pub fn strong_from_listed(listed: impl AsRef<str>) -> String {
234 format!("{SEAT_FP_PREFIX}{}", listed.as_ref().trim())
235 }
236
237 pub fn fingerprint(
244 account_id_prefix: Option<&str>,
245 identity: Option<&str>,
246 added_at: Option<u64>,
247 ) -> String {
248 if let Some(p) = account_id_prefix.map(str::trim).filter(|p| !p.is_empty()) {
249 return format!("id:{p}");
250 }
251 if let Some(i) = identity.map(str::trim).filter(|i| !i.is_empty()) {
252 use sha2::{Digest, Sha256};
253 let digest = Sha256::digest(i.as_bytes());
254 let hex: String = digest.iter().take(6).map(|b| format!("{b:02x}")).collect();
255 return format!("who:{hex}");
256 }
257 if let Some(t) = added_at {
258 return format!("added:{t}");
259 }
260 "unknown".to_string()
261 }
262}
263
264#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
268pub struct BankedResets {
269 #[serde(default, skip_serializing_if = "Option::is_none")]
271 pub available_count: Option<u64>,
272 #[serde(default, skip_serializing_if = "Option::is_none")]
274 pub earliest_expiry_ms: Option<u64>,
275 #[serde(default, skip_serializing_if = "Option::is_none")]
276 pub inventory_error: Option<String>,
277}
278
279#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
281pub struct UsageObservation {
282 pub observed_at_ms: u64,
284 pub outcome: ObservationOutcome,
285}
286
287#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
288#[serde(tag = "kind", rename_all = "snake_case")]
289pub enum ObservationOutcome {
290 Ok {
291 windows: Vec<WindowLimit>,
292 #[serde(default, skip_serializing_if = "Option::is_none")]
293 models: Option<Vec<ModelAvailability>>,
294 #[serde(default, skip_serializing_if = "Option::is_none")]
297 limit_reached: Option<bool>,
298 #[serde(default, skip_serializing_if = "Option::is_none")]
299 banked: Option<BankedResets>,
300 #[serde(default, skip_serializing_if = "Option::is_none")]
302 identity_prefix: Option<String>,
303 },
304 AuthError {
305 detail: String,
306 },
307 Transport {
309 detail: String,
310 },
311 Malformed {
312 detail: String,
313 },
314 Unsupported {
315 detail: String,
316 },
317}
318
319pub fn windows_with_overall_flag(
323 windows: &[WindowLimit],
324 limit_reached: Option<bool>,
325) -> Vec<WindowLimit> {
326 let mut out: Vec<WindowLimit> = windows.to_vec();
327 if limit_reached == Some(true) {
328 let mut any_account_wide = false;
329 for w in out.iter_mut().filter(|w| w.models.is_none()) {
330 any_account_wide = true;
331 w.limit_reached = Some(true);
332 }
333 if !any_account_wide {
334 out.push(WindowLimit {
335 id: "overall".into(),
336 duration_ms: None,
337 used_percent: None,
338 limit_reached: Some(true),
339 resets_at_ms: None,
340 models: None,
341 });
342 }
343 }
344 out
345}
346
347impl UsageObservation {
348 pub fn to_quota_observation(&self) -> QuotaObservation {
350 match &self.outcome {
351 ObservationOutcome::Ok {
352 windows,
353 models,
354 limit_reached,
355 ..
356 } => QuotaObservation::Ok {
357 windows: windows_with_overall_flag(windows, *limit_reached),
358 models: models.clone(),
359 },
360 ObservationOutcome::AuthError { .. } => QuotaObservation::AuthError,
361 ObservationOutcome::Transport { .. } => QuotaObservation::Unknown,
362 ObservationOutcome::Malformed { .. } => QuotaObservation::Malformed,
363 ObservationOutcome::Unsupported { .. } => QuotaObservation::Unsupported,
364 }
365 }
366}
367
368#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
374#[serde(tag = "kind", rename_all = "snake_case")]
375pub enum Attribution {
376 Observed,
378 AttemptCorrelated { attempt: u32 },
381}
382
383#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
388#[serde(rename_all = "snake_case")]
389pub enum NotSentStage {
390 Unsupported,
392 TokenVend,
394 AccountId,
396 IdentityMismatch,
402 RequestBuild,
404 Connect,
406}
407
408pub const REFUNDABLE_REJECT_STATUSES: &[u16] = &[400, 401, 403, 404, 413, 415, 422];
412
413#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
417#[serde(tag = "kind", rename_all = "snake_case")]
418pub enum AttemptOutcome {
419 Sent { http_status: u16 },
421 Rejected { http_status: u16 },
423 NotSent { stage: NotSentStage, reason: String },
425 Ambiguous { reason: String },
427}
428
429impl AttemptOutcome {
430 pub fn is_refundable(&self) -> bool {
432 match self {
433 Self::NotSent { .. } => true,
434 Self::Rejected { http_status } => REFUNDABLE_REJECT_STATUSES.contains(http_status),
435 Self::Sent { .. } | Self::Ambiguous { .. } => false,
436 }
437 }
438}
439
440#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
441#[serde(tag = "phase", rename_all = "snake_case")]
442pub enum Phase {
443 Unknown { reason: String },
445 AuthError { detail: String },
447 Exhausted {
450 generation: u64,
451 #[serde(default)]
452 reset_passed: bool,
453 },
454 Active {
456 generation: u64,
457 #[serde(default, skip_serializing_if = "Option::is_none")]
458 used_percent: Option<f64>,
459 verified_at_ms: u64,
460 attribution: Attribution,
461 },
462 AwaitingActivation { generation: u64, due_since_ms: u64 },
464 ActivationPending {
466 generation: u64,
467 attempt: u32,
468 started_ms: u64,
469 #[serde(default, skip_serializing_if = "Option::is_none")]
470 outcome: Option<AttemptOutcome>,
471 },
472 Unverified { generation: u64, attempts: u32 },
474}
475
476impl Phase {
477 pub fn label(&self) -> &'static str {
478 match self {
479 Self::Unknown { .. } => "unknown",
480 Self::AuthError { .. } => "auth_error",
481 Self::Exhausted { .. } => "exhausted",
482 Self::Active { .. } => "active",
483 Self::AwaitingActivation { .. } => "awaiting_activation",
484 Self::ActivationPending { .. } => "activation_pending",
485 Self::Unverified { .. } => "unverified",
486 }
487 }
488
489 pub fn generation(&self) -> Option<u64> {
491 match self {
492 Self::Unknown { .. } | Self::AuthError { .. } => None,
493 Self::Exhausted { generation, .. }
494 | Self::Active { generation, .. }
495 | Self::AwaitingActivation { generation, .. }
496 | Self::ActivationPending { generation, .. }
497 | Self::Unverified { generation, .. } => Some(*generation),
498 }
499 }
500}
501
502#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
504pub struct AttemptRecord {
505 pub attempt: u32,
506 pub generation: u64,
507 pub started_ms: u64,
508 #[serde(default, skip_serializing_if = "Option::is_none")]
509 pub finished_ms: Option<u64>,
510 #[serde(default, skip_serializing_if = "Option::is_none")]
511 pub outcome: Option<AttemptOutcome>,
512 #[serde(default, skip_serializing_if = "Option::is_none")]
514 pub verified_at_ms: Option<u64>,
515 #[serde(default, skip_serializing_if = "Option::is_none")]
516 pub new_generation: Option<u64>,
517}
518
519#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
520pub struct AccountState {
521 pub identity: AccountIdentity,
522 pub phase: Phase,
523 #[serde(default, skip_serializing_if = "Option::is_none")]
525 pub generation: Option<u64>,
526 #[serde(default)]
528 pub attempts_in_generation: u32,
529 #[serde(default)]
531 pub not_sent_in_generation: u32,
532 #[serde(default, skip_serializing_if = "Option::is_none")]
533 pub last_poll_ms: Option<u64>,
534 #[serde(default, skip_serializing_if = "Option::is_none")]
535 pub last_ok_observation_ms: Option<u64>,
536 #[serde(default)]
538 pub last_poll_ok: bool,
539 #[serde(default)]
541 pub last_windows: Vec<WindowLimit>,
542 #[serde(default, skip_serializing_if = "Option::is_none")]
543 pub last_models: Option<Vec<ModelAvailability>>,
544 #[serde(default, skip_serializing_if = "Option::is_none")]
545 pub last_limit_reached: Option<bool>,
546 #[serde(default, skip_serializing_if = "Option::is_none")]
547 pub last_error: Option<String>,
548 #[serde(default)]
549 pub consecutive_errors: u32,
550 #[serde(default)]
551 pub backoff_ms: u64,
552 #[serde(default)]
554 pub next_action_at_ms: u64,
555 #[serde(default)]
557 pub next_attempt_allowed_ms: u64,
558 #[serde(default, skip_serializing_if = "Option::is_none")]
560 pub last_idle_delay_ms: Option<u64>,
561 #[serde(default)]
564 pub last_idle_delay_is_upper_bound: bool,
565 #[serde(default, skip_serializing_if = "Option::is_none")]
566 pub weekly: Option<WindowLimit>,
567 #[serde(default, skip_serializing_if = "Option::is_none")]
568 pub banked: Option<BankedResets>,
569 #[serde(default)]
570 pub history: Vec<AttemptRecord>,
571}
572
573impl AccountState {
574 pub fn new(identity: AccountIdentity) -> Self {
575 Self {
576 identity,
577 phase: Phase::Unknown {
578 reason: "never polled".into(),
579 },
580 generation: None,
581 attempts_in_generation: 0,
582 not_sent_in_generation: 0,
583 last_poll_ms: None,
584 last_ok_observation_ms: None,
585 last_poll_ok: false,
586 last_windows: Vec::new(),
587 last_models: None,
588 last_limit_reached: None,
589 last_error: None,
590 consecutive_errors: 0,
591 backoff_ms: 0,
592 next_action_at_ms: 0,
593 next_attempt_allowed_ms: 0,
594 last_idle_delay_ms: None,
595 last_idle_delay_is_upper_bound: false,
596 weekly: None,
597 banked: None,
598 history: Vec::new(),
599 }
600 }
601
602 fn push_history(&mut self, rec: AttemptRecord) {
603 self.history.push(rec);
604 if self.history.len() > MAX_HISTORY {
605 let drop = self.history.len() - MAX_HISTORY;
606 self.history.drain(..drop);
607 }
608 }
609}
610
611#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
612pub struct KeeperState {
613 pub version: u32,
614 #[serde(default)]
615 pub updated_at_ms: u64,
616 #[serde(default)]
617 pub accounts: BTreeMap<String, AccountState>,
618}
619
620impl Default for KeeperState {
621 fn default() -> Self {
622 Self {
623 version: STATE_VERSION,
624 updated_at_ms: 0,
625 accounts: BTreeMap::new(),
626 }
627 }
628}
629
630impl KeeperState {
631 pub fn entry(&mut self, identity: &AccountIdentity) -> &mut AccountState {
635 let st = self
636 .accounts
637 .entry(identity.key())
638 .or_insert_with(|| AccountState::new(identity.clone()));
639 if st.identity != *identity {
640 st.identity = identity.clone();
641 }
642 st
643 }
644
645 pub fn prune_idle(&mut self, keep: &[AccountIdentity], now_ms: u64, max_idle_ms: u64) -> Vec<String> {
653 let keep: Vec<String> = keep.iter().map(AccountIdentity::key).collect();
654 let pruned: Vec<String> = self
655 .accounts
656 .iter()
657 .filter(|(k, a)| {
658 !keep.contains(k)
659 && a.history.is_empty()
660 && !matches!(
661 a.phase,
662 Phase::ActivationPending { .. } | Phase::Unverified { .. }
663 )
664 && now_ms.saturating_sub(a.last_poll_ms.unwrap_or(0)) > max_idle_ms
665 })
666 .map(|(k, _)| k.clone())
667 .collect();
668 for k in &pruned {
669 self.accounts.remove(k);
670 }
671 pruned
672 }
673
674 pub fn due(&self, now_ms: u64) -> Vec<String> {
676 self.accounts
677 .iter()
678 .filter(|(_, a)| a.next_action_at_ms <= now_ms)
679 .map(|(k, _)| k.clone())
680 .collect()
681 }
682
683 pub fn due_among(&self, now_ms: u64, keys: &[String]) -> Vec<String> {
685 self.due(now_ms)
686 .into_iter()
687 .filter(|k| keys.contains(k))
688 .collect()
689 }
690
691 pub fn next_wake_ms(&self) -> Option<u64> {
693 self.accounts.values().map(|a| a.next_action_at_ms).min()
694 }
695
696 pub fn next_wake_among(&self, keys: &[String]) -> Option<u64> {
698 self.accounts
699 .iter()
700 .filter(|(k, _)| keys.contains(k))
701 .map(|(_, a)| a.next_action_at_ms)
702 .min()
703 }
704}
705
706#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
710#[serde(tag = "event", rename_all = "snake_case")]
711pub enum KeeperEvent {
712 PhaseChanged {
713 key: String,
714 from: String,
715 to: String,
716 },
717 PollFailed {
718 key: String,
719 detail: String,
720 backoff_ms: u64,
721 },
722 NewWindow {
724 key: String,
725 from_generation: Option<u64>,
726 to_generation: u64,
727 #[serde(default, skip_serializing_if = "Option::is_none")]
728 idle_delay_ms: Option<u64>,
729 idle_delay_is_upper_bound: bool,
730 attribution: Attribution,
731 },
732 LimitAssertedAfterReset {
734 key: String,
735 generation: u64,
736 },
737 ActivationDue {
739 key: String,
740 generation: u64,
741 due_since_ms: u64,
742 },
743 AttemptStarted {
744 key: String,
745 generation: u64,
746 attempt: u32,
747 },
748 AttemptFinished {
749 key: String,
750 generation: u64,
751 attempt: u32,
752 outcome: AttemptOutcome,
753 refunded: bool,
754 },
755 Unverified {
756 key: String,
757 generation: u64,
758 attempts: u32,
759 },
760 Rearmed {
762 key: String,
763 generation: u64,
764 previous_attempts: u32,
765 },
766 AuthError {
767 key: String,
768 detail: String,
769 },
770 BankedResets {
771 key: String,
772 available_count: u64,
773 #[serde(default, skip_serializing_if = "Option::is_none")]
774 earliest_expiry_ms: Option<u64>,
775 expiring_soon: bool,
776 },
777}
778
779#[derive(Debug)]
782pub enum KeeperError {
783 UnknownAccount(String),
784 NotDue { key: String, phase: String },
786 IdentityRequired { key: String },
788 CorruptState { path: PathBuf, detail: String },
790 UnsupportedVersion { path: PathBuf, version: u32 },
792 Io { path: PathBuf, detail: String },
793 Locked { path: PathBuf, holder: Option<String> },
795}
796
797impl std::fmt::Display for KeeperError {
798 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
799 match self {
800 Self::UnknownAccount(k) => write!(f, "unknown keeper account '{k}'"),
801 Self::NotDue { key, phase } => {
802 write!(f, "account '{key}' is not awaiting activation (phase {phase})")
803 }
804 Self::IdentityRequired { key } => write!(
805 f,
806 "account '{key}' has no established seat identity; activation refused"
807 ),
808 Self::CorruptState { path, detail } => write!(
809 f,
810 "keeper state at {} is unreadable ({detail}); refusing to run — move it aside to reset",
811 path.display()
812 ),
813 Self::UnsupportedVersion { path, version } => write!(
814 f,
815 "keeper state at {} has version {version} (this build supports {STATE_VERSION})",
816 path.display()
817 ),
818 Self::Io { path, detail } => write!(f, "{}: {detail}", path.display()),
819 Self::Locked { path, holder } => match holder {
820 Some(h) => write!(f, "{} is held by {h}", path.display()),
821 None => write!(f, "{} is held by another process", path.display()),
822 },
823 }
824 }
825}
826
827impl std::error::Error for KeeperError {}
828
829fn set_phase(st: &mut AccountState, events: &mut Vec<KeeperEvent>, to: Phase) {
832 if st.phase != to {
833 let from = st.phase.label().to_string();
834 let to_label = to.label().to_string();
835 st.phase = to;
836 if from != to_label {
837 events.push(KeeperEvent::PhaseChanged {
838 key: st.identity.key(),
839 from,
840 to: to_label,
841 });
842 }
843 }
844}
845
846fn grow_backoff(st: &mut AccountState, cfg: &KeeperConfig) -> u64 {
847 st.backoff_ms = if st.backoff_ms == 0 {
848 cfg.poll_interval_ms
849 } else {
850 st.backoff_ms.saturating_mul(2).min(cfg.max_backoff_ms)
851 };
852 st.backoff_ms
853}
854
855fn schedule_regular(st: &mut AccountState, cfg: &KeeperConfig, now_ms: u64) {
859 let mut next = now_ms + cfg.poll_interval_ms;
860 if let Some(g) = st.generation {
861 if g > now_ms {
862 next = next.min(g + cfg.reset_grace_ms);
863 }
864 }
865 st.next_action_at_ms = next;
866}
867
868fn account_wide_headroom(windows: &[WindowLimit], limit_reached: Option<bool>, exhausted_at: f64) -> bool {
871 if limit_reached == Some(true) {
872 return false;
873 }
874 let mut seen = false;
875 for w in windows.iter().filter(|w| w.models.is_none()) {
876 seen = true;
877 if !matches!(w.verdict(exhausted_at), WindowVerdict::Headroom { .. }) {
878 return false;
879 }
880 }
881 seen
882}
883
884pub fn observe(
886 state: &mut KeeperState,
887 cfg: &KeeperConfig,
888 now_ms: u64,
889 key: &str,
890 obs: &UsageObservation,
891) -> Result<Vec<KeeperEvent>, KeeperError> {
892 let st = state
893 .accounts
894 .get_mut(key)
895 .ok_or_else(|| KeeperError::UnknownAccount(key.to_string()))?;
896 let mut events = Vec::new();
897 st.last_poll_ms = Some(now_ms);
898 st.last_poll_ok = false;
899 state.updated_at_ms = now_ms;
900
901 let future_stamped = obs.observed_at_ms > now_ms.saturating_add(MAX_FUTURE_OBSERVATION_SKEW_MS);
903
904 match &obs.outcome {
905 ObservationOutcome::Transport { detail } => {
906 st.last_error = Some(detail.clone());
907 st.consecutive_errors += 1;
908 let b = grow_backoff(st, cfg);
909 st.next_action_at_ms = now_ms + b;
910 events.push(KeeperEvent::PollFailed {
911 key: key.to_string(),
912 detail: detail.clone(),
913 backoff_ms: b,
914 });
915 expire_pending(st, cfg, now_ms, &mut events);
918 Ok(events)
919 }
920 ObservationOutcome::AuthError { detail } => {
921 st.last_error = Some(detail.clone());
922 st.consecutive_errors += 1;
923 st.backoff_ms = cfg.max_backoff_ms;
924 st.next_action_at_ms = now_ms + cfg.max_backoff_ms;
925 set_phase(
926 st,
927 &mut events,
928 Phase::AuthError {
929 detail: detail.clone(),
930 },
931 );
932 events.push(KeeperEvent::AuthError {
933 key: key.to_string(),
934 detail: detail.clone(),
935 });
936 Ok(events)
937 }
938 ObservationOutcome::Malformed { detail } | ObservationOutcome::Unsupported { detail } => {
939 st.last_error = Some(detail.clone());
940 st.consecutive_errors += 1;
941 let b = grow_backoff(st, cfg);
942 st.next_action_at_ms = now_ms + b;
943 set_phase(
944 st,
945 &mut events,
946 Phase::Unknown {
947 reason: detail.clone(),
948 },
949 );
950 events.push(KeeperEvent::PollFailed {
951 key: key.to_string(),
952 detail: detail.clone(),
953 backoff_ms: b,
954 });
955 expire_pending(st, cfg, now_ms, &mut events);
956 Ok(events)
957 }
958 ObservationOutcome::Ok { .. } if future_stamped => {
959 let detail = "observation timestamp is in the future".to_string();
960 st.last_error = Some(detail.clone());
961 st.consecutive_errors += 1;
962 let b = grow_backoff(st, cfg);
963 st.next_action_at_ms = now_ms + b;
964 set_phase(
965 st,
966 &mut events,
967 Phase::Unknown {
968 reason: detail.clone(),
969 },
970 );
971 events.push(KeeperEvent::PollFailed {
972 key: key.to_string(),
973 detail,
974 backoff_ms: b,
975 });
976 expire_pending(st, cfg, now_ms, &mut events);
977 Ok(events)
978 }
979 ObservationOutcome::Ok {
980 windows,
981 models,
982 limit_reached,
983 banked,
984 ..
985 } => {
986 st.last_error = None;
987 st.consecutive_errors = 0;
988 st.backoff_ms = 0;
989 st.last_poll_ok = true;
990 st.last_ok_observation_ms = Some(obs.observed_at_ms);
991 st.last_windows = windows.clone();
992 st.last_models = models.clone();
993 st.last_limit_reached = *limit_reached;
994 if let Some(b) = banked {
995 if let Some(count) = b.available_count.filter(|c| *c > 0) {
996 let expiring_soon = b
997 .earliest_expiry_ms
998 .is_some_and(|e| e <= now_ms.saturating_add(cfg.banked_expiry_alert_ms));
999 events.push(KeeperEvent::BankedResets {
1000 key: key.to_string(),
1001 available_count: count,
1002 earliest_expiry_ms: b.earliest_expiry_ms,
1003 expiring_soon,
1004 });
1005 }
1006 }
1007 st.banked = banked.clone();
1008 let weekly = weekly_window(windows).cloned();
1009 st.weekly = weekly.clone();
1010 let observed = obs.observed_at_ms;
1011 let headroom_now =
1012 account_wide_headroom(windows, *limit_reached, cfg.exhausted_at_percent);
1013
1014 match weekly.as_ref().and_then(|w| w.resets_at_ms.map(|r| (w, r))) {
1015 None => {
1016 let reason = if weekly.is_some() {
1018 "weekly window reported without a reset time"
1019 } else {
1020 "no weekly window evidence"
1021 };
1022 match st.generation {
1023 Some(g) if g <= now_ms && headroom_now => {
1024 window_ended(st, cfg, now_ms, g, &mut events);
1027 }
1028 Some(g) if g <= now_ms => {
1029 expire_pending(st, cfg, now_ms, &mut events);
1031 if !matches!(st.phase, Phase::ActivationPending { .. }) {
1032 set_phase(
1033 st,
1034 &mut events,
1035 Phase::Unknown {
1036 reason: format!("{reason}; no headroom evidence after reset"),
1037 },
1038 );
1039 }
1040 st.generation = Some(g);
1041 schedule_regular(st, cfg, now_ms);
1042 }
1043 _ => {
1044 set_phase(
1045 st,
1046 &mut events,
1047 Phase::Unknown {
1048 reason: reason.into(),
1049 },
1050 );
1051 schedule_regular(st, cfg, now_ms);
1052 }
1053 }
1054 }
1055 Some((w, r)) if r > now_ms => {
1056 let prev = st.generation;
1058 if let Some(p) = prev {
1059 if r < p {
1060 set_phase(
1062 st,
1063 &mut events,
1064 Phase::Unknown {
1065 reason: "reported reset earlier than the tracked generation".into(),
1066 },
1067 );
1068 schedule_regular(st, cfg, now_ms);
1069 return Ok(events);
1070 }
1071 }
1072 let is_new = prev != Some(r);
1073 let mut attribution = match &st.phase {
1074 Phase::Active { attribution, .. } if !is_new => *attribution,
1075 _ => Attribution::Observed,
1076 };
1077 if is_new {
1078 let attempt = match &st.phase {
1080 Phase::ActivationPending { attempt, .. } => Some(*attempt),
1081 Phase::Unverified { attempts, .. } if *attempts > 0 => Some(*attempts),
1082 _ => None,
1083 };
1084 if let Some(a) = attempt {
1085 attribution = Attribution::AttemptCorrelated { attempt: a };
1086 }
1087 let (idle, upper) = match (prev, w.duration_ms) {
1089 (Some(p), Some(d)) if r >= d => {
1090 let anchor = r - d;
1091 (Some(anchor.saturating_sub(p)), false)
1092 }
1093 (Some(p), _) => (Some(observed.saturating_sub(p)), true),
1094 (None, _) => (None, false),
1095 };
1096 st.last_idle_delay_ms = idle;
1097 st.last_idle_delay_is_upper_bound = upper;
1098 for rec in st.history.iter_mut().rev() {
1099 if rec.new_generation.is_none() && rec.generation != r {
1100 rec.verified_at_ms = Some(observed);
1101 rec.new_generation = Some(r);
1102 break;
1103 }
1104 }
1105 st.attempts_in_generation = 0;
1106 st.not_sent_in_generation = 0;
1107 st.next_attempt_allowed_ms = 0;
1108 events.push(KeeperEvent::NewWindow {
1109 key: key.to_string(),
1110 from_generation: prev,
1111 to_generation: r,
1112 idle_delay_ms: idle,
1113 idle_delay_is_upper_bound: upper,
1114 attribution,
1115 });
1116 }
1117 st.generation = Some(r);
1118 let phase = if *limit_reached == Some(true) {
1119 Phase::Exhausted {
1120 generation: r,
1121 reset_passed: false,
1122 }
1123 } else {
1124 match w.verdict(cfg.exhausted_at_percent) {
1125 WindowVerdict::Headroom { used_percent } => Phase::Active {
1126 generation: r,
1127 used_percent,
1128 verified_at_ms: observed,
1129 attribution,
1130 },
1131 WindowVerdict::Exhausted { .. } => Phase::Exhausted {
1132 generation: r,
1133 reset_passed: false,
1134 },
1135 WindowVerdict::Unknown => Phase::Unknown {
1136 reason: "weekly window has no usage evidence".into(),
1137 },
1138 }
1139 };
1140 set_phase(st, &mut events, phase);
1141 schedule_regular(st, cfg, now_ms);
1142 }
1143 Some((w, r)) => {
1144 if let Some(p) = st.generation {
1146 if r < p {
1147 set_phase(
1148 st,
1149 &mut events,
1150 Phase::Unknown {
1151 reason: "reported reset earlier than the tracked generation".into(),
1152 },
1153 );
1154 schedule_regular(st, cfg, now_ms);
1155 return Ok(events);
1156 }
1157 }
1158 if st.generation != Some(r) {
1159 st.generation = Some(r);
1160 st.attempts_in_generation = 0;
1161 st.not_sent_in_generation = 0;
1162 st.next_attempt_allowed_ms = 0;
1163 }
1164 let weekly_headroom =
1165 matches!(w.verdict(cfg.exhausted_at_percent), WindowVerdict::Headroom { .. });
1166 if headroom_now && weekly_headroom {
1167 window_ended(st, cfg, now_ms, r, &mut events);
1169 } else {
1170 expire_pending(st, cfg, now_ms, &mut events);
1174 if !matches!(st.phase, Phase::ActivationPending { .. }) {
1175 set_phase(
1176 st,
1177 &mut events,
1178 Phase::Exhausted {
1179 generation: r,
1180 reset_passed: true,
1181 },
1182 );
1183 }
1184 events.push(KeeperEvent::LimitAssertedAfterReset {
1185 key: key.to_string(),
1186 generation: r,
1187 });
1188 st.next_action_at_ms = now_ms + cfg.poll_interval_ms;
1189 }
1190 }
1191 }
1192 Ok(events)
1193 }
1194 }
1195}
1196
1197fn window_ended(
1200 st: &mut AccountState,
1201 cfg: &KeeperConfig,
1202 now_ms: u64,
1203 g: u64,
1204 events: &mut Vec<KeeperEvent>,
1205) {
1206 match st.phase.clone() {
1207 Phase::ActivationPending { generation, .. } if generation == g => {
1208 expire_pending(st, cfg, now_ms, events);
1210 if matches!(st.phase, Phase::ActivationPending { .. }) {
1211 st.next_action_at_ms = now_ms + cfg.verify_poll_ms;
1212 } else {
1213 st.next_action_at_ms = now_ms + cfg.poll_interval_ms;
1214 }
1215 }
1216 Phase::Unverified { generation, .. } if generation == g => {
1217 st.next_action_at_ms = now_ms + cfg.poll_interval_ms;
1218 }
1219 Phase::AwaitingActivation {
1220 generation,
1221 due_since_ms,
1222 } if generation == g => {
1223 events.push(KeeperEvent::ActivationDue {
1224 key: st.identity.key(),
1225 generation: g,
1226 due_since_ms,
1227 });
1228 st.next_action_at_ms = now_ms + cfg.poll_interval_ms;
1229 }
1230 _ => {
1231 if st.attempts_in_generation >= cfg.max_attempts_per_generation {
1232 let attempts = st.attempts_in_generation;
1233 set_phase(
1234 st,
1235 events,
1236 Phase::Unverified {
1237 generation: g,
1238 attempts,
1239 },
1240 );
1241 } else {
1242 set_phase(
1245 st,
1246 events,
1247 Phase::AwaitingActivation {
1248 generation: g,
1249 due_since_ms: g,
1250 },
1251 );
1252 events.push(KeeperEvent::ActivationDue {
1253 key: st.identity.key(),
1254 generation: g,
1255 due_since_ms: g,
1256 });
1257 }
1258 st.next_action_at_ms = now_ms + cfg.poll_interval_ms;
1259 }
1260 }
1261}
1262
1263fn expire_pending(
1266 st: &mut AccountState,
1267 cfg: &KeeperConfig,
1268 now_ms: u64,
1269 events: &mut Vec<KeeperEvent>,
1270) {
1271 if let Phase::ActivationPending {
1272 generation,
1273 started_ms,
1274 ..
1275 } = st.phase.clone()
1276 {
1277 if now_ms.saturating_sub(started_ms) >= cfg.verify_window_ms {
1278 let attempts = st.attempts_in_generation;
1279 if attempts >= cfg.max_attempts_per_generation {
1280 set_phase(
1281 st,
1282 events,
1283 Phase::Unverified {
1284 generation,
1285 attempts,
1286 },
1287 );
1288 events.push(KeeperEvent::Unverified {
1289 key: st.identity.key(),
1290 generation,
1291 attempts,
1292 });
1293 } else {
1294 set_phase(
1295 st,
1296 events,
1297 Phase::AwaitingActivation {
1298 generation,
1299 due_since_ms: generation,
1300 },
1301 );
1302 }
1303 }
1304 }
1305}
1306
1307#[derive(Debug, Clone, PartialEq, Eq)]
1309pub enum ActivationDecision {
1310 Activate { generation: u64, attempt: u32 },
1311 Skip { reason: String },
1312}
1313
1314pub fn capacity_view(st: &AccountState) -> Option<AccountCapacity> {
1316 let credential = CredentialRef::parse_storage_key(&st.identity.credential)?;
1317 Some(AccountCapacity {
1318 credential,
1319 observed_at_ms: st.last_ok_observation_ms,
1320 observation: QuotaObservation::Ok {
1321 windows: windows_with_overall_flag(&st.last_windows, st.last_limit_reached),
1322 models: st.last_models.clone(),
1323 },
1324 cooldown_until_ms: None,
1325 failure: None,
1326 })
1327}
1328
1329pub fn activation_decision(
1338 state: &KeeperState,
1339 cfg: &KeeperConfig,
1340 now_ms: u64,
1341 key: &str,
1342 opted_in: bool,
1343 model: &str,
1344) -> ActivationDecision {
1345 let skip = |r: &str| ActivationDecision::Skip {
1346 reason: r.to_string(),
1347 };
1348 let Some(st) = state.accounts.get(key) else {
1349 return skip("unknown account");
1350 };
1351 if !opted_in {
1352 return skip("not opted in (read-only)");
1353 }
1354 if !st.identity.is_strong() {
1355 return skip("seat identity not established (a metadata prefix is not an identity)");
1356 }
1357 let generation = match &st.phase {
1358 Phase::AwaitingActivation { generation, .. } => *generation,
1359 Phase::ActivationPending { .. } => return skip("attempt pending verification"),
1360 Phase::Unverified { .. } => return skip("attempt budget spent for this generation"),
1361 other => return skip(&format!("not due (phase {})", other.label())),
1362 };
1363 if st.attempts_in_generation >= cfg.max_attempts_per_generation {
1364 return skip("attempt budget spent for this generation");
1365 }
1366 if st.not_sent_in_generation >= MAX_NOT_SENT_PER_GENERATION {
1367 return skip("too many unsent attempts for this generation");
1368 }
1369 if now_ms < st.next_attempt_allowed_ms {
1370 return skip("backoff after an unsent attempt");
1371 }
1372 if !st.last_poll_ok {
1373 return skip("latest poll failed; no fresh evidence");
1374 }
1375 if st.last_limit_reached == Some(true) {
1376 return skip("provider asserts limit_reached");
1377 }
1378 let Some(cap) = capacity_view(st) else {
1379 return skip("unparseable credential key");
1380 };
1381 let req = SelectionRequest {
1382 model: Some(model),
1383 strategy: Strategy::PreferenceOrder,
1384 exhausted_at_percent: cfg.exhausted_at_percent,
1385 ..SelectionRequest::new(cap.credential.provider, now_ms, cfg.stale_after_ms)
1386 };
1387 if let Err(reason) = quota_policy::evaluate(&req, &cap) {
1388 return skip(&format!("no proven capacity: {reason}"));
1389 }
1390 ActivationDecision::Activate {
1391 generation,
1392 attempt: st.attempts_in_generation + 1,
1393 }
1394}
1395
1396#[derive(Debug, Clone, PartialEq, Eq)]
1399pub struct AttemptTicket {
1400 pub key: String,
1401 pub generation: u64,
1402 pub attempt: u32,
1403}
1404
1405pub fn begin_attempt(
1407 state: &mut KeeperState,
1408 now_ms: u64,
1409 key: &str,
1410) -> Result<(AttemptTicket, Vec<KeeperEvent>), KeeperError> {
1411 let st = state
1412 .accounts
1413 .get_mut(key)
1414 .ok_or_else(|| KeeperError::UnknownAccount(key.to_string()))?;
1415 if !st.identity.is_strong() {
1416 return Err(KeeperError::IdentityRequired {
1417 key: key.to_string(),
1418 });
1419 }
1420 let generation = match &st.phase {
1421 Phase::AwaitingActivation { generation, .. } => *generation,
1422 other => {
1423 return Err(KeeperError::NotDue {
1424 key: key.to_string(),
1425 phase: other.label().to_string(),
1426 })
1427 }
1428 };
1429 st.attempts_in_generation += 1;
1430 let attempt = st.attempts_in_generation;
1431 let mut events = Vec::new();
1432 set_phase(
1433 st,
1434 &mut events,
1435 Phase::ActivationPending {
1436 generation,
1437 attempt,
1438 started_ms: now_ms,
1439 outcome: None,
1440 },
1441 );
1442 st.push_history(AttemptRecord {
1443 attempt,
1444 generation,
1445 started_ms: now_ms,
1446 finished_ms: None,
1447 outcome: None,
1448 verified_at_ms: None,
1449 new_generation: None,
1450 });
1451 st.next_action_at_ms = now_ms;
1452 state.updated_at_ms = now_ms;
1453 events.push(KeeperEvent::AttemptStarted {
1454 key: key.to_string(),
1455 generation,
1456 attempt,
1457 });
1458 Ok((
1459 AttemptTicket {
1460 key: key.to_string(),
1461 generation,
1462 attempt,
1463 },
1464 events,
1465 ))
1466}
1467
1468pub fn finish_attempt(
1471 state: &mut KeeperState,
1472 cfg: &KeeperConfig,
1473 now_ms: u64,
1474 ticket: &AttemptTicket,
1475 outcome: AttemptOutcome,
1476) -> Result<Vec<KeeperEvent>, KeeperError> {
1477 let st = state
1478 .accounts
1479 .get_mut(&ticket.key)
1480 .ok_or_else(|| KeeperError::UnknownAccount(ticket.key.clone()))?;
1481 let mut events = Vec::new();
1482 if let Some(rec) = st
1483 .history
1484 .iter_mut()
1485 .rev()
1486 .find(|r| r.generation == ticket.generation && r.attempt == ticket.attempt)
1487 {
1488 rec.finished_ms = Some(now_ms);
1489 rec.outcome = Some(outcome.clone());
1490 }
1491 let refunded = outcome.is_refundable();
1492 match &st.phase {
1493 Phase::ActivationPending {
1494 generation,
1495 attempt,
1496 ..
1497 } if *generation == ticket.generation && *attempt == ticket.attempt => {}
1498 _ => {
1499 events.push(KeeperEvent::AttemptFinished {
1502 key: ticket.key.clone(),
1503 generation: ticket.generation,
1504 attempt: ticket.attempt,
1505 outcome,
1506 refunded: false,
1507 });
1508 state.updated_at_ms = now_ms;
1509 return Ok(events);
1510 }
1511 }
1512 if refunded {
1513 st.attempts_in_generation = st.attempts_in_generation.saturating_sub(1);
1514 st.not_sent_in_generation += 1;
1515 let b = grow_backoff(st, cfg);
1516 st.next_attempt_allowed_ms = now_ms + b;
1517 st.next_action_at_ms = now_ms + b;
1518 if let AttemptOutcome::Rejected { http_status } = &outcome {
1519 if matches!(http_status, 401 | 403) {
1520 set_phase(
1521 st,
1522 &mut events,
1523 Phase::AuthError {
1524 detail: format!("activation rejected with HTTP {http_status}"),
1525 },
1526 );
1527 events.push(KeeperEvent::AuthError {
1528 key: ticket.key.clone(),
1529 detail: format!("activation rejected with HTTP {http_status}"),
1530 });
1531 }
1532 }
1533 if !matches!(st.phase, Phase::AuthError { .. }) {
1534 set_phase(
1535 st,
1536 &mut events,
1537 Phase::AwaitingActivation {
1538 generation: ticket.generation,
1539 due_since_ms: ticket.generation,
1540 },
1541 );
1542 }
1543 st.last_error = Some(match &outcome {
1544 AttemptOutcome::NotSent { stage, reason } => {
1545 format!("activation not sent ({stage:?}): {reason}")
1546 }
1547 AttemptOutcome::Rejected { http_status } => {
1548 format!("activation rejected: HTTP {http_status}")
1549 }
1550 _ => "activation refunded".into(),
1551 });
1552 } else {
1553 st.phase = Phase::ActivationPending {
1554 generation: ticket.generation,
1555 attempt: ticket.attempt,
1556 started_ms: match &st.phase {
1557 Phase::ActivationPending { started_ms, .. } => *started_ms,
1558 _ => now_ms,
1559 },
1560 outcome: Some(outcome.clone()),
1561 };
1562 st.next_action_at_ms = now_ms;
1564 if let AttemptOutcome::Ambiguous { reason } = &outcome {
1565 st.last_error = Some(format!("activation ambiguous: {reason}"));
1566 }
1567 }
1568 state.updated_at_ms = now_ms;
1569 events.push(KeeperEvent::AttemptFinished {
1570 key: ticket.key.clone(),
1571 generation: ticket.generation,
1572 attempt: ticket.attempt,
1573 outcome,
1574 refunded,
1575 });
1576 Ok(events)
1577}
1578
1579pub fn recover_after_restart(state: &mut KeeperState, now_ms: u64) -> Vec<KeeperEvent> {
1582 let mut events = Vec::new();
1583 for (key, st) in state.accounts.iter_mut() {
1584 if let Phase::ActivationPending {
1585 generation,
1586 attempt,
1587 started_ms,
1588 outcome: None,
1589 } = st.phase.clone()
1590 {
1591 let outcome = AttemptOutcome::Ambiguous {
1592 reason: "process restarted before the attempt outcome was recorded".into(),
1593 };
1594 st.phase = Phase::ActivationPending {
1595 generation,
1596 attempt,
1597 started_ms,
1598 outcome: Some(outcome.clone()),
1599 };
1600 if let Some(rec) = st
1601 .history
1602 .iter_mut()
1603 .rev()
1604 .find(|r| r.generation == generation && r.attempt == attempt)
1605 {
1606 rec.finished_ms = Some(now_ms);
1607 rec.outcome = Some(outcome.clone());
1608 }
1609 st.next_action_at_ms = now_ms;
1610 events.push(KeeperEvent::AttemptFinished {
1611 key: key.clone(),
1612 generation,
1613 attempt,
1614 outcome,
1615 refunded: false,
1616 });
1617 }
1618 }
1619 events
1620}
1621
1622pub fn rearm(
1626 state: &mut KeeperState,
1627 now_ms: u64,
1628 key: &str,
1629) -> Result<Option<KeeperEvent>, KeeperError> {
1630 let st = state
1631 .accounts
1632 .get_mut(key)
1633 .ok_or_else(|| KeeperError::UnknownAccount(key.to_string()))?;
1634 let Phase::Unverified {
1635 generation,
1636 attempts,
1637 } = st.phase.clone()
1638 else {
1639 return Ok(None);
1640 };
1641 st.attempts_in_generation = 0;
1642 st.not_sent_in_generation = 0;
1643 st.next_attempt_allowed_ms = 0;
1644 st.phase = Phase::AwaitingActivation {
1645 generation,
1646 due_since_ms: generation,
1647 };
1648 st.next_action_at_ms = now_ms;
1649 state.updated_at_ms = now_ms;
1650 Ok(Some(KeeperEvent::Rearmed {
1651 key: key.to_string(),
1652 generation,
1653 previous_attempts: attempts,
1654 }))
1655}
1656
1657#[derive(Debug, Clone)]
1661pub struct StateStore {
1662 dir: PathBuf,
1663}
1664
1665impl StateStore {
1666 pub fn new(dir: &Path) -> Self {
1667 Self {
1668 dir: dir.to_path_buf(),
1669 }
1670 }
1671
1672 pub fn dir(&self) -> &Path {
1673 &self.dir
1674 }
1675
1676 pub fn path(&self) -> PathBuf {
1677 self.dir.join(STATE_FILE_NAME)
1678 }
1679
1680 pub fn load(&self) -> Result<KeeperState, KeeperError> {
1682 let path = self.path();
1683 let raw = match std::fs::read(&path) {
1684 Ok(b) => b,
1685 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
1686 return Ok(KeeperState::default())
1687 }
1688 Err(e) => {
1689 return Err(KeeperError::Io {
1690 path,
1691 detail: e.to_string(),
1692 })
1693 }
1694 };
1695 let state: KeeperState =
1696 serde_json::from_slice(&raw).map_err(|e| KeeperError::CorruptState {
1697 path: path.clone(),
1698 detail: e.to_string(),
1699 })?;
1700 if state.version > STATE_VERSION {
1701 return Err(KeeperError::UnsupportedVersion {
1702 path,
1703 version: state.version,
1704 });
1705 }
1706 Ok(state)
1707 }
1708
1709 pub fn save(&self, state: &KeeperState) -> Result<(), KeeperError> {
1710 crate::private_fs::ensure_private_dir(&self.dir).map_err(|e| KeeperError::Io {
1711 path: self.dir.clone(),
1712 detail: e.to_string(),
1713 })?;
1714 let mut out = KeeperState {
1715 version: STATE_VERSION,
1716 ..state.clone()
1717 };
1718 if out.updated_at_ms == 0 {
1719 out.updated_at_ms = state.updated_at_ms;
1720 }
1721 let bytes = serde_json::to_vec_pretty(&out).map_err(|e| KeeperError::Io {
1722 path: self.path(),
1723 detail: e.to_string(),
1724 })?;
1725 crate::private_fs::write_atomic_private(&self.path(), &bytes).map_err(|e| KeeperError::Io {
1726 path: self.path(),
1727 detail: e.to_string(),
1728 })
1729 }
1730}
1731
1732#[derive(Debug)]
1739pub struct KeeperLock {
1740 _file: File,
1741 path: PathBuf,
1742}
1743
1744impl KeeperLock {
1745 pub fn acquire(dir: &Path, name: &str, holder: &str) -> Result<Self, KeeperError> {
1746 use fs4::fs_std::FileExt;
1747 crate::private_fs::ensure_private_dir(dir).map_err(|e| KeeperError::Io {
1748 path: dir.to_path_buf(),
1749 detail: e.to_string(),
1750 })?;
1751 let path = dir.join(name);
1752 let file = OpenOptions::new()
1753 .create(true)
1754 .truncate(false)
1755 .read(true)
1756 .write(true)
1757 .open(&path)
1758 .map_err(|e| KeeperError::Io {
1759 path: path.clone(),
1760 detail: e.to_string(),
1761 })?;
1762 #[cfg(unix)]
1763 {
1764 use std::os::unix::fs::PermissionsExt;
1765 let _ = file.set_permissions(std::fs::Permissions::from_mode(0o600));
1766 }
1767 let locked = FileExt::try_lock_exclusive(&file).map_err(|e| KeeperError::Io {
1768 path: path.clone(),
1769 detail: e.to_string(),
1770 })?;
1771 if !locked {
1772 let body = std::fs::read_to_string(&path).unwrap_or_default();
1773 let holder = body.lines().next().map(str::to_string).filter(|s| !s.is_empty());
1774 return Err(KeeperError::Locked { path, holder });
1775 }
1776 use std::io::Write;
1777 let _ = file.set_len(0);
1778 let _ = (&file).write_all(format!("{holder}\n").as_bytes());
1779 Ok(Self { _file: file, path })
1780 }
1781
1782 pub fn path(&self) -> &Path {
1783 &self.path
1784 }
1785}
1786
1787pub fn account_lock_name(storage_key: &str) -> String {
1789 let safe: String = storage_key
1790 .chars()
1791 .map(|c| {
1792 if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-' | '@') {
1793 c
1794 } else {
1795 '_'
1796 }
1797 })
1798 .collect();
1799 format!("{safe}.lock")
1800}
1801
1802#[cfg(test)]
1803mod tests {
1804 use super::*;
1805 use crate::auth::quota_policy::ModelState;
1806
1807 const NOW: u64 = 1_800_000_000_000;
1808 const WEEK: u64 = 7 * DAY;
1809 const MODEL: &str = "gpt-5.4-mini";
1810
1811 fn ident(label: &str) -> AccountIdentity {
1814 AccountIdentity {
1815 source: "local:/tmp/auth.json".into(),
1816 credential: format!("openai-codex@{label}"),
1817 identity_fp: AccountIdentity::seat_fingerprint(
1818 OAuthProviderId::OpenAiCodex,
1819 &format!("2b2f1234-seat-{label}"),
1820 )
1821 .unwrap(),
1822 }
1823 }
1824
1825 fn weak_ident(label: &str) -> AccountIdentity {
1827 AccountIdentity {
1828 source: "local:/tmp/auth.json".into(),
1829 credential: format!("openai-codex@{label}"),
1830 identity_fp: "id:2b2f1234".into(),
1831 }
1832 }
1833
1834 fn cfg() -> KeeperConfig {
1835 KeeperConfig::default().bounded()
1836 }
1837
1838 fn five_hour(observed: u64, used: f64) -> WindowLimit {
1839 WindowLimit {
1840 id: "primary".into(),
1841 duration_ms: Some(5 * HOUR),
1842 used_percent: Some(used),
1843 limit_reached: Some(used >= 100.0),
1844 resets_at_ms: Some(observed + HOUR),
1845 models: None,
1846 }
1847 }
1848
1849 fn weekly(used: f64, resets: Option<u64>) -> WindowLimit {
1850 WindowLimit {
1851 id: "secondary".into(),
1852 duration_ms: Some(WEEK),
1853 used_percent: Some(used),
1854 limit_reached: Some(used >= 100.0),
1855 resets_at_ms: resets,
1856 models: None,
1857 }
1858 }
1859
1860 fn ok_obs(observed: u64, used: f64, resets: Option<u64>) -> UsageObservation {
1862 UsageObservation {
1863 observed_at_ms: observed,
1864 outcome: ObservationOutcome::Ok {
1865 windows: vec![five_hour(observed, 1.0), weekly(used, resets)],
1866 models: None,
1867 limit_reached: Some(used >= 100.0),
1868 banked: None,
1869 identity_prefix: Some("2b2f1234".into()),
1870 },
1871 }
1872 }
1873
1874 fn no_weekly_obs(observed: u64) -> UsageObservation {
1875 UsageObservation {
1876 observed_at_ms: observed,
1877 outcome: ObservationOutcome::Ok {
1878 windows: vec![five_hour(observed, 0.0)],
1879 models: None,
1880 limit_reached: Some(false),
1881 banked: None,
1882 identity_prefix: None,
1883 },
1884 }
1885 }
1886
1887 fn transport(t: u64) -> UsageObservation {
1888 UsageObservation {
1889 observed_at_ms: t,
1890 outcome: ObservationOutcome::Transport {
1891 detail: "connect".into(),
1892 },
1893 }
1894 }
1895
1896 fn fresh(label: &str) -> (KeeperState, String) {
1897 let mut s = KeeperState::default();
1898 let id = ident(label);
1899 s.entry(&id);
1900 (s, id.key())
1901 }
1902
1903 fn make_due(label: &str) -> (KeeperState, String, u64, u64) {
1907 let c = cfg();
1908 let (mut s, k) = fresh(label);
1909 let g1 = NOW + HOUR;
1910 observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
1911 assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { reset_passed: false, .. }));
1912 let t = g1 + MIN;
1913 observe(&mut s, &c, t, &k, &ok_obs(t, 0.0, Some(g1))).unwrap();
1914 assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { generation, due_since_ms } if generation == g1 && due_since_ms == g1));
1915 (s, k, g1, t)
1916 }
1917
1918 fn decide(s: &KeeperState, t: u64, k: &str, opted_in: bool) -> ActivationDecision {
1919 activation_decision(s, &cfg(), t, k, opted_in, MODEL)
1920 }
1921
1922 #[test]
1923 fn config_bounds_untrusted_intervals() {
1924 let c = KeeperConfig {
1925 poll_interval_ms: 1,
1926 max_backoff_ms: 0,
1927 stale_after_ms: u64::MAX,
1928 reset_grace_ms: u64::MAX,
1929 verify_poll_ms: 0,
1930 verify_window_ms: u64::MAX,
1931 max_attempts_per_generation: 99,
1932 attempt_timeout_ms: 0,
1933 usage_timeout_ms: u64::MAX,
1934 banked_expiry_alert_ms: 0,
1935 exhausted_at_percent: f64::NAN,
1936 }
1937 .bounded();
1938 assert_eq!(c.poll_interval_ms, MIN_POLL_INTERVAL_MS);
1939 assert_eq!(c.max_backoff_ms, MIN_POLL_INTERVAL_MS);
1940 assert_eq!(c.stale_after_ms, MAX_STALE_AFTER_MS);
1941 assert_eq!(c.reset_grace_ms, MAX_RESET_GRACE_MS);
1942 assert_eq!(c.max_attempts_per_generation, 1);
1943 assert_eq!(c.attempt_timeout_ms, MIN_ATTEMPT_TIMEOUT_MS);
1944 assert_eq!(c.exhausted_at_percent, DEFAULT_EXHAUSTED_AT_PERCENT);
1945 assert!(c.verify_window_ms <= MAX_VERIFY_WINDOW_MS);
1946 assert!(c.verify_poll_ms >= 5 * SEC);
1947 assert_eq!(KeeperConfig::default().bounded(), KeeperConfig::default());
1948 assert_eq!(KeeperConfig::default().poll_interval_ms, 5 * MIN);
1949 }
1950
1951 #[test]
1952 fn identity_fingerprint_prefers_account_id_and_changes_on_relogin() {
1953 assert_eq!(
1954 AccountIdentity::fingerprint(Some("2b2f1234"), Some("a@b"), Some(1)),
1955 "id:2b2f1234"
1956 );
1957 let a = AccountIdentity::fingerprint(None, Some("a@b"), Some(1));
1958 let b = AccountIdentity::fingerprint(None, Some("c@d"), Some(1));
1959 assert!(a.starts_with("who:") && b.starts_with("who:") && a != b);
1960 assert_eq!(AccountIdentity::fingerprint(None, None, Some(7)), "added:7");
1961 assert_eq!(AccountIdentity::fingerprint(None, None, None), "unknown");
1962 let mut i1 = weak_ident("x");
1963 let mut i2 = weak_ident("x");
1964 i1.identity_fp = "id:aaaa".into();
1965 i2.identity_fp = "id:bbbb".into();
1966 assert_ne!(i1.key(), i2.key());
1967 let (mut s, k, _g, _t) = make_due("x");
1970 let relogged = AccountIdentity {
1971 identity_fp: AccountIdentity::seat_fingerprint(OAuthProviderId::OpenAiCodex, "another-seat").unwrap(),
1972 ..ident("x")
1973 };
1974 s.entry(&relogged);
1975 assert!(matches!(s.accounts[&relogged.key()].phase, Phase::Unknown { .. }));
1976 assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { .. }));
1977 assert_eq!(s.accounts.len(), 2);
1978 }
1979
1980 #[test]
1981 fn strong_seat_identity_is_canonical_across_aliases_and_weak_is_label_scoped() {
1982 let codex = OAuthProviderId::OpenAiCodex;
1984 let fp = AccountIdentity::seat_fingerprint(codex, "2b2f1234-full-id").unwrap();
1985 assert!(fp.starts_with(SEAT_FP_PREFIX));
1986 assert!(!fp.contains("2b2f1234"), "state never carries the id itself");
1987 assert_ne!(fp, AccountIdentity::seat_fingerprint(codex, "2b2f1234-other").unwrap());
1988 assert_ne!(fp, AccountIdentity::seat_fingerprint(OAuthProviderId::Anthropic, "2b2f1234-full-id").unwrap());
1989 assert_eq!(fp, AccountIdentity::seat_fingerprint(codex, " 2b2f1234-full-id ").unwrap());
1990 assert_eq!(AccountIdentity::seat_fingerprint(codex, " "), None);
1991 let listed = super::super::account::seat_fingerprint(codex, "2b2f1234-full-id").unwrap();
1993 assert_eq!(fp, AccountIdentity::strong_from_listed(&listed));
1994 assert_eq!(fp, format!("seat:{listed}"));
1995 let a = AccountIdentity {
1996 source: "local:/tmp/auth.json".into(),
1997 credential: "openai-codex@a".into(),
1998 identity_fp: fp.clone(),
1999 };
2000 let b = AccountIdentity {
2001 credential: "openai-codex@b".into(),
2002 ..a.clone()
2003 };
2004 let default = AccountIdentity {
2005 credential: "openai-codex".into(),
2006 ..a.clone()
2007 };
2008 assert!(a.is_strong());
2009 assert_eq!(a.key(), b.key(), "aliases of one seat share one ledger");
2010 assert_eq!(a.key(), default.key());
2011 assert_eq!(a.ledger_scope(), "openai-codex");
2012 assert_eq!(a.key(), format!("local:/tmp/auth.json|openai-codex|{fp}"));
2013 let wa = weak_ident("a");
2015 let wb = weak_ident("b");
2016 assert!(!wa.is_strong());
2017 assert_ne!(wa.key(), wb.key());
2018 assert_eq!(wa.ledger_scope(), "openai-codex@a");
2019 let c = cfg();
2022 let mut s = KeeperState::default();
2023 s.entry(&a);
2024 let g1 = NOW + HOUR;
2025 observe(&mut s, &c, NOW, &a.key(), &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2026 let t = g1 + MIN;
2027 observe(&mut s, &c, t, &a.key(), &ok_obs(t, 0.0, Some(g1))).unwrap();
2028 let (ticket, _) = begin_attempt(&mut s, t, &a.key()).unwrap();
2029 finish_attempt(&mut s, &c, t, &ticket, AttemptOutcome::Ambiguous { reason: "timeout".into() }).unwrap();
2030 let st = s.entry(&b);
2031 assert_eq!(st.identity.credential, "openai-codex@b", "entry adopts the alias in use");
2032 assert_eq!(st.attempts_in_generation, 1);
2033 assert_eq!(s.accounts.len(), 1);
2034 assert!(matches!(decide(&s, t + SEC, &b.key(), true), ActivationDecision::Skip { .. }));
2035 assert!(begin_attempt(&mut s, t + SEC, &b.key()).is_err());
2036 }
2037
2038 #[test]
2039 fn activation_requires_a_strong_seat_identity() {
2040 let c = cfg();
2042 let mut s = KeeperState::default();
2043 let w = weak_ident("w");
2044 s.entry(&w);
2045 let k = w.key();
2046 let g1 = NOW + HOUR;
2047 observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2048 let t = g1 + MIN;
2049 observe(&mut s, &c, t, &k, &ok_obs(t, 0.0, Some(g1))).unwrap();
2050 assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { .. }), "due, alerts");
2051 match decide(&s, t, &k, true) {
2052 ActivationDecision::Skip { reason } => assert!(reason.contains("identity"), "{reason}"),
2053 other => panic!("weak identity must never activate: {other:?}"),
2054 }
2055 assert!(matches!(
2056 begin_attempt(&mut s, t, &k),
2057 Err(KeeperError::IdentityRequired { .. })
2058 ));
2059 assert_eq!(s.accounts[&k].attempts_in_generation, 0);
2060 let (s2, k2, _g, t2) = make_due("s");
2062 assert!(matches!(decide(&s2, t2, &k2, true), ActivationDecision::Activate { .. }));
2063 }
2064
2065 #[test]
2066 fn identity_mismatch_before_send_is_refunded_and_recorded() {
2067 let c = cfg();
2068 let (mut s, k, g1, t) = make_due("m");
2069 let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2070 let ev = finish_attempt(
2071 &mut s,
2072 &c,
2073 t + SEC,
2074 &ticket,
2075 AttemptOutcome::NotSent {
2076 stage: NotSentStage::IdentityMismatch,
2077 reason: "token belongs to another seat".into(),
2078 },
2079 )
2080 .unwrap();
2081 let st = &s.accounts[&k];
2082 assert_eq!(st.attempts_in_generation, 0, "nothing was sent → refunded");
2083 assert_eq!(st.not_sent_in_generation, 1);
2084 assert!(matches!(st.phase, Phase::AwaitingActivation { generation, .. } if generation == g1));
2085 assert!(st.last_error.as_deref().unwrap().contains("IdentityMismatch"));
2086 assert!(ev.iter().any(|e| matches!(
2087 e,
2088 KeeperEvent::AttemptFinished { refunded: true, outcome: AttemptOutcome::NotSent { stage: NotSentStage::IdentityMismatch, .. }, .. }
2089 )));
2090 assert!(st.history.last().unwrap().outcome.is_some());
2091 }
2092
2093 #[test]
2094 fn live_window_tracks_generation_and_schedules_reset_grace() {
2095 let (mut s, k) = fresh("a");
2096 let reset = NOW + 2 * MIN;
2097 let ev = observe(&mut s, &cfg(), NOW, &k, &ok_obs(NOW, 10.0, Some(reset))).unwrap();
2098 let st = &s.accounts[&k];
2099 assert!(matches!(st.phase, Phase::Active { generation, .. } if generation == reset));
2100 assert_eq!(st.generation, Some(reset));
2101 assert!(st.last_poll_ok);
2102 assert_eq!(st.next_action_at_ms, reset + cfg().reset_grace_ms);
2104 assert!(ev.iter().any(|e| matches!(e, KeeperEvent::NewWindow { from_generation: None, .. })));
2105 observe(&mut s, &cfg(), NOW + 1, &k, &ok_obs(NOW + 1, 100.0, Some(reset))).unwrap();
2107 assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { generation, reset_passed: false } if generation == reset));
2108 let mut obs = ok_obs(NOW + 1, 10.0, Some(reset));
2110 if let ObservationOutcome::Ok { limit_reached, .. } = &mut obs.outcome {
2111 *limit_reached = Some(true);
2112 }
2113 observe(&mut s, &cfg(), NOW + 1, &k, &obs).unwrap();
2114 assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { .. }));
2115 let far = NOW + 3 * DAY;
2117 observe(&mut s, &cfg(), NOW + 2, &k, &ok_obs(NOW + 2, 50.0, Some(far))).unwrap();
2118 assert_eq!(s.accounts[&k].next_action_at_ms, NOW + 2 + cfg().poll_interval_ms);
2119 }
2120
2121 #[test]
2122 fn weekly_from_duration_not_primary_name() {
2123 let obs = UsageObservation {
2125 observed_at_ms: NOW,
2126 outcome: ObservationOutcome::Ok {
2127 windows: vec![
2128 WindowLimit {
2129 id: "primary".into(),
2130 duration_ms: Some(WEEK),
2131 used_percent: Some(30.0),
2132 limit_reached: Some(false),
2133 resets_at_ms: Some(NOW + 3 * DAY),
2134 models: None,
2135 },
2136 WindowLimit {
2137 id: "secondary".into(),
2138 duration_ms: Some(5 * HOUR),
2139 used_percent: Some(100.0),
2140 limit_reached: Some(true),
2141 resets_at_ms: Some(NOW + HOUR),
2142 models: None,
2143 },
2144 ],
2145 models: None,
2146 limit_reached: Some(false),
2147 banked: None,
2148 identity_prefix: None,
2149 },
2150 };
2151 let (mut s, k) = fresh("a");
2152 observe(&mut s, &cfg(), NOW, &k, &obs).unwrap();
2153 assert!(matches!(s.accounts[&k].phase, Phase::Active { generation, .. } if generation == NOW + 3 * DAY));
2154 assert_eq!(s.accounts[&k].weekly.as_ref().unwrap().id, "primary");
2155 }
2156
2157 #[test]
2158 fn passed_reset_with_asserted_limit_is_exhausted_never_due() {
2159 let c = cfg();
2160 let (mut s, k) = fresh("a");
2161 let g1 = NOW + HOUR;
2162 observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2163 let t = g1 + MIN;
2165 let ev = observe(&mut s, &c, t, &k, &ok_obs(t, 100.0, Some(g1))).unwrap();
2166 assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { generation, reset_passed: true } if generation == g1));
2167 assert!(ev.iter().any(|e| matches!(e, KeeperEvent::LimitAssertedAfterReset { .. })));
2168 assert!(!ev.iter().any(|e| matches!(e, KeeperEvent::ActivationDue { .. })));
2169 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2170 assert!(matches!(begin_attempt(&mut s, t, &k), Err(KeeperError::NotDue { .. })));
2171 let mut obs = ok_obs(t, 0.0, Some(g1));
2173 if let ObservationOutcome::Ok { limit_reached, .. } = &mut obs.outcome {
2174 *limit_reached = Some(true);
2175 }
2176 observe(&mut s, &c, t, &k, &obs).unwrap();
2177 assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { reset_passed: true, .. }));
2178 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2179 let mut obs = ok_obs(t, 0.0, Some(g1));
2181 if let ObservationOutcome::Ok { windows, .. } = &mut obs.outcome {
2182 windows[0] = five_hour(t, 100.0);
2183 }
2184 observe(&mut s, &c, t, &k, &obs).unwrap();
2185 assert!(matches!(s.accounts[&k].phase, Phase::Exhausted { reset_passed: true, .. }));
2186 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2187 }
2188
2189 #[test]
2190 fn passed_reset_with_headroom_is_due_and_read_only_never_activates() {
2191 let (mut s, k, g1, t) = make_due("a");
2192 for _ in 0..5 {
2193 assert!(matches!(
2194 decide(&s, t, &k, false),
2195 ActivationDecision::Skip { reason } if reason.contains("read-only")
2196 ));
2197 }
2198 let t2 = t + 3 * cfg().poll_interval_ms;
2200 let ev = observe(&mut s, &cfg(), t2, &k, &ok_obs(t2, 0.0, Some(g1))).unwrap();
2201 assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { due_since_ms, .. } if due_since_ms == g1));
2202 assert!(ev.iter().any(|e| matches!(e, KeeperEvent::ActivationDue { due_since_ms, .. } if *due_since_ms == g1)));
2203 assert_eq!(
2205 decide(&s, t2, &k, true),
2206 ActivationDecision::Activate {
2207 generation: g1,
2208 attempt: 1
2209 }
2210 );
2211 }
2212
2213 #[test]
2214 fn missing_weekly_window_needs_headroom_evidence() {
2215 let c = cfg();
2216 let (mut s, k) = fresh("a");
2217 let g1 = NOW + HOUR;
2218 observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2219 let t = g1 + MIN;
2220 observe(&mut s, &c, t, &k, &no_weekly_obs(t)).unwrap();
2222 assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { generation, .. } if generation == g1));
2223 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Activate { .. }));
2224 let (mut s, k) = fresh("b");
2226 observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2227 let empty = UsageObservation {
2228 observed_at_ms: t,
2229 outcome: ObservationOutcome::Ok {
2230 windows: vec![],
2231 models: None,
2232 limit_reached: Some(false),
2233 banked: None,
2234 identity_prefix: None,
2235 },
2236 };
2237 observe(&mut s, &c, t, &k, &empty).unwrap();
2238 assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2239 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2240 let (mut s, k) = fresh("c");
2242 observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2243 let mut obs = no_weekly_obs(t);
2244 if let ObservationOutcome::Ok { limit_reached, .. } = &mut obs.outcome {
2245 *limit_reached = Some(true);
2246 }
2247 observe(&mut s, &c, t, &k, &obs).unwrap();
2248 assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2249 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2250 }
2251
2252 #[test]
2253 fn never_observed_account_with_no_reset_evidence_stays_unknown() {
2254 let (mut s, k) = fresh("a");
2255 observe(&mut s, &cfg(), NOW, &k, &no_weekly_obs(NOW)).unwrap();
2256 assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2257 assert!(matches!(decide(&s, NOW, &k, true), ActivationDecision::Skip { .. }));
2258 observe(&mut s, &cfg(), NOW, &k, &ok_obs(NOW, 0.0, None)).unwrap();
2260 assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2261 assert!(matches!(decide(&s, NOW, &k, true), ActivationDecision::Skip { .. }));
2262 }
2263
2264 #[test]
2265 fn activation_requires_latest_poll_ok_fresh_and_all_windows_for_model() {
2266 let c = cfg();
2267 let (mut s, k, g1, t) = make_due("a");
2269 observe(&mut s, &c, t + SEC, &k, &transport(t + SEC)).unwrap();
2270 assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { .. }));
2271 assert!(!s.accounts[&k].last_poll_ok);
2272 assert!(matches!(
2273 decide(&s, t + SEC, &k, true),
2274 ActivationDecision::Skip { reason } if reason.contains("latest poll failed")
2275 ));
2276 observe(&mut s, &c, t + 2 * SEC, &k, &ok_obs(t + 2 * SEC, 0.0, Some(g1))).unwrap();
2278 assert!(matches!(decide(&s, t + 2 * SEC, &k, true), ActivationDecision::Activate { .. }));
2279 let late = t + 2 * SEC + c.stale_after_ms + SEC;
2281 assert!(matches!(
2282 decide(&s, late, &k, true),
2283 ActivationDecision::Skip { reason } if reason.contains("stale")
2284 ));
2285 let (mut s, k, g1, t) = make_due("b");
2287 let fut = ok_obs(t + HOUR, 0.0, Some(g1));
2288 observe(&mut s, &c, t, &k, &fut).unwrap();
2289 assert!(!s.accounts[&k].last_poll_ok);
2290 assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2291 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2292 let (mut s, k, g1, t) = make_due("c");
2294 let mut obs = ok_obs(t, 0.0, Some(g1));
2295 if let ObservationOutcome::Ok { windows, .. } = &mut obs.outcome {
2296 windows.push(WindowLimit {
2297 id: "model_weekly".into(),
2298 duration_ms: Some(WEEK),
2299 used_percent: Some(100.0),
2300 limit_reached: Some(true),
2301 resets_at_ms: Some(g1 + DAY),
2302 models: Some(vec![MODEL.into()]),
2303 });
2304 }
2305 observe(&mut s, &c, t, &k, &obs).unwrap();
2306 assert!(matches!(
2307 decide(&s, t, &k, true),
2308 ActivationDecision::Skip { reason } if reason.contains("exhausted")
2309 ));
2310 let (mut s, k, g1, t) = make_due("d");
2312 for (state, ok) in [
2313 (ModelState::Exhausted, false),
2314 (ModelState::Unknown, false),
2315 (ModelState::Available, true),
2316 ] {
2317 let mut obs = ok_obs(t, 0.0, Some(g1));
2318 if let ObservationOutcome::Ok { models, .. } = &mut obs.outcome {
2319 *models = Some(vec![ModelAvailability {
2320 model: MODEL.into(),
2321 state,
2322 }]);
2323 }
2324 observe(&mut s, &c, t, &k, &obs).unwrap();
2325 assert_eq!(
2326 matches!(decide(&s, t, &k, true), ActivationDecision::Activate { .. }),
2327 ok,
2328 "{state:?}"
2329 );
2330 }
2331 let (mut s, k, g1, t) = make_due("e");
2333 let mut obs = ok_obs(t, 0.0, Some(g1));
2334 if let ObservationOutcome::Ok { windows, .. } = &mut obs.outcome {
2335 windows[0].used_percent = None;
2336 windows[0].limit_reached = None;
2337 }
2338 observe(&mut s, &c, t, &k, &obs).unwrap();
2339 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2340 }
2341
2342 #[test]
2343 fn one_activation_per_generation_verified_only_from_fresh_usage() {
2344 let c = cfg();
2345 let (mut s, k, g1, t) = make_due("a");
2346 assert_eq!(
2347 decide(&s, t, &k, true),
2348 ActivationDecision::Activate {
2349 generation: g1,
2350 attempt: 1
2351 }
2352 );
2353 let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2354 assert!(matches!(s.accounts[&k].phase, Phase::ActivationPending { attempt: 1, .. }));
2355 assert_eq!(s.accounts[&k].attempts_in_generation, 1);
2356 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2358 assert!(matches!(begin_attempt(&mut s, t, &k), Err(KeeperError::NotDue { .. })));
2359 finish_attempt(&mut s, &c, t + SEC, &ticket, AttemptOutcome::Sent { http_status: 200 }).unwrap();
2360 assert!(matches!(s.accounts[&k].phase, Phase::ActivationPending { .. }));
2362 observe(&mut s, &c, t + 2 * SEC, &k, &ok_obs(t + 2 * SEC, 0.0, Some(g1))).unwrap();
2364 assert!(matches!(s.accounts[&k].phase, Phase::ActivationPending { .. }));
2365 assert_eq!(s.accounts[&k].next_action_at_ms, t + 2 * SEC + c.verify_poll_ms);
2366 observe(&mut s, &c, t + 3 * SEC, &k, &ok_obs(t + 3 * SEC, 100.0, Some(g1))).unwrap();
2368 assert!(matches!(s.accounts[&k].phase, Phase::ActivationPending { .. }));
2369 let anchor = t + 30 * SEC;
2371 let g2 = anchor + WEEK;
2372 let ev = observe(&mut s, &c, t + MIN, &k, &ok_obs(t + MIN, 0.0, Some(g2))).unwrap();
2373 let st = &s.accounts[&k];
2374 assert!(matches!(
2375 st.phase,
2376 Phase::Active {
2377 generation,
2378 attribution: Attribution::AttemptCorrelated { attempt: 1 },
2379 ..
2380 } if generation == g2
2381 ));
2382 assert_eq!(st.generation, Some(g2));
2383 assert_eq!(st.attempts_in_generation, 0);
2384 assert_eq!(st.last_idle_delay_ms, Some(anchor - g1));
2385 assert!(!st.last_idle_delay_is_upper_bound);
2386 assert!(ev.iter().any(|e| matches!(
2387 e,
2388 KeeperEvent::NewWindow {
2389 from_generation: Some(f),
2390 to_generation,
2391 attribution: Attribution::AttemptCorrelated { attempt: 1 },
2392 ..
2393 } if *f == g1 && *to_generation == g2
2394 )));
2395 let rec = st.history.last().unwrap();
2396 assert_eq!(rec.new_generation, Some(g2));
2397 assert!(rec.verified_at_ms.is_some());
2398 assert!(matches!(decide(&s, t + MIN, &k, true), ActivationDecision::Skip { .. }));
2399 }
2400
2401 #[test]
2402 fn ambiguous_attempt_and_crash_never_burn_twice() {
2403 let c = cfg();
2404 let (mut s, k, g1, t) = make_due("a");
2405 let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2406 let json = serde_json::to_vec(&s).unwrap();
2408 let mut s2: KeeperState = serde_json::from_slice(&json).unwrap();
2409 let ev = recover_after_restart(&mut s2, t + 5 * MIN);
2410 assert!(ev.iter().any(|e| matches!(
2411 e,
2412 KeeperEvent::AttemptFinished {
2413 outcome: AttemptOutcome::Ambiguous { .. },
2414 refunded: false,
2415 ..
2416 }
2417 )));
2418 assert!(matches!(
2419 s2.accounts[&k].phase,
2420 Phase::ActivationPending {
2421 outcome: Some(AttemptOutcome::Ambiguous { .. }),
2422 ..
2423 }
2424 ));
2425 assert!(recover_after_restart(&mut s2, t + 6 * MIN).is_empty());
2427 let t2 = t + c.verify_window_ms + SEC;
2429 let ev = observe(&mut s2, &c, t2, &k, &ok_obs(t2, 0.0, Some(g1))).unwrap();
2430 assert!(matches!(s2.accounts[&k].phase, Phase::Unverified { attempts: 1, .. }));
2431 assert!(ev.iter().any(|e| matches!(e, KeeperEvent::Unverified { .. })));
2432 for i in 0..10u64 {
2434 let tt = t2 + i * c.poll_interval_ms;
2435 observe(&mut s2, &c, tt, &k, &ok_obs(tt, 0.0, Some(g1))).unwrap();
2436 assert!(matches!(decide(&s2, tt, &k, true), ActivationDecision::Skip { .. }));
2437 assert!(begin_attempt(&mut s2, tt, &k).is_err());
2438 }
2439 let ev = finish_attempt(&mut s2, &c, t2 + 1, &ticket, AttemptOutcome::Sent { http_status: 200 }).unwrap();
2441 assert!(matches!(s2.accounts[&k].phase, Phase::Unverified { .. }));
2442 assert!(ev.iter().any(|e| matches!(e, KeeperEvent::AttemptFinished { refunded: false, .. })));
2443 let g2 = t2 + DAY + WEEK;
2445 observe(&mut s2, &c, t2 + DAY, &k, &ok_obs(t2 + DAY, 0.0, Some(g2))).unwrap();
2446 assert!(matches!(
2447 s2.accounts[&k].phase,
2448 Phase::Active {
2449 attribution: Attribution::AttemptCorrelated { attempt: 1 },
2450 ..
2451 }
2452 ));
2453 }
2454
2455 #[test]
2456 fn not_sent_is_refunded_with_backoff_but_bounded() {
2457 let c = cfg();
2458 let (mut s, k, g1, mut t) = make_due("a");
2459 for i in 0..MAX_NOT_SENT_PER_GENERATION {
2460 assert_eq!(
2461 decide(&s, t, &k, true),
2462 ActivationDecision::Activate {
2463 generation: g1,
2464 attempt: 1
2465 },
2466 "iteration {i}"
2467 );
2468 let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2469 finish_attempt(
2470 &mut s,
2471 &c,
2472 t,
2473 &ticket,
2474 AttemptOutcome::NotSent {
2475 stage: NotSentStage::TokenVend,
2476 reason: "token vend failed".into(),
2477 },
2478 )
2479 .unwrap();
2480 let st = &s.accounts[&k];
2481 assert_eq!(st.attempts_in_generation, 0, "refunded");
2482 assert!(matches!(st.phase, Phase::AwaitingActivation { due_since_ms, .. } if due_since_ms == g1));
2483 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2485 t = st.next_attempt_allowed_ms;
2486 observe(&mut s, &c, t, &k, &ok_obs(t, 0.0, Some(g1))).unwrap();
2487 }
2488 assert!(matches!(
2489 decide(&s, t, &k, true),
2490 ActivationDecision::Skip { reason } if reason.contains("unsent")
2491 ));
2492 assert!(s.accounts[&k].backoff_ms <= c.max_backoff_ms);
2493 }
2494
2495 #[test]
2496 fn refund_whitelist_is_explicit() {
2497 for s in REFUNDABLE_REJECT_STATUSES {
2498 assert!(AttemptOutcome::Rejected { http_status: *s }.is_refundable());
2499 }
2500 for s in [402u16, 408, 409, 418, 429, 500, 502] {
2501 assert!(!AttemptOutcome::Rejected { http_status: s }.is_refundable(), "{s}");
2502 }
2503 assert!(AttemptOutcome::NotSent {
2504 stage: NotSentStage::Connect,
2505 reason: "x".into()
2506 }
2507 .is_refundable());
2508 assert!(!AttemptOutcome::Ambiguous { reason: "x".into() }.is_refundable());
2509 assert!(!AttemptOutcome::Sent { http_status: 200 }.is_refundable());
2510 }
2511
2512 #[test]
2513 fn rearm_is_explicit_and_only_from_unverified() {
2514 let c = cfg();
2515 let (mut s, k, g1, t) = make_due("a");
2516 assert_eq!(rearm(&mut s, t, &k).unwrap(), None);
2518 let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2519 finish_attempt(&mut s, &c, t, &ticket, AttemptOutcome::Ambiguous { reason: "timeout".into() }).unwrap();
2520 let t2 = t + c.verify_window_ms + SEC;
2521 observe(&mut s, &c, t2, &k, &ok_obs(t2, 0.0, Some(g1))).unwrap();
2522 assert!(matches!(s.accounts[&k].phase, Phase::Unverified { attempts: 1, .. }));
2523 assert!(matches!(decide(&s, t2, &k, true), ActivationDecision::Skip { .. }));
2524 let ev = rearm(&mut s, t2, &k).unwrap();
2525 assert!(matches!(ev, Some(KeeperEvent::Rearmed { previous_attempts: 1, .. })));
2526 assert!(matches!(s.accounts[&k].phase, Phase::AwaitingActivation { generation, due_since_ms } if generation == g1 && due_since_ms == g1));
2527 assert_eq!(decide(&s, t2, &k, true), ActivationDecision::Activate { generation: g1, attempt: 1 });
2528 assert_eq!(s.accounts[&k].history.len(), 1);
2530 assert_eq!(rearm(&mut s, t2, &k).unwrap(), None);
2532 }
2533
2534 #[test]
2535 fn rejected_401_becomes_auth_error_and_429_is_not_refunded() {
2536 let c = cfg();
2537 let (mut s, k, _g1, t) = make_due("a");
2538 let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2539 finish_attempt(&mut s, &c, t, &ticket, AttemptOutcome::Rejected { http_status: 401 }).unwrap();
2540 assert!(matches!(s.accounts[&k].phase, Phase::AuthError { .. }));
2541 assert_eq!(s.accounts[&k].attempts_in_generation, 0);
2542
2543 let (mut s, k, _g1, t) = make_due("b");
2544 let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2545 finish_attempt(&mut s, &c, t, &ticket, AttemptOutcome::Rejected { http_status: 429 }).unwrap();
2546 assert!(matches!(s.accounts[&k].phase, Phase::ActivationPending { .. }));
2547 assert_eq!(s.accounts[&k].attempts_in_generation, 1);
2548 }
2549
2550 #[test]
2551 fn transport_errors_back_off_exponentially_and_recover() {
2552 let c = cfg();
2553 let (mut s, k) = fresh("a");
2554 let mut t = NOW;
2555 let mut last = 0;
2556 for _ in 0..8 {
2557 observe(&mut s, &c, t, &k, &transport(t)).unwrap();
2558 let b = s.accounts[&k].backoff_ms;
2559 assert!(b >= last && b <= c.max_backoff_ms);
2560 assert_eq!(s.accounts[&k].next_action_at_ms, t + b);
2561 last = b;
2562 t += b;
2563 }
2564 assert_eq!(last, c.max_backoff_ms);
2565 observe(&mut s, &c, t, &k, &ok_obs(t, 1.0, Some(t + DAY))).unwrap();
2566 assert_eq!(s.accounts[&k].backoff_ms, 0);
2567 assert_eq!(s.accounts[&k].consecutive_errors, 0);
2568 }
2569
2570 #[test]
2571 fn auth_error_unsupported_and_malformed_are_labelled_not_activated() {
2572 let c = cfg();
2573 let (mut s, k, _g1, t) = make_due("a");
2574 observe(
2575 &mut s,
2576 &c,
2577 t,
2578 &k,
2579 &UsageObservation {
2580 observed_at_ms: t,
2581 outcome: ObservationOutcome::AuthError {
2582 detail: "401".into(),
2583 },
2584 },
2585 )
2586 .unwrap();
2587 assert!(matches!(s.accounts[&k].phase, Phase::AuthError { .. }));
2588 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2589 observe(
2590 &mut s,
2591 &c,
2592 t,
2593 &k,
2594 &UsageObservation {
2595 observed_at_ms: t,
2596 outcome: ObservationOutcome::Malformed {
2597 detail: "rate_limit missing".into(),
2598 },
2599 },
2600 )
2601 .unwrap();
2602 assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2603 observe(
2604 &mut s,
2605 &c,
2606 t,
2607 &k,
2608 &UsageObservation {
2609 observed_at_ms: t,
2610 outcome: ObservationOutcome::Unsupported {
2611 detail: "no adapter".into(),
2612 },
2613 },
2614 )
2615 .unwrap();
2616 assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2617 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2618 }
2619
2620 #[test]
2621 fn external_new_window_is_observed_not_attributed() {
2622 let c = cfg();
2623 let (mut s, k) = fresh("a");
2624 let g1 = NOW + HOUR;
2625 observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2626 let t = g1 + 2 * HOUR;
2627 let g2 = t + WEEK - 10 * MIN; let ev = observe(&mut s, &c, t, &k, &ok_obs(t, 1.0, Some(g2))).unwrap();
2629 assert!(matches!(
2630 s.accounts[&k].phase,
2631 Phase::Active {
2632 attribution: Attribution::Observed,
2633 ..
2634 }
2635 ));
2636 assert_eq!(s.accounts[&k].last_idle_delay_ms, Some(2 * HOUR - 10 * MIN));
2637 assert!(ev.iter().any(|e| matches!(e, KeeperEvent::NewWindow { attribution: Attribution::Observed, .. })));
2638 let (mut s, k) = fresh("b");
2640 observe(&mut s, &c, NOW, &k, &ok_obs(NOW, 100.0, Some(g1))).unwrap();
2641 observe(&mut s, &c, t, &k, &ok_obs(t, 1.0, Some(t + WEEK))).unwrap();
2642 observe(&mut s, &c, t + 1, &k, &ok_obs(t + 1, 1.0, Some(t + DAY))).unwrap();
2643 assert!(matches!(s.accounts[&k].phase, Phase::Unknown { .. }));
2644 }
2645
2646 #[test]
2647 fn banked_resets_alert_only() {
2648 let c = cfg();
2649 let (mut s, k) = fresh("a");
2650 let mut obs = ok_obs(NOW, 10.0, Some(NOW + DAY));
2651 if let ObservationOutcome::Ok { banked, .. } = &mut obs.outcome {
2652 *banked = Some(BankedResets {
2653 available_count: Some(2),
2654 earliest_expiry_ms: Some(NOW + DAY),
2655 inventory_error: None,
2656 });
2657 }
2658 let ev = observe(&mut s, &c, NOW, &k, &obs).unwrap();
2659 assert!(ev.iter().any(|e| matches!(
2660 e,
2661 KeeperEvent::BankedResets {
2662 available_count: 2,
2663 expiring_soon: true,
2664 ..
2665 }
2666 )));
2667 assert_eq!(s.accounts[&k].banked.as_ref().unwrap().available_count, Some(2));
2668 for count in [Some(0), None] {
2670 if let ObservationOutcome::Ok { banked, .. } = &mut obs.outcome {
2671 *banked = Some(BankedResets {
2672 available_count: count,
2673 earliest_expiry_ms: None,
2674 inventory_error: None,
2675 });
2676 }
2677 let ev = observe(&mut s, &c, NOW, &k, &obs).unwrap();
2678 assert!(!ev.iter().any(|e| matches!(e, KeeperEvent::BankedResets { .. })));
2679 }
2680 }
2681
2682 #[test]
2683 fn overall_limit_flag_folds_into_policy_view() {
2684 let w = vec![five_hour(NOW, 1.0)];
2685 let folded = windows_with_overall_flag(&w, Some(true));
2686 assert_eq!(folded[0].limit_reached, Some(true));
2687 let folded = windows_with_overall_flag(&[], Some(true));
2688 assert_eq!(folded.len(), 1);
2689 assert_eq!(folded[0].id, "overall");
2690 let folded = windows_with_overall_flag(&w, Some(false));
2691 assert_eq!(folded, w);
2692 }
2693
2694 #[test]
2695 fn prune_never_deletes_attempt_ledgers_and_scheduling_is_scoped() {
2696 let mut s = KeeperState::default();
2697 let a = ident("a");
2698 let mut b = ident("a");
2699 b.identity_fp = AccountIdentity::seat_fingerprint(OAuthProviderId::OpenAiCodex, "other-seat").unwrap();
2700 s.entry(&a);
2701 s.entry(&b);
2702 s.accounts.get_mut(&a.key()).unwrap().next_action_at_ms = NOW + 10;
2703 s.accounts.get_mut(&b.key()).unwrap().next_action_at_ms = NOW + 5;
2704 assert_eq!(s.next_wake_ms(), Some(NOW + 5));
2705 assert_eq!(s.due(NOW + 5), vec![b.key()]);
2706 assert_eq!(s.next_wake_among(&[a.key()]), Some(NOW + 10));
2707 assert!(s.due_among(NOW + 5, &[a.key()]).is_empty());
2708 s.accounts.get_mut(&a.key()).unwrap().last_poll_ms = Some(NOW);
2710 assert!(s.prune_idle(&[b.clone()], NOW + DAY, 30 * DAY).is_empty());
2711 assert_eq!(s.prune_idle(&[b.clone()], NOW + 31 * DAY, 30 * DAY), vec![a.key()]);
2712 let (mut s, k, _g, t) = make_due("c");
2714 let (ticket, _) = begin_attempt(&mut s, t, &k).unwrap();
2715 finish_attempt(&mut s, &cfg(), t, &ticket, AttemptOutcome::Sent { http_status: 200 }).unwrap();
2716 assert!(s.prune_idle(&[], t + 365 * DAY, 30 * DAY).is_empty());
2717 assert_eq!(s.accounts[&k].attempts_in_generation, 1);
2718 s.entry(&ident("c"));
2720 assert!(matches!(decide(&s, t, &k, true), ActivationDecision::Skip { .. }));
2721 }
2722
2723 #[test]
2724 fn state_store_is_private_atomic_and_fails_closed_on_corruption() {
2725 let dir = tempfile::tempdir().unwrap();
2726 let store = StateStore::new(&dir.path().join("keeper"));
2727 assert_eq!(store.load().unwrap(), KeeperState::default());
2728 let (mut s, k) = fresh("a");
2729 observe(&mut s, &cfg(), NOW, &k, &ok_obs(NOW, 1.0, Some(NOW + DAY))).unwrap();
2730 store.save(&s).unwrap();
2731 let back = store.load().unwrap();
2732 assert_eq!(back.accounts[&k].phase, s.accounts[&k].phase);
2733 #[cfg(unix)]
2734 {
2735 use std::os::unix::fs::PermissionsExt;
2736 let fm = std::fs::metadata(store.path()).unwrap().permissions().mode() & 0o777;
2737 let dm = std::fs::metadata(store.dir()).unwrap().permissions().mode() & 0o777;
2738 assert_eq!(fm, 0o600);
2739 assert_eq!(dm, 0o700);
2740 }
2741 let raw = std::fs::read_to_string(store.path()).unwrap();
2742 assert!(!raw.contains("Bearer") && !raw.contains("refresh"));
2743 std::fs::write(store.path(), b"{ not json").unwrap();
2744 assert!(matches!(store.load(), Err(KeeperError::CorruptState { .. })));
2745 std::fs::write(store.path(), b"{\"version\": 99, \"accounts\": {}}").unwrap();
2746 assert!(matches!(store.load(), Err(KeeperError::UnsupportedVersion { .. })));
2747 }
2748
2749 #[test]
2750 fn locks_are_exclusive_per_name_and_released_on_drop() {
2751 let dir = tempfile::tempdir().unwrap();
2752 let l1 = KeeperLock::acquire(dir.path(), SINGLETON_LOCK_NAME, "pid 1").unwrap();
2753 match KeeperLock::acquire(dir.path(), SINGLETON_LOCK_NAME, "pid 2") {
2754 Err(KeeperError::Locked { holder, .. }) => assert_eq!(holder.as_deref(), Some("pid 1")),
2755 other => panic!("expected lock conflict, got {other:?}"),
2756 }
2757 let name = account_lock_name("openai-codex@astra2");
2758 assert_eq!(name, "openai-codex@astra2.lock");
2759 assert_eq!(account_lock_name("weird/../key"), "weird_.._key.lock");
2760 let _l2 = KeeperLock::acquire(dir.path(), &name, "pid 1").unwrap();
2761 drop(l1);
2762 let _l3 = KeeperLock::acquire(dir.path(), SINGLETON_LOCK_NAME, "pid 3").unwrap();
2763 }
2764}