agent_core/core/auth/
auth_state.rs

1//! `synaps auth state`: what one broker remembers about its accounts (each
2//! account's cooldown, its last usage read and last token refresh exchange,
3//! and the last automatic selection), read from memory only.
4//!
5//! Building a report makes no network request and refreshes nothing; the
6//! account list (labels, access-token expiry, whether an account id is
7//! recorded) comes from the credential store. A report never carries token
8//! material, emails, account ids or identity hashes. `clear` drops the
9//! in-memory cooldowns, usage readings and stickiness of the matched
10//! accounts; it never writes the credential store.
11
12use serde::{Deserialize, Serialize};
13
14use super::quota_policy::{ObservationFailure, SelectionFailure};
15
16/// Version of [`AuthStateReport`]'s JSON shape (`synaps auth state --json`).
17pub const AUTH_STATE_SCHEMA_VERSION: u32 = 1;
18
19/// Which accounts to report on, and whether to clear their state first.
20#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
21pub struct AuthStateQuery {
22    /// Canonical provider id (`anthropic`); `None` = every provider that has
23    /// stored accounts.
24    #[serde(default, skip_serializing_if = "Option::is_none")]
25    pub provider: Option<String>,
26    /// One account label (`default` for the bare slot).
27    #[serde(default, skip_serializing_if = "Option::is_none")]
28    pub account: Option<String>,
29    /// Drop the matched accounts' in-memory state before reporting.
30    #[serde(default)]
31    pub clear: bool,
32}
33
34/// One broker's in-memory account state.
35#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
36pub struct AuthStateReport {
37    pub schema_version: u32,
38    /// Answered by the running daemon's broker (`true`), or built by the
39    /// asking process, which has served no request (`false`).
40    pub live: bool,
41    /// `false`: this process has no local broker yet (no request used
42    /// automatic selection or reported a limit), so nothing is recorded.
43    pub broker_active: bool,
44    pub generated_at_ms: u64,
45    /// Credential store the accounts were listed from.
46    pub store: String,
47    pub accounts: Vec<AccountState>,
48    /// Last automatic selection per provider and model, newest first.
49    #[serde(default, skip_serializing_if = "Vec::is_empty")]
50    pub selections: Vec<SelectionState>,
51    /// `provider:label` of every account whose state `clear` dropped.
52    #[serde(default, skip_serializing_if = "Vec::is_empty")]
53    pub cleared: Vec<String>,
54}
55
56/// How limits follow an account.
57#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
58#[serde(rename_all = "snake_case")]
59pub enum IdentityKind {
60    /// The slot records the provider account id.
61    Stable,
62    /// No account id recorded: the seat identity derives from refresh
63    /// material and changes when the token rotates (`synaps auth identify`
64    /// records the id).
65    Rotating,
66    /// A kind this build does not know (from a newer daemon).
67    #[serde(other)]
68    Unknown,
69}
70
71impl IdentityKind {
72    pub fn as_str(self) -> &'static str {
73        match self {
74            Self::Stable => "stable",
75            Self::Rotating => "rotating",
76            Self::Unknown => "unknown",
77        }
78    }
79}
80
81/// One stored account and what the broker remembers about it.
82#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
83pub struct AccountState {
84    pub provider: String,
85    pub label: String,
86    pub identity: IdentityKind,
87    /// Access-token expiry, epoch ms; `None` when unknown.
88    #[serde(default, skip_serializing_if = "Option::is_none")]
89    pub access_expires_ms: Option<u64>,
90    /// An active cooldown (a reported provider limit).
91    #[serde(default, skip_serializing_if = "Option::is_none")]
92    pub cooldown: Option<CooldownState>,
93    /// The last usage read of this account (cache hits are not reads).
94    #[serde(default, skip_serializing_if = "Option::is_none")]
95    pub last_usage: Option<ObservedOutcome>,
96    /// When a usage read of this account last succeeded, epoch ms.
97    #[serde(default, skip_serializing_if = "Option::is_none")]
98    pub last_usage_ok_ms: Option<u64>,
99    /// The last token refresh exchange for this account in this process.
100    #[serde(default, skip_serializing_if = "Option::is_none")]
101    pub last_refresh: Option<ObservedOutcome>,
102    /// A refused token refresh this broker holds: the account is not
103    /// refreshed again until its stored login changes or `until_ms`.
104    #[serde(default, skip_serializing_if = "Option::is_none")]
105    pub refresh_hold: Option<RefreshHoldState>,
106}
107
108/// A held refused refresh (`refresh_rejected` / `refresh_action_required`).
109#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
110pub struct RefreshHoldState {
111    pub failure: ObservationFailure,
112    /// When the provider refused the refresh, epoch ms.
113    pub refused_at_ms: u64,
114    /// When one refresh is tried again unless the stored login changes
115    /// first, epoch ms.
116    pub until_ms: u64,
117}
118
119/// A reported provider limit.
120#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
121pub struct CooldownState {
122    pub until_ms: u64,
123    /// Who reported it, as a fixed kind; never the reporter's own string.
124    pub reason: CooldownReason,
125}
126
127/// Why an account is cooling down: one of the known `report_cooldown`
128/// callers, or `other`. A report carries only this kind, never the string
129/// the reporter passed (which may come from another machine); the log line
130/// written when the cooldown is reported keeps that string. An unknown kind
131/// from a newer daemon reads as `other`.
132#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
133#[serde(rename_all = "snake_case", from = "String")]
134pub enum CooldownReason {
135    /// The Anthropic request path saw a quota-exhausted answer.
136    AnthropicQuotaExhausted,
137    /// The Codex request path saw a quota-exhausted answer.
138    CodexQuotaExhausted,
139    /// A broker client reported a limit without a reason.
140    ClientReport,
141    /// Any other reason.
142    Other,
143}
144
145impl CooldownReason {
146    /// The kind of a reporter's reason string.
147    pub fn from_reported(reason: &str) -> Self {
148        match reason {
149            "anthropic_quota_exhausted" => Self::AnthropicQuotaExhausted,
150            "codex_quota_exhausted" => Self::CodexQuotaExhausted,
151            "client_report" => Self::ClientReport,
152            _ => Self::Other,
153        }
154    }
155
156    pub fn as_str(self) -> &'static str {
157        match self {
158            Self::AnthropicQuotaExhausted => "anthropic_quota_exhausted",
159            Self::CodexQuotaExhausted => "codex_quota_exhausted",
160            Self::ClientReport => "client_report",
161            Self::Other => "other",
162        }
163    }
164}
165
166impl From<String> for CooldownReason {
167    fn from(reason: String) -> Self {
168        Self::from_reported(&reason)
169    }
170}
171
172impl std::fmt::Display for CooldownReason {
173    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
174        f.write_str(self.as_str())
175    }
176}
177
178/// One observation: when it happened and, if it failed, why.
179#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
180pub struct ObservedOutcome {
181    pub at_ms: u64,
182    /// `None`: it succeeded.
183    #[serde(default, skip_serializing_if = "Option::is_none")]
184    pub failure: Option<ObservationFailure>,
185}
186
187/// The last automatic selection for one provider and model.
188#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
189pub struct SelectionState {
190    pub provider: String,
191    /// `None`: any model.
192    #[serde(default, skip_serializing_if = "Option::is_none")]
193    pub model: Option<String>,
194    pub at_ms: u64,
195    /// The chosen account (also set when its token could not be vended).
196    #[serde(default, skip_serializing_if = "Option::is_none")]
197    pub chosen: Option<String>,
198    /// Why no account was chosen.
199    #[serde(default, skip_serializing_if = "Option::is_none")]
200    pub failure: Option<SelectionFailure>,
201    /// Why the chosen account's token could not be vended.
202    #[serde(default, skip_serializing_if = "Option::is_none")]
203    pub vend_failure: Option<ObservationFailure>,
204}
205
206fn span(ms: u64) -> String {
207    let secs = ms / 1000;
208    match secs {
209        s if s < 60 => format!("{s}s"),
210        s if s < 3600 => format!("{}m", s / 60),
211        s if s < 86_400 => format!("{}h{:02}m", s / 3600, (s % 3600) / 60),
212        s => format!("{}d{:02}h", s / 86_400, (s % 86_400) / 3600),
213    }
214}
215
216/// `12s ago` (or `in 3s` for a clock that moved backwards).
217fn ago(now_ms: u64, at_ms: u64) -> String {
218    if at_ms <= now_ms {
219        format!("{} ago", span(now_ms - at_ms))
220    } else {
221        format!("in {}", span(at_ms - now_ms))
222    }
223}
224
225fn utc(ms: u64) -> String {
226    chrono::DateTime::from_timestamp_millis(ms as i64)
227        .map(|t| t.to_rfc3339_opts(chrono::SecondsFormat::Secs, true))
228        .unwrap_or_else(|| ms.to_string())
229}
230
231fn outcome(o: &ObservedOutcome, now_ms: u64) -> String {
232    match &o.failure {
233        None => format!("ok, {}", ago(now_ms, o.at_ms)),
234        Some(f) => format!("{f}, {}", ago(now_ms, o.at_ms)),
235    }
236}
237
238/// Plain-text rendering, one block per account, then the last selections.
239pub fn render_text(report: &AuthStateReport) -> String {
240    use std::fmt::Write;
241    let now = report.generated_at_ms;
242    let mut out = String::new();
243    let source = if report.live {
244        "live: the running daemon's broker"
245    } else {
246        "NOT LIVE: built by this command, which has served no request"
247    };
248    let _ = writeln!(out, "auth state ({source})");
249    let _ = writeln!(out, "store: {}", report.store);
250    if !report.broker_active {
251        let _ = writeln!(
252            out,
253            "no automatic selection or limit report has reached this broker yet; nothing recorded"
254        );
255    }
256    if !report.cleared.is_empty() {
257        let _ = writeln!(
258            out,
259            "cleared (in memory only; credentials untouched): {}",
260            report.cleared.join(", ")
261        );
262    }
263    if report.accounts.is_empty() {
264        let _ = writeln!(out, "no stored accounts");
265    }
266    let mut provider = "";
267    for account in &report.accounts {
268        if account.provider != provider {
269            provider = &account.provider;
270            let _ = writeln!(out, "{provider}");
271        }
272        let expiry = match account.access_expires_ms {
273            Some(t) if t > now => format!("token expires in {}", span(t - now)),
274            Some(_) => "token expired (refreshed on next use)".to_string(),
275            None => "token expiry unknown".to_string(),
276        };
277        let _ = writeln!(
278            out,
279            "  {}  identity {}  {expiry}",
280            account.label,
281            account.identity.as_str()
282        );
283        match &account.cooldown {
284            Some(c) => {
285                let _ = writeln!(
286                    out,
287                    "    cooldown  {} left (until {}): {}",
288                    span(c.until_ms.saturating_sub(now)),
289                    utc(c.until_ms),
290                    c.reason
291                );
292            }
293            None => {
294                let _ = writeln!(out, "    cooldown  none");
295            }
296        }
297        match &account.last_usage {
298            Some(u) => {
299                let last_ok = match (u.failure.is_some(), account.last_usage_ok_ms) {
300                    (true, Some(ok)) => format!(" (last ok {})", ago(now, ok)),
301                    (true, None) => " (no successful read yet)".to_string(),
302                    _ => String::new(),
303                };
304                let _ = writeln!(out, "    usage     {}{last_ok}", outcome(u, now));
305            }
306            None => {
307                let _ = writeln!(out, "    usage     not read yet");
308            }
309        }
310        match &account.last_refresh {
311            Some(r) => {
312                let _ = writeln!(out, "    refresh   {}", outcome(r, now));
313            }
314            None => {
315                let _ = writeln!(out, "    refresh   none in this process");
316            }
317        }
318        if let Some(h) = &account.refresh_hold {
319            let _ = writeln!(
320                out,
321                "    held      refresh refused ({}), {}; held until the stored login changes or {}",
322                h.failure,
323                ago(now, h.refused_at_ms),
324                utc(h.until_ms)
325            );
326        }
327    }
328    if !report.selections.is_empty() {
329        let _ = writeln!(out, "last automatic selections");
330        for s in &report.selections {
331            let model = s.model.as_deref().unwrap_or("any model");
332            let what = match (&s.chosen, &s.failure, &s.vend_failure) {
333                (Some(label), _, Some(f)) => format!("chose {label}; token not vended: {f}"),
334                (Some(label), _, None) => format!("chose {label}"),
335                (None, Some(f), _) => {
336                    let accounts = f.accounts_field();
337                    if accounts.is_empty() {
338                        format!("failed: {}", f.kind)
339                    } else {
340                        format!("failed: {} ({accounts})", f.kind)
341                    }
342                }
343                (None, None, _) => "failed".to_string(),
344            };
345            let _ = writeln!(
346                out,
347                "  {} {model}: {what}, {}",
348                s.provider,
349                ago(now, s.at_ms)
350            );
351        }
352    }
353    out
354}
355
356#[cfg(test)]
357mod tests {
358    use super::*;
359    use crate::core::auth::quota_policy::{
360        AccountFailure, AccountFailureKind, SelectionFailureKind,
361    };
362
363    const NOW: u64 = 1_800_000_000_000;
364
365    fn report() -> AuthStateReport {
366        AuthStateReport {
367            schema_version: AUTH_STATE_SCHEMA_VERSION,
368            live: true,
369            broker_active: true,
370            generated_at_ms: NOW,
371            store: "/scratch/auth.json".into(),
372            accounts: vec![
373                AccountState {
374                    provider: "anthropic".into(),
375                    label: "claude5".into(),
376                    identity: IdentityKind::Stable,
377                    access_expires_ms: Some(NOW + 3 * 3600 * 1000),
378                    cooldown: Some(CooldownState {
379                        until_ms: NOW + 41 * 60 * 1000,
380                        reason: CooldownReason::AnthropicQuotaExhausted,
381                    }),
382                    last_usage: Some(ObservedOutcome {
383                        at_ms: NOW - 12_000,
384                        failure: Some(
385                            ObservationFailure::new(AccountFailureKind::UsageRateLimited)
386                                .with_status(429)
387                                .with_retry_after(Some(120)),
388                        ),
389                    }),
390                    last_usage_ok_ms: Some(NOW - 6 * 60 * 1000),
391                    last_refresh: Some(ObservedOutcome {
392                        at_ms: NOW - (2 * 3600 + 3 * 60) * 1000,
393                        failure: None,
394                    }),
395                    refresh_hold: None,
396                },
397                AccountState {
398                    provider: "anthropic".into(),
399                    label: "claude6".into(),
400                    identity: IdentityKind::Rotating,
401                    access_expires_ms: None,
402                    cooldown: None,
403                    last_usage: None,
404                    last_usage_ok_ms: None,
405                    last_refresh: None,
406                    refresh_hold: Some(RefreshHoldState {
407                        failure: ObservationFailure::new(AccountFailureKind::RefreshRejected)
408                            .with_status(400),
409                        refused_at_ms: NOW - 4 * 60 * 1000,
410                        until_ms: NOW + 11 * 60 * 1000,
411                    }),
412                },
413            ],
414            selections: vec![SelectionState {
415                provider: "anthropic".into(),
416                model: Some("claude-opus-4-1".into()),
417                at_ms: NOW - 3_000,
418                chosen: None,
419                failure: Some(SelectionFailure {
420                    kind: SelectionFailureKind::UsageReadFailed,
421                    accounts: vec![AccountFailure {
422                        account: "claude5".into(),
423                        kind: AccountFailureKind::UsageRateLimited,
424                        http_status: Some(429),
425                        retry_after_s: Some(120),
426                        until_ms: None,
427                        cooling_down: false,
428                    }],
429                    earliest_reset_ms: None,
430                    usage_read_retry_ms: None,
431                }),
432                vend_failure: None,
433            }],
434            cleared: vec![],
435        }
436    }
437
438    #[test]
439    fn text_names_each_kind_status_and_age() {
440        let text = render_text(&report());
441        for needle in [
442            "auth state (live: the running daemon's broker)",
443            "anthropic\n  claude5  identity stable  token expires in 3h00m",
444            "cooldown  41m left (until ",
445            "usage     usage_rate_limited, HTTP 429, retry after 120s, 12s ago (last ok 6m ago)",
446            "refresh   ok, 2h03m ago",
447            "claude6  identity rotating  token expiry unknown",
448            "usage     not read yet",
449            "anthropic claude-opus-4-1: failed: usage_read_failed (claude5:usage_rate_limited:429), 3s ago",
450        ] {
451            assert!(text.contains(needle), "{needle:?} missing from:\n{text}");
452        }
453        let held = format!(
454            "    held      refresh refused (refresh_rejected, HTTP 400), 4m ago; \
455             held until the stored login changes or {}\n",
456            utc(NOW + 11 * 60 * 1000)
457        );
458        assert!(text.contains(&held), "{held:?} missing from:\n{text}");
459        assert_eq!(text.matches("    held ").count(), 1, "only claude6 is held:\n{text}");
460        assert!(
461            text.contains("): anthropic_quota_exhausted\n"),
462            "the cooldown's reason kind:\n{text}"
463        );
464    }
465
466    /// r2 blocker 3: a reason is a fixed kind on the wire too; a reason
467    /// string this build does not know (a newer daemon's, or a raw string)
468    /// reads as `other`.
469    #[test]
470    fn cooldown_reasons_round_trip_as_fixed_kinds() {
471        for (wire, kind) in [
472            (
473                "anthropic_quota_exhausted",
474                CooldownReason::AnthropicQuotaExhausted,
475            ),
476            ("codex_quota_exhausted", CooldownReason::CodexQuotaExhausted),
477            ("client_report", CooldownReason::ClientReport),
478            ("other", CooldownReason::Other),
479            ("sk-ant-oat01-raw", CooldownReason::Other),
480        ] {
481            let state: CooldownState =
482                serde_json::from_value(serde_json::json!({"until_ms": 1, "reason": wire})).unwrap();
483            assert_eq!(state.reason, kind, "{wire}");
484            assert_eq!(
485                serde_json::to_value(&state).unwrap()["reason"],
486                kind.as_str(),
487                "{wire}"
488            );
489        }
490    }
491
492    /// A report from a daemon one build ahead must still decode: a kind
493    /// this build does not know reads as `unknown` instead of failing the
494    /// whole frame.
495    #[test]
496    fn kinds_from_a_newer_daemon_decode_as_unknown() {
497        let mut json = serde_json::to_value(report()).unwrap();
498        json["accounts"][0]["identity"] = "per_org".into();
499        json["accounts"][0]["last_usage"]["failure"]["kind"] = "usage_brand_new".into();
500        json["selections"][0]["failure"]["kind"] = "brand_new_headline".into();
501        json["selections"][0]["failure"]["accounts"][0]["kind"] = "usage_brand_new".into();
502        let decoded: AuthStateReport = serde_json::from_value(json).unwrap();
503        assert_eq!(decoded.accounts[0].identity, IdentityKind::Unknown);
504        let usage = decoded.accounts[0].last_usage.as_ref().unwrap();
505        assert_eq!(
506            usage.failure.as_ref().unwrap().kind,
507            AccountFailureKind::Unknown
508        );
509        let failure = decoded.selections[0].failure.as_ref().unwrap();
510        assert_eq!(failure.kind, SelectionFailureKind::Unknown);
511        assert_eq!(failure.accounts[0].kind, AccountFailureKind::Unknown);
512        // Known kinds are untouched, and an unknown one never claims a
513        // remedy it can't know.
514        assert_eq!(decoded.accounts[1].identity, IdentityKind::Rotating);
515        let k = AccountFailureKind::Unknown;
516        assert!(!k.is_transient() && !k.needs_login() && !k.needs_action() && !k.is_limited());
517        let text = render_text(&decoded);
518        assert!(text.contains("identity unknown"), "{text}");
519        assert!(text.contains("failed: unknown (claude5:unknown:429)"), "{text}");
520    }
521
522    #[test]
523    fn a_report_not_from_the_daemon_says_so() {
524        let mut r = report();
525        r.live = false;
526        r.broker_active = false;
527        let text = render_text(&r);
528        assert!(text.starts_with("auth state (NOT LIVE"), "{text}");
529        assert!(text.contains("nothing recorded"), "{text}");
530    }
531
532    #[test]
533    fn json_round_trips_with_its_schema_version() {
534        let r = report();
535        let value = serde_json::to_value(&r).unwrap();
536        assert_eq!(value["schema_version"], AUTH_STATE_SCHEMA_VERSION);
537        assert_eq!(value["accounts"][0]["identity"], "stable");
538        assert_eq!(
539            value["accounts"][0]["last_usage"]["failure"],
540            serde_json::json!({"kind": "usage_rate_limited", "http_status": 429, "retry_after_s": 120})
541        );
542        let back: AuthStateReport = serde_json::from_value(value).unwrap();
543        assert_eq!(back, r);
544    }
545}
agent_core/core/auth/auth_state.rs

agent_core/core/auth/
auth_state.rs
