Skip to main content

agent_core/core/auth/
broker.rs

1//! The credential broker — the mandatory boundary between runtimes and
2//! provider secrets (Checkpoint 1 decision, `docs/decisions/credential-broker-checkpoint-1.md`).
3//!
4//! ## Invariants (enforced by construction and tests)
5//!
6//! * Runtime/TUI code talks to a [`CredentialBroker`] and never reads
7//!   `auth.json`, provider-key config, or credential environment variables.
8//!   All of that discovery lives inside [`LocalBroker`] — behind the boundary.
9//! * OAuth responses vend **access token + expiry only** ([`AccessToken`] has
10//!   no refresh field to put one in). Refresh tokens stay broker-owned.
11//! * Static API keys are never vended, locally or remotely. They are applied
12//!   broker-side via the typed request proxy ([`ProxyRequest`]): the broker
13//!   pins the destination URL from its own [`static_providers`] table and
14//!   attaches the bearer key itself.
15//! * Capability/status queries ([`ProviderStatus`]) expose configured-ness
16//!   only — never key material.
17//! * Fail closed: a missing credential is an error, never a fallback to a
18//!   direct read at the call site.
19//!
20//! [`static_providers`]: super::static_providers
21
22use std::collections::BTreeMap;
23use std::pin::Pin;
24use std::sync::Arc;
25use std::time::Duration;
26
27use async_trait::async_trait;
28use futures::Stream;
29use serde::{Deserialize, Serialize};
30
31use super::account::{
32    seat_fingerprint, Account, AccountPolicy, AccountSelector, AccountSummary, CredentialRef,
33    MaterialDigest, SeatIdentity,
34};
35use super::cloud::{CloudProviderId, InvokeRequest};
36use super::provider::OAuthProviderId;
37use super::static_providers::{
38    allowed_proxy_paths, static_provider, StaticProviderSpec, LOCAL_DEFAULT_BASE_URL,
39    LOCAL_PROVIDER_KEY, STATIC_PROVIDERS,
40};
41use super::storage;
42
43// ── Buffering / time limits ──────────────────────────────────────────────────
44//
45// Streaming (SSE) bodies are never buffered — they flow chunk-by-chunk with
46// backpressure. These caps bound everything the broker (or a broker client)
47// must hold in memory, so a hostile or broken upstream cannot balloon the
48// process or smuggle unbounded content through error strings.
49
50/// Maximum serialized size of a proxy request body.
51pub const MAX_PROXY_REQUEST_BYTES: usize = 2 * 1024 * 1024;
52/// Maximum buffered (non-streaming) response body size.
53pub const MAX_PROXY_RESPONSE_BYTES: usize = 8 * 1024 * 1024;
54/// Historical cap for upstream error-body snippets. Upstream/broker error
55/// bodies are no longer read at all (spec §5.1: they may echo the request);
56/// the constant remains only because it is re-exported from `auth::mod`.
57pub const MAX_UPSTREAM_ERROR_BYTES: usize = 2 * 1024;
58/// Total time budget for a buffered (non-streaming) broker-executed request.
59pub const PROXY_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
60
61/// Fixed Anthropic usage endpoint — a typed broker operation, not a general
62/// proxy target. Callers cannot vary the URL or path.
63const ANTHROPIC_USAGE_URL: &str = "https://api.anthropic.com/api/oauth/usage";
64const ANTHROPIC_OAUTH_BETA: &str = "oauth-2025-04-20";
65
66/// Pinned Code Assist host for the broker-only Google Gemini proxy runtime.
67/// The service is treated as experimental and only the exact `v1internal`
68/// method paths reviewed in the spec are permitted (setup + streaming).
69pub const GOOGLE_GEMINI_CODE_ASSIST_BASE_URL: &str = "https://cloudcode-pa.googleapis.com";
70
71/// Exhaustive allowlist of `cloudcode-pa` methods the broker may proxy.
72/// Deliberately narrow: setup uses loadCodeAssist + onboardUser + operations
73/// polling, runtime uses streamGenerateContent. Anything else is denied.
74pub(crate) fn is_allowed_google_gemini_path(path: &str) -> bool {
75    matches!(
76        path,
77        "/v1internal:loadCodeAssist"
78            | "/v1internal:onboardUser"
79            | "/v1internal:streamGenerateContent"
80            | "/v1internal:countTokens"
81    ) || path.starts_with("/v1internal/operations/")
82}
83
84/// Pinned ChatGPT backend host for OpenAI Codex catalog traffic.
85pub const OPENAI_CODEX_BACKEND_BASE_URL: &str = "https://chatgpt.com/backend-api";
86
87/// Allow only the official Codex model-catalog path:
88/// `GET /codex/models?client_version=<non-empty>`.
89///
90/// Query is constrained to a single non-empty `client_version` key — no extra
91/// params, no empty version, no bare `/codex/models`. Same-host inference
92/// paths (`/codex/responses`, …) stay out of this catalog-only allowlist, as
93/// does the ChatGPT web picker at `/models` (different schema, not a Codex
94/// catalog).
95pub(crate) const OPENAI_CODEX_MODELS_PATH: &str = "/codex/models";
96
97pub(crate) fn is_allowed_openai_codex_path(path: &str) -> bool {
98    let Some((base, query)) = path.split_once('?') else {
99        return false;
100    };
101    if base != OPENAI_CODEX_MODELS_PATH {
102        return false;
103    }
104    let mut client_version: Option<&str> = None;
105    for pair in query.split('&') {
106        let Some((k, v)) = pair.split_once('=') else {
107            return false;
108        };
109        if k != "client_version" || client_version.is_some() {
110            return false;
111        }
112        if v.trim().is_empty() {
113            return false;
114        }
115        client_version = Some(v);
116    }
117    client_version.is_some()
118}
119
120/// Allow only Anthropic's paginated models catalog path:
121/// `/v1/models` or `/v1/models?limit=…` with optional `after_id=…`.
122///
123/// No other same-host endpoints (messages, keys, admin, …) are permitted.
124pub(crate) fn is_allowed_anthropic_path(path: &str) -> bool {
125    if path == "/v1/models" {
126        return true;
127    }
128    let Some((base, query)) = path.split_once('?') else {
129        return false;
130    };
131    if base != "/v1/models" {
132        return false;
133    }
134    if query.is_empty() {
135        return false;
136    }
137    let mut saw_limit = false;
138    let mut saw_after_id = false;
139    for pair in query.split('&') {
140        let Some((k, v)) = pair.split_once('=') else {
141            return false;
142        };
143        if v.is_empty() {
144            return false;
145        }
146        match k {
147            "limit" if !saw_limit => saw_limit = true,
148            "after_id" if !saw_after_id => saw_after_id = true,
149            _ => return false,
150        }
151    }
152    // limit may be omitted only for bare /v1/models; when a query is present
153    // at least one allowed key must appear (already enforced by the loop).
154    true
155}
156
157/// Headers applied by the broker for Anthropic OAuth catalog proxy requests.
158/// Bearer is set separately via `bearer_auth`; this list must NOT include
159/// `x-api-key` (OAuth tokens are never sent as API keys).
160pub(crate) fn anthropic_oauth_catalog_request_headers() -> &'static [(&'static str, &'static str)] {
161    &[
162        ("anthropic-version", "2023-06-01"),
163        ("anthropic-beta", ANTHROPIC_OAUTH_BETA),
164        ("accept", "application/json"),
165    ]
166}
167
168// ── Errors ───────────────────────────────────────────────────────────────────
169
170/// Broker failures. Messages never contain credential values.
171/// Non-exhaustive: new failure kinds are added as brokers gain operations.
172#[derive(Debug, Clone, PartialEq, Eq)]
173#[non_exhaustive]
174pub enum BrokerError {
175    /// The named provider is not in any broker registry.
176    UnknownProvider(String),
177    /// A required public-client registration is not configured.
178    RegistrationRequired {
179        provider: String,
180        remediation: String,
181    },
182    /// The provider is known but has no credential configured.
183    NotConfigured(String),
184    /// The caller failed broker (machine) authentication.
185    Unauthorized,
186    /// The request was rejected by broker policy (e.g. malformed proxy path).
187    Denied(String),
188    /// The route honestly lacks a capability the request needs (e.g. tools on
189    /// a text-only cloud route). Raised pre-flight, before credentials/network.
190    UnsupportedCapability {
191        provider: String,
192        capability: String,
193    },
194    /// Transport-level failure talking to the broker or the provider.
195    Transport(String),
196    /// Credential storage/refresh failure (message is already secret-free).
197    Credential(String),
198    /// An explicitly requested account slot does not exist. Never a fallback.
199    UnknownAccount { provider: String, label: String },
200    /// This broker implementation cannot address named account slots.
201    UnsupportedAccount { provider: String, label: String },
202    /// An account label failed validation (CLI/env/config/HTTP boundary).
203    InvalidAccount(String),
204    /// Automatic selection found no account with proven capacity. `failure`
205    /// is the typed explanation (top-level kind plus every account's kind
206    /// and HTTP status) when the broker knows it; `reason` its text.
207    NoAccountAvailable {
208        provider: String,
209        reason: String,
210        failure: Option<super::quota_policy::SelectionFailure>,
211    },
212    /// The remote broker predates an operation this client needs; the
213    /// payload names what is unavailable. Upgrade the broker host.
214    BrokerTooOld(String),
215    /// A locked token refresh failed; the kind tells a per-account cooldown,
216    /// a busy refresh lock or a transient network/provider failure apart
217    /// from an unusable login. Message is secret-free.
218    Refresh(super::token::RefreshFailure),
219}
220
221impl std::fmt::Display for BrokerError {
222    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
223        match self {
224            Self::UnknownProvider(p) => write!(f, "unknown provider: {p}"),
225            Self::RegistrationRequired {
226                provider,
227                remediation,
228            } => {
229                write!(f, "registration required for '{provider}': {remediation}")
230            }
231            Self::NotConfigured(p) => write!(
232                f,
233                "no credential configured for '{p}'. Run `synaps login` to add one."
234            ),
235            Self::Unauthorized => f.write_str(&super::broker_token::unauthorized_hint()),
236            Self::Denied(msg) => write!(f, "broker denied request: {msg}"),
237            Self::UnsupportedCapability {
238                provider,
239                capability,
240            } => write!(
241                f,
242                "cloud provider '{provider}' is text-only: {capability} are not supported yet \
243                 (no credential was used and no network request was made)"
244            ),
245            Self::Transport(msg) => write!(f, "broker transport error: {msg}"),
246            Self::Credential(msg) => write!(f, "credential error: {msg}"),
247            Self::Refresh(failure) => {
248                use super::token::RefreshFailureKind as K;
249                let hint = match failure.kind {
250                    K::Transient => " (network or provider error, not a login problem; retry shortly)",
251                    K::LockBusy => " (another refresh of this account is still running; retry shortly)",
252                    K::Other => " (retry shortly)",
253                    K::Credential | K::Cooldown { .. } | K::ActionRequired => "",
254                };
255                write!(f, "credential refresh failed{hint}: {failure}")
256            }
257            Self::UnknownAccount { provider, label } => write!(
258                f,
259                "unknown account '{label}' for provider '{provider}' (no fallback; run \
260                 `synaps auth list` or `synaps login --provider {provider} --account {label}`)"
261            ),
262            Self::UnsupportedAccount { provider, label } => write!(
263                f,
264                "this credential broker cannot address account '{label}' for provider '{provider}'"
265            ),
266            Self::InvalidAccount(msg) => write!(f, "invalid account: {msg}"),
267            Self::NoAccountAvailable {
268                provider,
269                reason,
270                failure: Some(failure),
271            } => write!(
272                f,
273                "no account with proven capacity for '{provider}' ({}): {reason}",
274                failure.kind
275            ),
276            Self::NoAccountAvailable {
277                provider, reason, ..
278            } => {
279                write!(
280                    f,
281                    "no account with proven capacity for '{provider}': {reason}"
282                )
283            }
284            Self::BrokerTooOld(what) => write!(
285                f,
286                "broker is older than this client; {what} unavailable (upgrade the broker host)"
287            ),
288        }
289    }
290}
291
292impl std::error::Error for BrokerError {}
293
294#[cfg(test)]
295#[path = "broker_remote_compat_tests.rs"]
296mod remote_compat_tests;
297
298// ── Vended types ─────────────────────────────────────────────────────────────
299
300/// An OAuth access token vended by the broker.
301///
302/// Deliberately has **no** refresh-token field — the invariant made
303/// structural. There is nowhere to put a refresh token even by mistake.
304#[derive(Debug, Clone, Serialize, Deserialize)]
305pub struct AccessToken {
306    pub token: String,
307    /// Absolute expiry, unix-epoch milliseconds.
308    pub expires: u64,
309}
310
311/// The selected/pinned pair a runtime works with for one attempt: the
312/// credential the broker chose and the token vended for exactly that
313/// credential. Account-specific headers (e.g. the Codex account id) must be
314/// derived from `token`, never from a second lookup.
315#[derive(Debug, Clone)]
316pub struct PinnedToken {
317    pub credential: CredentialRef,
318    pub token: AccessToken,
319}
320
321impl PinnedToken {
322    /// Seat fingerprint derived from THIS token's own identity claim (Codex:
323    /// the JWT `chatgpt_account_id`), comparable with
324    /// [`AccountSummary::seat_fingerprint`]. `None` when the token carries no
325    /// account identity — callers that must spend on exactly one listed seat
326    /// treat `None` or a mismatch as "do not send".
327    pub fn seat_fingerprint(&self) -> Option<String> {
328        match self.credential.provider {
329            OAuthProviderId::OpenAiCodex => super::extract_codex_account_id(&self.token.token)
330                .and_then(|id| seat_fingerprint(self.credential.provider, &id)),
331            _ => None,
332        }
333    }
334}
335
336/// HTTP method subset the proxy protocol supports.
337#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
338#[serde(rename_all = "lowercase")]
339pub enum ProxyMethod {
340    Get,
341    Post,
342}
343
344/// A typed, credential-free request the broker executes on the caller's
345/// behalf against an OpenAI-compatible provider endpoint.
346///
347/// The caller names a provider and a relative path; the broker derives the
348/// destination from its pinned provider table and applies the (broker-owned)
349/// credential. Callers can never supply an absolute URL for a static-key
350/// provider, so a key can never be coaxed toward an attacker-chosen host.
351#[derive(Debug, Clone, Serialize, Deserialize)]
352pub struct ProxyRequest {
353    /// Static provider key (e.g. `groq`), `local`, or an OAuth-proxied
354    /// provider. For OAuth providers an explicit account slot may be pinned
355    /// as `<provider>@<label>` or `<provider>@default`; a bare provider means
356    /// "the broker's account policy". Validated by [`validate`](Self::validate);
357    /// an explicit unknown slot is an error, never a fallback.
358    pub provider: String,
359    pub method: ProxyMethod,
360    /// Relative path joined onto the pinned base URL, e.g. `/chat/completions`.
361    pub path: String,
362    /// JSON request body (POST only).
363    #[serde(default, skip_serializing_if = "Option::is_none")]
364    pub body: Option<serde_json::Value>,
365    /// Request an SSE byte stream (`proxy_stream`) instead of a buffered body.
366    #[serde(default)]
367    pub stream: bool,
368    /// Typed exact-byte handoff (request-trace spec §6.2): when present this
369    /// MUST be the exact serialization of `body`, produced once by the
370    /// caller. [`LocalBroker`] sends these very bytes upstream verbatim, so
371    /// a caller-computed digest over them is a digest of the true wire body.
372    /// The field never crosses the remote broker HTTP boundary
373    /// (`serde(skip)`): a remote broker daemon re-serializes `body` on its
374    /// side, so callers on a remote path must NOT claim exact upstream
375    /// bytes. Never logged, never part of the broker wire schema.
376    #[serde(skip)]
377    pub body_bytes: Option<bytes::Bytes>,
378}
379
380impl ProxyRequest {
381    /// Validate provider identity and path shape. Fail closed on anything
382    /// that could redirect a broker-owned credential.
383    pub fn validate(&self) -> Result<(), BrokerError> {
384        let (provider, _account) = self.split_provider()?;
385        let provider = provider.as_str();
386        if provider != LOCAL_PROVIDER_KEY
387            && provider != "xai-auth"
388            && provider != "github-copilot"
389            && provider != "google-gemini"
390            && provider != "openai-codex"
391            && provider != "anthropic"
392            && provider != "kimi-code"
393            && provider != "muse"
394            && static_provider(provider).is_none()
395        {
396            return Err(BrokerError::UnknownProvider(self.provider.clone()));
397        }
398        if !self.path.starts_with('/') {
399            return Err(BrokerError::Denied("proxy path must be relative".into()));
400        }
401        if self.path.contains("://") || self.path.contains("..") {
402            return Err(BrokerError::Denied(
403                "proxy path is not a plain relative path".into(),
404            ));
405        }
406        // Per-provider endpoint allowlist: a signed proxy request can only
407        // reach the cataloged inference/model paths, never other same-host
408        // endpoints (key management, billing, admin, …).
409        let oauth_path_allowed = provider == "xai-auth" && self.path == "/responses"
410            || provider == "github-copilot"
411                && matches!(
412                    self.path.as_str(),
413                    "/models" | "/chat/completions" | "/responses"
414                )
415            || provider == "google-gemini" && is_allowed_google_gemini_path(&self.path)
416            // Managed Kimi Code endpoint allowlist: chat inference plus the
417            // read-only catalog/profile/quota surfaces the official CLI uses.
418            || provider == "kimi-code"
419                && (self.path == "/chat/completions"
420                    || self.method == ProxyMethod::Get
421                        && matches!(self.path.as_str(), "/models" | "/me" | "/usages"))
422            // Meta Muse: Responses inference plus the read-only model catalog.
423            // Never the key mint/logout endpoints on the same host.
424            || provider == "muse"
425                && (self.method == ProxyMethod::Post && self.path == "/v1/responses"
426                    || self.method == ProxyMethod::Get && self.path == "/muse-code/models")
427            || provider == "openai-codex"
428                && self.method == ProxyMethod::Get
429                && is_allowed_openai_codex_path(&self.path)
430            || provider == "anthropic"
431                && self.method == ProxyMethod::Get
432                && is_allowed_anthropic_path(&self.path);
433        if !oauth_path_allowed && !allowed_proxy_paths(provider).contains(&self.path.as_str()) {
434            return Err(BrokerError::Denied(format!(
435                "proxy path '{}' is not in the provider's endpoint allowlist",
436                self.path
437            )));
438        }
439        // Size cap over the effective wire body. The exact-byte handoff is
440        // authoritative whenever present — even if `body` is unset, those
441        // bytes are what `LocalBroker` would send upstream, so they must
442        // never bypass the cap (fail closed).
443        let size = match (&self.body_bytes, &self.body) {
444            // Exact-byte handoff: the bytes ARE the wire body.
445            (Some(bytes), _) => Some(bytes.len()),
446            (None, Some(body)) => Some(
447                serde_json::to_vec(body)
448                    .map(|v| v.len())
449                    .unwrap_or(usize::MAX),
450            ),
451            (None, None) => None,
452        };
453        if let Some(size) = size {
454            if size > MAX_PROXY_REQUEST_BYTES {
455                return Err(BrokerError::Denied(format!(
456                    "request body exceeds the {MAX_PROXY_REQUEST_BYTES}-byte broker limit"
457                )));
458            }
459        }
460        // Coherence invariant (debug/test only — never a release hot-path
461        // re-serialization): when both representations are present,
462        // `body_bytes` MUST parse back to exactly `body`. A divergence
463        // means a caller bypassed [`ProxyRequest::post_json_exact`] and the
464        // digested bytes would not describe the JSON the broker validated.
465        #[cfg(any(test, debug_assertions))]
466        if let (Some(bytes), Some(body)) = (&self.body_bytes, &self.body) {
467            let parsed = serde_json::from_slice::<serde_json::Value>(bytes).ok();
468            if parsed.as_ref() != Some(body) {
469                return Err(BrokerError::Denied(
470                    "body_bytes does not match the JSON body (exact-byte handoff incoherent)"
471                        .into(),
472                ));
473            }
474        }
475        Ok(())
476    }
477
478    /// Build a POST proxy request whose JSON body is serialized **exactly
479    /// once**: the returned [`bytes::Bytes`] handle shares the same buffer
480    /// stored in `body_bytes` (ref-counted clone, no copy), so a
481    /// caller-side digest over it is a digest of the true wire body on the
482    /// local-broker path. This is the sanctioned way to populate
483    /// `body_bytes` — going through it makes a `body`/`body_bytes`
484    /// semantic divergence unconstructible, and it never re-serializes on
485    /// the release hot path.
486    pub fn post_json_exact(
487        provider: impl Into<String>,
488        path: impl Into<String>,
489        body: serde_json::Value,
490        stream: bool,
491    ) -> Result<(Self, bytes::Bytes), BrokerError> {
492        let bytes =
493            bytes::Bytes::from(serde_json::to_vec(&body).map_err(|e| {
494                BrokerError::Denied(format!("request body failed to serialize: {e}"))
495            })?);
496        let request = Self {
497            provider: provider.into(),
498            method: ProxyMethod::Post,
499            path: path.into(),
500            body: Some(body),
501            stream,
502            body_bytes: Some(bytes.clone()),
503        };
504        Ok((request, bytes))
505    }
506
507    /// Pin the request to an explicit account slot (`<provider>@<label>` /
508    /// `<provider>@default`), replacing any previous pin.
509    pub fn with_account(mut self, account: &Account) -> Self {
510        let base = self
511            .provider
512            .split_once('@')
513            .map(|(p, _)| p.to_string())
514            .unwrap_or_else(|| self.provider.clone());
515        self.provider = format!("{base}@{}", account.label_str());
516        self
517    }
518
519    /// Split `provider` into the base provider key and the explicit account
520    /// pin, validating the label. Static/local providers never carry a pin.
521    pub fn split_provider(&self) -> Result<(String, Option<Account>), BrokerError> {
522        match self.provider.split_once('@') {
523            None => Ok((self.provider.clone(), None)),
524            Some((base, account)) => {
525                if base.parse::<OAuthProviderId>().is_err() {
526                    return Err(BrokerError::UnknownProvider(self.provider.clone()));
527                }
528                let account = Account::parse(account).map_err(BrokerError::InvalidAccount)?;
529                Ok((base.to_string(), Some(account)))
530            }
531        }
532    }
533
534    /// Base provider key without any account pin (after validation).
535    pub fn base_provider(&self) -> Result<String, BrokerError> {
536        Ok(self.split_provider()?.0)
537    }
538
539    /// The explicit account pin, if any. `None` → the policy decides.
540    pub fn explicit_account(&self) -> Result<Option<Account>, BrokerError> {
541        Ok(self.split_provider()?.1)
542    }
543}
544
545/// Buffered (non-streaming) proxy result: upstream status + body text.
546#[derive(Debug, Clone, Serialize, Deserialize)]
547pub struct ProxyResponse {
548    pub status: u16,
549    pub body: String,
550}
551
552/// Streaming proxy result: raw upstream bytes (SSE), post-authorization.
553pub type ProxyByteStream = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, BrokerError>> + Send>>;
554
555/// What kind of credential a provider uses. Never carries key material.
556#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
557#[serde(rename_all = "snake_case")]
558pub enum CredentialKind {
559    OAuth,
560    StaticKey,
561    LocalEndpoint,
562}
563
564/// Non-secret capability/status row for one provider.
565#[derive(Debug, Clone, Serialize, Deserialize)]
566pub struct ProviderStatus {
567    pub key: String,
568    pub name: String,
569    pub kind: CredentialKind,
570    /// Whether a credential is available through the broker (for OAuth rows:
571    /// the policy-selected account exists, or any account under `auto`).
572    pub configured: bool,
573    /// Stored account slots (OAuth providers only; non-secret rows).
574    #[serde(default, skip_serializing_if = "Vec::is_empty")]
575    pub accounts: Vec<AccountSummary>,
576}
577
578/// Non-secret display status for one static key (settings UI).
579#[derive(Debug, Clone, PartialEq, Eq)]
580pub enum StaticKeyStatus {
581    NotSet,
582    /// Configured via broker storage or login config; carries a masked,
583    /// non-reversible preview like `gsk-…7f2a`.
584    Configured {
585        masked: String,
586    },
587    /// Available from the process environment (behind the broker boundary).
588    FromEnv,
589}
590
591impl StaticKeyStatus {
592    pub fn is_configured(&self) -> bool {
593        !matches!(self, StaticKeyStatus::NotSet)
594    }
595}
596
597// ── The broker contract ──────────────────────────────────────────────────────
598
599/// Normalized dynamic cloud model returned across local/remote broker boundaries.
600#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
601pub struct CloudCatalogEntry {
602    pub provider: CloudProviderId,
603    pub id: String,
604    pub display_name: String,
605    /// Opaque broker route identifier; never a provider, account, project, or resource name.
606    pub context_ref: String,
607    pub context_label: String,
608    pub stale: bool,
609    /// Unix epoch milliseconds when this catalog snapshot was fetched.
610    pub fetched_at: u64,
611}
612
613/// Credential-free normalized cloud invocation event.
614#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
615#[serde(tag = "type", rename_all = "snake_case")]
616pub enum CloudEvent {
617    TextDelta {
618        delta: String,
619    },
620    ToolArguments {
621        id: String,
622        name: Option<String>,
623        delta: String,
624    },
625    Usage {
626        input_tokens: u64,
627        output_tokens: u64,
628    },
629    Done,
630}
631pub type CloudEventStream = Pin<Box<dyn Stream<Item = Result<CloudEvent, BrokerError>> + Send>>;
632
633/// The typed credential broker protocol. One implementation runs in-process
634/// ([`LocalBroker`]) so normal local use needs no daemon; the other talks to a
635/// remote `synaps auth-broker` over authenticated HTTP(S) ([`RemoteBroker`]).
636#[async_trait]
637pub trait CredentialBroker: Send + Sync {
638    /// Vend a fresh OAuth access token (token + expiry ONLY) for the
639    /// policy-selected account of `provider`.
640    async fn access_token(&self, provider: OAuthProviderId) -> Result<AccessToken, BrokerError>;
641
642    /// Effective account selector for `provider` (env > config > injected
643    /// policy). No I/O beyond reading configuration; no secrets.
644    fn account_selector(&self, provider: OAuthProviderId) -> AccountSelector {
645        let _ = provider;
646        AccountSelector::Account(Account::Default)
647    }
648
649    /// Vend a token for an explicitly addressed credential. The default
650    /// implementation serves only the default slot (via [`access_token`]);
651    /// a named slot is an error — NEVER a fallback to another account.
652    ///
653    /// [`access_token`]: Self::access_token
654    async fn access_token_for(&self, cred: &CredentialRef) -> Result<AccessToken, BrokerError> {
655        match &cred.account {
656            Account::Default => self.access_token(cred.provider).await,
657            Account::Named(label) => Err(BrokerError::UnsupportedAccount {
658                provider: cred.provider.as_str().to_string(),
659                label: label.as_str().to_string(),
660            }),
661        }
662    }
663
664    /// Re-vend the exact credential whose access token was rejected. Local
665    /// brokers force refresh only if storage still holds that rejected token.
666    /// Remote/custom brokers keep their ordinary explicit-slot semantics.
667    async fn recover_rejected_token(
668        &self,
669        cred: &CredentialRef,
670        _rejected: &str,
671    ) -> Result<AccessToken, BrokerError> {
672        self.access_token_for(cred).await
673    }
674
675    /// Policy-resolved pair: the credential the broker selected for
676    /// `provider` and its token. Default implementation: the default slot.
677    async fn access_token_pinned(
678        &self,
679        provider: OAuthProviderId,
680    ) -> Result<PinnedToken, BrokerError> {
681        let token = self.access_token(provider).await?;
682        Ok(PinnedToken {
683            credential: CredentialRef::default_for(provider),
684            token,
685        })
686    }
687
688    /// Model-aware variant of [`access_token_pinned`]: under an `auto`
689    /// selector only accounts with proven capacity *for `model`* are
690    /// eligible. Default implementation ignores the model.
691    ///
692    /// [`access_token_pinned`]: Self::access_token_pinned
693    async fn access_token_pinned_for(
694        &self,
695        provider: OAuthProviderId,
696        model: Option<&str>,
697    ) -> Result<PinnedToken, BrokerError> {
698        let _ = model;
699        self.access_token_pinned(provider).await
700    }
701
702    /// [`access_token_pinned_for`] that never answers with an account in
703    /// `exclude` (the accounts one logical request already used). Default
704    /// implementation: the ordinary selection, with an excluded answer
705    /// refused as "no account available"; a broker that can pass the
706    /// exclusion into its own selection overrides this.
707    ///
708    /// [`access_token_pinned_for`]: Self::access_token_pinned_for
709    async fn access_token_pinned_excluding(
710        &self,
711        provider: OAuthProviderId,
712        model: Option<&str>,
713        exclude: &[CredentialRef],
714    ) -> Result<PinnedToken, BrokerError> {
715        let pinned = self.access_token_pinned_for(provider, model).await?;
716        if exclude.contains(&pinned.credential) {
717            return Err(BrokerError::NoAccountAvailable {
718                provider: provider.as_str().to_string(),
719                reason: format!(
720                    "the broker chose {}, which this request already used",
721                    pinned.credential.account
722                ),
723                failure: None,
724            });
725        }
726        Ok(pinned)
727    }
728
729    /// Non-secret account listing for `provider`. Default: unsupported.
730    async fn accounts(&self, provider: OAuthProviderId) -> Result<Vec<AccountSummary>, BrokerError> {
731        Err(BrokerError::UnsupportedCapability {
732            provider: provider.as_str().to_string(),
733            capability: "accounts".into(),
734        })
735    }
736
737    /// Typed, read-only usage snapshot for one credential. The token is
738    /// resolved and used behind the boundary; callers receive normalized,
739    /// secret-free data only. Default: unsupported.
740    async fn usage(&self, cred: &CredentialRef) -> Result<super::usage::UsageSnapshot, BrokerError> {
741        Err(BrokerError::UnsupportedCapability {
742            provider: cred.provider.as_str().to_string(),
743            capability: "usage".into(),
744        })
745    }
746
747    /// Report that `cred` hit a provider limit so automatic selection stops
748    /// advertising it until `until_ms` (epoch ms; `None` = a short default).
749    /// Best effort; default implementation is a no-op.
750    async fn report_cooldown(
751        &self,
752        cred: &CredentialRef,
753        until_ms: Option<u64>,
754        reason: &str,
755    ) -> Result<(), BrokerError> {
756        let _ = (cred, until_ms, reason);
757        Ok(())
758    }
759
760    /// When `cred`'s active cooldown ends (epoch ms), if this broker knows
761    /// of one. No network request and no token refresh. Default: none.
762    fn active_cooldown(&self, cred: &CredentialRef) -> Option<u64> {
763        let _ = cred;
764        None
765    }
766
767    /// A request sent on `cred` at `sent_at_ms` (epoch ms) was accepted:
768    /// drop its cooldown if that was reported no later than the request
769    /// left (the same millisecond counts: the caller saw the limit first), so
770    /// a limit that has passed does not keep moving turns off the account.
771    /// A cooldown reported later than `sent_at_ms` stays (that request proves
772    /// nothing about it). Memory only. Default: no-op.
773    fn clear_cooldown(&self, cred: &CredentialRef, sent_at_ms: u64) {
774        let _ = (cred, sent_at_ms);
775    }
776
777    /// Re-vend `cred` for automatic selection (a turn keeping the account
778    /// it is on). Unlike [`access_token_for`], which serves an account the
779    /// user chose, this honours and records a refused refresh's hold, the
780    /// same as the automatic choice's own vend. Default:
781    /// [`access_token_for`].
782    ///
783    /// [`access_token_for`]: Self::access_token_for
784    async fn access_token_for_auto(&self, cred: &CredentialRef) -> Result<AccessToken, BrokerError> {
785        self.access_token_for(cred).await
786    }
787
788    /// Execute a non-streaming provider request with the broker-owned key.
789    async fn proxy(&self, request: ProxyRequest) -> Result<ProxyResponse, BrokerError>;
790
791    /// Execute a streaming (SSE) provider request with the broker-owned key.
792    /// Returns the byte stream only when the upstream response is successful;
793    /// error statuses are read (bounded) and surfaced as [`BrokerError`].
794    async fn proxy_stream(&self, request: ProxyRequest) -> Result<ProxyByteStream, BrokerError>;
795
796    /// Typed operation: fetch the Anthropic account-usage summary.
797    ///
798    /// The destination URL is pinned broker-side and the OAuth access token
799    /// is resolved and attached behind the boundary — callers receive usage
800    /// JSON only and never see a token or `auth.json`. This is deliberately
801    /// NOT a generic OAuth proxy: one operation, one fixed endpoint.
802    async fn anthropic_usage(&self) -> Result<serde_json::Value, BrokerError>;
803
804    /// Typed cloud catalog. Credentials and provider authority remain broker-owned.
805    async fn cloud_catalog(
806        &self,
807        provider: CloudProviderId,
808        context_ref: &str,
809        allow_stale: bool,
810    ) -> Result<Vec<CloudCatalogEntry>, BrokerError> {
811        let _ = (context_ref, allow_stale);
812        Err(BrokerError::NotConfigured(provider.to_string()))
813    }
814
815    /// Typed cloud invocation. The canonical model and normalized request are the
816    /// only caller-controlled inputs; hosts, auth and signing remain broker-owned.
817    async fn cloud_invoke(
818        &self,
819        provider: CloudProviderId,
820        context_ref: &str,
821        model_id: &str,
822        request: InvokeRequest,
823    ) -> Result<CloudEventStream, BrokerError> {
824        let _ = (context_ref, model_id, request);
825        Err(BrokerError::NotConfigured(provider.to_string()))
826    }
827
828    /// Non-secret provider capability/status list.
829    async fn capabilities(&self) -> Result<Vec<ProviderStatus>, BrokerError>;
830}
831
832// ── Local (in-process) broker ────────────────────────────────────────────────
833
834#[async_trait]
835pub trait CloudBackend: Send + Sync {
836    async fn catalog(
837        &self,
838        provider: CloudProviderId,
839        context_ref: &str,
840        allow_stale: bool,
841    ) -> Result<Vec<CloudCatalogEntry>, BrokerError>;
842    async fn invoke(
843        &self,
844        provider: CloudProviderId,
845        context_ref: &str,
846        model_id: &str,
847        request: InvokeRequest,
848    ) -> Result<CloudEventStream, BrokerError>;
849}
850
851#[derive(Clone)]
852struct ProductionCloudBackend {
853    /// Dedicated credential-bearing client: redirects are always disabled and
854    /// both connect and whole-request time are bounded, independent of callers.
855    http: reqwest::Client,
856    refresh_lock: Arc<tokio::sync::Mutex<()>>,
857}
858impl ProductionCloudBackend {
859    fn new(_http: reqwest::Client) -> Self {
860        let http = reqwest::Client::builder()
861            .redirect(reqwest::redirect::Policy::none())
862            .connect_timeout(Duration::from_secs(10))
863            .timeout(Duration::from_secs(90))
864            .build()
865            .expect("cloud HTTPS client configuration is valid");
866        Self {
867            http,
868            refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
869        }
870    }
871
872    fn state(&self, provider: CloudProviderId) -> Result<serde_json::Value, BrokerError> {
873        storage::load_cloud_state(provider.as_str())
874            .map_err(BrokerError::Credential)?
875            .ok_or_else(|| BrokerError::NotConfigured(provider.to_string()))
876    }
877
878    fn context_ref(&self, provider: CloudProviderId) -> Result<String, BrokerError> {
879        use sha2::{Digest, Sha256};
880        let state = self.state(provider)?;
881        let public = serde_json::to_vec(&state["config"])
882            .map_err(|_| BrokerError::Credential("invalid cloud context".into()))?;
883        let digest = Sha256::digest([provider.as_str().as_bytes(), &public].concat());
884        Ok(format!(
885            "ctx-{}",
886            digest[..16]
887                .iter()
888                .map(|b| format!("{b:02x}"))
889                .collect::<String>()
890        ))
891    }
892
893    fn validate_context(
894        &self,
895        provider: CloudProviderId,
896        supplied: &str,
897    ) -> Result<String, BrokerError> {
898        let opaque = self.context_ref(provider)?;
899        // The canonical provider id is accepted only as the catalog bootstrap
900        // selector. Every returned entry and persisted model route uses opaque.
901        if supplied == provider.as_str() || supplied == opaque {
902            Ok(opaque)
903        } else {
904            Err(BrokerError::Denied(
905                "cloud context does not match stored provider".into(),
906            ))
907        }
908    }
909
910    async fn aws(
911        &self,
912    ) -> Result<super::aws_bedrock::AwsBedrockBroker<super::aws_bedrock::AwsHttpApi>, BrokerError>
913    {
914        #[derive(Deserialize)]
915        struct State {
916            config: super::cloud::AwsBedrockConfig,
917            access_key: String,
918            secret_key: String,
919            session_token: String,
920            expires_at: u64,
921            registered_client: serde_json::Value,
922            sso_access_token: String,
923            sso_refresh_token: Option<String>,
924            sso_expires_at: u64,
925        }
926        let decode = |value| {
927            serde_json::from_value::<State>(value)
928                .map_err(|_| BrokerError::Credential("invalid AWS broker state".into()))
929        };
930        let mut raw = self.state(CloudProviderId::AwsBedrock)?;
931        let now = std::time::SystemTime::now()
932            .duration_since(std::time::UNIX_EPOCH)
933            .unwrap_or_default()
934            .as_millis() as u64;
935        let mut s = decode(raw.clone())?;
936        if s.expires_at <= now + 60_000 {
937            // Role credentials are shared mutable broker state. Serialize refresh,
938            // then reload so waiters consume the winner's atomic commit.
939            let _guard = self.refresh_lock.lock().await;
940            raw = self.state(CloudProviderId::AwsBedrock)?;
941            s = decode(raw.clone())?;
942            if s.expires_at <= now + 60_000 {
943                let api =
944                    super::aws_bedrock::AwsHttpApi::new(self.http.clone(), &s.config.sso_region);
945                if s.sso_expires_at <= now + 60_000 {
946                    let refresh = s
947                        .sso_refresh_token
948                        .as_deref()
949                        .filter(|v| !v.is_empty())
950                        .ok_or_else(|| {
951                            BrokerError::Credential("AWS SSO session expired; login again".into())
952                        })?;
953                    let client = super::aws_bedrock::RegisteredClient::new(
954                        s.registered_client["id"].as_str().ok_or_else(|| {
955                            BrokerError::Credential("invalid AWS client registration".into())
956                        })?,
957                        s.registered_client["secret"].as_str().ok_or_else(|| {
958                            BrokerError::Credential("invalid AWS client registration".into())
959                        })?,
960                        s.registered_client["expires_at"].as_u64().ok_or_else(|| {
961                            BrokerError::Credential("invalid AWS client registration".into())
962                        })?,
963                    );
964                    if client.expires_at <= now / 1000 {
965                        return Err(BrokerError::Credential(
966                            "AWS client registration expired; login again".into(),
967                        ));
968                    }
969                    use super::aws_bedrock::AwsApi;
970                    let token = api
971                        .create_token(
972                            &client,
973                            &s.config.sso_region,
974                            super::aws_bedrock::TokenGrant::RefreshToken(refresh),
975                        )
976                        .await
977                        .map_err(|_| {
978                            BrokerError::Credential("AWS SSO refresh rejected; login again".into())
979                        })?;
980                    raw["sso_access_token"] = token.access().into();
981                    if let Some(rotated) = token.refresh() {
982                        raw["sso_refresh_token"] = rotated.into();
983                    }
984                    raw["sso_expires_at"] = (now + token.expires_in * 1000).into();
985                    // A refresh token is single-use and may rotate. Commit the
986                    // refreshed SSO session before the independent role fetch;
987                    // otherwise a transient GetRoleCredentials failure loses
988                    // the only usable token and strands the persisted login.
989                    storage::save_cloud_state("aws-bedrock", &raw)
990                        .map_err(BrokerError::Credential)?;
991                    s = decode(raw.clone())?;
992                }
993                use super::aws_bedrock::AwsApi;
994                let credentials = api
995                    .get_role_credentials(
996                        &s.config.sso_region,
997                        &s.sso_access_token,
998                        &s.config.account_id,
999                        &s.config.role_name,
1000                    )
1001                    .await
1002                    .map_err(|_| {
1003                        BrokerError::Credential("AWS role refresh rejected; login again".into())
1004                    })?;
1005                raw["access_key"] = credentials.access_key().into();
1006                raw["secret_key"] = credentials.secret_key().into();
1007                raw["session_token"] = credentials.session_token().into();
1008                raw["expires_at"] = credentials.expires_at.into();
1009                storage::save_cloud_state("aws-bedrock", &raw).map_err(BrokerError::Credential)?;
1010                s = decode(raw)?;
1011            }
1012        }
1013        let api = super::aws_bedrock::AwsHttpApi::new(self.http.clone(), &s.config.sso_region);
1014        Ok(super::aws_bedrock::AwsBedrockBroker::from_credentials(
1015            api,
1016            s.config,
1017            super::aws_bedrock::RoleCredentials::new(
1018                s.access_key,
1019                s.secret_key,
1020                s.session_token,
1021                s.expires_at,
1022            ),
1023        ))
1024    }
1025    async fn azure_request(
1026        &self,
1027        state: &mut serde_json::Value,
1028        audience: super::azure_openai::AzureAudience,
1029    ) -> Result<String, BrokerError> {
1030        let now = std::time::SystemTime::now()
1031            .duration_since(std::time::UNIX_EPOCH)
1032            .unwrap_or_default()
1033            .as_millis() as u64;
1034        let key = match audience {
1035            super::azure_openai::AzureAudience::Arm => "arm",
1036            super::azure_openai::AzureAudience::Inference => "inference",
1037        };
1038        if state[key]["expires_at"].as_u64().unwrap_or(0) > now + 60_000 {
1039            return state[key]["access_token"]
1040                .as_str()
1041                .map(str::to_owned)
1042                .ok_or_else(|| BrokerError::Credential("invalid Azure token state".into()));
1043        }
1044        // Serialize refresh and re-read after waiting so concurrent catalog and
1045        // invocation calls reuse the single atomically persisted rotation.
1046        let _refresh = self.refresh_lock.lock().await;
1047        *state = self.state(CloudProviderId::AzureOpenAi)?;
1048        if state[key]["expires_at"].as_u64().unwrap_or(0) > now + 60_000 {
1049            return state[key]["access_token"]
1050                .as_str()
1051                .map(str::to_owned)
1052                .ok_or_else(|| BrokerError::Credential("invalid Azure token state".into()));
1053        }
1054        let config: super::cloud::AzureOpenAiConfig =
1055            serde_json::from_value(state["config"].clone())
1056                .map_err(|_| BrokerError::Credential("invalid Azure broker state".into()))?;
1057        let client_id = state["client_id"]
1058            .as_str()
1059            .ok_or_else(|| BrokerError::Credential("invalid Azure client registration".into()))?;
1060        let refresh = state["refresh_token"]
1061            .as_str()
1062            .ok_or_else(|| BrokerError::Credential("invalid Azure refresh state".into()))?;
1063        let reg = super::azure_openai::AzureRegistration::production(Some(client_id.into()))
1064            .map_err(|e| BrokerError::Credential(e.to_string()))?;
1065        let r = super::azure_openai::refresh_request(&config, &reg, audience, refresh)
1066            .map_err(|e| BrokerError::Credential(e.to_string()))?;
1067        let response = self
1068            .http
1069            .post(r.url)
1070            .form(&r.form)
1071            .send()
1072            .await
1073            .map_err(|_| BrokerError::Transport("Azure refresh failed".into()))?;
1074        if !response.status().is_success() {
1075            return Err(BrokerError::Credential(
1076                "Azure refresh rejected; login again".into(),
1077            ));
1078        }
1079        let wire: serde_json::Value = response
1080            .json()
1081            .await
1082            .map_err(|_| BrokerError::Transport("invalid Azure token response".into()))?;
1083        let access = wire["access_token"]
1084            .as_str()
1085            .filter(|v| !v.is_empty())
1086            .ok_or_else(|| BrokerError::Transport("invalid Azure token response".into()))?
1087            .to_owned();
1088        if let Some(r) = wire["refresh_token"].as_str().filter(|v| !v.is_empty()) {
1089            state["refresh_token"] = r.into();
1090        }
1091        state[key] = serde_json::json!({"access_token": access, "expires_at": now + wire["expires_in"].as_u64().unwrap_or(3600) * 1000});
1092        storage::save_cloud_state("azure-openai", state).map_err(BrokerError::Credential)?;
1093        Ok(access)
1094    }
1095
1096    async fn vertex_request(&self, state: &mut serde_json::Value) -> Result<String, BrokerError> {
1097        let now = std::time::SystemTime::now()
1098            .duration_since(std::time::UNIX_EPOCH)
1099            .unwrap_or_default()
1100            .as_millis() as u64;
1101        if state["expires_at"].as_u64().unwrap_or(0) > now + 60_000 {
1102            return state["access_token"]
1103                .as_str()
1104                .map(str::to_owned)
1105                .ok_or_else(|| BrokerError::Credential("invalid Vertex token state".into()));
1106        }
1107        let _refresh = self.refresh_lock.lock().await;
1108        *state = self.state(CloudProviderId::GoogleVertex)?;
1109        if state["expires_at"].as_u64().unwrap_or(0) > now + 60_000 {
1110            return state["access_token"]
1111                .as_str()
1112                .map(str::to_owned)
1113                .ok_or_else(|| BrokerError::Credential("invalid Vertex token state".into()));
1114        }
1115        let client_id = state["client_id"]
1116            .as_str()
1117            .ok_or_else(|| BrokerError::Credential("invalid Vertex registration".into()))?;
1118        let refresh = state["refresh_token"]
1119            .as_str()
1120            .ok_or_else(|| BrokerError::Credential("invalid Vertex refresh state".into()))?;
1121        let response = self
1122            .http
1123            .post(super::google_vertex::TOKEN_URL)
1124            .form(&[
1125                ("client_id", client_id),
1126                ("grant_type", "refresh_token"),
1127                ("refresh_token", refresh),
1128            ])
1129            .send()
1130            .await
1131            .map_err(|_| BrokerError::Transport("Vertex refresh failed".into()))?;
1132        if !response.status().is_success() {
1133            return Err(BrokerError::Credential(
1134                "Vertex refresh rejected; login again".into(),
1135            ));
1136        }
1137        let wire: serde_json::Value = response
1138            .json()
1139            .await
1140            .map_err(|_| BrokerError::Transport("invalid Vertex token response".into()))?;
1141        let access = wire["access_token"]
1142            .as_str()
1143            .filter(|v| !v.is_empty())
1144            .ok_or_else(|| BrokerError::Transport("invalid Vertex token response".into()))?
1145            .to_owned();
1146        if let Some(r) = wire["refresh_token"].as_str().filter(|v| !v.is_empty()) {
1147            state["refresh_token"] = r.into();
1148        }
1149        state["access_token"] = access.clone().into();
1150        state["expires_at"] = (now + wire["expires_in"].as_u64().unwrap_or(3600) * 1000).into();
1151        storage::save_cloud_state("google-vertex", state).map_err(BrokerError::Credential)?;
1152        Ok(access)
1153    }
1154}
1155
1156#[async_trait]
1157impl CloudBackend for ProductionCloudBackend {
1158    async fn catalog(
1159        &self,
1160        provider: CloudProviderId,
1161        context_ref: &str,
1162        _allow_stale: bool,
1163    ) -> Result<Vec<CloudCatalogEntry>, BrokerError> {
1164        let opaque_context = self.validate_context(provider, context_ref)?;
1165        let fetched_at = std::time::SystemTime::now()
1166            .duration_since(std::time::UNIX_EPOCH)
1167            .unwrap_or_default()
1168            .as_millis() as u64;
1169        match provider {
1170            CloudProviderId::AwsBedrock => {
1171                let broker = self.aws().await?;
1172                let label = broker.public_context().bedrock_region;
1173                broker
1174                    .catalog()
1175                    .await
1176                    .map_err(|_| {
1177                        BrokerError::Transport("AWS catalog failed (details redacted)".into())
1178                    })
1179                    .map(|xs| {
1180                        xs.into_iter()
1181                            .map(|x| CloudCatalogEntry {
1182                                provider,
1183                                id: x.id,
1184                                display_name: x.display_name,
1185                                context_ref: opaque_context.clone(),
1186                                context_label: label.clone(),
1187                                stale: false,
1188                                fetched_at,
1189                            })
1190                            .collect()
1191                    })
1192            }
1193            CloudProviderId::AzureOpenAi => {
1194                let mut state = self.state(provider)?;
1195                let config: super::cloud::AzureOpenAiConfig =
1196                    serde_json::from_value(state["config"].clone()).map_err(|_| {
1197                        BrokerError::Credential("invalid Azure broker state".into())
1198                    })?;
1199                let token = self
1200                    .azure_request(&mut state, super::azure_openai::AzureAudience::Arm)
1201                    .await?;
1202                let mut discovery =
1203                    super::azure_openai::DeploymentDiscovery::new(config.clone(), 20, 1000);
1204                let mut url = Some(discovery.initial_url());
1205                while let Some(next) = url {
1206                    let r = self
1207                        .http
1208                        .get(next)
1209                        .bearer_auth(&token)
1210                        .send()
1211                        .await
1212                        .map_err(|_| BrokerError::Transport("Azure catalog failed".into()))?;
1213                    if !r.status().is_success() {
1214                        return Err(BrokerError::Transport("Azure catalog rejected".into()));
1215                    }
1216                    let body = read_body_capped(r, MAX_CLOUD_CATALOG_BODY_BYTES)
1217                        .await
1218                        .map_err(|_| {
1219                            BrokerError::Transport("Azure catalog exceeded body limit".into())
1220                        })?;
1221                    url = discovery
1222                        .accept_page(&body)
1223                        .map_err(|_| BrokerError::Transport("invalid Azure catalog".into()))?;
1224                }
1225                discovery
1226                    .finish()
1227                    .map_err(|_| BrokerError::Transport("Azure catalog unavailable".into()))
1228                    .map(|xs| {
1229                        xs.into_iter()
1230                            .map(|x| CloudCatalogEntry {
1231                                provider,
1232                                id: x.id,
1233                                display_name: x.display_name,
1234                                context_ref: opaque_context.clone(),
1235                                context_label: format!(
1236                                    "{}/{}",
1237                                    config.resource_name, config.resource_group
1238                                ),
1239                                stale: false,
1240                                fetched_at,
1241                            })
1242                            .collect()
1243                    })
1244            }
1245            CloudProviderId::GoogleVertex => {
1246                let mut state = self.state(provider)?;
1247                let token = self.vertex_request(&mut state).await?;
1248                let config: super::cloud::GoogleVertexConfig =
1249                    serde_json::from_value(state["config"].clone()).map_err(|_| {
1250                        BrokerError::Credential("invalid Vertex broker state".into())
1251                    })?;
1252                let mut page: Option<String> = None;
1253                let mut out = Vec::new();
1254                let mut seen = std::collections::HashSet::new();
1255                for _ in 0..20 {
1256                    let mut url=format!("https://{}-aiplatform.googleapis.com/v1/projects/{}/locations/{}/publishers/google/models",config.location,config.project_id,config.location);
1257                    if let Some(p) = &page {
1258                        let mut parsed = url::Url::parse(&url).map_err(|_| {
1259                            BrokerError::Transport("invalid Vertex catalog URL".into())
1260                        })?;
1261                        parsed.query_pairs_mut().append_pair("pageToken", p);
1262                        url = parsed.into();
1263                    }
1264                    let r = self
1265                        .http
1266                        .get(url)
1267                        .bearer_auth(&token)
1268                        .send()
1269                        .await
1270                        .map_err(|_| BrokerError::Transport("Vertex catalog failed".into()))?;
1271                    if !r.status().is_success() {
1272                        return Err(BrokerError::Transport("Vertex catalog rejected".into()));
1273                    }
1274                    let v = cloud_catalog_json(r).await?;
1275                    for m in v["publisherModels"].as_array().into_iter().flatten() {
1276                        if let Some(name) = m["name"]
1277                            .as_str()
1278                            .filter(|n| n.starts_with("publishers/google/models/"))
1279                        {
1280                            if out.len() >= MAX_CLOUD_CATALOG_ENTRIES {
1281                                return Err(BrokerError::Transport(
1282                                    "Vertex catalog exceeded entry limit".into(),
1283                                ));
1284                            }
1285                            if !out.iter().any(|entry: &CloudCatalogEntry| {
1286                                entry.id == format!("google-vertex/{name}")
1287                            }) {
1288                                out.push(CloudCatalogEntry {
1289                                    provider,
1290                                    id: format!("google-vertex/{name}"),
1291                                    display_name: m["displayName"].as_str().unwrap_or(name).into(),
1292                                    context_ref: opaque_context.clone(),
1293                                    context_label: format!(
1294                                        "{}/{}",
1295                                        config.project_id, config.location
1296                                    ),
1297                                    stale: false,
1298                                    fetched_at,
1299                                });
1300                            }
1301                        }
1302                    }
1303                    page = v["nextPageToken"].as_str().map(str::to_owned);
1304                    match &page {
1305                        Some(p) if seen.insert(p.clone()) => {}
1306                        Some(_) => {
1307                            return Err(BrokerError::Transport("Vertex pagination loop".into()))
1308                        }
1309                        None => break,
1310                    }
1311                }
1312                if out.is_empty() {
1313                    Err(BrokerError::Transport("Vertex catalog is empty".into()))
1314                } else {
1315                    Ok(out)
1316                }
1317            }
1318        }
1319    }
1320    async fn invoke(
1321        &self,
1322        provider: CloudProviderId,
1323        context_ref: &str,
1324        model_id: &str,
1325        request: InvokeRequest,
1326    ) -> Result<CloudEventStream, BrokerError> {
1327        self.validate_context(provider, context_ref)?;
1328        if !request.tools.is_empty() {
1329            return Err(BrokerError::Denied(
1330                "tools are not yet supported by cloud providers".into(),
1331            ));
1332        }
1333        let authorized = self.catalog(provider, context_ref, false).await?;
1334        if !authorized.iter().any(|entry| entry.id == model_id) {
1335            return Err(BrokerError::Denied(
1336                "model is not present in the current provider catalog".into(),
1337            ));
1338        }
1339        match provider {
1340            CloudProviderId::AwsBedrock => {
1341                let broker = self.aws().await?;
1342                if request.stream {
1343                    use futures::StreamExt;
1344                    let stream = broker
1345                        .converse_stream(model_id, request)
1346                        .await
1347                        .map_err(|_| {
1348                            BrokerError::Transport(
1349                                "AWS invocation failed (details redacted)".into(),
1350                            )
1351                        })?
1352                        .map(|event| {
1353                            event
1354                                .map_err(|_| {
1355                                    BrokerError::Transport(
1356                                        "AWS stream failed (details redacted)".into(),
1357                                    )
1358                                })
1359                                .map(|e| match e {
1360                                    super::aws_bedrock::ConverseEvent::TextDelta(delta) => {
1361                                        CloudEvent::TextDelta { delta }
1362                                    }
1363                                    super::aws_bedrock::ConverseEvent::ToolArguments {
1364                                        id,
1365                                        delta,
1366                                    } => CloudEvent::ToolArguments {
1367                                        id,
1368                                        name: None,
1369                                        delta,
1370                                    },
1371                                    super::aws_bedrock::ConverseEvent::Usage(u) => {
1372                                        CloudEvent::Usage {
1373                                            input_tokens: u.input_tokens,
1374                                            output_tokens: u.output_tokens,
1375                                        }
1376                                    }
1377                                    super::aws_bedrock::ConverseEvent::Done => CloudEvent::Done,
1378                                })
1379                        });
1380                    Ok(Box::pin(stream))
1381                } else {
1382                    let o = broker.converse(model_id, request).await.map_err(|_| {
1383                        BrokerError::Transport("AWS invocation failed (details redacted)".into())
1384                    })?;
1385                    Ok(Box::pin(futures::stream::iter(vec![
1386                        Ok(CloudEvent::TextDelta { delta: o.text }),
1387                        Ok(CloudEvent::Usage {
1388                            input_tokens: o.usage.input_tokens,
1389                            output_tokens: o.usage.output_tokens,
1390                        }),
1391                        Ok(CloudEvent::Done),
1392                    ])))
1393                }
1394            }
1395            CloudProviderId::AzureOpenAi => {
1396                let mut state = self.state(provider)?;
1397                let token = self
1398                    .azure_request(&mut state, super::azure_openai::AzureAudience::Inference)
1399                    .await?;
1400                let endpoint = super::azure_openai::AzureEndpoint::parse(
1401                    state["endpoint"]
1402                        .as_str()
1403                        .ok_or_else(|| BrokerError::Credential("invalid Azure endpoint".into()))?,
1404                )
1405                .map_err(|_| BrokerError::Credential("invalid Azure endpoint".into()))?;
1406                let deployment = model_id
1407                    .strip_prefix("azure-openai/")
1408                    .ok_or_else(|| BrokerError::Denied("invalid Azure model".into()))?;
1409                let body = serde_json::json!({"input":request.messages.into_iter().map(|m|serde_json::json!({"role":match m.role{super::cloud::MessageRole::Assistant=>"assistant",super::cloud::MessageRole::System=>"system",_=>"user"},"content":m.content})).collect::<Vec<_>>(),"stream":request.stream});
1410                let rr = super::azure_openai::responses_request(
1411                    &endpoint,
1412                    deployment,
1413                    &serde_json::to_vec(&body).unwrap(),
1414                )
1415                .map_err(|_| BrokerError::Denied("invalid Azure request".into()))?;
1416                let r = self
1417                    .http
1418                    .post(rr.url)
1419                    .bearer_auth(token)
1420                    .header("content-type", "application/json")
1421                    .body(rr.body)
1422                    .send()
1423                    .await
1424                    .map_err(|_| BrokerError::Transport("Azure invocation failed".into()))?;
1425                if !r.status().is_success() {
1426                    return Err(BrokerError::Transport("Azure invocation rejected".into()));
1427                }
1428                if request.stream {
1429                    use futures::StreamExt;
1430                    use std::sync::{
1431                        atomic::{AtomicBool, Ordering},
1432                        Arc,
1433                    };
1434                    let terminal = Arc::new(AtomicBool::new(false));
1435                    let seen = terminal.clone();
1436                    let events = sse_json_stream(r).filter_map(move |item| {
1437                        let seen = seen.clone();
1438                        async move {
1439                            match item {
1440                                Ok(Some(v)) => v["delta"]
1441                                    .as_str()
1442                                    .or_else(|| v["text"].as_str())
1443                                    .map(|delta| {
1444                                        Ok(CloudEvent::TextDelta {
1445                                            delta: delta.into(),
1446                                        })
1447                                    }),
1448                                Ok(None) => {
1449                                    seen.store(true, Ordering::SeqCst);
1450                                    Some(Ok(CloudEvent::Done))
1451                                }
1452                                Err(error) => Some(Err(error)),
1453                            }
1454                        }
1455                    });
1456                    let stream = events.chain(futures::stream::unfold(
1457                        (terminal, false),
1458                        |(terminal, emitted)| async move {
1459                            if terminal.load(Ordering::SeqCst) || emitted {
1460                                None
1461                            } else {
1462                                Some((
1463                                    Err(BrokerError::Transport(
1464                                        "Azure stream ended without [DONE]".into(),
1465                                    )),
1466                                    (terminal, true),
1467                                ))
1468                            }
1469                        },
1470                    ));
1471                    Ok(Box::pin(stream))
1472                } else {
1473                    let text = read_body_capped(r, MAX_CLOUD_STREAM_EVENT_BYTES).await?;
1474                    let v: serde_json::Value = serde_json::from_str(&text)
1475                        .map_err(|_| BrokerError::Transport("invalid Azure response".into()))?;
1476                    let delta = v["output_text"]
1477                        .as_str()
1478                        .ok_or_else(|| {
1479                            BrokerError::Transport("Azure response omitted output".into())
1480                        })?
1481                        .to_owned();
1482                    Ok(Box::pin(futures::stream::iter(vec![
1483                        Ok(CloudEvent::TextDelta { delta }),
1484                        Ok(CloudEvent::Done),
1485                    ])))
1486                }
1487            }
1488            CloudProviderId::GoogleVertex => {
1489                let mut state = self.state(provider)?;
1490                let token = self.vertex_request(&mut state).await?;
1491                let config: super::cloud::GoogleVertexConfig =
1492                    serde_json::from_value(state["config"].clone())
1493                        .map_err(|_| BrokerError::Credential("invalid Vertex state".into()))?;
1494                let model = model_id
1495                    .strip_prefix("google-vertex/publishers/google/models/")
1496                    .filter(|m| {
1497                        m.bytes()
1498                            .all(|b| b.is_ascii_alphanumeric() || b"-._".contains(&b))
1499                    })
1500                    .ok_or_else(|| BrokerError::Denied("invalid Vertex model".into()))?;
1501                let action = if request.stream {
1502                    "streamGenerateContent?alt=sse"
1503                } else {
1504                    "generateContent"
1505                };
1506                let url=format!("https://{}-aiplatform.googleapis.com/v1/projects/{}/locations/{}/publishers/google/models/{}:{}",config.location,config.project_id,config.location,model,action);
1507                let body = serde_json::json!({"contents":request.messages.into_iter().map(|m|serde_json::json!({"role":if matches!(m.role,super::cloud::MessageRole::Assistant){"model"}else{"user"},"parts":[{"text":m.content}]})).collect::<Vec<_>>()});
1508                let r = self
1509                    .http
1510                    .post(url)
1511                    .bearer_auth(token)
1512                    .json(&body)
1513                    .send()
1514                    .await
1515                    .map_err(|_| BrokerError::Transport("Vertex invocation failed".into()))?;
1516                if !r.status().is_success() {
1517                    return Err(BrokerError::Transport("Vertex invocation rejected".into()));
1518                }
1519                if request.stream {
1520                    use futures::StreamExt;
1521                    let events = sse_json_stream(r).map(|item| {
1522                        item.and_then(|value| {
1523                            value.ok_or_else(|| {
1524                                BrokerError::Transport(
1525                                    "Vertex stream terminated without a provider event".into(),
1526                                )
1527                            })
1528                        })
1529                    });
1530                    use std::sync::{
1531                        atomic::{AtomicBool, Ordering},
1532                        Arc,
1533                    };
1534                    let terminal = Arc::new(AtomicBool::new(false));
1535                    let terminal_in_events = terminal.clone();
1536                    let stream = events
1537                        .flat_map(move |item| {
1538                            let mut out = Vec::new();
1539                            match item {
1540                                Ok(v) => {
1541                                    if terminal_in_events.load(Ordering::SeqCst) {
1542                                        out.push(Err(BrokerError::Transport(
1543                                            "Vertex emitted data after terminal metadata".into(),
1544                                        )));
1545                                        return futures::stream::iter(out);
1546                                    }
1547                                    for c in v["candidates"].as_array().into_iter().flatten() {
1548                                        for p in
1549                                            c["content"]["parts"].as_array().into_iter().flatten()
1550                                        {
1551                                            if let Some(delta) = p["text"].as_str() {
1552                                                out.push(Ok(CloudEvent::TextDelta {
1553                                                    delta: delta.into(),
1554                                                }));
1555                                            }
1556                                        }
1557                                        if let Some(reason) = c["finishReason"].as_str() {
1558                                            const VALID: &[&str] = &[
1559                                                "STOP",
1560                                                "MAX_TOKENS",
1561                                                "SAFETY",
1562                                                "RECITATION",
1563                                                "OTHER",
1564                                                "BLOCKLIST",
1565                                                "PROHIBITED_CONTENT",
1566                                                "SPII",
1567                                                "MALFORMED_FUNCTION_CALL",
1568                                            ];
1569                                            if !VALID.contains(&reason) {
1570                                                out.push(Err(BrokerError::Transport(
1571                                                    "Vertex returned an invalid finish reason"
1572                                                        .into(),
1573                                                )));
1574                                            } else if terminal_in_events
1575                                                .swap(true, Ordering::SeqCst)
1576                                            {
1577                                                out.push(Err(BrokerError::Transport(
1578                                                    "Vertex returned duplicate terminal metadata"
1579                                                        .into(),
1580                                                )));
1581                                            } else {
1582                                                out.push(Ok(CloudEvent::Done));
1583                                            }
1584                                        }
1585                                    }
1586                                    if let Some(u) = v.get("usageMetadata") {
1587                                        out.push(Ok(CloudEvent::Usage {
1588                                            input_tokens: u["promptTokenCount"]
1589                                                .as_u64()
1590                                                .unwrap_or(0),
1591                                            output_tokens: u["candidatesTokenCount"]
1592                                                .as_u64()
1593                                                .unwrap_or(0),
1594                                        }));
1595                                    }
1596                                }
1597                                Err(error) => out.push(Err(error)),
1598                            }
1599                            futures::stream::iter(out)
1600                        })
1601                        .chain(futures::stream::unfold(
1602                            (terminal, false),
1603                            |(terminal, emitted)| async move {
1604                                if terminal.load(Ordering::SeqCst) || emitted {
1605                                    None
1606                                } else {
1607                                    Some((
1608                                        Err(BrokerError::Transport(
1609                                            "Vertex stream ended without finish metadata".into(),
1610                                        )),
1611                                        (terminal, true),
1612                                    ))
1613                                }
1614                            },
1615                        ));
1616                    Ok(Box::pin(stream))
1617                } else {
1618                    let text = read_body_capped(r, MAX_CLOUD_STREAM_EVENT_BYTES).await?;
1619                    let v: serde_json::Value = serde_json::from_str(&text)
1620                        .map_err(|_| BrokerError::Transport("invalid Vertex response".into()))?;
1621                    let mut out = Vec::new();
1622                    for c in v["candidates"].as_array().into_iter().flatten() {
1623                        for p in c["content"]["parts"].as_array().into_iter().flatten() {
1624                            if let Some(delta) = p["text"].as_str() {
1625                                out.push(Ok(CloudEvent::TextDelta {
1626                                    delta: delta.into(),
1627                                }));
1628                            }
1629                        }
1630                    }
1631                    out.push(Ok(CloudEvent::Done));
1632                    Ok(Box::pin(futures::stream::iter(out)))
1633                }
1634            }
1635        }
1636    }
1637}
1638
1639const MAX_CLOUD_CATALOG_BODY_BYTES: usize = 2 * 1024 * 1024;
1640const MAX_CLOUD_CATALOG_ENTRIES: usize = 10_000;
1641const MAX_CLOUD_STREAM_EVENT_BYTES: usize = 1024 * 1024;
1642
1643/// Incremental SSE parser. It retains only one bounded event, yields as soon as
1644/// an event delimiter arrives, and owns the response so dropping the consumer
1645/// cancels the upstream request.
1646fn sse_json_stream(
1647    resp: reqwest::Response,
1648) -> Pin<Box<dyn Stream<Item = Result<Option<serde_json::Value>, BrokerError>> + Send>> {
1649    use futures::StreamExt;
1650    let chunks = Box::pin(resp.bytes_stream());
1651    Box::pin(futures::stream::unfold(
1652        (chunks, Vec::<u8>::new(), false),
1653        |(mut chunks, mut buf, done)| async move {
1654            if done {
1655                return None;
1656            }
1657            loop {
1658                if let Some(end) = buf.windows(2).position(|w| w == b"\n\n") {
1659                    let frame: Vec<u8> = buf.drain(..end + 2).collect();
1660                    let text = match std::str::from_utf8(&frame[..end]) {
1661                        Ok(v) => v,
1662                        Err(_) => {
1663                            return Some((
1664                                Err(BrokerError::Transport("invalid SSE encoding".into())),
1665                                (chunks, buf, true),
1666                            ))
1667                        }
1668                    };
1669                    let data = text
1670                        .lines()
1671                        .filter_map(|line| line.strip_prefix("data:"))
1672                        .map(str::trim_start)
1673                        .collect::<Vec<_>>()
1674                        .join("\n");
1675                    if data.is_empty() {
1676                        continue;
1677                    }
1678                    if data == "[DONE]" {
1679                        return Some((Ok(None), (chunks, buf, true)));
1680                    }
1681                    let value = serde_json::from_str(&data)
1682                        .map_err(|_| BrokerError::Transport("invalid SSE event".into()));
1683                    let terminal = value.is_err();
1684                    return Some((value.map(Some), (chunks, buf, terminal)));
1685                }
1686                match chunks.next().await {
1687                    Some(Ok(chunk)) if buf.len() + chunk.len() <= MAX_CLOUD_STREAM_EVENT_BYTES => {
1688                        buf.extend_from_slice(&chunk)
1689                    }
1690                    Some(_) => {
1691                        return Some((
1692                            Err(BrokerError::Transport(
1693                                "cloud stream event exceeded limit".into(),
1694                            )),
1695                            (chunks, buf, true),
1696                        ))
1697                    }
1698                    None if buf.is_empty() => return None,
1699                    None => {
1700                        return Some((
1701                            Err(BrokerError::Transport("truncated SSE event".into())),
1702                            (chunks, buf, true),
1703                        ))
1704                    }
1705                }
1706            }
1707        },
1708    ))
1709}
1710
1711/// Read one catalog page without allowing an upstream to allocate an unbounded body.
1712async fn cloud_catalog_json(resp: reqwest::Response) -> Result<serde_json::Value, BrokerError> {
1713    let body = read_body_capped(resp, MAX_CLOUD_CATALOG_BODY_BYTES).await?;
1714    serde_json::from_str(&body)
1715        .map_err(|_| BrokerError::Transport("invalid cloud catalog response".into()))
1716}
1717
1718/// In-process credential broker. This module is the ONLY place runtime-serving
1719/// code may read `auth.json`, `provider.<key>` config values, or credential
1720/// environment variables.
1721pub struct LocalBroker {
1722    http: reqwest::Client,
1723    /// Test seam: overrides the local endpoint URL without env/config.
1724    local_base_url: Option<String>,
1725    /// Test seam: overrides the pinned Anthropic usage URL.
1726    anthropic_usage_url: Option<String>,
1727    /// Test seam: overrides the pinned cloudcode-pa base URL.
1728    google_gemini_base_url: Option<String>,
1729    /// Test seam: overrides the pinned ChatGPT backend base URL.
1730    openai_codex_base_url: Option<String>,
1731    /// Test seam: overrides the pinned Meta Muse API origin.
1732    muse_base_url: Option<String>,
1733    /// Test seam: refresh endpoint overrides (the Muse key mint).
1734    refresh_endpoints: super::provider::RefreshEndpoints,
1735    /// Idle (between-bytes) timeout for the per-request no-redirect client.
1736    stream_idle_timeout: Duration,
1737    /// Time budget for buffered (non-streaming) requests.
1738    request_timeout: Duration,
1739    /// Buffered response size cap.
1740    max_response_bytes: usize,
1741    cloud_backend: Option<Arc<dyn CloudBackend>>,
1742    /// Injected account policy. `None` → resolved from env + config on every
1743    /// selection (no global mutation, picks up `synaps auth use` changes).
1744    account_policy: Option<AccountPolicy>,
1745    auto_policy: Option<super::AutoSelectionPolicy>,
1746    auto_policy_cache: std::sync::Mutex<
1747        Option<(
1748            std::path::PathBuf,
1749            Option<std::time::SystemTime>,
1750            super::AutoSelectionPolicy,
1751        )>,
1752    >,
1753    /// Successful vends only, scoped by model and immutable seat identity.
1754    current_accounts: SharedMap<(CredentialRef, SeatIdentity)>,
1755    /// In-memory cooldowns for automatic selection, keyed by storage key.
1756    cooldowns: Arc<std::sync::Mutex<BTreeMap<String, Cooldown>>>,
1757    /// Test seam: overrides the usage endpoint for typed usage snapshots.
1758    usage_endpoint_override: Option<String>,
1759    /// Max age of a usage observation that still counts as capacity.
1760    max_snapshot_age: Duration,
1761    /// Short-lived cache of the last usage snapshot per *seat key* (credential
1762    /// file + storage key + [`SeatIdentity`] of the stored credential) so
1763    /// automatic selection does not re-poll every provider on every token
1764    /// vend. A re-login, removal or profile switch changes the key, so a
1765    /// reading can never be attributed to a different seat than the one it
1766    /// was taken from. Entries older than `max_snapshot_age` are never reused.
1767    snapshots: SharedMap<super::usage::UsageSnapshot>,
1768    /// Process-wide usage-read table (single-flight per slot, held failed
1769    /// reads, last good readings); see [`super::usage_reads`].
1770    usage_reads: Arc<super::usage_reads::UsageReads>,
1771    /// Small per-process number for the usage-read log line (not secret).
1772    instance: u64,
1773    /// Cached account policy keyed by the config file's modification time
1774    /// (env overlay is applied on every read; it is cheap and has no I/O).
1775    policy_cache: Arc<std::sync::Mutex<Option<CachedPolicy>>>,
1776    /// What `synaps auth state` shows per slot (credential file + storage
1777    /// key): the last usage read and the last successful one. Diagnostic
1778    /// only; selection never reads it.
1779    observations: SharedMap<SlotObservations>,
1780    /// Last automatic selection per [`Self::current_key`] (provider + model).
1781    selections: SharedMap<super::auth_state::SelectionState>,
1782    /// Refused token refreshes held per [`Self::state_key`] (see
1783    /// [`REFUSED_REFRESH_HOLD`]). Memory only.
1784    refresh_holds: SharedMap<RefreshHold>,
1785    /// Stored seats for cooldown lookups, valid for one version of the
1786    /// credential file ([`Self::cooldown_seat`]).
1787    seat_cache: Arc<std::sync::Mutex<Option<SeatCache>>>,
1788    /// Stored-login reads [`Self::cooldown_seat`] made (test seam).
1789    seat_loads: Arc<std::sync::atomic::AtomicU64>,
1790}
1791
1792/// One version of the credential file: modification time, length and
1793/// inode (writes replace the file by rename, so each one gets a new inode;
1794/// the coarse mtime alone could miss two writes in one clock tick).
1795type FileStamp = (std::time::SystemTime, u64, u64);
1796
1797/// The version of the file at `path`, if it can be stat'ed.
1798fn file_stamp(path: &std::path::Path) -> Option<FileStamp> {
1799    let meta = std::fs::metadata(path).ok()?;
1800    #[cfg(unix)]
1801    let inode = std::os::unix::fs::MetadataExt::ino(&meta);
1802    #[cfg(not(unix))]
1803    let inode = 0;
1804    Some((meta.modified().ok()?, meta.len(), inode))
1805}
1806
1807/// The stored seat of each slot looked up, read from one version of the
1808/// credential file.
1809struct SeatCache {
1810    path: std::path::PathBuf,
1811    stamp: FileStamp,
1812    seats: BTreeMap<String, Option<SeatIdentity>>,
1813}
1814
1815/// One slot's recorded usage reads (`synaps auth state`).
1816#[derive(Debug, Clone, Default)]
1817struct SlotObservations {
1818    last_usage: Option<super::auth_state::ObservedOutcome>,
1819    last_usage_ok_ms: Option<u64>,
1820}
1821
1822/// Process-shared map keyed by storage key or seat key.
1823type SharedMap<T> = Arc<std::sync::Mutex<BTreeMap<String, T>>>;
1824
1825/// The accounts that failed earlier in one automatic selection (storage key
1826/// → why): a failed refresh of the chosen seat or a failed usage read. They
1827/// count as failed for the rest of that selection, without a new request.
1828type FailedThisSelection = BTreeMap<String, super::quota_policy::ObservationFailure>;
1829
1830/// Config-derived policy tagged with the config file mtime it was read at.
1831type CachedPolicy = (Option<std::time::SystemTime>, AccountPolicy);
1832
1833/// A reported provider limit for one account (non-secret).
1834#[derive(Debug, Clone, PartialEq, Eq)]
1835struct Cooldown {
1836    until_ms: u64,
1837    reason: String,
1838    /// When it was reported (epoch ms): only a request sent after this
1839    /// clears it ([`CredentialBroker::clear_cooldown`]).
1840    reported_at_ms: u64,
1841}
1842
1843/// The keys one account's cooldown is kept under
1844/// ([`LocalBroker::cooldown_keys`]). Non-secret: the seat part is an
1845/// identity digest, never a token.
1846struct CooldownKeys {
1847    /// The stored seat's key; None when the slot has no readable login.
1848    seat: Option<String>,
1849    /// The slot's key.
1850    slot: String,
1851    /// The stored seat names a provider account id.
1852    identified: bool,
1853}
1854
1855impl CooldownKeys {
1856    /// Keys a report writes: the seat's, and the slot's unless the seat
1857    /// names an account id.
1858    fn written(&self) -> impl Iterator<Item = &str> + '_ {
1859        let slot = (!self.identified).then_some(self.slot.as_str());
1860        self.seat.as_deref().into_iter().chain(slot)
1861    }
1862
1863    /// Keys a lookup or a clear reads: the seat's and the slot's.
1864    fn read(&self) -> impl Iterator<Item = &str> + '_ {
1865        self.seat.as_deref().into_iter().chain([self.slot.as_str()])
1866    }
1867}
1868
1869/// Cooldown applied when a limit is reported without a reset hint.
1870const DEFAULT_COOLDOWN: Duration = Duration::from_secs(15 * 60);
1871
1872/// A refused token refresh of one slot. No Debug: `material` is
1873/// secret-derived.
1874struct RefreshHold {
1875    /// The stored login the provider refused.
1876    material: MaterialDigest,
1877    failure: super::token::RefreshFailure,
1878    at_ms: u64,
1879}
1880
1881/// How long a refused refresh (`refresh_rejected` /
1882/// `refresh_action_required`) of an unchanged stored login is held. While
1883/// held, neither refresh site (the chosen account's vend, the usage read's
1884/// pre-read refresh) asks the provider again: the account fails with the
1885/// held kind. The hold ends when the stored login changes (a new login, or a
1886/// refresh by another process), on `synaps auth state --clear`, on a restart,
1887/// or after this window; then ONE refresh is tried and a new refusal holds
1888/// it again. Bounded so that a provider wrongly refusing during an outage
1889/// cannot leave every account dead until a login or a restart; one probe per
1890/// account per window is no storm. Retryable refresh failures are never
1891/// held.
1892pub const REFUSED_REFRESH_HOLD: Duration = Duration::from_secs(15 * 60);
1893
1894/// One slot's refused-refresh hold as the token helper sees it
1895/// ([`super::token::RefusalMemory`]): checked after the stored login is
1896/// re-read under the slot's refresh gate and lock, and written there from
1897/// the exchange's answer. So concurrent callers of one slot make one
1898/// decision: one exchange, and every waiter takes the held refusal. The
1899/// digest is of the login actually refused. No Debug (the holds are
1900/// secret-derived).
1901struct SlotHold {
1902    holds: SharedMap<RefreshHold>,
1903    /// [`LocalBroker::state_key`] of the slot.
1904    key: String,
1905    provider: OAuthProviderId,
1906    /// Set when this call was answered from the hold (no exchange).
1907    answered: std::sync::atomic::AtomicBool,
1908}
1909
1910impl SlotHold {
1911    fn answered(&self) -> bool {
1912        self.answered.load(std::sync::atomic::Ordering::SeqCst)
1913    }
1914}
1915
1916impl super::token::RefusalMemory for SlotHold {
1917    /// The hold of this slot while it is inside [`REFUSED_REFRESH_HOLD`]
1918    /// and is for exactly `creds`; anything else ends it.
1919    fn held(&self, creds: &super::OAuthCredentials) -> Option<super::token::RefreshFailure> {
1920        let material = MaterialDigest::of(self.provider, creds);
1921        let window_ms = REFUSED_REFRESH_HOLD.as_millis() as u64;
1922        let mut holds = self.holds.lock().unwrap_or_else(|e| e.into_inner());
1923        let hold = holds.get(&self.key)?;
1924        if crate::epoch_millis() < hold.at_ms.saturating_add(window_ms)
1925            && hold.material == material
1926        {
1927            self.answered.store(true, std::sync::atomic::Ordering::SeqCst);
1928            return Some(hold.failure.clone());
1929        }
1930        holds.remove(&self.key);
1931        None
1932    }
1933
1934    /// Holds a refusal (`Credential` / `ActionRequired`) of `creds`; a
1935    /// retryable kind is never held. Store failures never arrive here.
1936    fn refused(&self, creds: &super::OAuthCredentials, failure: &super::token::RefreshFailure) {
1937        use super::token::RefreshFailureKind as R;
1938        if !matches!(failure.kind, R::Credential | R::ActionRequired) {
1939            return;
1940        }
1941        let hold = RefreshHold {
1942            material: MaterialDigest::of(self.provider, creds),
1943            failure: failure.clone(),
1944            at_ms: crate::epoch_millis(),
1945        };
1946        self.holds
1947            .lock()
1948            .unwrap_or_else(|e| e.into_inner())
1949            .insert(self.key.clone(), hold);
1950    }
1951}
1952
1953/// Longest cooldown a client report may impose (a weekly window plus slack);
1954/// anything longer is clamped so a bad report cannot bench a seat forever.
1955const MAX_COOLDOWN: Duration = Duration::from_secs(8 * 24 * 60 * 60);
1956/// Default staleness bound for automatic selection. Short on purpose: a
1957/// selection is a spend decision and should rest on a fresh reading.
1958pub const DEFAULT_MAX_SNAPSHOT_AGE: Duration = Duration::from_secs(60);
1959
1960/// Idle timeout between bytes on the broker's per-request no-redirect
1961/// client (Muse, Kimi, Gemini, Codex/Anthropic catalog). Matches the runtime
1962/// client's 90s read timeout that xAI streams already get.
1963pub const BROKER_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(90);
1964
1965/// Gemini (Code Assist) sends no data chunk while it thinks (thought
1966/// streaming is off), and a hard prompt can think silently for minutes;
1967/// its stream has no retry, so it gets a longer idle limit.
1968pub const GEMINI_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
1969
1970/// Next per-process broker number for the usage-read log line.
1971fn next_broker_instance() -> u64 {
1972    static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
1973    NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
1974}
1975
1976impl LocalBroker {
1977    pub fn new(http: reqwest::Client) -> Self {
1978        let cloud_backend = Arc::new(ProductionCloudBackend::new(http.clone()));
1979        Self {
1980            http,
1981            local_base_url: None,
1982            anthropic_usage_url: None,
1983            google_gemini_base_url: None,
1984            openai_codex_base_url: None,
1985            muse_base_url: None,
1986            refresh_endpoints: Default::default(),
1987            stream_idle_timeout: BROKER_STREAM_IDLE_TIMEOUT,
1988            request_timeout: PROXY_REQUEST_TIMEOUT,
1989            max_response_bytes: MAX_PROXY_RESPONSE_BYTES,
1990            cloud_backend: Some(cloud_backend),
1991            account_policy: None,
1992            auto_policy: None,
1993            auto_policy_cache: std::sync::Mutex::new(None),
1994            current_accounts: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
1995            cooldowns: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
1996            usage_endpoint_override: None,
1997            max_snapshot_age: DEFAULT_MAX_SNAPSHOT_AGE,
1998            snapshots: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
1999            usage_reads: super::usage_reads::process_table(),
2000            instance: next_broker_instance(),
2001            policy_cache: Arc::new(std::sync::Mutex::new(None)),
2002            observations: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
2003            selections: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
2004            refresh_holds: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
2005            seat_cache: Arc::new(std::sync::Mutex::new(None)),
2006            seat_loads: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2007        }
2008    }
2009
2010    pub fn with_cloud_backend(mut self, backend: Arc<dyn CloudBackend>) -> Self {
2011        self.cloud_backend = Some(backend);
2012        self
2013    }
2014
2015    /// Pin the account policy instead of reading env + config per call.
2016    pub fn with_account_policy(mut self, policy: AccountPolicy) -> Self {
2017        self.account_policy = Some(policy);
2018        self
2019    }
2020
2021    /// Pin the automatic ranking policy (tests/embedding).
2022    pub fn with_auto_policy(mut self, policy: super::AutoSelectionPolicy) -> Self {
2023        self.auto_policy = Some(policy);
2024        self
2025    }
2026
2027    fn auto_policy(&self) -> super::AutoSelectionPolicy {
2028        if let Some(policy) = &self.auto_policy {
2029            return policy.clone();
2030        }
2031        let path = crate::config::resolve_read_path("config");
2032        let mtime = std::fs::metadata(&path).and_then(|m| m.modified()).ok();
2033        let mut cache = self
2034            .auto_policy_cache
2035            .lock()
2036            .unwrap_or_else(|e| e.into_inner());
2037        if let Some((cached_path, cached_time, policy)) = cache.as_ref() {
2038            if cached_path == &path && *cached_time == mtime {
2039                return policy.clone();
2040            }
2041        }
2042        let (policy, warnings) =
2043            super::AutoSelectionPolicy::from_config_map(&crate::config::load_config().auth.auto);
2044        for warning in warnings {
2045            tracing::warn!(%warning, "automatic account policy");
2046        }
2047        *cache = Some((path, mtime, policy.clone()));
2048        policy
2049    }
2050
2051    fn current_key(provider: OAuthProviderId, model: Option<&str>) -> String {
2052        format!(
2053            "{}|{}|{}",
2054            storage::auth_file_path().display(),
2055            provider,
2056            model.unwrap_or("")
2057        )
2058    }
2059
2060    /// Test seam: point the pinned ChatGPT backend host at a loopback fake.
2061    /// Only relaxes the base URL; allowlists, bearer/header pairing and
2062    /// redirect denial are unchanged.
2063    #[doc(hidden)]
2064    pub fn with_openai_codex_base_url_for_tests(mut self, base_url: impl Into<String>) -> Self {
2065        self.openai_codex_base_url = Some(base_url.into().trim_end_matches('/').to_string());
2066        self
2067    }
2068
2069    /// Test seam: point the pinned Meta Muse origin at a loopback fake. Only
2070    /// relaxes the origin; the path allowlist, bearer resolution, 401
2071    /// recovery and redirect denial are unchanged.
2072    #[doc(hidden)]
2073    pub fn with_muse_base_url_for_tests(mut self, base_url: impl Into<String>) -> Self {
2074        self.muse_base_url = Some(base_url.into().trim_end_matches('/').to_string());
2075        self
2076    }
2077
2078    /// Idle (between-bytes) limit for `provider_key`'s no-redirect client.
2079    fn stream_idle_timeout_for(&self, provider_key: &str) -> Duration {
2080        if provider_key == "google-gemini" {
2081            GEMINI_STREAM_IDLE_TIMEOUT.max(self.stream_idle_timeout)
2082        } else {
2083            self.stream_idle_timeout
2084        }
2085    }
2086
2087    /// Test seam: shorten [`BROKER_STREAM_IDLE_TIMEOUT`] (not Gemini's).
2088    #[doc(hidden)]
2089    pub fn with_stream_idle_timeout_for_tests(mut self, idle: Duration) -> Self {
2090        self.stream_idle_timeout = idle;
2091        self
2092    }
2093
2094    /// Test seam: send Muse key re-mints to a loopback fake mint
2095    /// (`http://127.0.0.1:<port>/...`) instead of the pinned
2096    /// `https://api.meta.ai/muse-code/key`. The request itself (headers,
2097    /// body, OAuth bearer) is built exactly as in production.
2098    #[doc(hidden)]
2099    pub fn with_muse_mint_url_for_tests(mut self, mint_url: impl Into<String>) -> Self {
2100        self.refresh_endpoints =
2101            std::mem::take(&mut self.refresh_endpoints).with_muse_mint_url_for_tests(mint_url);
2102        self
2103    }
2104
2105    /// Test seam: send Anthropic token refreshes to a loopback fake token
2106    /// endpoint instead of the real one; a non-loopback URL is refused at
2107    /// refresh time. The request is built exactly as in production.
2108    #[doc(hidden)]
2109    pub fn with_anthropic_token_url_for_tests(mut self, url: impl Into<String>) -> Self {
2110        self.refresh_endpoints =
2111            std::mem::take(&mut self.refresh_endpoints).with_anthropic_token_url_for_tests(url);
2112        self
2113    }
2114
2115    /// Capacity-based selection for `provider` regardless of the configured
2116    /// selector (used by the broker daemon when a remote client's own policy
2117    /// is `auto`). Fails closed like [`access_token_pinned_for`].
2118    ///
2119    /// [`access_token_pinned_for`]: CredentialBroker::access_token_pinned_for
2120    pub async fn access_token_auto(
2121        &self,
2122        provider: OAuthProviderId,
2123        model: Option<&str>,
2124    ) -> Result<PinnedToken, BrokerError> {
2125        self.vend_auto(provider, model, &[]).await
2126    }
2127
2128    /// Test seam: point typed usage fetches at a fake server.
2129    #[doc(hidden)]
2130    pub fn with_usage_endpoint_override(mut self, url: impl Into<String>) -> Self {
2131        self.usage_endpoint_override = Some(url.into());
2132        self
2133    }
2134
2135    /// Bound on usage-snapshot age for automatic selection.
2136    pub fn with_max_snapshot_age(mut self, age: Duration) -> Self {
2137        self.max_snapshot_age = age;
2138        self
2139    }
2140
2141    /// Effective policy: injected, else env > config > default. The config
2142    /// file is re-parsed only when its modification time changes, so a vend
2143    /// never re-reads config (and never re-emits warnings) needlessly.
2144    fn policy(&self) -> AccountPolicy {
2145        if let Some(policy) = &self.account_policy {
2146            return policy.clone();
2147        }
2148        let mtime = std::fs::metadata(crate::config::resolve_read_path("config"))
2149            .and_then(|m| m.modified())
2150            .ok();
2151        let mut cache = self.policy_cache.lock().unwrap_or_else(|e| e.into_inner());
2152        let config_policy = match cache.as_ref() {
2153            Some((cached_mtime, policy)) if *cached_mtime == mtime => policy.clone(),
2154            _ => {
2155                let (policy, _warnings) = AccountPolicy::from_config_map(
2156                    &crate::config::load_config().auth.accounts,
2157                );
2158                *cache = Some((mtime, policy.clone()));
2159                policy
2160            }
2161        };
2162        config_policy.with_env_overlay()
2163    }
2164
2165    /// Where `cred`'s cooldown is kept. Limits follow the seat, not an
2166    /// alias that can be re-used: aliases of one seat share its cooldown.
2167    /// A login without a provider account id has a seat derived from its
2168    /// token, which a refresh rotates, so its cooldown is also kept under
2169    /// its slot and survives the rotation. The slot can't tell that login
2170    /// gaining an id from another account logging into the label, so any
2171    /// new login into the slot inherits it too (under-use only; `auth state
2172    /// --clear` drops it). The cooldown of a login WITH an account id is
2173    /// kept under that id only: a login with another id does not inherit it.
2174    fn cooldown_keys(&self, cred: &CredentialRef) -> CooldownKeys {
2175        let store = storage::auth_file_path().display().to_string();
2176        let slot = format!("{store}|{}|{}", cred.provider, cred.storage_key());
2177        let seat = self.cooldown_seat(cred);
2178        CooldownKeys {
2179            identified: seat.as_ref().is_some_and(|s| s.seat_fingerprint().is_some()),
2180            seat: seat.map(|s| format!("{store}|{}|{}", cred.provider, s.as_str())),
2181            slot,
2182        }
2183    }
2184
2185    /// The seat stored in `cred`'s slot, for cooldown keys. A turn asks for
2186    /// its account's cooldown before every request while any cooldown is
2187    /// active, so the stored login is read once per version of the
2188    /// credential file ([`FileStamp`]), not per lookup:
2189    /// a later lookup costs one stat. A write by this or another process
2190    /// (a login, a refresh) changes the version, so a rotated or replaced
2191    /// login is read again. Never used for spending (vends and usage reads
2192    /// load the slot themselves).
2193    fn cooldown_seat(&self, cred: &CredentialRef) -> Option<SeatIdentity> {
2194        let path = storage::auth_file_path();
2195        let stamp = file_stamp(&path);
2196        let key = cred.storage_key();
2197        let mut cache = self.seat_cache.lock().unwrap_or_else(|e| e.into_inner());
2198        if let (Some(stamp), Some(c)) = (stamp, cache.as_ref()) {
2199            if c.path == path && c.stamp == stamp {
2200                if let Some(seat) = c.seats.get(&key) {
2201                    return seat.clone();
2202                }
2203            }
2204        }
2205        self.seat_loads
2206            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2207        let seat = self.stored_seat(cred).ok();
2208        match stamp {
2209            Some(stamp) => {
2210                let fresh = cache
2211                    .as_ref()
2212                    .is_some_and(|c| c.path == path && c.stamp == stamp);
2213                if !fresh {
2214                    *cache = Some(SeatCache {
2215                        path,
2216                        stamp,
2217                        seats: BTreeMap::new(),
2218                    });
2219                }
2220                if let Some(c) = cache.as_mut() {
2221                    c.seats.insert(key, seat.clone());
2222                }
2223            }
2224            None => *cache = None,
2225        }
2226        seat
2227    }
2228
2229    /// Test seam: how many times cooldown lookups read a stored login.
2230    #[doc(hidden)]
2231    pub fn cooldown_seat_loads_for_tests(&self) -> u64 {
2232        self.seat_loads.load(std::sync::atomic::Ordering::Relaxed)
2233    }
2234
2235    /// Whether any cooldown is active at `now_ms`.
2236    fn any_cooldown_active(&self, now_ms: u64) -> bool {
2237        self.cooldowns
2238            .lock()
2239            .unwrap_or_else(|e| e.into_inner())
2240            .values()
2241            .any(|c| c.until_ms > now_ms)
2242    }
2243
2244    /// `cred`'s active cooldown: the latest one under its keys.
2245    fn active_cooldown_at(&self, cred: &CredentialRef, now_ms: u64) -> Option<Cooldown> {
2246        // Nothing cooling, nothing to look up: no read of the stored login
2247        // (a turn asks before every request).
2248        if !self.any_cooldown_active(now_ms) {
2249            return None;
2250        }
2251        self.cooldown_for_keys(&self.cooldown_keys(cred), now_ms)
2252    }
2253
2254    /// The latest active cooldown under `keys`.
2255    fn cooldown_for_keys(&self, keys: &CooldownKeys, now_ms: u64) -> Option<Cooldown> {
2256        let cooldowns = self.cooldowns.lock().unwrap_or_else(|e| e.into_inner());
2257        keys.read()
2258            .filter_map(|key| cooldowns.get(key))
2259            .filter(|c| c.until_ms > now_ms)
2260            .max_by_key(|c| c.until_ms)
2261            .cloned()
2262    }
2263
2264    /// Reading an expired cooldown is non-mutating (preview shares this path).
2265    fn cooldown_until(&self, storage_key: &str, now_ms: u64) -> Option<u64> {
2266        let cred = CredentialRef::parse_storage_key(storage_key)?;
2267        self.active_cooldown_at(&cred, now_ms).map(|c| c.until_ms)
2268    }
2269
2270    // ── `synaps auth state` ─────────────────────────────────────────────
2271
2272    /// What this broker remembers about the accounts `query` selects: no
2273    /// network request, no refresh; the account list comes from the store.
2274    /// With `query.clear`, their in-memory state is dropped first.
2275    pub fn auth_state(
2276        &self,
2277        query: &super::auth_state::AuthStateQuery,
2278    ) -> Result<super::auth_state::AuthStateReport, BrokerError> {
2279        auth_state_report(Some(self), query)
2280    }
2281
2282    /// The active cooldown of `cred`'s current seat, with its reason.
2283    fn cooldown_state(
2284        &self,
2285        cred: &CredentialRef,
2286        now_ms: u64,
2287    ) -> Option<super::auth_state::CooldownState> {
2288        self.active_cooldown_at(cred, now_ms)
2289            .map(|c| super::auth_state::CooldownState {
2290                until_ms: c.until_ms,
2291                reason: super::auth_state::CooldownReason::from_reported(&c.reason),
2292            })
2293    }
2294
2295    /// Drop the in-memory state of `creds`: cooldowns, cached usage
2296    /// readings, held usage-read failures and last good readings in the
2297    /// process-wide table (its gates stay), `auth state` records, held
2298    /// refused refreshes and stickiness. With
2299    /// `whole_providers` (no single account was named), also every cooldown
2300    /// and selection record of those providers, including ones a previous
2301    /// occupant of a slot left. Never touches the credential store.
2302    fn clear_state(&self, creds: &[CredentialRef], whole_providers: &[OAuthProviderId]) {
2303        let store = storage::auth_file_path().display().to_string();
2304        let keys: Vec<CooldownKeys> = creds.iter().map(|c| self.cooldown_keys(c)).collect();
2305        {
2306            let mut cooldowns = self.cooldowns.lock().unwrap_or_else(|e| e.into_inner());
2307            for key in keys.iter().flat_map(|k| k.read()) {
2308                cooldowns.remove(key);
2309            }
2310            for provider in whole_providers {
2311                let prefix = format!("{store}|{provider}|");
2312                cooldowns.retain(|k, _| !k.starts_with(&prefix));
2313            }
2314        }
2315        for cred in creds {
2316            self.forget_snapshots(cred);
2317            self.usage_reads.forget(&self.read_key(cred));
2318        }
2319        {
2320            let mut observations = self.observations.lock().unwrap_or_else(|e| e.into_inner());
2321            for cred in creds {
2322                observations.remove(&Self::state_key(cred));
2323            }
2324        }
2325        {
2326            let mut holds = self.refresh_holds.lock().unwrap_or_else(|e| e.into_inner());
2327            for cred in creds {
2328                holds.remove(&Self::state_key(cred));
2329            }
2330            for provider in whole_providers {
2331                let slot = format!("{store}|{}", provider.as_str());
2332                let named = format!("{slot}@");
2333                holds.retain(|k, _| *k != slot && !k.starts_with(&named));
2334            }
2335        }
2336        self.current_accounts
2337            .lock()
2338            .unwrap_or_else(|e| e.into_inner())
2339            .retain(|_, (current, _)| !creds.contains(current));
2340        let mut selections = self.selections.lock().unwrap_or_else(|e| e.into_inner());
2341        for provider in whole_providers {
2342            let prefix = format!("{store}|{provider}|");
2343            selections.retain(|k, _| !k.starts_with(&prefix));
2344        }
2345    }
2346
2347    /// Verify an explicitly addressed slot exists. Named slots that are
2348    /// missing are `UnknownAccount` (never a fallback); the default slot
2349    /// keeps the historical load-miss `Credential` error from the refresh
2350    /// path so existing "not logged in" classifiers keep working.
2351    fn check_slot_exists(&self, cred: &CredentialRef) -> Result<(), BrokerError> {
2352        if let Account::Named(label) = &cred.account {
2353            match storage::load_credential(cred) {
2354                Ok(Some(_)) => {}
2355                Ok(None) => {
2356                    return Err(BrokerError::UnknownAccount {
2357                        provider: cred.provider.as_str().to_string(),
2358                        label: label.as_str().to_string(),
2359                    })
2360                }
2361                Err(e) => return Err(BrokerError::Credential(e)),
2362            }
2363        }
2364        Ok(())
2365    }
2366
2367    /// Resolve and vend for a request: explicit account (validated, never a
2368    /// fallback) or the policy; `auto` goes through capacity selection with
2369    /// the token paired to the proven seat. `exclude` lists the accounts the
2370    /// request already used: `auto` selects among the rest, and an explicit
2371    /// account in it is refused.
2372    async fn resolve_and_vend(
2373        &self,
2374        provider: OAuthProviderId,
2375        explicit: Option<Account>,
2376        model: Option<&str>,
2377        exclude: &[CredentialRef],
2378    ) -> Result<PinnedToken, BrokerError> {
2379        let selector = match explicit {
2380            Some(account) => AccountSelector::Account(account),
2381            None => self.policy().selector(provider),
2382        };
2383        match selector {
2384            AccountSelector::Account(account) => {
2385                let credential = CredentialRef::new(provider, account);
2386                if exclude.contains(&credential) {
2387                    return Err(BrokerError::NoAccountAvailable {
2388                        provider: provider.as_str().to_string(),
2389                        reason: format!(
2390                            "the selected account {} was already used by this request",
2391                            credential.account
2392                        ),
2393                        failure: None,
2394                    });
2395                }
2396                let token = self.access_token_for(&credential).await?;
2397                Ok(PinnedToken { credential, token })
2398            }
2399            AccountSelector::Auto => self.vend_auto(provider, model, exclude).await,
2400            AccountSelector::Invalid { source, reason } => {
2401                Err(BrokerError::InvalidAccount(format!("{source}: {reason}")))
2402            }
2403        }
2404    }
2405
2406    /// Bound on select → seat-pinned vend rounds when the chosen slot is
2407    /// replaced underneath us (re-login storm / rotation by another process).
2408    const MAX_AUTO_VEND_ROUNDS: usize = 2;
2409
2410    /// `auto`: select on fresh usage, then vend a token for exactly the seat
2411    /// that usage was read from. If the slot changed in between (re-login,
2412    /// removal, rotation by another party) the reading is discarded and the
2413    /// selection re-runs on the new content, at most
2414    /// [`Self::MAX_AUTO_VEND_ROUNDS`] times; never a token for a seat whose
2415    /// capacity was not proven.
2416    ///
2417    /// A refused refresh of the chosen seat (`refresh_rejected`,
2418    /// `refresh_action_required`) does not end the selection while another
2419    /// account has proven capacity: the account counts as failed, with its
2420    /// refresh kind, for the rest of this selection and the selection runs
2421    /// again without it. A retryable refresh failure (transient, lock busy,
2422    /// other) ends the selection with that kind ([`BrokerError::Refresh`]):
2423    /// it says nothing against the account, and moving would cost the turn
2424    /// its prompt cache. A failed usage
2425    /// read counts the same way (a held transient read is answered by the
2426    /// usage-read table without a request). So each account gets at most one
2427    /// refresh attempt and one failed read per selection, and the rounds are
2428    /// bounded by the number of accounts. When none is left, the selection
2429    /// fails with every account's kind (S1's classified failure). A refused
2430    /// refresh is also held across selections ([`REFUSED_REFRESH_HOLD`]): a
2431    /// held account fails with its held kind and no exchange.
2432    async fn vend_auto(
2433        &self,
2434        provider: OAuthProviderId,
2435        model: Option<&str>,
2436        exclude: &[CredentialRef],
2437    ) -> Result<PinnedToken, BrokerError> {
2438        use super::token::{ensure_fresh_credential_for_seat, SeatVendError};
2439        let mut failed = FailedThisSelection::new();
2440        let mut seat_changes = 0;
2441        loop {
2442            let (credential, seat) = match self
2443                .select_auto(provider, model, exclude, &mut failed)
2444                .await
2445            {
2446                Ok(choice) => choice,
2447                Err(e) => {
2448                    let failure = match &e {
2449                        BrokerError::NoAccountAvailable { failure, .. } => failure.clone(),
2450                        // The account list could not be read (select_auto
2451                        // logged it).
2452                        _ => Some(super::quota_policy::SelectionFailure::new(
2453                            super::quota_policy::SelectionFailureKind::CredentialStore,
2454                        )),
2455                    };
2456                    self.record_selection(provider, model, None, failure, None);
2457                    return Err(e);
2458                }
2459            };
2460            // A refused refresh of this login is held: no new exchange.
2461            if let Some(held) = self.held_refresh(&credential) {
2462                let cause = refresh_failure(&held);
2463                tracing::info!(
2464                    credential = %credential,
2465                    kind = %cause.kind,
2466                    "automatic choice not vended: its refused token refresh is held until the stored login changes; selecting again"
2467                );
2468                failed.insert(credential.storage_key(), cause);
2469                continue;
2470            }
2471            // The hold is checked again, and a refusal recorded, under the
2472            // slot's refresh gate: concurrent turns make one decision.
2473            let hold = self.slot_hold(&credential);
2474            let memory: Arc<dyn super::token::RefusalMemory> = hold.clone();
2475            match ensure_fresh_credential_for_seat(
2476                &self.http,
2477                &credential,
2478                &seat,
2479                &self.refresh_endpoints,
2480                Some(memory),
2481            )
2482            .await
2483            {
2484                Ok(creds) => {
2485                    self.current_accounts
2486                        .lock()
2487                        .unwrap_or_else(|e| e.into_inner())
2488                        .insert(
2489                            Self::current_key(provider, model),
2490                            (credential.clone(), seat),
2491                        );
2492                    self.record_selection(provider, model, Some(&credential), None, None);
2493                    return Ok(PinnedToken {
2494                        credential,
2495                        token: AccessToken {
2496                            token: creds.access,
2497                            expires: creds.expires,
2498                        },
2499                    });
2500                }
2501                Err(SeatVendError::SeatChanged) => {
2502                    tracing::info!(credential = %credential, "slot changed after selection; re-selecting");
2503                    self.forget_snapshots(&credential);
2504                    seat_changes += 1;
2505                    if seat_changes >= Self::MAX_AUTO_VEND_ROUNDS {
2506                        break;
2507                    }
2508                }
2509                // Typed, so a transient refresh failure of the chosen seat
2510                // is never reported as an unusable login.
2511                Err(SeatVendError::Other(f)) => {
2512                    let cause = refresh_failure(&f);
2513                    // Only a refusal (a login or an action fixes it) moves
2514                    // the selection on. A retryable failure (the token
2515                    // endpoint down, the refresh lock busy) says nothing
2516                    // against this account, and another account would cost
2517                    // the turn its prompt cache for nothing: the selection
2518                    // fails with the refresh's own kind, and the next turn
2519                    // tries again.
2520                    if !matches!(
2521                        f.kind,
2522                        super::token::RefreshFailureKind::Credential
2523                            | super::token::RefreshFailureKind::ActionRequired
2524                    ) {
2525                        tracing::warn!(
2526                            credential = %credential,
2527                            kind = %cause.kind,
2528                            "automatic choice could not be vended: token refresh failed"
2529                        );
2530                        self.record_selection(provider, model, Some(&credential), None, Some(cause));
2531                        return Err(BrokerError::Refresh(auto_vend_failure(&credential, f)));
2532                    }
2533                    if hold.answered() {
2534                        // A refusal another caller just got, under the gate.
2535                        tracing::info!(
2536                            credential = %credential,
2537                            kind = %cause.kind,
2538                            "automatic choice not vended: its refused token refresh is held until the stored login changes; selecting again"
2539                        );
2540                    } else {
2541                        tracing::warn!(
2542                            credential = %credential,
2543                            kind = %cause.kind,
2544                            "automatic choice could not be vended: token refresh failed; selecting again"
2545                        );
2546                    }
2547                    failed.insert(credential.storage_key(), cause);
2548                }
2549            }
2550        }
2551        let failure = super::quota_policy::SelectionFailure::new(
2552            super::quota_policy::SelectionFailureKind::SeatChurn,
2553        );
2554        self.record_selection(provider, model, None, Some(failure.clone()), None);
2555        Err(selection_failed(
2556            provider,
2557            model,
2558            "selected credential kept changing while its capacity was being proven".into(),
2559            failure,
2560        ))
2561    }
2562
2563    /// Record the outcome of an automatic selection for `synaps auth state`.
2564    /// `vend_failure`: the chosen account's token refresh failed for a
2565    /// retryable reason, which ended the selection. (A refused refresh is
2566    /// no outcome: the selection runs again without that account.)
2567    fn record_selection(
2568        &self,
2569        provider: OAuthProviderId,
2570        model: Option<&str>,
2571        chosen: Option<&CredentialRef>,
2572        failure: Option<super::quota_policy::SelectionFailure>,
2573        vend_failure: Option<super::quota_policy::ObservationFailure>,
2574    ) {
2575        let state = super::auth_state::SelectionState {
2576            provider: provider.as_str().to_string(),
2577            model: model.map(str::to_string),
2578            at_ms: crate::epoch_millis(),
2579            chosen: chosen.map(|c| c.account.label_str().to_string()),
2580            failure,
2581            vend_failure,
2582        };
2583        self.selections
2584            .lock()
2585            .unwrap_or_else(|e| e.into_inner())
2586            .insert(Self::current_key(provider, model), state);
2587    }
2588
2589    /// Cache key for one stored seat: credential file + slot + identity of
2590    /// the material stored there. Never contains token material.
2591    fn seat_key(cred: &CredentialRef, seat: &SeatIdentity) -> String {
2592        format!(
2593            "{}|{}|{}",
2594            storage::auth_file_path().display(),
2595            cred.storage_key(),
2596            seat.as_str()
2597        )
2598    }
2599
2600    /// Identity of the credential currently stored in `cred`'s slot.
2601    fn stored_seat(&self, cred: &CredentialRef) -> Result<SeatIdentity, BrokerError> {
2602        match storage::load_credential(cred).map_err(BrokerError::Credential)? {
2603            Some(creds) => Ok(SeatIdentity::of(cred.provider, &creds)),
2604            None => Err(BrokerError::UnknownAccount {
2605                provider: cred.provider.as_str().to_string(),
2606                label: cred.account.label_str().to_string(),
2607            }),
2608        }
2609    }
2610
2611    fn cached_snapshot(&self, key: &str) -> Option<super::usage::UsageSnapshot> {
2612        let max_age_ms = self.max_snapshot_age.as_millis() as u64;
2613        let now_ms = crate::epoch_millis();
2614        self.snapshots
2615            .lock()
2616            .unwrap_or_else(|e| e.into_inner())
2617            .get(key)
2618            .filter(|s| !s.is_stale(now_ms, max_age_ms))
2619            .cloned()
2620    }
2621
2622    /// Usage-read table key for `cred`'s slot: credential file + storage key
2623    /// + usage endpoint. Stable across token rotation; no token material.
2624    fn read_key(&self, cred: &CredentialRef) -> String {
2625        format!(
2626            "{}|{}|{}",
2627            storage::auth_file_path().display(),
2628            cred.storage_key(),
2629            self.usage_endpoint_override.as_deref().unwrap_or("")
2630        )
2631    }
2632
2633    /// Drop every cached reading for `cred`'s slot (any seat). The
2634    /// single-flight gate lives in the process-wide usage-read table and is
2635    /// never dropped, so this cannot open a second flight.
2636    fn forget_snapshots(&self, cred: &CredentialRef) {
2637        let prefix = format!("|{}|", cred.storage_key());
2638        self.snapshots
2639            .lock()
2640            .unwrap_or_else(|e| e.into_inner())
2641            .retain(|k, _| !k.contains(&prefix));
2642    }
2643
2644    /// Fresh usage snapshot for `cred` plus the seat it was read from: the
2645    /// short-lived cache is consulted first (never past `max_snapshot_age`,
2646    /// and only for the seat currently stored in the slot), otherwise the
2647    /// process-wide usage-read table: ONE read per slot at a time across
2648    /// every broker in the process, and a held failed read is answered
2649    /// without a request. The token for the read and the reported seat come
2650    /// from the same load, so the pair is exact even if the slot changes
2651    /// concurrently.
2652    async fn fresh_usage(
2653        &self,
2654        cred: &CredentialRef,
2655    ) -> Result<(super::usage::UsageSnapshot, SeatIdentity), super::usage_reads::ReadError> {
2656        if let Some(held) = self.held_refresh(cred) {
2657            return Err(Self::held_read(cred, held).into_read_error());
2658        }
2659        let seat = self
2660            .stored_seat(cred)
2661            .map_err(|e| UsageReadError::slot(e).into_read_error())?;
2662        let key = Self::seat_key(cred, &seat);
2663        if let Some(cached) = self.cached_snapshot(&key) {
2664            return Ok((cached, seat));
2665        }
2666        let (snapshot, used) = self.shared_read(cred).await?;
2667        // One slot holds one seat: a reading for its current content
2668        // supersedes anything cached for a previous occupant.
2669        self.forget_snapshots(cred);
2670        let used_key = Self::seat_key(cred, &used);
2671        self.snapshots
2672            .lock()
2673            .unwrap_or_else(|e| e.into_inner())
2674            .insert(used_key, snapshot.clone());
2675        Ok((snapshot, used))
2676    }
2677
2678    /// Read-only usage for `cred` through the process-wide usage-read table,
2679    /// reporting the seat whose token made the request. A failure keeps its
2680    /// typed cause. A held refused refresh answers without a request.
2681    async fn usage_with_seat(
2682        &self,
2683        cred: &CredentialRef,
2684    ) -> Result<(super::usage::UsageSnapshot, SeatIdentity), UsageReadError> {
2685        if let Some(held) = self.held_refresh(cred) {
2686            return Err(Self::held_read(cred, held));
2687        }
2688        self.shared_read(cred)
2689            .await
2690            .map_err(|e| UsageReadError::from_read(&e))
2691    }
2692
2693    /// Key of a slot's `auth state` records: credential file + storage key.
2694    /// Never contains token material.
2695    fn state_key(cred: &CredentialRef) -> String {
2696        format!(
2697            "{}|{}",
2698            storage::auth_file_path().display(),
2699            cred.storage_key()
2700        )
2701    }
2702
2703    /// Digest of the login stored in `cred`'s slot now; `None` when the
2704    /// slot is empty or the store can't be read.
2705    fn stored_material(cred: &CredentialRef) -> Option<MaterialDigest> {
2706        storage::load_credential(cred)
2707            .ok()
2708            .flatten()
2709            .map(|creds| MaterialDigest::of(cred.provider, &creds))
2710    }
2711
2712    /// The held refused refresh of `cred`'s slot, if the provider refused
2713    /// the login stored there now and [`REFUSED_REFRESH_HOLD`] has not
2714    /// passed. A changed login or an ended window drops the hold, so the
2715    /// next refresh asks the provider again. Without a hold for the slot
2716    /// nothing is loaded.
2717    fn held_refresh(&self, cred: &CredentialRef) -> Option<super::token::RefreshFailure> {
2718        let key = Self::state_key(cred);
2719        let (material, failure, at_ms) = {
2720            let holds = self.refresh_holds.lock().unwrap_or_else(|e| e.into_inner());
2721            let hold = holds.get(&key)?;
2722            (hold.material.clone(), hold.failure.clone(), hold.at_ms)
2723        };
2724        let window_ms = REFUSED_REFRESH_HOLD.as_millis() as u64;
2725        let in_window = crate::epoch_millis() < at_ms.saturating_add(window_ms);
2726        if in_window && Self::stored_material(cred).as_ref() == Some(&material) {
2727            return Some(failure);
2728        }
2729        let mut holds = self.refresh_holds.lock().unwrap_or_else(|e| e.into_inner());
2730        if holds.get(&key).is_some_and(|h| h.at_ms == at_ms) {
2731            holds.remove(&key);
2732        }
2733        None
2734    }
2735
2736    /// The refused-refresh hold of `cred`'s slot, for the token helper to
2737    /// check and write under the slot's refresh gate ([`SlotHold`]).
2738    fn slot_hold(&self, cred: &CredentialRef) -> Arc<SlotHold> {
2739        Arc::new(SlotHold {
2740            holds: self.refresh_holds.clone(),
2741            key: Self::state_key(cred),
2742            provider: cred.provider,
2743            answered: std::sync::atomic::AtomicBool::new(false),
2744        })
2745    }
2746
2747    /// A usage read of an account whose refused refresh is held: no refresh
2748    /// and no request, so not recorded as a read; the failure is the held
2749    /// refusal (it names the login command or remedy).
2750    fn held_read(cred: &CredentialRef, held: super::token::RefreshFailure) -> UsageReadError {
2751        let err = UsageReadError::refresh(held);
2752        tracing::debug!(
2753            credential = %cred,
2754            kind = %err.failure.kind,
2755            "usage read not sent: the refused token refresh is held until the stored login changes"
2756        );
2757        err
2758    }
2759
2760    /// What `auth state` shows of `cred`'s held refused refresh (the same
2761    /// test as [`Self::held_refresh`], dropping nothing). Never the digest.
2762    fn refresh_hold_state(
2763        &self,
2764        cred: &CredentialRef,
2765        now_ms: u64,
2766    ) -> Option<super::auth_state::RefreshHoldState> {
2767        let (material, failure, at_ms) = {
2768            let holds = self.refresh_holds.lock().unwrap_or_else(|e| e.into_inner());
2769            let hold = holds.get(&Self::state_key(cred))?;
2770            (hold.material.clone(), hold.failure.clone(), hold.at_ms)
2771        };
2772        let until_ms = at_ms.saturating_add(REFUSED_REFRESH_HOLD.as_millis() as u64);
2773        (now_ms < until_ms && Self::stored_material(cred).as_ref() == Some(&material)).then(|| {
2774            super::auth_state::RefreshHoldState {
2775                failure: refresh_failure(&failure),
2776                refused_at_ms: at_ms,
2777                until_ms,
2778            }
2779        })
2780    }
2781
2782    /// Test seam: move every held refused refresh `by` into the past (the
2783    /// hold window's injected clock).
2784    #[doc(hidden)]
2785    pub fn age_refresh_holds_for_tests(&self, by: Duration) {
2786        let by_ms = by.as_millis() as u64;
2787        for hold in self
2788            .refresh_holds
2789            .lock()
2790            .unwrap_or_else(|e| e.into_inner())
2791            .values_mut()
2792        {
2793            hold.at_ms = hold.at_ms.saturating_sub(by_ms);
2794        }
2795    }
2796
2797    /// Test seam: the digest a held refusal of `cred` keeps, so a test can
2798    /// check that it appears in no output.
2799    #[doc(hidden)]
2800    pub fn refresh_hold_digest_for_tests(&self, cred: &CredentialRef) -> Option<String> {
2801        self.refresh_holds
2802            .lock()
2803            .unwrap_or_else(|e| e.into_inner())
2804            .get(&Self::state_key(cred))
2805            .map(|h| h.material.expose_for_tests().to_string())
2806    }
2807
2808    fn record_usage(
2809        &self,
2810        cred: &CredentialRef,
2811        failure: Option<super::quota_policy::ObservationFailure>,
2812    ) {
2813        let at_ms = crate::epoch_millis();
2814        let mut observations = self.observations.lock().unwrap_or_else(|e| e.into_inner());
2815        let slot = observations.entry(Self::state_key(cred)).or_default();
2816        if failure.is_none() {
2817            slot.last_usage_ok_ms = Some(at_ms);
2818        }
2819        slot.last_usage = Some(super::auth_state::ObservedOutcome { at_ms, failure });
2820    }
2821
2822    /// One usage read for `cred` through the process-wide table, with its
2823    /// INFO line (account label, outcome, failure kind/status; no token).
2824    /// A read that went to the network (or joined one in flight) is
2825    /// recorded for `synaps auth state`; a held read sends nothing and is
2826    /// not a new read.
2827    async fn shared_read(&self, cred: &CredentialRef) -> super::usage_reads::ReadResult {
2828        use super::usage_reads::{ReadError, ReadPath, USAGE_READ_TARGET};
2829        let key = self.read_key(cred);
2830        let (result, path) = self
2831            .usage_reads
2832            .read(&key, || self.network_read(cred))
2833            .await;
2834        if path != ReadPath::NegativeCacheHit {
2835            self.record_usage(cred, result.as_ref().err().map(ReadError::cause));
2836        }
2837        let outcome = match (path, &result) {
2838            (ReadPath::Network, Ok(_)) => "ok",
2839            (ReadPath::Network, Err(_)) => "failed",
2840            (ReadPath::Joined, _) => "joined",
2841            (ReadPath::NegativeCacheHit, _) => "negative_cache_hit",
2842        };
2843        let now_ms = crate::epoch_millis();
2844        let (kind, status, retry_in_s) = match &result {
2845            Ok(_) => ("none", String::new(), 0),
2846            Err(ReadError::Transient {
2847                failure,
2848                retry_at_ms,
2849            }) => (
2850                failure.kind,
2851                failure.status.map(|s| s.to_string()).unwrap_or_default(),
2852                retry_at_ms.saturating_sub(now_ms) / 1000,
2853            ),
2854            Err(ReadError::Other { cause, .. }) => (
2855                cause.kind.as_str(),
2856                cause.http_status.map(|s| s.to_string()).unwrap_or_default(),
2857                0,
2858            ),
2859        };
2860        tracing::info!(
2861            target: USAGE_READ_TARGET,
2862            broker = self.instance,
2863            account = %cred.account.label_str(),
2864            outcome = %outcome,
2865            kind = %kind,
2866            status = %status,
2867            retry_in_s,
2868            "usage read"
2869        );
2870        result
2871    }
2872
2873    /// One network usage read for `cred`, reporting the seat whose token
2874    /// made the request. Token resolution happens HERE, behind the boundary,
2875    /// for exactly this credential; the usage helper pairs any account
2876    /// header with the same token. Only [`Self::shared_read`] calls it.
2877    async fn network_read(&self, cred: &CredentialRef) -> super::usage_reads::Fetched {
2878        use super::usage::{fetch_usage, supports_usage, UsageClient, UsageError, UsageFetchOptions};
2879        use super::usage_reads::Fetched;
2880        if !supports_usage(cred.provider) {
2881            return Fetched::Usage(UsageError::UnsupportedProvider {
2882                provider: cred.provider.as_str().to_string(),
2883            });
2884        }
2885        if let Err(e) = self.check_slot_exists(cred) {
2886            return UsageReadError::slot(e).into_fetched();
2887        }
2888        // The hold is checked again, and a refusal recorded, under the
2889        // slot's refresh gate ([`SlotHold`]).
2890        let memory: Arc<dyn super::token::RefusalMemory> = self.slot_hold(cred);
2891        let creds = match super::token::ensure_fresh_credential_via(
2892            &self.http,
2893            cred,
2894            &self.refresh_endpoints,
2895            Some(memory),
2896        )
2897        .await
2898        {
2899            Ok(creds) => creds,
2900            Err(f) => return UsageReadError::refresh(f).into_fetched(),
2901        };
2902        let seat = SeatIdentity::of(cred.provider, &creds);
2903        let client = match UsageClient::new() {
2904            Ok(client) => client,
2905            Err(e) => return Fetched::Usage(e),
2906        };
2907        let opts = UsageFetchOptions {
2908            endpoint_override: self.usage_endpoint_override.clone(),
2909            ..UsageFetchOptions::default()
2910        };
2911        match fetch_usage(
2912            &client,
2913            cred.provider,
2914            cred.account.label_str(),
2915            &creds.access,
2916            &opts,
2917        )
2918        .await
2919        {
2920            Ok(snapshot) => Fetched::Ok(snapshot, seat),
2921            Err(e) => Fetched::Usage(e),
2922        }
2923    }
2924
2925    /// Capacity view for one account from a fresh, read-only usage snapshot,
2926    /// with the seat it was read from. Anything that is not a well-formed,
2927    /// fresh reading is NOT capacity; a failed read keeps its typed cause. A
2928    /// read that failed transiently also reports that failure (and when
2929    /// reads resume) and, when the slot has one read from the account stored
2930    /// there now, its last good reading for the policy's grace, paired with
2931    /// that stored seat.
2932    async fn capacity_for(&self, cred: &CredentialRef) -> CandidateRead {
2933        use super::quota_policy::AccountCapacity;
2934        use super::usage_reads::ReadError;
2935        match self.fresh_usage(cred).await {
2936            Ok((snapshot, seat)) => CandidateRead {
2937                capacity: capacity_from_snapshot(cred, &snapshot),
2938                seat: Some(seat),
2939                transient: None,
2940                grace: None,
2941            },
2942            Err(read_err) => {
2943                let err = UsageReadError::from_read(&read_err);
2944                if err.failure.kind.needs_action() {
2945                    // Only the user can fix this, outside synaps; the
2946                    // selection failure names just the kind, so say here
2947                    // what and where.
2948                    tracing::warn!(
2949                        credential = %cred,
2950                        kind = %err.failure.kind,
2951                        detail = %action_detail(cred, &err),
2952                        "usage read needs action"
2953                    );
2954                } else {
2955                    tracing::debug!(credential = %cred, cause = %err.failure, "usage read failed");
2956                }
2957                let capacity = AccountCapacity::failed(cred.clone(), err.failure);
2958                let ReadError::Transient {
2959                    failure,
2960                    retry_at_ms,
2961                } = read_err
2962                else {
2963                    return CandidateRead {
2964                        capacity,
2965                        seat: None,
2966                        transient: None,
2967                        grace: None,
2968                    };
2969                };
2970                // Grace only on a reading of the account in the slot now: a
2971                // re-login to another account id never inherits it.
2972                let graced = self.stored_seat(cred).ok().and_then(|seat| {
2973                    let good = self
2974                        .usage_reads
2975                        .last_good_for(&self.read_key(cred), &seat)?;
2976                    Some((good, seat))
2977                });
2978                let (grace, seat) = match graced {
2979                    Some((good, seat)) => (Some(capacity_from_snapshot(cred, &good)), Some(seat)),
2980                    None => (None, None),
2981                };
2982                CandidateRead {
2983                    capacity,
2984                    seat,
2985                    transient: Some((failure, retry_at_ms)),
2986                    grace,
2987                }
2988            }
2989        }
2990    }
2991
2992    /// Preview the exact ranking without vending inference credentials or
2993    /// changing stickiness/cooldowns. Usage reads may refresh expired OAuth
2994    /// tokens through the ordinary rotation-safe credential path.
2995    pub async fn plan(
2996        &self,
2997        provider: OAuthProviderId,
2998        model: Option<&str>,
2999    ) -> Result<super::SelectionPlan, BrokerError> {
3000        self.selection_plan(provider, model, &[], &mut FailedThisSelection::new())
3001            .await
3002            .map(|(plan, ..)| plan)
3003    }
3004
3005    /// Read every account's capacity and rank them; an account whose
3006    /// cooldown is active is not read. `exclude` lists the
3007    /// accounts the request already used. An account in `failed` failed
3008    /// earlier in this selection: it counts as failed with that kind and is
3009    /// not read again; a read that fails now (other than a held transient
3010    /// read, which the usage-read table answers without a request) is added.
3011    #[allow(clippy::type_complexity)]
3012    async fn selection_plan(
3013        &self,
3014        provider: OAuthProviderId,
3015        model: Option<&str>,
3016        exclude: &[CredentialRef],
3017        failed: &mut FailedThisSelection,
3018    ) -> Result<
3019        (
3020            super::SelectionPlan,
3021            BTreeMap<String, SeatIdentity>,
3022            BTreeMap<String, (super::usage_reads::TransientReadFailure, u64)>,
3023        ),
3024        BrokerError,
3025    > {
3026        use super::quota_policy::{AccountCapacity, SelectionRequest};
3027        let summaries = storage::list_accounts(provider).map_err(BrokerError::Credential)?;
3028        let mut candidates = Vec::with_capacity(summaries.len());
3029        let mut seats = BTreeMap::new();
3030        let mut transient = BTreeMap::new();
3031        let mut grace = Vec::new();
3032        let mut seen = std::collections::BTreeSet::new();
3033        let mut exclude = exclude.to_vec();
3034        // Each account's cooldown keys, worked out once for both lookups
3035        // below (and only while some cooldown is active: they read the
3036        // stored login).
3037        let cooling_any = self.any_cooldown_active(crate::epoch_millis());
3038        let mut keys = BTreeMap::new();
3039        for summary in &summaries {
3040            let Some(cred) = summary.credential_ref() else {
3041                continue;
3042            };
3043            if cooling_any {
3044                keys.insert(cred.storage_key(), self.cooldown_keys(&cred));
3045            }
3046            if let Some(failure) = failed.get(&cred.storage_key()) {
3047                candidates.push(AccountCapacity::failed(cred, failure.clone()));
3048                continue;
3049            }
3050            // An account whose cooldown is active can't be chosen, so its
3051            // usage is not read (no usage request, no token refresh). It
3052            // keeps its stored seat, for stickiness and the duplicate-seat
3053            // check; the policy rejects it as cooling down.
3054            let read = if keys
3055                .get(&cred.storage_key())
3056                .and_then(|k| self.cooldown_for_keys(k, crate::epoch_millis()))
3057                .is_some()
3058            {
3059                CandidateRead {
3060                    capacity: AccountCapacity::unobserved(cred.clone()),
3061                    seat: self.stored_seat(&cred).ok(),
3062                    transient: None,
3063                    grace: None,
3064                }
3065            } else {
3066                self.capacity_for(&cred).await
3067            };
3068            let (mut capacity, seat) = (read.capacity, read.seat);
3069            match (read.transient, &capacity.failure) {
3070                (Some(held), _) => {
3071                    transient.insert(cred.storage_key(), held);
3072                }
3073                (None, Some(failure)) => {
3074                    failed.insert(cred.storage_key(), failure.clone());
3075                }
3076                (None, None) => {}
3077            }
3078            // The same entitlement guard for a fresh reading and for a last
3079            // good reading standing in for a failed read.
3080            require_requested_model(provider, model, &mut capacity);
3081            grace.extend(read.grace.map(|mut good| {
3082                require_requested_model(provider, model, &mut good);
3083                good
3084            }));
3085            if let Some(seat) = seat {
3086                if !seen.insert(seat.as_str().to_string()) {
3087                    exclude.push(cred.clone());
3088                }
3089                seats.insert(cred.storage_key(), seat);
3090            }
3091            candidates.push(capacity);
3092        }
3093        let now_ms = crate::epoch_millis();
3094        // A cooldown reported while the reads ran counts too.
3095        if !cooling_any && self.any_cooldown_active(now_ms) {
3096            for capacity in &candidates {
3097                let cred = &capacity.credential;
3098                keys.insert(cred.storage_key(), self.cooldown_keys(cred));
3099            }
3100        }
3101        for capacity in &mut candidates {
3102            capacity.cooldown_until_ms = keys
3103                .get(&capacity.credential.storage_key())
3104                .and_then(|k| self.cooldown_for_keys(k, now_ms))
3105                .map(|c| c.until_ms);
3106        }
3107        let previous = self
3108            .current_accounts
3109            .lock()
3110            .unwrap_or_else(|e| e.into_inner())
3111            .get(&Self::current_key(provider, model))
3112            .cloned();
3113        let current = previous
3114            .filter(|(cred, seat)| seats.get(&cred.storage_key()) == Some(seat))
3115            .map(|(cred, _)| cred);
3116        let policy = self.auto_policy();
3117        let preference: Vec<_> = summaries.iter().map(|s| s.label.clone()).collect();
3118        let request = SelectionRequest {
3119            model,
3120            preference: &preference,
3121            exclude: &exclude,
3122            strategy: policy.strategy_for(provider),
3123            current: current.as_ref(),
3124            urgent_horizon_ms: policy.urgent_horizon_ms,
3125            sticky: policy.sticky,
3126            grace: &grace,
3127            ..SelectionRequest::new(provider, now_ms, self.max_snapshot_age.as_millis() as u64)
3128        };
3129        Ok((
3130            super::SelectionPlan::from_candidates(&request, &candidates),
3131            seats,
3132            transient,
3133        ))
3134    }
3135
3136    async fn select_auto(
3137        &self,
3138        provider: OAuthProviderId,
3139        model: Option<&str>,
3140        exclude: &[CredentialRef],
3141        failed: &mut FailedThisSelection,
3142    ) -> Result<(CredentialRef, SeatIdentity), BrokerError> {
3143        use super::quota_policy::{Selection, SelectionFailure, SelectionFailureKind};
3144        use super::usage_reads::USAGE_READ_TARGET;
3145        let (plan, mut seats, transient) = match self
3146            .selection_plan(provider, model, exclude, failed)
3147            .await
3148        {
3149            Ok(planned) => planned,
3150            // The account list itself could not be read (the only error
3151            // `selection_plan` returns): still exactly one WARN for this
3152            // failed selection, and the caller gets the store's own error.
3153            Err(e) => {
3154                log_selection_failure(
3155                    provider,
3156                    model,
3157                    &SelectionFailure::new(SelectionFailureKind::CredentialStore),
3158                );
3159                return Err(e);
3160            }
3161        };
3162        match plan.selection {
3163            Selection::Selected {
3164                credential,
3165                tier,
3166                budget_reset_ms,
3167                utilization,
3168                ..
3169            } => {
3170                // Every account whose usage was read has its seat, so a
3171                // selected account always has one.
3172                let seat = seats.remove(&credential.storage_key());
3173                debug_assert!(
3174                    seat.is_some(),
3175                    "selected account has no paired seat identity"
3176                );
3177                let Some(seat) = seat else {
3178                    return Err(selection_failed(
3179                        provider,
3180                        model,
3181                        "selected account has no paired seat identity".into(),
3182                        SelectionFailure::new(SelectionFailureKind::SeatChurn),
3183                    ));
3184                };
3185                // A chosen account whose read failed was chosen on its last
3186                // good reading (the policy's grace).
3187                if let Some((failure, retry_at_ms)) = transient.get(&credential.storage_key()) {
3188                    tracing::info!(
3189                        target: USAGE_READ_TARGET,
3190                        broker = self.instance,
3191                        account = %credential.account.label_str(),
3192                        outcome = %"grace",
3193                        kind = %failure.kind,
3194                        status = %failure.status.map(|s| s.to_string()).unwrap_or_default(),
3195                        retry_in_s = retry_at_ms.saturating_sub(crate::epoch_millis()) / 1000,
3196                        "usage read"
3197                    );
3198                }
3199                tracing::info!(%credential, ?plan.strategy, tier, ?budget_reset_ms, ?utilization, "automatic subscription choice");
3200                Ok((credential, seat))
3201            }
3202            Selection::NoCapacity {
3203                rejections,
3204                earliest_reset_ms,
3205            } => {
3206                let detail: Vec<_> = rejections
3207                    .iter()
3208                    .map(|r| format!("{}: {}", r.credential.account, r.describe()))
3209                    .collect();
3210                let reset = earliest_reset_ms
3211                    .map(|t| format!("; earliest reset at {t}"))
3212                    .unwrap_or_default();
3213                let mut failure = super::quota_policy::SelectionFailure::from_rejections(
3214                    &rejections,
3215                    plan.now_ms,
3216                );
3217                // A held usage read goes to the network again at the next
3218                // selection after its hold ends: say when.
3219                if failure.kind == SelectionFailureKind::UsageReadFailed {
3220                    failure.usage_read_retry_ms = rejections
3221                        .iter()
3222                        .filter_map(|r| transient.get(&r.credential.storage_key()))
3223                        .map(|(_, at)| *at)
3224                        .min();
3225                }
3226                let reason = if detail.is_empty() {
3227                    format!("no {provider} account is logged in")
3228                } else {
3229                    format!("{}{reset}", detail.join(", "))
3230                };
3231                Err(selection_failed(provider, model, reason, failure))
3232            }
3233        }
3234    }
3235
3236    /// Test/embedding seam: pin the `local` provider endpoint explicitly.
3237    pub fn with_local_base_url(http: reqwest::Client, base_url: impl Into<String>) -> Self {
3238        Self {
3239            local_base_url: Some(base_url.into()),
3240            ..Self::new(http)
3241        }
3242    }
3243
3244    /// Test seam: point the pinned Google Code Assist host at a loopback fake.
3245    /// Production code must never call this — it only relaxes the base URL,
3246    /// not any of the path allowlist / bearer / redirect-denial invariants.
3247    #[doc(hidden)]
3248    pub fn with_google_gemini_base_url_for_tests(
3249        http: reqwest::Client,
3250        base_url: impl Into<String>,
3251    ) -> Self {
3252        Self {
3253            google_gemini_base_url: Some(base_url.into().trim_end_matches('/').to_string()),
3254            ..Self::new(http)
3255        }
3256    }
3257
3258    /// Test seam: point the pinned Anthropic usage operation at a fake server.
3259    #[doc(hidden)]
3260    pub fn with_anthropic_usage_url(mut self, url: impl Into<String>) -> Self {
3261        self.anthropic_usage_url = Some(url.into());
3262        self
3263    }
3264
3265    /// Test seam: shrink the buffered-request time budget.
3266    #[doc(hidden)]
3267    pub fn with_request_timeout(mut self, timeout: Duration) -> Self {
3268        self.request_timeout = timeout;
3269        self
3270    }
3271
3272    /// Test seam: shrink the buffered-response size cap.
3273    #[doc(hidden)]
3274    pub fn with_max_response_bytes(mut self, cap: usize) -> Self {
3275        self.max_response_bytes = cap;
3276        self
3277    }
3278
3279    /// Resolve (and migrate) a static provider key. Broker-owned storage wins;
3280    /// login config and env are legacy discovery surfaces that are migrated
3281    /// into broker storage on first use.
3282    fn resolve_static_key(&self, provider: &str) -> Result<String, BrokerError> {
3283        if provider == LOCAL_PROVIDER_KEY {
3284            return Ok(self.resolve_local_key());
3285        }
3286        let spec = static_provider(provider)
3287            .ok_or_else(|| BrokerError::UnknownProvider(provider.to_string()))?;
3288        if let Ok(Some(key)) = storage::load_static_key(provider) {
3289            return Ok(key);
3290        }
3291        if let Some(key) = discover_legacy_static_key(spec) {
3292            // Migration: persist into broker-owned storage. Best-effort — a
3293            // read-only home dir must not break the request itself.
3294            if let Err(e) = storage::save_static_key(provider, &key) {
3295                tracing::warn!(provider, "static key migration failed: {e}");
3296            }
3297            return Ok(key);
3298        }
3299        Err(BrokerError::NotConfigured(provider.to_string()))
3300    }
3301
3302    fn resolve_local_key(&self) -> String {
3303        storage::load_static_key(LOCAL_PROVIDER_KEY)
3304            .ok()
3305            .flatten()
3306            .or_else(|| {
3307                crate::config::get_provider_keys()
3308                    .get(LOCAL_PROVIDER_KEY)
3309                    .filter(|s| !s.is_empty())
3310                    .cloned()
3311            })
3312            .or_else(|| {
3313                std::env::var("LOCAL_API_KEY")
3314                    .ok()
3315                    .filter(|s| !s.is_empty())
3316            })
3317            .unwrap_or_else(|| "local".to_string())
3318    }
3319
3320    fn base_url_for(&self, provider: &str) -> Result<String, BrokerError> {
3321        if provider == LOCAL_PROVIDER_KEY {
3322            if let Some(url) = &self.local_base_url {
3323                return Ok(url.trim_end_matches('/').to_string());
3324            }
3325            return Ok(local_endpoint_url());
3326        }
3327        static_provider(provider)
3328            .map(|s| s.base_url.trim_end_matches('/').to_string())
3329            .ok_or_else(|| BrokerError::UnknownProvider(provider.to_string()))
3330    }
3331
3332    async fn send(&self, request: &ProxyRequest) -> Result<reqwest::Response, BrokerError> {
3333        request.validate()?;
3334        let (provider_key, explicit_account) = request.split_provider()?;
3335        let provider_key = provider_key.as_str();
3336        // OAuth-proxied providers: resolve ONE credential (explicit account
3337        // or policy) and derive both the bearer and any account-specific
3338        // header from that same credential.
3339        let oauth_provider: Option<OAuthProviderId> = provider_key.parse().ok();
3340        let mut pinned: Option<PinnedToken> = None;
3341        // Muse honours the official CLI's `META_API_KEY` override: a Model
3342        // API key from the environment wins over the account login (only for
3343        // policy-selected requests — an explicit account stays explicit).
3344        let muse_env_key = (provider_key == "muse" && explicit_account.is_none())
3345            .then(super::muse::api_key_from_env)
3346            .flatten();
3347        if let Some(provider) = oauth_provider.filter(|_| muse_env_key.is_none()) {
3348            if provider_key != LOCAL_PROVIDER_KEY && static_provider(provider_key).is_none() {
3349                pinned = Some(
3350                    self.resolve_and_vend(provider, explicit_account, None, &[])
3351                        .await?,
3352                );
3353            }
3354        }
3355        let bearer = |pinned: &Option<PinnedToken>| -> Result<String, BrokerError> {
3356            pinned
3357                .as_ref()
3358                .map(|p| p.token.token.clone())
3359                .ok_or_else(|| BrokerError::UnknownProvider(request.provider.clone()))
3360        };
3361        let (key, base) = if provider_key == "xai-auth" {
3362            (bearer(&pinned)?, "https://api.x.ai/v1".to_string())
3363        } else if provider_key == "github-copilot" {
3364            // Catalog-only OAuth proxy: short-lived Copilot session token only.
3365            // Never attach the GitHub user token (stored as OAuth refresh).
3366            (
3367                bearer(&pinned)?,
3368                super::github_copilot_models_base_url().to_string(),
3369            )
3370        } else if provider_key == "google-gemini" {
3371            // Google Gemini (Code Assist) is broker-proxy-only. Refresh stays
3372            // broker-owned; runtime never receives it.
3373            let base = self
3374                .google_gemini_base_url
3375                .clone()
3376                .unwrap_or_else(|| GOOGLE_GEMINI_CODE_ASSIST_BASE_URL.to_string());
3377            (bearer(&pinned)?, base)
3378        } else if provider_key == "openai-codex" {
3379            // Catalog-only OAuth proxy for ChatGPT backend models. Access token
3380            // never leaves the broker; account header is derived broker-side.
3381            let base = self
3382                .openai_codex_base_url
3383                .clone()
3384                .unwrap_or_else(|| OPENAI_CODEX_BACKEND_BASE_URL.to_string());
3385            (bearer(&pinned)?, base)
3386        } else if provider_key == "kimi-code" {
3387            // Managed Kimi Code OAuth proxy: short-lived (~15 min) access
3388            // token resolved broker-side; the rotating refresh token never
3389            // leaves the boundary. Base is pinned to the managed endpoint.
3390            (bearer(&pinned)?, super::kimi_code::API_BASE_URL.to_string())
3391        } else if provider_key == "muse" {
3392            // Meta Muse: the bearer is a Model API key (env override, or the
3393            // key minted from the broker-owned Meta OAuth login). The OAuth
3394            // token itself never leaves the boundary. Origin is pinned; the
3395            // allowlist pins the paths.
3396            let key = match &muse_env_key {
3397                Some(key) => key.clone(),
3398                None => bearer(&pinned)?,
3399            };
3400            let base = self
3401                .muse_base_url
3402                .clone()
3403                .unwrap_or_else(|| super::muse::API_ORIGIN.to_string());
3404            (key, base)
3405        } else if provider_key == "anthropic" {
3406            // Catalog-only OAuth proxy for Anthropic /v1/models pagination.
3407            // Keeps the access token broker-owned (no runtime token vending).
3408            (bearer(&pinned)?, "https://api.anthropic.com".to_string())
3409        } else {
3410            (
3411                self.resolve_static_key(provider_key)?,
3412                self.base_url_for(provider_key)?,
3413            )
3414        };
3415        let response = self
3416            .dispatch(request, provider_key, &key, &base, &pinned)
3417            .await?;
3418        // Muse mirrors the official CLI: a 401 on a minted key re-mints ONCE
3419        // from the stored OAuth login (under the credential's refresh locks;
3420        // a peer's fresh key is adopted instead) and the request is retried
3421        // once. An env-supplied key is never re-minted.
3422        //
3423        // If the re-mint is refused (forced-refresh cooldown) or the mint
3424        // rejects the stored login, the caller gets the ORIGINAL upstream 401
3425        // — status and body untouched, exactly like the env-key path — not a
3426        // broker error (the authenticator-gives-up convention). A transient,
3427        // lock-busy or local failure is a retryable `BrokerError::Refresh`
3428        // instead. Only the status has been read, so `response` is still
3429        // unconsumed here. This covers `proxy` and `proxy_stream`, which both
3430        // go through `send`.
3431        if provider_key == "muse" && response.status() == reqwest::StatusCode::UNAUTHORIZED {
3432            if let Some(rejected) = pinned.as_ref() {
3433                tracing::info!(credential = %rejected.credential, "muse key rejected (401); re-minting once");
3434                match self.recover_rejected_token(&rejected.credential, &key).await {
3435                    Ok(fresh) => {
3436                        let pinned = Some(PinnedToken {
3437                            credential: rejected.credential.clone(),
3438                            token: fresh,
3439                        });
3440                        let key = bearer(&pinned)?;
3441                        return self
3442                            .dispatch(request, provider_key, &key, &base, &pinned)
3443                            .await;
3444                    }
3445                    // A transient mint failure, a busy refresh lock, a local
3446                    // failure or something the user must fix elsewhere (a
3447                    // corrupt store) says nothing about the login: surface
3448                    // the refresh error, not the upstream 401 (which the
3449                    // runtime shows as "sign in again").
3450                    Err(BrokerError::Refresh(f))
3451                        if f.kind.is_retryable()
3452                            || f.kind == super::token::RefreshFailureKind::ActionRequired =>
3453                    {
3454                        tracing::warn!(
3455                            credential = %rejected.credential,
3456                            reason = %f,
3457                            "muse key re-mint failed; returning a retryable error"
3458                        );
3459                        return Err(BrokerError::Refresh(f));
3460                    }
3461                    Err(e) => {
3462                        // Token-free: recovery errors carry sanitized reasons
3463                        // (cooldown, MuseAuthError labels, lock timeouts).
3464                        tracing::warn!(
3465                            credential = %rejected.credential,
3466                            reason = %e,
3467                            "muse key re-mint refused or rejected; returning the upstream 401"
3468                        );
3469                    }
3470                }
3471            }
3472        }
3473        Ok(response)
3474    }
3475
3476    /// Build and send one upstream request with an already-resolved bearer.
3477    async fn dispatch(
3478        &self,
3479        request: &ProxyRequest,
3480        provider_key: &str,
3481        key: &str,
3482        base: &str,
3483        pinned: &Option<PinnedToken>,
3484    ) -> Result<reqwest::Response, BrokerError> {
3485        let url = format!("{base}{}", request.path);
3486        // Deny redirects for credential-bearing OAuth catalog traffic and
3487        // google-gemini: a 3xx must not replay the bearer token off-origin.
3488        let mut builder = if provider_key == "google-gemini"
3489            || provider_key == "openai-codex"
3490            || provider_key == "anthropic"
3491            || provider_key == "kimi-code"
3492            || provider_key == "muse"
3493        {
3494            // `read_timeout` is an IDLE timer (reset on every read), so a
3495            // long SSE stream that keeps sending is never cut; a stalled one
3496            // (silent NAT drop, hung edge) fails instead of hanging the turn.
3497            let no_redirect = reqwest::Client::builder()
3498                .redirect(reqwest::redirect::Policy::none())
3499                .connect_timeout(Duration::from_secs(10))
3500                .read_timeout(self.stream_idle_timeout_for(provider_key))
3501                .build()
3502                .map_err(|e| BrokerError::Transport(format!("client build failed: {e}")))?;
3503            match request.method {
3504                ProxyMethod::Get => no_redirect.get(&url),
3505                ProxyMethod::Post => no_redirect.post(&url),
3506            }
3507        } else {
3508            match request.method {
3509                ProxyMethod::Get => self.http.get(&url),
3510                ProxyMethod::Post => self.http.post(&url),
3511            }
3512        };
3513        builder = builder.bearer_auth(key);
3514        if provider_key == "openai-codex" {
3515            // ChatGPT backend requires the account id. Derive it from the SAME
3516            // credential that produced the bearer: the JWT claim first, then
3517            // that slot's stored `accountId`. Never another slot, never omitted.
3518            let cred = pinned
3519                .as_ref()
3520                .map(|p| p.credential.clone())
3521                .unwrap_or_else(|| CredentialRef::default_for(OAuthProviderId::OpenAiCodex));
3522            let account_id = super::extract_codex_account_id(key)
3523                .or_else(|| {
3524                    storage::load_credential(&cred)
3525                        .ok()
3526                        .flatten()
3527                        .and_then(|c| c.account_id)
3528                        .filter(|s| !s.trim().is_empty())
3529                })
3530                .ok_or_else(|| {
3531                    BrokerError::Credential(
3532                        "openai-codex credential is missing chatgpt account id".into(),
3533                    )
3534                })?;
3535            builder = builder
3536                .header("chatgpt-account-id", account_id)
3537                .header("originator", "synaps")
3538                .header("OpenAI-Beta", "responses=experimental")
3539                .header("accept", "application/json");
3540        }
3541        if provider_key == "anthropic" {
3542            // OAuth catalog: Bearer + anthropic-version + anthropic-beta.
3543            // Do NOT send the OAuth token as x-api-key.
3544            for (name, value) in anthropic_oauth_catalog_request_headers() {
3545                builder = builder.header(*name, *value);
3546            }
3547        }
3548        if provider_key == "github-copilot" {
3549            for (name, value) in super::github_copilot_models_request_headers() {
3550                builder = builder.header(*name, *value);
3551            }
3552            if request.path != "/models" {
3553                builder = builder
3554                    .header("Openai-Intent", "conversation-edits")
3555                    .header("X-Initiator", "agent");
3556            }
3557        }
3558        if provider_key == "google-gemini" {
3559            // The Code Assist reference client uses `?alt=sse` for streaming.
3560            // Match that so upstream returns line-delimited SSE frames rather
3561            // than JSON-in-one-response.
3562            if request.stream && request.path == "/v1internal:streamGenerateContent" {
3563                builder = builder.query(&[("alt", "sse")]);
3564            }
3565            builder = builder.header("user-agent", "SynapsCLI/0.6.0 (google-gemini)");
3566        }
3567        if provider_key == "muse" {
3568            // API-key requests need only Authorization (verified); identify
3569            // Synaps honestly instead of mimicking the official client.
3570            builder = builder.header("user-agent", super::muse::USER_AGENT);
3571        }
3572        if provider_key == "kimi-code" {
3573            // Conventional Kimi device-identity surface (mirrors the official
3574            // CLI's `X-Msh-*` headers). Values are non-secret; the User-Agent
3575            // identifies Synaps honestly.
3576            let device_id = super::kimi_code::load_or_create_device_id();
3577            for (name, value) in super::kimi_code::identity_request_headers(&device_id) {
3578                builder = builder.header(name, value);
3579            }
3580        }
3581        if let Some(bytes) = &request.body_bytes {
3582            // Exact-byte handoff (request-trace spec §6.2): send the very
3583            // buffer the caller serialized (and digested) — never a
3584            // re-serialization of the JSON value.
3585            builder = builder
3586                .header("content-type", "application/json")
3587                .body(bytes.clone());
3588        } else if let Some(body) = &request.body {
3589            builder = builder.json(body);
3590        }
3591        if request.stream {
3592            builder = builder.header("accept", "text/event-stream");
3593        } else {
3594            // Buffered requests get an explicit total time budget; streaming
3595            // responses are open-ended by design (SSE) and rely on the
3596            // client's connect timeout plus consumer-side backpressure.
3597            builder = builder.timeout(self.request_timeout);
3598        }
3599        builder.send().await.map_err(|e| {
3600            if e.is_connect() && provider_key == LOCAL_PROVIDER_KEY {
3601                BrokerError::Transport(format!(
3602                    "can't reach local endpoint at {url} — is Ollama/LM Studio running?"
3603                ))
3604            } else {
3605                // reqwest errors do not include the bearer header.
3606                BrokerError::Transport(format!("request to {} failed: {e}", request.provider))
3607            }
3608        })
3609    }
3610}
3611
3612// ── Usage → capacity mapping (fail closed) ───────────────────────────────────
3613
3614/// Map a typed usage snapshot onto the pure capacity policy's input. Only
3615/// account-wide and model-scoped quota windows constrain inference;
3616/// feature-scoped windows (e.g. code review) are not inference limits.
3617pub fn capacity_from_snapshot(
3618    cred: &CredentialRef,
3619    snapshot: &super::usage::UsageSnapshot,
3620) -> super::quota_policy::AccountCapacity {
3621    use super::quota_policy::{
3622        AccountCapacity, ModelAvailability, ModelState, QuotaObservation, WindowLimit,
3623    };
3624    use super::usage::{Availability, WindowScope};
3625    let mut windows: Vec<WindowLimit> = snapshot
3626        .windows
3627        .iter()
3628        .filter_map(|w| {
3629            let models = match &w.scope {
3630                WindowScope::Account => None,
3631                WindowScope::Model { model } => Some(vec![model.clone()]),
3632                WindowScope::Feature { .. } => return None,
3633            };
3634            Some(WindowLimit {
3635                id: w.id.clone(),
3636                duration_ms: w.duration_secs.map(|s| s.saturating_mul(1000)),
3637                used_percent: w.used_percent.valid(),
3638                limit_reached: w.limit_reached,
3639                resets_at_ms: w.reset_at,
3640                models,
3641            })
3642        })
3643        .collect();
3644    if snapshot.limit_reached == Some(true) {
3645        // Provider-asserted overall exhaustion applies to every model.
3646        windows.push(WindowLimit {
3647            id: "limit_reached".into(),
3648            duration_ms: None,
3649            used_percent: None,
3650            limit_reached: Some(true),
3651            resets_at_ms: snapshot.earliest_reset_at(),
3652            models: None,
3653        });
3654    }
3655    let models = if snapshot.model_availability.is_empty() {
3656        None
3657    } else {
3658        Some(
3659            snapshot
3660                .model_availability
3661                .iter()
3662                .map(|m| ModelAvailability {
3663                    model: m.model.clone(),
3664                    state: match m.availability {
3665                        Availability::Available => ModelState::Available,
3666                        Availability::Exhausted => ModelState::Exhausted,
3667                        Availability::Unknown => ModelState::Unknown,
3668                    },
3669                })
3670                .collect(),
3671        )
3672    };
3673    AccountCapacity {
3674        credential: cred.clone(),
3675        observed_at_ms: Some(snapshot.observed_at),
3676        observation: QuotaObservation::Ok { windows, models },
3677        cooldown_until_ms: None,
3678        failure: None,
3679    }
3680}
3681
3682/// A failed usage read: the broker error `usage()` callers see (unchanged
3683/// vocabulary) plus the typed cause the selection report keeps.
3684#[derive(Debug)]
3685struct UsageReadError {
3686    error: BrokerError,
3687    failure: super::quota_policy::ObservationFailure,
3688}
3689
3690/// One account's read for automatic selection.
3691struct CandidateRead {
3692    capacity: super::quota_policy::AccountCapacity,
3693    /// Seat the reading belongs to (for a graced account: the seat stored now).
3694    seat: Option<SeatIdentity>,
3695    /// The read failed transiently: the failure and when reads resume.
3696    transient: Option<(super::usage_reads::TransientReadFailure, u64)>,
3697    /// Last good reading offered to the policy's grace.
3698    grace: Option<super::quota_policy::AccountCapacity>,
3699}
3700
3701impl UsageReadError {
3702    /// A read answered by the process-wide usage-read table (a network,
3703    /// joined or held read): the same error and cause the read itself had.
3704    fn from_read(e: &super::usage_reads::ReadError) -> Self {
3705        Self {
3706            error: e.to_broker_error(),
3707            failure: e.cause(),
3708        }
3709    }
3710
3711    /// This failure as the table's error for a read that never reached the
3712    /// usage request (slot check).
3713    fn into_read_error(self) -> super::usage_reads::ReadError {
3714        super::usage_reads::ReadError::Other {
3715            error: self.error,
3716            cause: self.failure,
3717        }
3718    }
3719
3720    /// This failure as a network read's outcome (slot check, refresh).
3721    fn into_fetched(self) -> super::usage_reads::Fetched {
3722        super::usage_reads::Fetched::Failed {
3723            error: self.error,
3724            cause: self.failure,
3725        }
3726    }
3727
3728    /// The `usage()` text stays the historical credential error; the cause
3729    /// keeps the refresh kind (transient vs rejected login).
3730    fn refresh(f: super::token::RefreshFailure) -> Self {
3731        Self {
3732            failure: refresh_failure(&f),
3733            error: BrokerError::Credential(f.into()),
3734        }
3735    }
3736
3737    /// Slot checks: a slot that vanished vs a store that can't be read.
3738    fn slot(error: BrokerError) -> Self {
3739        use super::quota_policy::{AccountFailureKind as K, ObservationFailure};
3740        let kind = match &error {
3741            BrokerError::UnknownAccount { .. } => K::CredentialMissing,
3742            _ => K::CredentialStore,
3743        };
3744        Self {
3745            failure: ObservationFailure::new(kind),
3746            error,
3747        }
3748    }
3749
3750    /// A failed usage request as the selection sees it: through the
3751    /// usage-read table, which keeps the request's own error and cause.
3752    #[cfg(test)]
3753    fn usage(e: super::usage::UsageError) -> Self {
3754        let table = super::usage_reads::UsageReads::default();
3755        match table.record_at("slot", super::usage_reads::Fetched::Usage(e), 0) {
3756            Err(read) => Self::from_read(&read),
3757            Ok(_) => unreachable!("a usage error is never a reading"),
3758        }
3759    }
3760}
3761
3762/// A generic Codex window (including a free 30-day plan) does not prove
3763/// entitlement to Astra or any other requested model: a requested model the
3764/// reading does not list counts as unknown. Applied to every reading the
3765/// policy judges, fresh or graced.
3766fn require_requested_model(
3767    provider: OAuthProviderId,
3768    model: Option<&str>,
3769    capacity: &mut super::quota_policy::AccountCapacity,
3770) {
3771    use super::quota_policy::{model_matches, ModelAvailability, ModelState, QuotaObservation};
3772    if provider != OAuthProviderId::OpenAiCodex {
3773        return;
3774    }
3775    if let (Some(model), QuotaObservation::Ok { models, .. }) = (model, &mut capacity.observation) {
3776        let availability = models.get_or_insert_with(Vec::new);
3777        if !availability.iter().any(|m| model_matches(&m.model, model)) {
3778            availability.push(ModelAvailability {
3779                model: model.to_string(),
3780                state: ModelState::Unknown,
3781            });
3782        }
3783    }
3784}
3785
3786/// What a usage read that needs action asks of the user. A refresh that
3787/// needs action carries our own instructions (paths, what to move back);
3788/// an unreadable stored login names the store, not the parser's message
3789/// (which can quote stored values).
3790fn action_detail(cred: &CredentialRef, err: &UsageReadError) -> String {
3791    use super::quota_policy::AccountFailureKind as K;
3792    match err.failure.kind {
3793        K::CredentialStore => format!(
3794            "the stored login of '{}' could not be read from {}; repair or restore the \
3795             credential store",
3796            cred.account.label_str(),
3797            storage::auth_file_path().display()
3798        ),
3799        _ => err.error.to_string(),
3800    }
3801}
3802
3803/// The observation-failure kind of a token refresh failure, with the token
3804/// endpoint's HTTP status when it answered with one.
3805fn refresh_failure(f: &super::token::RefreshFailure) -> super::quota_policy::ObservationFailure {
3806    let failure = refresh_kind_failure(f.kind);
3807    match f.http_status() {
3808        Some(status) => failure.with_status(status),
3809        None => failure,
3810    }
3811}
3812
3813/// The observation-failure kind of a token refresh failure kind.
3814fn refresh_kind_failure(
3815    kind: super::token::RefreshFailureKind,
3816) -> super::quota_policy::ObservationFailure {
3817    use super::quota_policy::{AccountFailureKind as K, ObservationFailure};
3818    use super::token::RefreshFailureKind as R;
3819    match kind {
3820        R::Transient => ObservationFailure::new(K::RefreshTransient),
3821        R::LockBusy => ObservationFailure::new(K::RefreshLockBusy),
3822        R::Other => ObservationFailure::new(K::RefreshOther),
3823        R::Cooldown { retry_in_secs } => {
3824            ObservationFailure::new(K::RefreshOther).with_retry_after(Some(retry_in_secs))
3825        }
3826        R::Credential => ObservationFailure::new(K::RefreshRejected),
3827        R::ActionRequired => ObservationFailure::new(K::RefreshActionRequired),
3828    }
3829}
3830
3831/// The refresh failure of an automatically chosen account, naming it (and,
3832/// for a refused login, the command that logs it in again).
3833fn auto_vend_failure(
3834    credential: &CredentialRef,
3835    f: super::token::RefreshFailure,
3836) -> super::token::RefreshFailure {
3837    let label = credential.account.label_str();
3838    let mut message = format!("account '{label}': {}", f.message());
3839    if f.kind == super::token::RefreshFailureKind::Credential && !message.contains("synaps login") {
3840        let account = if credential.account.is_default() {
3841            String::new()
3842        } else {
3843            format!(" --account {label}")
3844        };
3845        message.push_str(&format!(
3846            "; log it in again with `synaps login --provider {}{account}`",
3847            credential.provider
3848        ));
3849    }
3850    super::token::RefreshFailure::new(f.kind, message).with_http_status(f.http_status())
3851}
3852
3853/// A failed automatic selection: ONE WARN (see [`log_selection_failure`]),
3854/// then the typed error.
3855fn selection_failed(
3856    provider: OAuthProviderId,
3857    model: Option<&str>,
3858    reason: String,
3859    failure: super::quota_policy::SelectionFailure,
3860) -> BrokerError {
3861    log_selection_failure(provider, model, &failure);
3862    BrokerError::NoAccountAvailable {
3863        provider: provider.to_string(),
3864        reason,
3865        failure: Some(failure),
3866    }
3867}
3868
3869/// The one `auth_selection_failed` WARN of a failed automatic selection:
3870/// the top-level kind and every account's `label:kind[:status][:cooling]`
3871/// (labels only, never identities or token material).
3872fn log_selection_failure(
3873    provider: OAuthProviderId,
3874    model: Option<&str>,
3875    failure: &super::quota_policy::SelectionFailure,
3876) {
3877    tracing::warn!(
3878        provider = %provider,
3879        model = model.unwrap_or("any"),
3880        kind = %failure.kind,
3881        accounts = %failure.accounts_field(),
3882        earliest_reset_ms = ?failure.earliest_reset_ms,
3883        "auth_selection_failed"
3884    );
3885}
3886
3887// ── Bounded body handling ────────────────────────────────────────────────────
3888
3889/// Read a response body up to `cap` bytes; fail closed (no truncated JSON
3890/// masquerading as a full payload) if the upstream exceeds the cap.
3891async fn read_body_capped(resp: reqwest::Response, cap: usize) -> Result<String, BrokerError> {
3892    use futures::StreamExt;
3893    let mut buf: Vec<u8> = Vec::new();
3894    let mut stream = resp.bytes_stream();
3895    while let Some(chunk) = stream.next().await {
3896        let chunk =
3897            chunk.map_err(|e| BrokerError::Transport(format!("failed to read response: {e}")))?;
3898        if buf.len() + chunk.len() > cap {
3899            return Err(BrokerError::Transport(format!(
3900                "response body exceeded the {cap}-byte broker buffering limit"
3901            )));
3902        }
3903        buf.extend_from_slice(&chunk);
3904    }
3905    String::from_utf8(buf)
3906        .map_err(|_| BrokerError::Transport("response body was not valid UTF-8".into()))
3907}
3908
3909/// Cap on how much of a non-2xx upstream body is read for *classification*
3910/// only (spec §5.1). Error envelopes are tiny; anything larger is cut off
3911/// and simply fails to classify.
3912const MAX_PROXY_ERROR_CLASSIFY_BYTES: usize = 8 * 1024;
3913
3914/// Read at most `cap` bytes of an untrusted error body for classification.
3915/// Never fails: overflow truncates (so JSON no longer parses → no label),
3916/// transport errors and non-UTF-8 yield `None`. The returned text must only
3917/// ever be fed to a vetted classifier — never to an error string or log.
3918async fn read_error_body_for_classification(resp: reqwest::Response, cap: usize) -> Option<String> {
3919    use futures::StreamExt;
3920    let mut buf: Vec<u8> = Vec::with_capacity(cap.min(1024));
3921    let mut stream = resp.bytes_stream();
3922    while let Some(chunk) = stream.next().await {
3923        let chunk = chunk.ok()?;
3924        let room = cap.saturating_sub(buf.len());
3925        if chunk.len() >= room {
3926            buf.extend_from_slice(&chunk[..room]);
3927            // Drop the rest unread: truncated JSON cannot classify, which is
3928            // the intended fail-closed outcome for oversized bodies.
3929            return String::from_utf8(buf).ok();
3930        }
3931        buf.extend_from_slice(&chunk);
3932    }
3933    String::from_utf8(buf).ok()
3934}
3935
3936/// Legacy discovery for a static key: login config first, then env vars.
3937/// Only callable from inside the broker boundary.
3938fn discover_legacy_static_key(spec: &StaticProviderSpec) -> Option<String> {
3939    if let Some(v) = crate::config::get_provider_keys().get(spec.key) {
3940        if !v.is_empty() {
3941            return Some(v.clone());
3942        }
3943    }
3944    spec.env_vars
3945        .iter()
3946        .find_map(|var| std::env::var(var).ok().filter(|v| !v.is_empty()))
3947}
3948
3949/// True if a static key is available through the broker (storage, login
3950/// config, or env). Never returns the key.
3951pub fn static_key_configured(provider: &str) -> bool {
3952    if provider == LOCAL_PROVIDER_KEY {
3953        return true; // the local endpoint never requires a key
3954    }
3955    let Some(spec) = static_provider(provider) else {
3956        return false;
3957    };
3958    storage::load_static_key(provider).ok().flatten().is_some()
3959        || discover_legacy_static_key(spec).is_some()
3960}
3961
3962/// Non-secret display status for a static key: configured (with masked
3963/// preview), from-env, or not set. The mask keeps at most the first 4 and
3964/// last 4 characters and is safe to render.
3965pub fn static_key_status(provider: &str) -> StaticKeyStatus {
3966    let stored = storage::load_static_key(provider)
3967        .ok()
3968        .flatten()
3969        .or_else(|| {
3970            crate::config::get_provider_keys()
3971                .get(provider)
3972                .filter(|s| !s.is_empty())
3973                .cloned()
3974        });
3975    if let Some(key) = stored {
3976        return StaticKeyStatus::Configured {
3977            masked: mask_key(&key),
3978        };
3979    }
3980    if let Some(spec) = static_provider(provider) {
3981        if spec
3982            .env_vars
3983            .iter()
3984            .any(|var| std::env::var(var).is_ok_and(|v| !v.is_empty()))
3985        {
3986            return StaticKeyStatus::FromEnv;
3987        }
3988    }
3989    StaticKeyStatus::NotSet
3990}
3991
3992/// Masked, non-reversible preview of a key (`gsk-…7f2a`).
3993fn mask_key(key: &str) -> String {
3994    let chars: Vec<char> = key.chars().collect();
3995    if chars.len() <= 8 {
3996        return "…".to_string();
3997    }
3998    let head: String = chars[..4].iter().collect();
3999    let tail: String = chars[chars.len() - 4..].iter().collect();
4000    format!("{head}…{tail}")
4001}
4002
4003/// The set of static providers with an available credential. Non-secret.
4004pub fn configured_static_provider_keys() -> std::collections::BTreeSet<String> {
4005    STATIC_PROVIDERS
4006        .iter()
4007        .filter(|s| static_key_configured(s.key))
4008        .map(|s| s.key.to_string())
4009        .collect()
4010}
4011
4012/// The local endpoint URL (non-secret configuration: `provider.local.url`
4013/// config → `LOCAL_ENDPOINT` env → default).
4014pub fn local_endpoint_url() -> String {
4015    local_endpoint_config()
4016        .unwrap_or_else(|| LOCAL_DEFAULT_BASE_URL.to_string())
4017        .trim_end_matches('/')
4018        .to_string()
4019}
4020
4021/// The explicitly-configured local endpoint, if any (None → default in use).
4022pub fn local_endpoint_config() -> Option<String> {
4023    crate::config::get_provider_keys()
4024        .get("local.url")
4025        .filter(|s| !s.is_empty())
4026        .cloned()
4027        .or_else(|| {
4028            std::env::var("LOCAL_ENDPOINT")
4029                .ok()
4030                .filter(|s| !s.is_empty())
4031        })
4032}
4033
4034/// Comma-separated local model ids from non-secret config (`provider.local.models`).
4035pub fn local_model_ids() -> Vec<String> {
4036    crate::config::get_provider_keys()
4037        .get("local.models")
4038        .map(|value| {
4039            value
4040                .split(',')
4041                .map(|s| s.trim().to_string())
4042                .filter(|s| !s.is_empty())
4043                .collect()
4044        })
4045        .unwrap_or_default()
4046}
4047
4048/// True if any Anthropic credential is available through the broker
4049/// (OAuth login under the active account policy, or `ANTHROPIC_API_KEY`).
4050/// Non-secret answer for first-run UX.
4051pub fn anthropic_credential_available() -> bool {
4052    oauth_provider_logged_in(OAuthProviderId::Anthropic)
4053        || std::env::var("ANTHROPIC_API_KEY").is_ok_and(|v| !v.is_empty())
4054}
4055
4056/// True if the OAuth provider has a stored, refreshable credential the
4057/// active account policy (env > config > default) can reach. Non-secret
4058/// availability query for UI surfaces; an expired access token is still a
4059/// valid login when the broker can refresh it on first use.
4060///
4061/// * An explicitly selected named account counts only if THAT slot exists
4062///   (explicit selection never falls back, so neither does this answer).
4063/// * Otherwise any usable slot of the provider counts — a named-only
4064///   installation is logged in, not hidden. Which slot a vend uses, and
4065///   whether a malformed selector fails it, is reported by the vend itself.
4066pub fn oauth_provider_logged_in(provider: OAuthProviderId) -> bool {
4067    oauth_provider_logged_in_at(
4068        &storage::auth_file_path(),
4069        &AccountPolicy::from_environment(),
4070        provider,
4071    )
4072}
4073
4074/// Path/policy-explicit core of [`oauth_provider_logged_in`].
4075pub(crate) fn oauth_provider_logged_in_at(
4076    path: &std::path::Path,
4077    policy: &AccountPolicy,
4078    provider: OAuthProviderId,
4079) -> bool {
4080    let Ok(inventory) = storage::list_accounts_detailed_at(path, Some(provider)) else {
4081        return false;
4082    };
4083    match policy.selector(provider) {
4084        AccountSelector::Account(Account::Named(label)) => inventory
4085            .accounts
4086            .iter()
4087            .any(|row| row.label == label.as_str()),
4088        AccountSelector::Account(Account::Default)
4089        | AccountSelector::Auto
4090        | AccountSelector::Invalid { .. } => !inventory.accounts.is_empty(),
4091    }
4092}
4093
4094#[async_trait]
4095impl CredentialBroker for LocalBroker {
4096    async fn recover_rejected_token(
4097        &self,
4098        cred: &CredentialRef,
4099        rejected: &str,
4100    ) -> Result<AccessToken, BrokerError> {
4101        self.check_slot_exists(cred)?;
4102        let creds = super::token::recover_rejected_credential(
4103            &self.http,
4104            cred,
4105            rejected,
4106            &self.refresh_endpoints,
4107        )
4108        .await
4109        .map_err(BrokerError::Refresh)?;
4110        Ok(AccessToken {
4111            token: creds.access,
4112            expires: creds.expires,
4113        })
4114    }
4115
4116    async fn access_token(&self, provider: OAuthProviderId) -> Result<AccessToken, BrokerError> {
4117        Ok(self.access_token_pinned(provider).await?.token)
4118    }
4119
4120    fn account_selector(&self, provider: OAuthProviderId) -> AccountSelector {
4121        self.policy().selector(provider)
4122    }
4123
4124    async fn access_token_for(&self, cred: &CredentialRef) -> Result<AccessToken, BrokerError> {
4125        self.check_slot_exists(cred)?;
4126        // An account the user chose is never held (no memory).
4127        let creds = super::token::ensure_fresh_credential_via(
4128            &self.http,
4129            cred,
4130            &self.refresh_endpoints,
4131            None,
4132        )
4133        .await
4134        .map_err(|f| match f.kind {
4135            // Typed so the runtime doesn't send the user to a login
4136            // that would fail on the same store.
4137            super::token::RefreshFailureKind::ActionRequired => BrokerError::Refresh(f),
4138            _ => BrokerError::Credential(f.into()),
4139        })?;
4140        // Strip to token + expiry: the refresh token stays behind the boundary.
4141        Ok(AccessToken {
4142            token: creds.access,
4143            expires: creds.expires,
4144        })
4145    }
4146
4147    async fn access_token_pinned(
4148        &self,
4149        provider: OAuthProviderId,
4150    ) -> Result<PinnedToken, BrokerError> {
4151        self.access_token_pinned_for(provider, None).await
4152    }
4153
4154    async fn access_token_pinned_for(
4155        &self,
4156        provider: OAuthProviderId,
4157        model: Option<&str>,
4158    ) -> Result<PinnedToken, BrokerError> {
4159        self.resolve_and_vend(provider, None, model, &[]).await
4160    }
4161
4162    /// The exclusion goes into the selection itself: `auto` selects among
4163    /// the accounts the request hasn't used (one that is cooling down still
4164    /// reads as cooling down in a failure).
4165    async fn access_token_pinned_excluding(
4166        &self,
4167        provider: OAuthProviderId,
4168        model: Option<&str>,
4169        exclude: &[CredentialRef],
4170    ) -> Result<PinnedToken, BrokerError> {
4171        self.resolve_and_vend(provider, None, model, exclude).await
4172    }
4173
4174    async fn accounts(&self, provider: OAuthProviderId) -> Result<Vec<AccountSummary>, BrokerError> {
4175        let mut rows = storage::list_accounts(provider).map_err(BrokerError::Credential)?;
4176        let selector = self.policy().selector(provider);
4177        let now_ms = crate::epoch_millis();
4178        for row in &mut rows {
4179            row.selected = matches!(&selector, AccountSelector::Account(a) if a.label_str() == row.label);
4180            if let Some(cred) = row.credential_ref() {
4181                row.cooldown_until = self.cooldown_until(&cred.storage_key(), now_ms);
4182            }
4183        }
4184        Ok(rows)
4185    }
4186
4187    async fn usage(&self, cred: &CredentialRef) -> Result<super::usage::UsageSnapshot, BrokerError> {
4188        self.usage_with_seat(cred)
4189            .await
4190            .map(|(snapshot, _seat)| snapshot)
4191            .map_err(|e| e.error)
4192    }
4193
4194    fn active_cooldown(&self, cred: &CredentialRef) -> Option<u64> {
4195        self.active_cooldown_at(cred, crate::epoch_millis())
4196            .map(|c| c.until_ms)
4197    }
4198
4199    fn clear_cooldown(&self, cred: &CredentialRef, sent_at_ms: u64) {
4200        let now_ms = crate::epoch_millis();
4201        if !self.any_cooldown_active(now_ms) {
4202            return;
4203        }
4204        let keys = self.cooldown_keys(cred);
4205        let mut cleared = None;
4206        {
4207            let mut cooldowns = self.cooldowns.lock().unwrap_or_else(|e| e.into_inner());
4208            for key in keys.read() {
4209                if cooldowns
4210                    .get(key)
4211                    .is_some_and(|c| c.until_ms > now_ms && c.reported_at_ms <= sent_at_ms)
4212                {
4213                    cleared = cooldowns.remove(key).or(cleared);
4214                }
4215            }
4216        }
4217        if let Some(c) = cleared {
4218            tracing::info!(
4219                credential = %cred,
4220                until_ms = c.until_ms,
4221                reason = %c.reason,
4222                "account cooldown cleared: a request sent after it was reported was accepted"
4223            );
4224        }
4225    }
4226
4227    async fn access_token_for_auto(&self, cred: &CredentialRef) -> Result<AccessToken, BrokerError> {
4228        self.check_slot_exists(cred)?;
4229        // A refused refresh of this login is held: no new exchange.
4230        if let Some(held) = self.held_refresh(cred) {
4231            return Err(BrokerError::Refresh(auto_vend_failure(cred, held)));
4232        }
4233        let memory: Arc<dyn super::token::RefusalMemory> = self.slot_hold(cred);
4234        let creds = super::token::ensure_fresh_credential_via(
4235            &self.http,
4236            cred,
4237            &self.refresh_endpoints,
4238            Some(memory),
4239        )
4240        .await
4241        .map_err(|f| BrokerError::Refresh(auto_vend_failure(cred, f)))?;
4242        Ok(AccessToken {
4243            token: creds.access,
4244            expires: creds.expires,
4245        })
4246    }
4247
4248    async fn report_cooldown(
4249        &self,
4250        cred: &CredentialRef,
4251        until_ms: Option<u64>,
4252        reason: &str,
4253    ) -> Result<(), BrokerError> {
4254        let now_ms = crate::epoch_millis();
4255        let max_until = now_ms.saturating_add(MAX_COOLDOWN.as_millis() as u64);
4256        let until_ms = until_ms
4257            .filter(|t| *t > now_ms)
4258            .unwrap_or(now_ms + DEFAULT_COOLDOWN.as_millis() as u64)
4259            .min(max_until);
4260        let reason = crate::truncate_str(reason, 64).to_string();
4261        tracing::info!(credential = %cred, until_ms, reason = %reason, "account cooldown reported");
4262        let keys = self.cooldown_keys(cred);
4263        {
4264            let mut cooldowns = self.cooldowns.lock().unwrap_or_else(|e| e.into_inner());
4265            for key in keys.written() {
4266                cooldowns.insert(
4267                    key.to_string(),
4268                    Cooldown {
4269                        until_ms,
4270                        reason: reason.clone(),
4271                        reported_at_ms: now_ms,
4272                    },
4273                );
4274            }
4275        }
4276        // The cached reading predates the limit report; drop it so the next
4277        // selection re-reads instead of trusting stale headroom, and drop
4278        // the slot's last good reading so grace can never outlive the limit.
4279        self.forget_snapshots(cred);
4280        self.usage_reads.forget_last_good(&self.read_key(cred));
4281        Ok(())
4282    }
4283
4284    async fn proxy(&self, request: ProxyRequest) -> Result<ProxyResponse, BrokerError> {
4285        let resp = self.send(&request).await?;
4286        let status = resp.status().as_u16();
4287        let body = read_body_capped(resp, self.max_response_bytes).await?;
4288        Ok(ProxyResponse { status, body })
4289    }
4290
4291    async fn proxy_stream(&self, request: ProxyRequest) -> Result<ProxyByteStream, BrokerError> {
4292        let mut request = request;
4293        request.stream = true;
4294        let resp = self.send(&request).await?;
4295        let status = resp.status();
4296        if !status.is_success() {
4297            // Spec §5.1: the upstream may echo the full request (prompts,
4298            // tool schemas, credentials) in its error body. The body is read
4299            // (bounded) for *classification only*: it selects one of OUR
4300            // static labels or nothing — no provider bytes reach the error
4301            // or the log. The stable `provider request failed: {status}`
4302            // prefix (with canonical reason phrase, e.g. `429 Too Many
4303            // Requests`) is the transport contract engine-side classifiers
4304            // parse; the optional ` [label]` suffix follows it and contains
4305            // no ':' so `redact_provider_proxy_error` keeps it intact.
4306            let label = read_error_body_for_classification(resp, MAX_PROXY_ERROR_CLASSIFY_BYTES)
4307                .await
4308                .as_deref()
4309                .and_then(crate::error::vetted_proxy_error_label);
4310            tracing::warn!(
4311                provider = %request.provider,
4312                status = status.as_u16(),
4313                label = label.unwrap_or("-"),
4314                "provider request failed"
4315            );
4316            let label_suffix = label.map(|l| format!(" [{l}]")).unwrap_or_default();
4317            return Err(BrokerError::Transport(format!(
4318                "provider request failed: {status}{label_suffix}"
4319            )));
4320        }
4321        use futures::StreamExt;
4322        let stream = resp
4323            .bytes_stream()
4324            .map(|chunk| chunk.map_err(|e| BrokerError::Transport(format!("stream error: {e}"))));
4325        Ok(Box::pin(stream))
4326    }
4327
4328    async fn anthropic_usage(&self) -> Result<serde_json::Value, BrokerError> {
4329        // Token resolution happens HERE, behind the boundary — the caller
4330        // never touches auth.json or sees the access token.
4331        let token = self.access_token(OAuthProviderId::Anthropic).await?;
4332        let url = self
4333            .anthropic_usage_url
4334            .as_deref()
4335            .unwrap_or(ANTHROPIC_USAGE_URL);
4336        let resp = self
4337            .http
4338            .get(url)
4339            .timeout(self.request_timeout)
4340            .bearer_auth(&token.token)
4341            .header("anthropic-beta", ANTHROPIC_OAUTH_BETA)
4342            .send()
4343            .await
4344            .map_err(|e| BrokerError::Transport(format!("usage request failed: {e}")))?;
4345        let status = resp.status();
4346        if !status.is_success() {
4347            // The error body is upstream-controlled and may echo the request;
4348            // it never enters the error (spec §5.1).
4349            drop(resp);
4350            return Err(BrokerError::Transport(format!(
4351                "usage request failed: {status}"
4352            )));
4353        }
4354        let body = read_body_capped(resp, self.max_response_bytes).await?;
4355        serde_json::from_str(&body)
4356            .map_err(|e| BrokerError::Transport(format!("invalid usage response: {e}")))
4357    }
4358
4359    async fn cloud_catalog(
4360        &self,
4361        provider: CloudProviderId,
4362        context_ref: &str,
4363        allow_stale: bool,
4364    ) -> Result<Vec<CloudCatalogEntry>, BrokerError> {
4365        self.cloud_backend
4366            .as_ref()
4367            .ok_or_else(|| BrokerError::NotConfigured(provider.to_string()))?
4368            .catalog(provider, context_ref, allow_stale)
4369            .await
4370    }
4371
4372    async fn cloud_invoke(
4373        &self,
4374        provider: CloudProviderId,
4375        context_ref: &str,
4376        model_id: &str,
4377        request: InvokeRequest,
4378    ) -> Result<CloudEventStream, BrokerError> {
4379        self.cloud_backend
4380            .as_ref()
4381            .ok_or_else(|| BrokerError::NotConfigured(provider.to_string()))?
4382            .invoke(provider, context_ref, model_id, request)
4383            .await
4384    }
4385
4386    async fn capabilities(&self) -> Result<Vec<ProviderStatus>, BrokerError> {
4387        let mut out = Vec::new();
4388        let policy = self.policy();
4389        for descriptor in super::provider::registry().iter() {
4390            let accounts = self.accounts(descriptor.id).await.unwrap_or_default();
4391            let configured = match policy.selector(descriptor.id) {
4392                AccountSelector::Account(account) => accounts
4393                    .iter()
4394                    .any(|a| a.label == account.label_str()),
4395                AccountSelector::Auto => !accounts.is_empty(),
4396                AccountSelector::Invalid { .. } => false,
4397            };
4398            out.push(ProviderStatus {
4399                key: descriptor.id.as_str().to_string(),
4400                name: descriptor.display_name.to_string(),
4401                kind: CredentialKind::OAuth,
4402                configured,
4403                accounts,
4404            });
4405        }
4406        for spec in STATIC_PROVIDERS {
4407            out.push(ProviderStatus {
4408                key: spec.key.to_string(),
4409                name: spec.name.to_string(),
4410                kind: CredentialKind::StaticKey,
4411                configured: static_key_configured(spec.key),
4412                accounts: Vec::new(),
4413            });
4414        }
4415        out.push(ProviderStatus {
4416            key: LOCAL_PROVIDER_KEY.to_string(),
4417            name: "Local endpoint".to_string(),
4418            kind: CredentialKind::LocalEndpoint,
4419            configured: true,
4420            accounts: Vec::new(),
4421        });
4422        Ok(out)
4423    }
4424}
4425
4426// ── Remote broker client ─────────────────────────────────────────────────────
4427
4428/// Client for a remote `synaps auth-broker`. Presents the machine's own
4429/// bearer token; receives access tokens (OAuth) and proxied responses
4430/// (static-key providers). Never receives refresh tokens or raw keys.
4431pub struct RemoteBroker {
4432    http: reqwest::Client,
4433    endpoint: String,
4434    /// Reloadable from the local broker token file after a 401 when the
4435    /// endpoint is loopback (prA I2).
4436    machine_token: super::MachineToken,
4437    cache: super::TokenCache,
4438    /// Injected account policy; `None` → env + config per call.
4439    account_policy: Option<AccountPolicy>,
4440}
4441
4442/// A 404 from a route the broker does not have: 0.10.2's axum router
4443/// answers an empty, untyped body, while a current broker's errors are JSON.
4444/// Only emptiness and the content type are inspected, never the content.
4445async fn route_missing(resp: reqwest::Response) -> bool {
4446    let json = resp
4447        .headers()
4448        .get(reqwest::header::CONTENT_TYPE)
4449        .and_then(|v| v.to_str().ok())
4450        .is_some_and(|v| v.contains("json"));
4451    !json
4452        && read_body_capped(resp, 1024)
4453            .await
4454            .is_ok_and(|body| body.trim().is_empty())
4455}
4456
4457impl RemoteBroker {
4458    /// True if the broker has no `/usage/snapshot` (0.10.2 and older). The
4459    /// probe sends an empty account, which a current broker rejects with a
4460    /// 400 before touching any credential.
4461    async fn predates_usage_snapshot(&self) -> Result<bool, BrokerError> {
4462        let resp = self
4463            .machine_token
4464            .send(|t| {
4465                self.http
4466                    .get(format!("{}/usage/snapshot", self.endpoint))
4467                    .query(&[("provider", "anthropic"), ("account", "")])
4468                    .bearer_auth(t)
4469            })
4470            .await
4471            .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4472        match resp.status().as_u16() {
4473            401 => Err(BrokerError::Unauthorized),
4474            404 => Ok(route_missing(resp).await),
4475            _ => Ok(false),
4476        }
4477    }
4478
4479    pub fn new(
4480        endpoint: impl Into<String>,
4481        machine_token: impl Into<String>,
4482        http: reqwest::Client,
4483        cache: super::TokenCache,
4484    ) -> Self {
4485        let endpoint = endpoint.into().trim_end_matches('/').to_string();
4486        // Never trust a caller-supplied redirect policy for credential-bearing
4487        // broker RPC. A redirect could otherwise replay machine auth off-origin.
4488        let http = reqwest::Client::builder()
4489            .redirect(reqwest::redirect::Policy::none())
4490            .connect_timeout(Duration::from_secs(10))
4491            .timeout(Duration::from_secs(90))
4492            .build()
4493            .unwrap_or(http);
4494        let machine_token = super::MachineToken::for_endpoint(&endpoint, machine_token);
4495        Self {
4496            http,
4497            endpoint,
4498            machine_token,
4499            cache,
4500            account_policy: None,
4501        }
4502    }
4503
4504    /// Pin the account policy instead of reading env + config per call.
4505    pub fn with_account_policy(mut self, policy: AccountPolicy) -> Self {
4506        self.account_policy = Some(policy);
4507        self
4508    }
4509
4510    fn policy(&self) -> AccountPolicy {
4511        self.account_policy
4512            .clone()
4513            .unwrap_or_else(AccountPolicy::from_environment)
4514    }
4515
4516    fn fetcher(&self) -> super::BrokerClient {
4517        super::BrokerClient::with_machine_token(
4518            self.endpoint.clone(),
4519            self.machine_token.clone(),
4520            self.http.clone(),
4521        )
4522    }
4523
4524    /// Cache scope: endpoint + machine-principal digest (never the token).
4525    fn scope(&self) -> String {
4526        super::credential_source::source_scope(&self.endpoint, &self.machine_token.get())
4527    }
4528
4529    /// Resolve the credential for a token request on the client side. An
4530    /// `auto` selector is delegated to the broker daemon (`account=auto`),
4531    /// which holds the usage/cooldown state; the daemon reports which slot
4532    /// it chose.
4533    async fn resolve_pinned(
4534        &self,
4535        provider: OAuthProviderId,
4536        model: Option<&str>,
4537    ) -> Result<PinnedToken, BrokerError> {
4538        match self.policy().selector(provider) {
4539            AccountSelector::Account(account) => {
4540                let credential = CredentialRef::new(provider, account);
4541                let token = self.access_token_for(&credential).await?;
4542                Ok(PinnedToken { credential, token })
4543            }
4544            AccountSelector::Auto => self.fetch_auto(provider, model).await,
4545            AccountSelector::Invalid { source, reason } => {
4546                Err(BrokerError::InvalidAccount(format!("{source}: {reason}")))
4547            }
4548        }
4549    }
4550
4551    /// `GET /token?provider=X&account=auto[&model=M]` — the daemon selects;
4552    /// the response's `account` field names the chosen slot. Never cached
4553    /// (the selection is the daemon's, per call).
4554    async fn fetch_auto(
4555        &self,
4556        provider: OAuthProviderId,
4557        model: Option<&str>,
4558    ) -> Result<PinnedToken, BrokerError> {
4559        let mut query: Vec<(&str, &str)> = vec![
4560            ("provider", provider.as_str()),
4561            ("account", super::account::AUTO_ACCOUNT_NAME),
4562        ];
4563        if let Some(model) = model {
4564            query.push(("model", model));
4565        }
4566        let resp = self
4567            .machine_token
4568            .send(|t| {
4569                self.http
4570                .get(format!("{}/token", self.endpoint))
4571                .query(&query)
4572                .bearer_auth(t)
4573            })
4574            .await
4575            .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4576        match resp.status().as_u16() {
4577            401 => return Err(BrokerError::Unauthorized),
4578            503 => {
4579                drop(resp);
4580                return Err(BrokerError::NoAccountAvailable {
4581                    provider: provider.as_str().to_string(),
4582                    reason: "broker reported no account with proven capacity".into(),
4583                    // The broker host's own log has the per-account kinds.
4584                    failure: Some(super::quota_policy::SelectionFailure::new(
4585                        super::quota_policy::SelectionFailureKind::RemoteNoCapacity,
4586                    )),
4587                });
4588            }
4589            s if !(200..300).contains(&s) => {
4590                drop(resp);
4591                return Err(BrokerError::Transport(format!("broker returned HTTP {s}")));
4592            }
4593            _ => {}
4594        }
4595        #[derive(Deserialize)]
4596        struct AutoToken {
4597            access_token: String,
4598            expires: u64,
4599            #[serde(default)]
4600            ttl_ms: Option<u64>,
4601            account: Option<String>,
4602        }
4603        let body = read_body_capped(resp, MAX_PROXY_RESPONSE_BYTES).await?;
4604        let tok: AutoToken = serde_json::from_str(&body)
4605            .map_err(|e| BrokerError::Transport(format!("invalid broker token response: {e}")))?;
4606        let account = tok
4607            .account
4608            .as_deref()
4609            .ok_or_else(|| BrokerError::Transport("broker did not report the selected account".into()))
4610            .and_then(|a| Account::parse(a).map_err(BrokerError::InvalidAccount))?;
4611        if tok.access_token.is_empty() {
4612            return Err(BrokerError::Transport("broker returned an empty access_token".into()));
4613        }
4614        let expires = match tok.ttl_ms {
4615            Some(ttl) => crate::epoch_millis().saturating_add(ttl),
4616            None => tok.expires,
4617        };
4618        Ok(PinnedToken {
4619            credential: CredentialRef::new(provider, account),
4620            token: AccessToken {
4621                token: tok.access_token,
4622                expires,
4623            },
4624        })
4625    }
4626
4627    /// A non-2xx `/proxy` answer. A 502 whose `error` is a retryable
4628    /// `BrokerError::Refresh` from the daemon (it maps those to 502) becomes
4629    /// the same typed error, so the client keeps its "retry shortly" hint.
4630    /// Only the KIND is read from the body, by matching the exact prefix our
4631    /// own Display produces for that kind; the message is static, so no
4632    /// broker-controlled byte reaches the Display (spec §5.1).
4633    async fn proxy_failure(resp: reqwest::Response, fallback: String) -> BrokerError {
4634        if resp.status() != reqwest::StatusCode::BAD_GATEWAY {
4635            return BrokerError::Transport(fallback);
4636        }
4637        let Ok(body) = read_body_capped(resp, 4096).await else {
4638            return BrokerError::Transport(fallback);
4639        };
4640        let text = serde_json::from_str::<serde_json::Value>(&body)
4641            .ok()
4642            .and_then(|v| v.get("error")?.as_str().map(str::to_string))
4643            .unwrap_or_default();
4644        use super::token::{RefreshFailure, RefreshFailureKind as K};
4645        for kind in [K::Transient, K::LockBusy, K::Other] {
4646            let prefix = BrokerError::Refresh(RefreshFailure::new(kind, "")).to_string();
4647            if text.starts_with(&prefix) {
4648                return BrokerError::Refresh(RefreshFailure::new(
4649                    kind,
4650                    "the remote credential broker could not refresh this credential",
4651                ));
4652            }
4653        }
4654        BrokerError::Transport(fallback)
4655    }
4656
4657    async fn post_proxy(&self, request: &ProxyRequest) -> Result<reqwest::Response, BrokerError> {
4658        request.validate()?;
4659        let resp = self
4660            .machine_token
4661            .send(|t| {
4662                self.http
4663                .post(format!("{}/proxy", self.endpoint))
4664                .bearer_auth(t)
4665                .json(request)
4666            })
4667            .await
4668            .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4669        match resp.status().as_u16() {
4670            401 => Err(BrokerError::Unauthorized),
4671            s @ (400 | 403) => {
4672                // A compromised broker controls every byte of its response —
4673                // including JSON `error` fields. Static status-only message;
4674                // the body is dropped unread (spec §5.1).
4675                drop(resp);
4676                Err(BrokerError::Denied(format!(
4677                    "broker rejected the request (HTTP {s})"
4678                )))
4679            }
4680            _ => Ok(resp),
4681        }
4682    }
4683}
4684
4685#[async_trait]
4686impl CredentialBroker for RemoteBroker {
4687    async fn access_token(&self, provider: OAuthProviderId) -> Result<AccessToken, BrokerError> {
4688        Ok(self.access_token_pinned(provider).await?.token)
4689    }
4690
4691    fn account_selector(&self, provider: OAuthProviderId) -> AccountSelector {
4692        self.policy().selector(provider)
4693    }
4694
4695    async fn access_token_for(&self, cred: &CredentialRef) -> Result<AccessToken, BrokerError> {
4696        let fetcher = self.fetcher();
4697        let tok = super::credential_source::resolve_remote_credential(
4698            &fetcher,
4699            &self.cache,
4700            &self.scope(),
4701            cred,
4702            super::DEFAULT_MARGIN_MS,
4703        )
4704        .await
4705        .map_err(|e| e.into_broker_error(cred))?;
4706        Ok(AccessToken {
4707            token: tok.access_token,
4708            expires: tok.expires,
4709        })
4710    }
4711
4712    async fn access_token_pinned(
4713        &self,
4714        provider: OAuthProviderId,
4715    ) -> Result<PinnedToken, BrokerError> {
4716        self.resolve_pinned(provider, None).await
4717    }
4718
4719    async fn access_token_pinned_for(
4720        &self,
4721        provider: OAuthProviderId,
4722        model: Option<&str>,
4723    ) -> Result<PinnedToken, BrokerError> {
4724        self.resolve_pinned(provider, model).await
4725    }
4726
4727    async fn accounts(&self, provider: OAuthProviderId) -> Result<Vec<AccountSummary>, BrokerError> {
4728        let caps = self.capabilities().await?;
4729        let accounts = caps
4730            .into_iter()
4731            .find(|c| c.key == provider.as_str())
4732            .map(|c| c.accounts)
4733            .unwrap_or_default();
4734        // A 0.10.2 broker lists no accounts at all; don't report that as
4735        // "nothing logged in".
4736        if accounts.is_empty() && self.predates_usage_snapshot().await? {
4737            return Err(BrokerError::BrokerTooOld("account listing".into()));
4738        }
4739        Ok(accounts)
4740    }
4741
4742    async fn usage(&self, cred: &CredentialRef) -> Result<super::usage::UsageSnapshot, BrokerError> {
4743        let resp = self
4744            .machine_token
4745            .send(|t| {
4746                self.http
4747                .get(format!("{}/usage/snapshot", self.endpoint))
4748                .query(&[
4749                    ("provider", cred.provider.as_str()),
4750                    ("account", cred.account.label_str()),
4751                ])
4752                .bearer_auth(t)
4753            })
4754            .await
4755            .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4756        match resp.status().as_u16() {
4757            401 => Err(BrokerError::Unauthorized),
4758            404 => {
4759                if !route_missing(resp).await {
4760                    return Err(BrokerError::UnknownAccount {
4761                        provider: cred.provider.as_str().to_string(),
4762                        label: cred.account.label_str().to_string(),
4763                    });
4764                }
4765                if cred.provider != OAuthProviderId::Anthropic || !cred.account.is_default() {
4766                    return Err(BrokerError::BrokerTooOld("per-account usage".into()));
4767                }
4768                // The old broker's `/usage` serves its default Anthropic slot.
4769                let body = self.anthropic_usage().await?.to_string();
4770                super::usage::parse_anthropic_usage(
4771                    &body,
4772                    cred.account.label_str(),
4773                    crate::epoch_millis(),
4774                )
4775                .map_err(super::usage::UsageError::into_broker_error)
4776            }
4777            s if !(200..300).contains(&s) => {
4778                // Broker-controlled error body: dropped unread (spec §5.1).
4779                drop(resp);
4780                Err(BrokerError::Transport(format!(
4781                    "broker usage snapshot returned HTTP {s}"
4782                )))
4783            }
4784            _ => {
4785                let body = read_body_capped(resp, MAX_PROXY_RESPONSE_BYTES).await?;
4786                let snapshot: super::usage::UsageSnapshot = serde_json::from_str(&body)
4787                    .map_err(|e| BrokerError::Transport(format!("invalid usage snapshot: {e}")))?;
4788                if snapshot.provider != cred.provider.as_str()
4789                    || snapshot.account != cred.account.label_str()
4790                {
4791                    return Err(BrokerError::Transport(
4792                        "broker returned a usage snapshot for a different account".into(),
4793                    ));
4794                }
4795                Ok(snapshot)
4796            }
4797        }
4798    }
4799
4800    async fn report_cooldown(
4801        &self,
4802        cred: &CredentialRef,
4803        until_ms: Option<u64>,
4804        reason: &str,
4805    ) -> Result<(), BrokerError> {
4806        let resp = self
4807            .machine_token
4808            .send(|t| {
4809                self.http
4810                .post(format!("{}/accounts/cooldown", self.endpoint))
4811                .bearer_auth(t)
4812                .json(&serde_json::json!({
4813                    "provider": cred.provider.as_str(),
4814                    "account": cred.account.label_str(),
4815                    "until_ms": until_ms,
4816                    "reason": crate::truncate_str(reason, 64),
4817                }))
4818            })
4819            .await
4820            .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4821        match resp.status().as_u16() {
4822            401 => Err(BrokerError::Unauthorized),
4823            s if !(200..300).contains(&s) => {
4824                Err(BrokerError::Transport(format!("broker returned HTTP {s}")))
4825            }
4826            _ => Ok(()),
4827        }
4828    }
4829
4830    async fn proxy(&self, request: ProxyRequest) -> Result<ProxyResponse, BrokerError> {
4831        let mut request = request;
4832        request.stream = false;
4833        let resp = self.post_proxy(&request).await?;
4834        if !resp.status().is_success() {
4835            let fallback = format!("broker proxy returned HTTP {}", resp.status());
4836            return Err(Self::proxy_failure(resp, fallback).await);
4837        }
4838        let body = read_body_capped(resp, MAX_PROXY_RESPONSE_BYTES).await?;
4839        serde_json::from_str::<ProxyResponse>(&body)
4840            .map_err(|e| BrokerError::Transport(format!("invalid broker proxy response: {e}")))
4841    }
4842
4843    async fn proxy_stream(&self, request: ProxyRequest) -> Result<ProxyByteStream, BrokerError> {
4844        let mut request = request;
4845        request.stream = true;
4846        let resp = self.post_proxy(&request).await?;
4847        let status = resp.status();
4848        if !status.is_success() {
4849            // Broker-controlled error body: only a refresh-failure kind is
4850            // recovered from it, never its text (spec §5.1).
4851            let fallback = format!("broker proxy stream failed: {status}");
4852            return Err(Self::proxy_failure(resp, fallback).await);
4853        }
4854        use futures::StreamExt;
4855        let stream = resp
4856            .bytes_stream()
4857            .map(|chunk| chunk.map_err(|e| BrokerError::Transport(format!("stream error: {e}"))));
4858        Ok(Box::pin(stream))
4859    }
4860
4861    async fn anthropic_usage(&self) -> Result<serde_json::Value, BrokerError> {
4862        // Machine-authenticated typed operation: the remote broker resolves
4863        // the OAuth token on its side and returns usage JSON only.
4864        let resp = self
4865            .machine_token
4866            .send(|t| {
4867                self.http
4868                .get(format!("{}/usage", self.endpoint))
4869                .bearer_auth(t)
4870            })
4871            .await
4872            .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4873        match resp.status().as_u16() {
4874            401 => Err(BrokerError::Unauthorized),
4875            s if !(200..300).contains(&s) => {
4876                // Broker-controlled error body: dropped unread (spec §5.1).
4877                drop(resp);
4878                Err(BrokerError::Transport(format!(
4879                    "broker usage returned HTTP {s}"
4880                )))
4881            }
4882            _ => {
4883                let body = read_body_capped(resp, MAX_PROXY_RESPONSE_BYTES).await?;
4884                serde_json::from_str(&body)
4885                    .map_err(|e| BrokerError::Transport(format!("invalid usage response: {e}")))
4886            }
4887        }
4888    }
4889
4890    async fn cloud_catalog(
4891        &self,
4892        provider: CloudProviderId,
4893        context_ref: &str,
4894        allow_stale: bool,
4895    ) -> Result<Vec<CloudCatalogEntry>, BrokerError> {
4896        let resp = self.machine_token.send(|t| self.http.post(format!("{}/cloud/catalog", self.endpoint)).bearer_auth(t).json(&serde_json::json!({"provider":provider,"context_ref":context_ref,"allow_stale":allow_stale}))).await.map_err(|e| BrokerError::Transport(e.to_string()))?;
4897        if resp.status() == reqwest::StatusCode::UNAUTHORIZED {
4898            return Err(BrokerError::Unauthorized);
4899        }
4900        if !resp.status().is_success() {
4901            return Err(BrokerError::Transport(format!(
4902                "cloud catalog returned HTTP {}",
4903                resp.status()
4904            )));
4905        }
4906        let body = read_body_capped(resp, MAX_CLOUD_CATALOG_BODY_BYTES).await?;
4907        let entries: Vec<CloudCatalogEntry> = serde_json::from_str(&body)
4908            .map_err(|e| BrokerError::Transport(format!("invalid cloud catalog: {e}")))?;
4909        if entries.len() > MAX_CLOUD_CATALOG_ENTRIES
4910            || entries.iter().any(|e| {
4911                e.provider != provider || !e.context_ref.starts_with("ctx-") || e.id.len() > 512
4912            })
4913        {
4914            return Err(BrokerError::Transport(
4915                "broker returned an invalid cloud catalog".into(),
4916            ));
4917        }
4918        Ok(entries)
4919    }
4920
4921    async fn cloud_invoke(
4922        &self,
4923        provider: CloudProviderId,
4924        context_ref: &str,
4925        model_id: &str,
4926        request: InvokeRequest,
4927    ) -> Result<CloudEventStream, BrokerError> {
4928        let resp = self.machine_token.send(|t| self.http.post(format!("{}/cloud/invoke", self.endpoint)).bearer_auth(t).json(&serde_json::json!({"provider":provider,"context_ref":context_ref,"model_id":model_id,"request":request}))).await.map_err(|e| BrokerError::Transport(e.to_string()))?;
4929        if !resp.status().is_success() {
4930            return Err(if resp.status() == reqwest::StatusCode::UNAUTHORIZED {
4931                BrokerError::Unauthorized
4932            } else {
4933                BrokerError::Transport(format!("cloud invoke returned HTTP {}", resp.status()))
4934            });
4935        }
4936        use futures::StreamExt;
4937        let chunks = Box::pin(resp.bytes_stream());
4938        let stream = futures::stream::unfold(
4939            (chunks, Vec::<u8>::new(), false, false),
4940            |(mut chunks, mut buffer, done, failed)| async move {
4941                if failed {
4942                    return None;
4943                }
4944                loop {
4945                    if let Some(end) = buffer.iter().position(|b| *b == b'\n') {
4946                        let line: Vec<u8> = buffer.drain(..=end).collect();
4947                        let parsed = serde_json::from_slice::<CloudEvent>(&line[..end])
4948                            .map_err(|_| BrokerError::Transport("invalid cloud event".into()));
4949                        let event = match parsed {
4950                            Ok(CloudEvent::Done) if done => Err(BrokerError::Transport(
4951                                "duplicate cloud terminal event".into(),
4952                            )),
4953                            Ok(CloudEvent::Done) => Ok(CloudEvent::Done),
4954                            Ok(_) if done => Err(BrokerError::Transport(
4955                                "cloud data followed terminal event".into(),
4956                            )),
4957                            other => other,
4958                        };
4959                        let next_done = done || matches!(event, Ok(CloudEvent::Done));
4960                        let next_failed = event.is_err();
4961                        return Some((event, (chunks, buffer, next_done, next_failed)));
4962                    }
4963                    match chunks.next().await {
4964                        Some(Ok(chunk))
4965                            if buffer.len() + chunk.len() <= MAX_CLOUD_STREAM_EVENT_BYTES =>
4966                        {
4967                            buffer.extend_from_slice(&chunk)
4968                        }
4969                        Some(Ok(_)) => {
4970                            return Some((
4971                                Err(BrokerError::Transport("cloud event exceeded limit".into())),
4972                                (chunks, Vec::new(), done, true),
4973                            ))
4974                        }
4975                        Some(Err(_)) => {
4976                            return Some((
4977                                Err(BrokerError::Transport("broker stream failed".into())),
4978                                (chunks, Vec::new(), done, true),
4979                            ))
4980                        }
4981                        None if buffer.is_empty() && done => return None,
4982                        None if buffer.is_empty() => {
4983                            return Some((
4984                                Err(BrokerError::Transport(
4985                                    "cloud stream ended without terminal event".into(),
4986                                )),
4987                                (chunks, Vec::new(), done, true),
4988                            ))
4989                        }
4990                        None => {
4991                            return Some((
4992                                Err(BrokerError::Transport("truncated cloud event".into())),
4993                                (chunks, Vec::new(), done, true),
4994                            ))
4995                        }
4996                    }
4997                }
4998            },
4999        );
5000        Ok(Box::pin(stream))
5001    }
5002
5003    async fn capabilities(&self) -> Result<Vec<ProviderStatus>, BrokerError> {
5004        let resp = self
5005            .machine_token
5006            .send(|t| {
5007                self.http
5008                .get(format!("{}/capabilities", self.endpoint))
5009                .bearer_auth(t)
5010            })
5011            .await
5012            .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
5013        match resp.status().as_u16() {
5014            401 => Err(BrokerError::Unauthorized),
5015            s if !(200..300).contains(&s) => {
5016                Err(BrokerError::Transport(format!("broker returned HTTP {s}")))
5017            }
5018            _ => {
5019                let body = read_body_capped(resp, MAX_PROXY_RESPONSE_BYTES).await?;
5020                serde_json::from_str::<Vec<ProviderStatus>>(&body).map_err(|e| {
5021                    BrokerError::Transport(format!("invalid capabilities response: {e}"))
5022                })
5023            }
5024        }
5025    }
5026}
5027
5028// ── Construction and process-wide handle ─────────────────────────────────────
5029
5030/// Build the right broker for a credential source. Local sources get the
5031/// in-process broker (no daemon needed); remote sources get the authenticated
5032/// Pre-flight capability check for a cloud route (spec §5.5). Pure function:
5033/// callers MUST run it before constructing a broker, looking up credentials,
5034/// or opening any connection. The invoke-time guard inside the broker remains
5035/// in place as defense in depth.
5036pub fn preflight_cloud_capability(
5037    provider: CloudProviderId,
5038    needs_tools: bool,
5039) -> Result<(), BrokerError> {
5040    if needs_tools && !provider.supports_tools() {
5041        return Err(BrokerError::UnsupportedCapability {
5042            provider: provider.to_string(),
5043            capability: "tools".into(),
5044        });
5045    }
5046    Ok(())
5047}
5048
5049/// `synaps auth state` for the local broker `cache` holds for the current
5050/// credential store and config (the one requests use) or, when no request
5051/// has built one yet, the account list with nothing recorded. `live`: the
5052/// caller is the process that serves requests (the daemon). No network
5053/// request, no refresh; `query.clear` never writes the store.
5054pub fn local_auth_state(
5055    cache: &super::TokenCache,
5056    query: &super::auth_state::AuthStateQuery,
5057    live: bool,
5058) -> Result<super::auth_state::AuthStateReport, BrokerError> {
5059    let key = (
5060        storage::auth_file_path(),
5061        crate::config::resolve_read_path("config"),
5062    );
5063    let broker = cache
5064        .local_brokers
5065        .lock()
5066        .unwrap_or_else(|e| e.into_inner())
5067        .get(&key)
5068        .cloned();
5069    let mut report = auth_state_report(broker.as_deref(), query)?;
5070    report.live = live;
5071    Ok(report)
5072}
5073
5074fn auth_state_report(
5075    broker: Option<&LocalBroker>,
5076    query: &super::auth_state::AuthStateQuery,
5077) -> Result<super::auth_state::AuthStateReport, BrokerError> {
5078    use super::auth_state::{
5079        AccountState, AuthStateReport, IdentityKind, ObservedOutcome, AUTH_STATE_SCHEMA_VERSION,
5080    };
5081    let provider = match query.provider.as_deref() {
5082        Some(p) => Some(
5083            p.parse::<OAuthProviderId>()
5084                .map_err(|_| BrokerError::UnknownProvider(p.to_string()))?,
5085        ),
5086        None => None,
5087    };
5088    let mut summaries = match provider {
5089        Some(p) => storage::list_accounts(p),
5090        None => storage::list_all_accounts(),
5091    }
5092    .map_err(BrokerError::Credential)?;
5093    if let Some(label) = query.account.as_deref() {
5094        summaries.retain(|s| s.label == label);
5095        if summaries.is_empty() {
5096            return Err(BrokerError::UnknownAccount {
5097                provider: provider.map_or("any provider", |p| p.as_str()).to_string(),
5098                label: label.to_string(),
5099            });
5100        }
5101    }
5102    let creds: Vec<CredentialRef> = summaries
5103        .iter()
5104        .filter_map(AccountSummary::credential_ref)
5105        .collect();
5106    let mut cleared = Vec::new();
5107    if query.clear {
5108        if let Some(broker) = broker {
5109            let whole: Vec<OAuthProviderId> = match (&query.account, provider) {
5110                (Some(_), _) => Vec::new(),
5111                (None, Some(p)) => vec![p],
5112                (None, None) => super::provider::DESCRIPTORS.iter().map(|d| d.id).collect(),
5113            };
5114            broker.clear_state(&creds, &whole);
5115        }
5116        cleared = creds
5117            .iter()
5118            .map(|c| format!("{}:{}", c.provider, c.account.label_str()))
5119            .collect();
5120        tracing::info!(
5121            accounts = %cleared.join(","),
5122            broker_active = broker.is_some(),
5123            "auth_state_cleared"
5124        );
5125    }
5126    let store = storage::auth_file_path();
5127    let now_ms = crate::epoch_millis();
5128    let mut accounts = Vec::with_capacity(summaries.len());
5129    for summary in &summaries {
5130        let Some(cred) = summary.credential_ref() else {
5131            continue;
5132        };
5133        let observed = broker
5134            .and_then(|b| {
5135                b.observations
5136                    .lock()
5137                    .unwrap_or_else(|e| e.into_inner())
5138                    .get(&LocalBroker::state_key(&cred))
5139                    .cloned()
5140            })
5141            .unwrap_or_default();
5142        accounts.push(AccountState {
5143            provider: summary.provider.clone(),
5144            label: summary.label.clone(),
5145            identity: if summary.seat_fingerprint.is_some() {
5146                IdentityKind::Stable
5147            } else {
5148                IdentityKind::Rotating
5149            },
5150            access_expires_ms: (summary.expires != 0).then_some(summary.expires),
5151            cooldown: broker.and_then(|b| b.cooldown_state(&cred, now_ms)),
5152            last_usage: observed.last_usage,
5153            last_usage_ok_ms: observed.last_usage_ok_ms,
5154            last_refresh: super::token::last_refresh_exchange(&store, &cred.storage_key()).map(
5155                |x| ObservedOutcome {
5156                    at_ms: x.at_ms,
5157                    failure: x.failure.map(|kind| {
5158                        let failure = refresh_kind_failure(kind);
5159                        match x.http_status {
5160                            Some(status) => failure.with_status(status),
5161                            None => failure,
5162                        }
5163                    }),
5164                },
5165            ),
5166            refresh_hold: broker.and_then(|b| b.refresh_hold_state(&cred, now_ms)),
5167        });
5168    }
5169    let store_prefix = format!("{}|", store.display());
5170    let mut selections: Vec<_> = broker
5171        .map(|b| {
5172            b.selections
5173                .lock()
5174                .unwrap_or_else(|e| e.into_inner())
5175                .iter()
5176                .filter(|(k, s)| {
5177                    k.starts_with(&store_prefix)
5178                        && provider.is_none_or(|p| s.provider == p.as_str())
5179                        && query
5180                            .account
5181                            .as_deref()
5182                            .is_none_or(|label| selection_involves(s, label))
5183                })
5184                .map(|(_, s)| s.clone())
5185                .collect()
5186        })
5187        .unwrap_or_default();
5188    selections.sort_by_key(|s| std::cmp::Reverse(s.at_ms));
5189    Ok(AuthStateReport {
5190        schema_version: AUTH_STATE_SCHEMA_VERSION,
5191        live: false,
5192        broker_active: broker.is_some(),
5193        generated_at_ms: now_ms,
5194        store: store.display().to_string(),
5195        accounts,
5196        selections,
5197        cleared,
5198    })
5199}
5200
5201/// `auth state --account LABEL` keeps a selection that chose `label` or
5202/// lists it among the accounts that failed.
5203fn selection_involves(s: &super::auth_state::SelectionState, label: &str) -> bool {
5204    s.chosen.as_deref() == Some(label)
5205        || s.failure
5206            .as_ref()
5207            .is_some_and(|f| f.accounts.iter().any(|a| a.account == label))
5208}
5209
5210/// remote transport. There is no third option — and no direct-read fallback.
5211pub fn broker_from_source(
5212    source: &super::CredentialSource,
5213    cache: &super::TokenCache,
5214    http: reqwest::Client,
5215) -> Arc<dyn CredentialBroker> {
5216    match source {
5217        super::CredentialSource::Local => {
5218            // Runtime refresh and failover build adapters repeatedly. Keep
5219            // their local authority on the runtime-owned TokenCache, not a
5220            // global broker (which may belong to another remote principal).
5221            let key = (
5222                storage::auth_file_path(),
5223                crate::config::resolve_read_path("config"),
5224            );
5225            let mut brokers = cache
5226                .local_brokers
5227                .lock()
5228                .unwrap_or_else(|e| e.into_inner());
5229            brokers
5230                .entry(key)
5231                .or_insert_with(|| Arc::new(LocalBroker::new(http)))
5232                .clone()
5233        }
5234        super::CredentialSource::Remote {
5235            endpoint,
5236            machine_token,
5237        } => Arc::new(RemoteBroker::new(
5238            endpoint.clone(),
5239            machine_token.clone(),
5240            http,
5241            cache.clone(),
5242        )),
5243    }
5244}
5245
5246static GLOBAL_BROKER: std::sync::RwLock<Option<Arc<dyn CredentialBroker>>> =
5247    std::sync::RwLock::new(None);
5248
5249static GLOBAL_BROKER_INSTALLS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
5250
5251/// Install the process-wide broker (called once from runtime configuration).
5252/// Every call is counted ([`global_broker_install_count`]) and, when
5253/// `SYNAPS_MEM_TRACE=1`, logged — the daemon-mode S7 gate asserts the count
5254/// stays at 1 across subagent spawns.
5255pub fn set_global_broker(broker: Arc<dyn CredentialBroker>) {
5256    *GLOBAL_BROKER.write().expect("broker registry poisoned") = Some(broker);
5257    let n = GLOBAL_BROKER_INSTALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
5258    if crate::core::memstat::mem_trace_enabled() {
5259        tracing::info!(target: "agent_core::memstat", installs = n, "global broker installed");
5260    }
5261}
5262
5263/// How many times [`set_global_broker`] has run in this process.
5264pub fn global_broker_install_count() -> u64 {
5265    GLOBAL_BROKER_INSTALLS.load(std::sync::atomic::Ordering::Relaxed)
5266}
5267
5268/// The process-wide broker. Defaults to the in-process [`LocalBroker`] so
5269/// normal local use never requires a separately launched daemon.
5270pub fn global_broker() -> Arc<dyn CredentialBroker> {
5271    if let Some(b) = GLOBAL_BROKER
5272        .read()
5273        .expect("broker registry poisoned")
5274        .clone()
5275    {
5276        return b;
5277    }
5278    let default: Arc<dyn CredentialBroker> = Arc::new(LocalBroker::new(
5279        reqwest::Client::builder()
5280            .connect_timeout(std::time::Duration::from_secs(10))
5281            .timeout(std::time::Duration::from_secs(300))
5282            .build()
5283            .unwrap_or_default(),
5284    ));
5285    let mut guard = GLOBAL_BROKER.write().expect("broker registry poisoned");
5286    if let Some(b) = guard.clone() {
5287        return b;
5288    }
5289    *guard = Some(default.clone());
5290    default
5291}
5292
5293/// Legacy signature bridge: resolve an access token via the appropriate
5294/// broker for `source`. Kept for CLI paths (`synaps status`).
5295pub async fn broker_access_token(
5296    provider: OAuthProviderId,
5297    source: &super::CredentialSource,
5298    cache: &super::TokenCache,
5299    http: &reqwest::Client,
5300) -> Result<AccessToken, BrokerError> {
5301    broker_from_source(source, cache, http.clone())
5302        .access_token(provider)
5303        .await
5304}
5305
5306/// Non-secret map of static key display statuses for every provider,
5307/// plus the local endpoint. For settings UI snapshots.
5308pub fn static_key_status_map() -> BTreeMap<String, StaticKeyStatus> {
5309    STATIC_PROVIDERS
5310        .iter()
5311        .map(|s| (s.key.to_string(), static_key_status(s.key)))
5312        .collect()
5313}
5314
5315#[cfg(test)]
5316mod tests {
5317    use super::*;
5318
5319    #[test]
5320    fn login_availability_handles_named_only_and_explicit_policy() {
5321        let dir = tempfile::tempdir().unwrap();
5322        let path = dir.path().join("auth.json");
5323        let provider = OAuthProviderId::Anthropic;
5324        let empty = AccountPolicy::new();
5325        assert!(!oauth_provider_logged_in_at(&path, &empty, provider));
5326        std::fs::write(&path, serde_json::json!({
5327            "anthropic@work": {"type":"oauth", "access":"synthetic-access",
5328                "refresh":"synthetic-refresh", "expires":1}
5329        }).to_string()).unwrap();
5330        // This is login inventory for UI, not an authorization/capacity check.
5331        assert!(oauth_provider_logged_in_at(&path, &empty, provider));
5332        for selector in [AccountSelector::Auto,
5333            AccountSelector::parse_or_invalid("../bad", "test"),
5334            AccountSelector::Account(Account::named("work").unwrap())] {
5335            let policy = AccountPolicy::new().with(provider, selector);
5336            assert!(oauth_provider_logged_in_at(&path, &policy, provider));
5337        }
5338        let missing = AccountPolicy::new().with(provider,
5339            AccountSelector::Account(Account::named("missing").unwrap()));
5340        assert!(!oauth_provider_logged_in_at(&path, &missing, provider));
5341        assert!(!oauth_provider_logged_in_at(&path, &empty, OAuthProviderId::OpenAiCodex));
5342        std::fs::write(&path, "{broken").unwrap();
5343        assert!(!oauth_provider_logged_in_at(&path, &empty, provider));
5344    }
5345
5346    /// Spec §5.5: the pre-flight capability check is a pure function — it can
5347    /// be (and is) called before any credential lookup or network access. A
5348    /// tool-requiring request against any text-only cloud route must yield the
5349    /// typed unsupported-capability error; text-only requests pass.
5350    #[test]
5351    fn preflight_rejects_tool_requiring_cloud_routes_with_typed_error() {
5352        for provider in [
5353            CloudProviderId::AzureOpenAi,
5354            CloudProviderId::AwsBedrock,
5355            CloudProviderId::GoogleVertex,
5356        ] {
5357            let err = preflight_cloud_capability(provider, true)
5358                .expect_err("tool-requiring cloud route must fail pre-flight");
5359            assert_eq!(
5360                err,
5361                BrokerError::UnsupportedCapability {
5362                    provider: provider.to_string(),
5363                    capability: "tools".into(),
5364                }
5365            );
5366            assert!(err.to_string().contains("text-only"));
5367            assert_eq!(preflight_cloud_capability(provider, false), Ok(()));
5368        }
5369    }
5370
5371    async fn spawn_ndjson(body: &'static str) -> String {
5372        use axum::{body::Body, routing::post, Router};
5373        use bytes::Bytes;
5374        let app = Router::new().route(
5375            "/cloud/invoke",
5376            post(move || async move {
5377                let chunks = body
5378                    .as_bytes()
5379                    .chunks(3)
5380                    .map(|chunk| Ok::<_, std::convert::Infallible>(Bytes::copy_from_slice(chunk)));
5381                axum::response::Response::builder()
5382                    .header("content-type", "application/x-ndjson")
5383                    .body(Body::from_stream(futures::stream::iter(chunks)))
5384                    .unwrap()
5385            }),
5386        );
5387        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5388        let addr = listener.local_addr().unwrap();
5389        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
5390        format!("http://{addr}")
5391    }
5392
5393    async fn remote_events(body: &'static str) -> Vec<Result<CloudEvent, BrokerError>> {
5394        use futures::StreamExt;
5395        let broker = RemoteBroker::new(
5396            spawn_ndjson(body).await,
5397            "opaque-machine-token",
5398            reqwest::Client::new(),
5399            super::super::TokenCache::new(),
5400        );
5401        broker
5402            .cloud_invoke(
5403                CloudProviderId::GoogleVertex,
5404                "ctx-opaque",
5405                "google-vertex/publishers/google/models/test",
5406                InvokeRequest {
5407                    messages: vec![],
5408                    tools: vec![],
5409                    stream: true,
5410                    options: Default::default(),
5411                },
5412            )
5413            .await
5414            .unwrap()
5415            .collect()
5416            .await
5417    }
5418
5419    #[tokio::test]
5420    async fn remote_ndjson_requires_exactly_one_terminal_and_stops_after_error() {
5421        let valid =
5422            remote_events("{\"type\":\"text_delta\",\"delta\":\"hi\"}\n{\"type\":\"done\"}\n")
5423                .await;
5424        assert_eq!(valid.len(), 2);
5425        assert!(matches!(valid[1], Ok(CloudEvent::Done)));
5426
5427        for body in [
5428            "{\"type\":\"text_delta\",\"delta\":\"hi\"}\n",
5429            "{\"type\":\"done\"}\n{\"type\":\"done\"}\n",
5430            "{\"type\":\"done\"}\n{\"type\":\"text_delta\",\"delta\":\"late\"}\n",
5431        ] {
5432            let events = remote_events(body).await;
5433            assert_eq!(events.iter().filter(|event| event.is_err()).count(), 1);
5434            assert!(events.last().unwrap().is_err());
5435        }
5436    }
5437
5438    /// The per-provider endpoint allowlist: a signed proxy request cannot be
5439    /// steered at other same-host endpoints (key management, billing, …).
5440    #[test]
5441    fn proxy_rejects_unlisted_same_host_paths() {
5442        for (provider, path) in [
5443            ("groq", "/v1/keys"),
5444            ("groq", "/admin"),
5445            ("openrouter", "/api/v1/auth/keys"),
5446            ("local", "/audio/speech"),
5447        ] {
5448            let req = ProxyRequest {
5449                provider: provider.into(),
5450                method: ProxyMethod::Get,
5451                path: path.into(),
5452                body: None,
5453                stream: false,
5454                body_bytes: None,
5455            };
5456            match req.validate() {
5457                Err(BrokerError::Denied(msg)) => {
5458                    assert!(msg.contains("allowlist"), "got: {msg}")
5459                }
5460                other => panic!("{provider} {path} must be denied, got {other:?}"),
5461            }
5462        }
5463        // The cataloged paths remain reachable.
5464        for path in ["/models", "/chat/completions"] {
5465            let req = ProxyRequest {
5466                provider: "groq".into(),
5467                method: ProxyMethod::Get,
5468                path: path.into(),
5469                body: None,
5470                stream: false,
5471                body_bytes: None,
5472            };
5473            assert!(req.validate().is_ok(), "{path} must be allowed");
5474        }
5475    }
5476
5477    /// Request bodies above the broker buffering limit are rejected before
5478    /// any credential resolution or upstream contact.
5479    #[test]
5480    fn proxy_rejects_oversize_request_body() {
5481        let req = ProxyRequest {
5482            provider: "groq".into(),
5483            method: ProxyMethod::Post,
5484            path: "/chat/completions".into(),
5485            body: Some(serde_json::json!({
5486                "blob": "x".repeat(MAX_PROXY_REQUEST_BYTES + 1)
5487            })),
5488            stream: false,
5489            body_bytes: None,
5490        };
5491        match req.validate() {
5492            Err(BrokerError::Denied(msg)) => assert!(msg.contains("byte"), "got: {msg}"),
5493            other => panic!("oversize body must be denied, got {other:?}"),
5494        }
5495    }
5496
5497    /// Regression: the size cap must hold over the exact-byte handoff even
5498    /// when the JSON `body` is unset — `body_bytes` alone is what
5499    /// `LocalBroker` would send upstream, so it can never bypass the limit.
5500    #[test]
5501    fn proxy_rejects_oversize_body_bytes_without_json_body() {
5502        let req = ProxyRequest {
5503            provider: "groq".into(),
5504            method: ProxyMethod::Post,
5505            path: "/chat/completions".into(),
5506            body: None,
5507            stream: false,
5508            body_bytes: Some(bytes::Bytes::from(vec![b'x'; MAX_PROXY_REQUEST_BYTES + 1])),
5509        };
5510        match req.validate() {
5511            Err(BrokerError::Denied(msg)) => assert!(msg.contains("byte"), "got: {msg}"),
5512            other => panic!("oversize body_bytes must be denied, got {other:?}"),
5513        }
5514    }
5515
5516    /// The sanctioned constructor serializes once and keeps `body` and
5517    /// `body_bytes` coherent: same buffer returned to the caller (for
5518    /// digesting) and stored on the request, parsing back to the very value.
5519    #[test]
5520    fn post_json_exact_sets_coherent_body_and_bytes() {
5521        let value = serde_json::json!({"model": "m", "stream": true});
5522        let (req, digest_bytes) =
5523            ProxyRequest::post_json_exact("groq", "/chat/completions", value.clone(), true)
5524                .expect("serializable body must construct");
5525        assert_eq!(req.method, ProxyMethod::Post);
5526        assert!(req.stream);
5527        assert_eq!(req.body.as_ref(), Some(&value));
5528        let stored = req.body_bytes.as_ref().expect("bytes must be set");
5529        assert_eq!(stored, &digest_bytes, "caller digest bytes == wire bytes");
5530        assert_eq!(
5531            serde_json::from_slice::<serde_json::Value>(stored).unwrap(),
5532            value
5533        );
5534        req.validate().expect("coherent request must validate");
5535    }
5536
5537    /// A semantically divergent `body`/`body_bytes` pair (constructed by
5538    /// hand, bypassing `post_json_exact`) is rejected by validation in
5539    /// debug/test builds — the digest would not describe the claimed body.
5540    #[test]
5541    fn mismatched_body_bytes_rejected_by_debug_validation() {
5542        let req = ProxyRequest {
5543            provider: "groq".into(),
5544            method: ProxyMethod::Post,
5545            path: "/chat/completions".into(),
5546            body: Some(serde_json::json!({"model": "claimed"})),
5547            stream: true,
5548            body_bytes: Some(bytes::Bytes::from_static(b"{\"model\":\"actually-sent\"}")),
5549        };
5550        match req.validate() {
5551            Err(BrokerError::Denied(msg)) => {
5552                assert!(msg.contains("body_bytes"), "got: {msg}");
5553            }
5554            other => panic!("incoherent handoff must be denied, got {other:?}"),
5555        }
5556    }
5557
5558    #[test]
5559    fn proxy_request_validation_fails_closed() {
5560        let ok = ProxyRequest {
5561            provider: "groq".into(),
5562            method: ProxyMethod::Post,
5563            path: "/chat/completions".into(),
5564            body: None,
5565            stream: false,
5566            body_bytes: None,
5567        };
5568        assert!(ok.validate().is_ok());
5569
5570        let unknown = ProxyRequest {
5571            provider: "evil".into(),
5572            ..ok.clone()
5573        };
5574        assert!(matches!(
5575            unknown.validate(),
5576            Err(BrokerError::UnknownProvider(_))
5577        ));
5578
5579        let absolute = ProxyRequest {
5580            path: "https://evil.example/x".into(),
5581            ..ok.clone()
5582        };
5583        assert!(matches!(absolute.validate(), Err(BrokerError::Denied(_))));
5584
5585        let traversal = ProxyRequest {
5586            path: "/../secrets".into(),
5587            ..ok.clone()
5588        };
5589        assert!(matches!(traversal.validate(), Err(BrokerError::Denied(_))));
5590
5591        let relative = ProxyRequest {
5592            path: "chat/completions".into(),
5593            ..ok
5594        };
5595        assert!(matches!(relative.validate(), Err(BrokerError::Denied(_))));
5596    }
5597
5598    /// Managed Kimi Code proxy: chat inference plus the read-only catalog,
5599    /// profile, and quota GETs the official CLI uses — nothing else.
5600    #[test]
5601    fn proxy_allows_only_pinned_kimi_code_paths() {
5602        let chat = ProxyRequest {
5603            provider: "kimi-code".into(),
5604            method: ProxyMethod::Post,
5605            path: "/chat/completions".into(),
5606            body: None,
5607            stream: true,
5608            body_bytes: None,
5609        };
5610        assert!(chat.validate().is_ok(), "kimi-code chat must be allowed");
5611
5612        for path in ["/models", "/me", "/usages"] {
5613            let get = ProxyRequest {
5614                provider: "kimi-code".into(),
5615                method: ProxyMethod::Get,
5616                path: path.into(),
5617                body: None,
5618                stream: false,
5619                body_bytes: None,
5620            };
5621            assert!(get.validate().is_ok(), "kimi-code GET {path} must pass");
5622            // The same surface must not accept POST (read-only endpoints).
5623            let post = ProxyRequest {
5624                method: ProxyMethod::Post,
5625                ..get
5626            };
5627            assert!(
5628                matches!(post.validate(), Err(BrokerError::Denied(_))),
5629                "kimi-code POST {path} must be denied"
5630            );
5631        }
5632
5633        // Arbitrary same-host endpoints (feedback/upload/admin) stay denied.
5634        for path in ["/feedback", "/search", "/fetch", "/completions"] {
5635            let denied = ProxyRequest {
5636                provider: "kimi-code".into(),
5637                method: ProxyMethod::Post,
5638                path: path.into(),
5639                body: None,
5640                stream: false,
5641                body_bytes: None,
5642            };
5643            assert!(
5644                matches!(denied.validate(), Err(BrokerError::Denied(_))),
5645                "kimi-code {path} must be denied"
5646            );
5647        }
5648
5649        // `kimi` stays the static Moonshot API-key provider key — validating
5650        // it as an OAuth proxy id must keep resolving via static_provider.
5651        let static_kimi = ProxyRequest {
5652            provider: "kimi".into(),
5653            method: ProxyMethod::Post,
5654            path: "/chat/completions".into(),
5655            body: None,
5656            stream: false,
5657            body_bytes: None,
5658        };
5659        assert!(static_kimi.validate().is_ok());
5660    }
5661
5662    /// OAuth providers stay fail-closed except reviewed catalog paths.
5663    /// Claude alias remains unknown; anthropic/openai-codex only accept
5664    /// their exact catalog allowlists (not bare `/models`).
5665    #[test]
5666    fn proxy_rejects_oauth_providers_except_codex_catalog() {
5667        // Unrecognized OAuth alias is still unknown.
5668        let claude = ProxyRequest {
5669            provider: "claude".into(),
5670            method: ProxyMethod::Get,
5671            path: "/models".into(),
5672            body: None,
5673            stream: false,
5674            body_bytes: None,
5675        };
5676        assert!(
5677            matches!(claude.validate(), Err(BrokerError::UnknownProvider(_))),
5678            "claude alias must not be proxyable"
5679        );
5680        // Anthropic bare OpenAI-style /models is denied (only /v1/models…).
5681        let anth = ProxyRequest {
5682            provider: "anthropic".into(),
5683            method: ProxyMethod::Get,
5684            path: "/models".into(),
5685            body: None,
5686            stream: false,
5687            body_bytes: None,
5688        };
5689        assert!(
5690            matches!(anth.validate(), Err(BrokerError::Denied(_))),
5691            "anthropic /models must be denied"
5692        );
5693        // openai-codex bare /models (no client_version) must be denied.
5694        let bare = ProxyRequest {
5695            provider: "openai-codex".into(),
5696            method: ProxyMethod::Get,
5697            path: "/models".into(),
5698            body: None,
5699            stream: false,
5700            body_bytes: None,
5701        };
5702        assert!(
5703            matches!(bare.validate(), Err(BrokerError::Denied(_))),
5704            "openai-codex /models without client_version must be denied"
5705        );
5706    }
5707
5708    /// Codex catalog proxy is pinned to GET /codex/models?client_version=…
5709    /// only. Arbitrary same-host paths and query shapes are denied — including
5710    /// the ChatGPT web picker at `/models`, which is not a Codex catalog.
5711    #[test]
5712    fn proxy_allows_only_pinned_openai_codex_models_path() {
5713        let allowed = ProxyRequest {
5714            provider: "openai-codex".into(),
5715            method: ProxyMethod::Get,
5716            path: "/codex/models?client_version=0.153.3".into(),
5717            body: None,
5718            stream: false,
5719            body_bytes: None,
5720        };
5721        assert!(
5722            allowed.validate().is_ok(),
5723            "exact codex models path must be allowed"
5724        );
5725
5726        for path in [
5727            "/codex/models",
5728            "/codex/models?client_version=",
5729            "/codex/models?foo=1",
5730            "/codex/models?client_version=0.153.3&extra=1",
5731            "/codex/responses",
5732            "/models",
5733            "/models?client_version=0.153.3",
5734            "/backend-api/codex/models?client_version=0.153.3",
5735            "/v1/models?client_version=0.153.3",
5736            "/chat/completions",
5737        ] {
5738            let req = ProxyRequest {
5739                provider: "openai-codex".into(),
5740                method: ProxyMethod::Get,
5741                path: path.into(),
5742                body: None,
5743                stream: false,
5744                body_bytes: None,
5745            };
5746            assert!(
5747                matches!(req.validate(), Err(BrokerError::Denied(_))),
5748                "openai-codex path {path} must be denied"
5749            );
5750        }
5751
5752        // POST is never allowed for the catalog path.
5753        let post = ProxyRequest {
5754            provider: "openai-codex".into(),
5755            method: ProxyMethod::Post,
5756            path: "/codex/models?client_version=0.153.3".into(),
5757            body: None,
5758            stream: false,
5759            body_bytes: None,
5760        };
5761        assert!(
5762            matches!(post.validate(), Err(BrokerError::Denied(_))),
5763            "POST openai-codex models must be denied"
5764        );
5765    }
5766
5767    /// Copilot is pinned to its catalog and two reviewed inference paths.
5768    #[test]
5769    fn proxy_allows_only_pinned_github_copilot_paths() {
5770        let models = ProxyRequest {
5771            provider: "github-copilot".into(),
5772            method: ProxyMethod::Get,
5773            path: "/models".into(),
5774            body: None,
5775            stream: false,
5776            body_bytes: None,
5777        };
5778        assert!(models.validate().is_ok());
5779
5780        let chat = ProxyRequest {
5781            provider: "github-copilot".into(),
5782            method: ProxyMethod::Post,
5783            path: "/chat/completions".into(),
5784            body: None,
5785            stream: false,
5786            body_bytes: None,
5787        };
5788        assert!(chat.validate().is_ok());
5789        let responses = ProxyRequest {
5790            provider: "github-copilot".into(),
5791            method: ProxyMethod::Post,
5792            path: "/responses".into(),
5793            body: None,
5794            stream: false,
5795            body_bytes: None,
5796        };
5797        assert!(responses.validate().is_ok());
5798        for path in ["/v1/messages", "/models?x=1", "/embeddings"] {
5799            let request = ProxyRequest {
5800                provider: "github-copilot".into(),
5801                method: ProxyMethod::Post,
5802                path: path.into(),
5803                body: None,
5804                stream: false,
5805                body_bytes: None,
5806            };
5807            assert!(matches!(request.validate(), Err(BrokerError::Denied(_))));
5808        }
5809    }
5810
5811    /// Anthropic catalog proxy is pinned to /v1/models (+ limit/after_id) only.
5812    #[test]
5813    fn proxy_allows_only_pinned_anthropic_models_path() {
5814        for path in [
5815            "/v1/models",
5816            "/v1/models?limit=100",
5817            "/v1/models?limit=100&after_id=claude-opus-4-7",
5818        ] {
5819            let req = ProxyRequest {
5820                provider: "anthropic".into(),
5821                method: ProxyMethod::Get,
5822                path: path.into(),
5823                body: None,
5824                stream: false,
5825                body_bytes: None,
5826            };
5827            assert!(req.validate().is_ok(), "{path} must be allowed");
5828        }
5829        for path in [
5830            "/v1/messages",
5831            "/models",
5832            "/v1/models?foo=1",
5833            "/v1/models?limit=",
5834            "/api/oauth/usage",
5835            "/v1/models?limit=100&evil=1",
5836            "/v1/models?limit=100&limit=50",
5837            "/v1/models?after_id=a&after_id=b",
5838            "/v1/models?limit=100&after_id=a&limit=50",
5839        ] {
5840            let req = ProxyRequest {
5841                provider: "anthropic".into(),
5842                method: ProxyMethod::Get,
5843                path: path.into(),
5844                body: None,
5845                stream: false,
5846                body_bytes: None,
5847            };
5848            assert!(
5849                matches!(req.validate(), Err(BrokerError::Denied(_))),
5850                "anthropic path {path} must be denied"
5851            );
5852        }
5853    }
5854
5855    #[test]
5856    fn anthropic_oauth_catalog_headers_are_bearer_version_beta_not_x_api_key() {
5857        let headers = anthropic_oauth_catalog_request_headers();
5858        let names: Vec<_> = headers.iter().map(|(n, _)| *n).collect();
5859        assert!(names.contains(&"anthropic-version"));
5860        assert!(names.contains(&"anthropic-beta"));
5861        assert!(names.contains(&"accept"));
5862        assert!(!names.iter().any(|n| n.eq_ignore_ascii_case("x-api-key")));
5863        assert!(!names
5864            .iter()
5865            .any(|n| n.eq_ignore_ascii_case("authorization")));
5866        let beta = headers
5867            .iter()
5868            .find(|(n, _)| *n == "anthropic-beta")
5869            .map(|(_, v)| *v)
5870            .expect("beta");
5871        assert_eq!(beta, "oauth-2025-04-20");
5872    }
5873
5874    #[test]
5875    fn anthropic_models_path_round_trip_and_rejects_duplicates() {
5876        assert!(is_allowed_anthropic_path("/v1/models"));
5877        assert!(is_allowed_anthropic_path("/v1/models?limit=100"));
5878        assert!(is_allowed_anthropic_path(
5879            "/v1/models?limit=100&after_id=claude-opus-4-7"
5880        ));
5881        assert!(!is_allowed_anthropic_path("/v1/models?limit=100&limit=50"));
5882        assert!(!is_allowed_anthropic_path(
5883            "/v1/models?after_id=a&after_id=b"
5884        ));
5885        // Round-trip the catalog helper path shape used by the engine.
5886        let page0 = "/v1/models?limit=100";
5887        let page1 = "/v1/models?limit=100&after_id=model-1";
5888        assert!(is_allowed_anthropic_path(page0));
5889        assert!(is_allowed_anthropic_path(page1));
5890    }
5891
5892    /// google-gemini is pinned to the reviewed cloudcode-pa v1internal methods.
5893    #[test]
5894    fn proxy_allows_only_pinned_google_gemini_paths() {
5895        for path in [
5896            "/v1internal:loadCodeAssist",
5897            "/v1internal:onboardUser",
5898            "/v1internal:streamGenerateContent",
5899            "/v1internal:countTokens",
5900            "/v1internal/operations/op-12345",
5901        ] {
5902            let req = ProxyRequest {
5903                provider: "google-gemini".into(),
5904                method: ProxyMethod::Post,
5905                path: path.into(),
5906                body: None,
5907                stream: false,
5908                body_bytes: None,
5909            };
5910            assert!(req.validate().is_ok(), "{path} must be allowed");
5911        }
5912        for path in [
5913            // No arbitrary same-host methods.
5914            "/v1internal:listExperiments",
5915            "/v1internal:fetchAdminControls",
5916            "/v1internal:setCodeAssistGlobalUserSetting",
5917            "/v1internal:generateContent",
5918            // No unrelated versions.
5919            "/v2/models",
5920            "/v1beta/models",
5921            // No path traversal or root probe.
5922            "/",
5923            "/v1internal:",
5924        ] {
5925            let req = ProxyRequest {
5926                provider: "google-gemini".into(),
5927                method: ProxyMethod::Post,
5928                path: path.into(),
5929                body: None,
5930                stream: false,
5931                body_bytes: None,
5932            };
5933            assert!(
5934                matches!(req.validate(), Err(BrokerError::Denied(_))),
5935                "{path} must be denied"
5936            );
5937        }
5938    }
5939
5940    #[test]
5941    fn access_token_type_has_no_refresh_field() {
5942        // Structural invariant: deserializing a broker response that includes
5943        // a refresh token silently drops it — there is no field to hold one.
5944        let t: AccessToken =
5945            serde_json::from_str(r#"{"token":"sk-x","expires":123,"refresh":"MUST-NOT-EXIST"}"#)
5946                .unwrap();
5947        let round = serde_json::to_value(&t).unwrap();
5948        assert_eq!(round.get("refresh"), None);
5949        assert_eq!(round["token"], "sk-x");
5950    }
5951
5952    #[test]
5953    fn broker_error_display_never_echoes_values() {
5954        // Errors carry provider names and status text only.
5955        let e = BrokerError::NotConfigured("groq".into());
5956        assert!(!format!("{e}").contains("sk-"));
5957        let e = BrokerError::Unauthorized;
5958        let shown = format!("{e}");
5959        assert!(shown.starts_with("broker rejected machine auth"), "{shown}");
5960        // Actionable: names the config keys and the token file location.
5961        assert!(shown.contains("auth.machine_token") && shown.contains("SYNAPS_MACHINE_TOKEN"), "{shown}");
5962        assert!(shown.contains("auth-broker.token") && shown.contains("restart"), "{shown}");
5963    }
5964
5965    #[test]
5966    fn mask_key_is_short_and_lossy() {
5967        assert_eq!(mask_key("gsk-live-1234567890abcdef"), "gsk-…cdef");
5968        assert_eq!(mask_key("short"), "…");
5969    }
5970
5971    #[test]
5972    fn proxy_request_serde_roundtrip() {
5973        let req = ProxyRequest {
5974            provider: "openrouter".into(),
5975            method: ProxyMethod::Get,
5976            path: "/models".into(),
5977            body: Some(serde_json::json!({"a": 1})),
5978            stream: true,
5979            body_bytes: None,
5980        };
5981        let json = serde_json::to_string(&req).unwrap();
5982        let back: ProxyRequest = serde_json::from_str(&json).unwrap();
5983        assert_eq!(back.provider, "openrouter");
5984        assert!(back.stream);
5985        assert!(matches!(back.method, ProxyMethod::Get));
5986    }
5987
5988    // ── LocalBroker streaming/proxy behavior against a fake upstream ─────────
5989
5990    async fn spawn_upstream(app: axum::Router) -> String {
5991        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5992        let addr = listener.local_addr().unwrap();
5993        tokio::spawn(async move {
5994            axum::serve(listener, app).await.unwrap();
5995        });
5996        format!("http://{addr}")
5997    }
5998
5999    /// Streaming forwarding: SSE bytes from the upstream arrive unmodified
6000    /// through `proxy_stream`, and the Authorization header applied is the
6001    /// broker's — the caller never supplied one.
6002    #[tokio::test]
6003    async fn local_broker_streams_local_endpoint_and_applies_key() {
6004        use axum::routing::post;
6005        let seen_auth = Arc::new(std::sync::Mutex::new(String::new()));
6006        let seen = seen_auth.clone();
6007        let app = axum::Router::new().route(
6008            "/chat/completions",
6009            post(move |headers: axum::http::HeaderMap| {
6010                let seen = seen.clone();
6011                async move {
6012                    *seen.lock().unwrap() = headers
6013                        .get("authorization")
6014                        .and_then(|v| v.to_str().ok())
6015                        .unwrap_or("")
6016                        .to_string();
6017                    (
6018                        [("content-type", "text/event-stream")],
6019                        "data: {\"x\":1}\n\ndata: [DONE]\n\n",
6020                    )
6021                }
6022            }),
6023        );
6024        let url = spawn_upstream(app).await;
6025        let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6026        let mut stream = broker
6027            .proxy_stream(ProxyRequest {
6028                provider: LOCAL_PROVIDER_KEY.into(),
6029                method: ProxyMethod::Post,
6030                path: "/chat/completions".into(),
6031                body: Some(serde_json::json!({"model": "m"})),
6032                stream: true,
6033                body_bytes: None,
6034            })
6035            .await
6036            .expect("stream must open");
6037        use futures::StreamExt;
6038        let mut collected = Vec::new();
6039        while let Some(chunk) = stream.next().await {
6040            collected.extend_from_slice(&chunk.unwrap());
6041        }
6042        let text = String::from_utf8(collected).unwrap();
6043        assert!(text.contains("data: {\"x\":1}"));
6044        assert!(text.contains("[DONE]"));
6045        // Key applied broker-side (default local key), not by the caller.
6046        assert_eq!(&*seen_auth.lock().unwrap(), "Bearer local");
6047    }
6048
6049    /// A non-2xx upstream response never yields a stream — it becomes a typed
6050    /// error whose text is the provider's status/body (no key material).
6051    #[tokio::test]
6052    async fn local_broker_stream_error_is_typed_and_keyless() {
6053        use axum::routing::post;
6054        let app = axum::Router::new().route(
6055            "/chat/completions",
6056            post(|| async {
6057                (
6058                    axum::http::StatusCode::UNAUTHORIZED,
6059                    "{\"error\":\"bad key\"}",
6060                )
6061            }),
6062        );
6063        let url = spawn_upstream(app).await;
6064        let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6065        let err = match broker
6066            .proxy_stream(ProxyRequest {
6067                provider: LOCAL_PROVIDER_KEY.into(),
6068                method: ProxyMethod::Post,
6069                path: "/chat/completions".into(),
6070                body: None,
6071                stream: true,
6072                body_bytes: None,
6073            })
6074            .await
6075        {
6076            Err(e) => e,
6077            Ok(_) => panic!("non-2xx upstream must not yield a stream"),
6078        };
6079        let msg = format!("{err}");
6080        assert!(msg.contains("401"), "got: {msg}");
6081        assert!(
6082            !msg.to_lowercase().contains("bearer"),
6083            "no auth material in errors"
6084        );
6085    }
6086
6087    /// Non-streaming proxy returns upstream status + body verbatim.
6088    #[tokio::test]
6089    async fn local_broker_proxy_returns_status_and_body() {
6090        use axum::routing::get;
6091        let app =
6092            axum::Router::new().route("/models", get(|| async { "{\"data\":[{\"id\":\"m1\"}]}" }));
6093        let url = spawn_upstream(app).await;
6094        let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6095        let resp = broker
6096            .proxy(ProxyRequest {
6097                provider: LOCAL_PROVIDER_KEY.into(),
6098                method: ProxyMethod::Get,
6099                path: "/models".into(),
6100                body: None,
6101                stream: false,
6102                body_bytes: None,
6103            })
6104            .await
6105            .unwrap();
6106        assert_eq!(resp.status, 200);
6107        assert!(resp.body.contains("m1"));
6108    }
6109
6110    /// A buffered response above the broker cap fails closed instead of
6111    /// ballooning memory or returning silently truncated JSON.
6112    #[tokio::test]
6113    async fn local_broker_proxy_rejects_oversize_response() {
6114        use axum::routing::get;
6115        let app = axum::Router::new().route("/models", get(|| async { "x".repeat(4096) }));
6116        let url = spawn_upstream(app).await;
6117        let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url)
6118            .with_max_response_bytes(64);
6119        let err = broker
6120            .proxy(ProxyRequest {
6121                provider: LOCAL_PROVIDER_KEY.into(),
6122                method: ProxyMethod::Get,
6123                path: "/models".into(),
6124                body: None,
6125                stream: false,
6126                body_bytes: None,
6127            })
6128            .await
6129            .expect_err("oversize body must be rejected");
6130        let msg = format!("{err}");
6131        assert!(msg.contains("limit"), "got: {msg}");
6132    }
6133
6134    /// Upstream error bodies never reach the caller — a hostile provider
6135    /// cannot flood the caller or inject terminal escapes via error text.
6136    #[tokio::test]
6137    async fn local_broker_stream_error_is_status_only() {
6138        use axum::routing::post;
6139        let app = axum::Router::new().route(
6140            "/chat/completions",
6141            post(|| async {
6142                (
6143                    axum::http::StatusCode::INTERNAL_SERVER_ERROR,
6144                    format!("\x1b[2J{}", "E".repeat(64 * 1024)),
6145                )
6146            }),
6147        );
6148        let url = spawn_upstream(app).await;
6149        let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6150        let err = broker
6151            .proxy_stream(ProxyRequest {
6152                provider: LOCAL_PROVIDER_KEY.into(),
6153                method: ProxyMethod::Post,
6154                path: "/chat/completions".into(),
6155                body: None,
6156                stream: true,
6157                body_bytes: None,
6158            })
6159            .await
6160            .err()
6161            .expect("non-2xx upstream must not yield a stream");
6162        let msg = format!("{err}");
6163        assert!(
6164            msg.len() < 700,
6165            "error must be bounded, got {} bytes",
6166            msg.len()
6167        );
6168        assert!(!msg.contains('\x1b'), "control chars must be stripped");
6169        assert!(msg.contains("500"));
6170    }
6171
6172    /// Buffered proxy requests carry an explicit time budget: a hung upstream
6173    /// becomes a typed transport error, not an indefinite stall.
6174    #[tokio::test]
6175    async fn local_broker_buffered_request_times_out() {
6176        use axum::routing::get;
6177        let app = axum::Router::new().route(
6178            "/models",
6179            get(|| async {
6180                tokio::time::sleep(Duration::from_secs(30)).await;
6181                "too late"
6182            }),
6183        );
6184        let url = spawn_upstream(app).await;
6185        let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url)
6186            .with_request_timeout(Duration::from_millis(200));
6187        let started = std::time::Instant::now();
6188        let err = broker
6189            .proxy(ProxyRequest {
6190                provider: LOCAL_PROVIDER_KEY.into(),
6191                method: ProxyMethod::Get,
6192                path: "/models".into(),
6193                body: None,
6194                stream: false,
6195                body_bytes: None,
6196            })
6197            .await
6198            .expect_err("hung upstream must time out");
6199        assert!(matches!(err, BrokerError::Transport(_)));
6200        assert!(
6201            started.elapsed() < Duration::from_secs(5),
6202            "timeout must be enforced promptly"
6203        );
6204    }
6205
6206    /// Local in-process behavior: the process-wide broker exists without any
6207    /// daemon being launched and answers capability queries immediately.
6208    #[tokio::test]
6209    async fn global_broker_defaults_to_in_process_local() {
6210        let broker = global_broker();
6211        let caps = broker
6212            .capabilities()
6213            .await
6214            .expect("in-process broker needs no daemon");
6215        assert!(caps.iter().any(|c| c.key == LOCAL_PROVIDER_KEY));
6216        // Idempotent: repeated calls return an installed instance.
6217        let again = global_broker();
6218        assert!(again.capabilities().await.is_ok());
6219    }
6220
6221    // ── Upstream/broker error-body disclosure (spec §5.1) ────────────────────
6222    //
6223    // An upstream may echo the full request (prompts, tool input_schema,
6224    // credentials) in its error body, and a compromised remote broker controls
6225    // every byte of its responses including JSON `error` fields. No
6226    // provider/broker-controlled bytes may reach BrokerError Display.
6227
6228    /// Unique sentinel: if this appears in any error Display, provider bytes
6229    /// leaked across the broker boundary.
6230    const HOSTILE_SENTINEL: &str = "ZX9-HOSTILE-SENTINEL-7Q";
6231
6232    /// A hostile error body shaped like an echoed request: sentinel, marker,
6233    /// and request-shaped fields a provider could reflect back.
6234    fn hostile_body() -> String {
6235        format!(
6236            "{{\"error\":{{\"message\":\"ECHOED {HOSTILE_SENTINEL} \
6237             {{\\\"messages\\\":[{{\\\"content\\\":\\\"my secret prompt\\\"}}],\
6238             \\\"tools\\\":[{{\\\"input_schema\\\":{{}}}}]}}\"}}}}"
6239        )
6240    }
6241
6242    /// A hostile broker-style JSON body whose top-level `error` field is
6243    /// attacker-controlled (compromised remote broker).
6244    fn hostile_broker_json() -> String {
6245        format!(
6246            "{{\"error\":\"ECHOED {HOSTILE_SENTINEL} input_schema \
6247             my secret prompt\"}}"
6248        )
6249    }
6250
6251    fn assert_no_hostile_bytes(msg: &str) {
6252        assert!(
6253            !msg.contains(HOSTILE_SENTINEL),
6254            "sentinel leaked into error: {msg}"
6255        );
6256        assert!(!msg.contains("ECHOED"), "echoed body leaked: {msg}");
6257        assert!(
6258            !msg.contains("input_schema"),
6259            "request-shaped data leaked: {msg}"
6260        );
6261        assert!(
6262            !msg.contains("secret prompt"),
6263            "prompt content leaked: {msg}"
6264        );
6265    }
6266
6267    /// LocalBroker::proxy_stream: a non-2xx upstream body must never enter the
6268    /// error, while the stable `provider request failed: {status}` prefix and
6269    /// typed Transport category are retained for engine-side classification
6270    /// (including Gemini 429 detection on the status reason phrase).
6271    #[tokio::test]
6272    async fn local_broker_stream_error_omits_upstream_body() {
6273        use axum::routing::post;
6274        let app = axum::Router::new().route(
6275            "/chat/completions",
6276            post(|| async {
6277                (
6278                    axum::http::StatusCode::TOO_MANY_REQUESTS,
6279                    format!("\x1b[2J{}", hostile_body()),
6280                )
6281            }),
6282        );
6283        let url = spawn_upstream(app).await;
6284        let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6285        let err = broker
6286            .proxy_stream(ProxyRequest {
6287                provider: LOCAL_PROVIDER_KEY.into(),
6288                method: ProxyMethod::Post,
6289                path: "/chat/completions".into(),
6290                body: None,
6291                stream: true,
6292                body_bytes: None,
6293            })
6294            .await
6295            .err()
6296            .expect("non-2xx upstream must not yield a stream");
6297        assert!(matches!(err, BrokerError::Transport(_)));
6298        let msg = format!("{err}");
6299        assert!(
6300            msg.contains("provider request failed: 429 Too Many Requests"),
6301            "status prefix must survive for engine classification: {msg}"
6302        );
6303        assert!(
6304            !msg.contains('\x1b'),
6305            "escapes must not reach errors: {msg}"
6306        );
6307        assert_no_hostile_bytes(&msg);
6308    }
6309
6310    /// Drive `proxy_stream` against an upstream that answers `/chat/completions`
6311    /// with `status` + `body`, returning the Transport error message.
6312    async fn stream_error_message(status: axum::http::StatusCode, body: String) -> String {
6313        use axum::routing::post;
6314        let app = axum::Router::new().route(
6315            "/chat/completions",
6316            post(move || async move { (status, body) }),
6317        );
6318        let url = spawn_upstream(app).await;
6319        let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6320        let err = broker
6321            .proxy_stream(ProxyRequest {
6322                provider: LOCAL_PROVIDER_KEY.into(),
6323                method: ProxyMethod::Post,
6324                path: "/chat/completions".into(),
6325                body: None,
6326                stream: true,
6327                body_bytes: None,
6328            })
6329            .await
6330            .err()
6331            .expect("non-2xx upstream must not yield a stream");
6332        assert!(matches!(err, BrokerError::Transport(_)));
6333        format!("{err}")
6334    }
6335
6336    /// Everything after the `provider request failed: ` marker.
6337    fn after_marker(msg: &str) -> &str {
6338        let marker = "provider request failed: ";
6339        let idx = msg.find(marker).expect("marker present");
6340        &msg[idx + marker.len()..]
6341    }
6342
6343    /// Kimi Code weekly-quota 403: the vetted `access_terminated_error`
6344    /// class is surfaced as OUR static label so callers can tell quota
6345    /// exhaustion from an auth failure — while none of the body's text
6346    /// reaches the error, and the suffix stays ':'-free so engine-side
6347    /// `redact_provider_proxy_error` / `broker_error_status` keep parsing.
6348    #[tokio::test]
6349    async fn local_broker_stream_403_kimi_quota_surfaces_vetted_label_only() {
6350        let kimi_body = "{\"error\":{\"type\":\"access_terminated_error\",\"message\":\
6351                         \"You've reached your weekly (7-day) usage limit. Your quota will \
6352                         reset when the current 7-day window ends. \"}}";
6353        let msg = stream_error_message(axum::http::StatusCode::FORBIDDEN, kimi_body.into()).await;
6354        assert!(
6355            msg.ends_with("provider request failed: 403 Forbidden [access_terminated_error]"),
6356            "expected exact labelled prefix: {msg}"
6357        );
6358        assert!(!msg.contains("weekly"), "body text leaked: {msg}");
6359        assert!(!msg.contains("quota"), "body text leaked: {msg}");
6360        assert!(!msg.contains('{'), "raw JSON leaked: {msg}");
6361        assert!(
6362            !after_marker(&msg).contains(':'),
6363            "suffix must not contain ':' (engine truncates there): {msg}"
6364        );
6365    }
6366
6367    /// Hostile/unvetted `error.type` never becomes a label and its bytes
6368    /// never reach the error.
6369    #[tokio::test]
6370    async fn local_broker_stream_429_unvetted_type_yields_no_label() {
6371        let body = format!(
6372            "{{\"error\":{{\"type\":\"<script>ECHO:secret\",\"message\":\"ECHOED {HOSTILE_SENTINEL}\"}}}}"
6373        );
6374        let msg = stream_error_message(axum::http::StatusCode::TOO_MANY_REQUESTS, body).await;
6375        assert!(
6376            msg.ends_with("provider request failed: 429 Too Many Requests"),
6377            "no label expected: {msg}"
6378        );
6379        assert!(!msg.contains('['), "unexpected label: {msg}");
6380        assert!(!msg.contains("script"), "hostile type leaked: {msg}");
6381        assert!(!msg.contains("secret"), "hostile type leaked: {msg}");
6382        assert!(!after_marker(&msg).contains(':'), "got: {msg}");
6383        assert_no_hostile_bytes(&msg);
6384    }
6385
6386    /// Non-JSON (HTML) error page → no label, no leak.
6387    #[tokio::test]
6388    async fn local_broker_stream_html_body_yields_no_label() {
6389        let body =
6390            format!("<html><body>ECHOED {HOSTILE_SENTINEL} access_terminated_error</body></html>");
6391        let msg = stream_error_message(axum::http::StatusCode::BAD_GATEWAY, body).await;
6392        assert!(
6393            msg.ends_with("provider request failed: 502 Bad Gateway"),
6394            "no label expected: {msg}"
6395        );
6396        assert!(!msg.contains("html"), "body leaked: {msg}");
6397        assert!(!msg.contains('['), "unexpected label: {msg}");
6398        assert_no_hostile_bytes(&msg);
6399    }
6400
6401    /// Body larger than the classification cap: truncated read, no label,
6402    /// no panic, no error from the reader itself.
6403    #[tokio::test]
6404    async fn local_broker_stream_oversized_body_yields_no_label_without_panic() {
6405        // Valid vetted envelope, but padded far past the cap so the read is
6406        // cut off before the closing braces → cannot parse → no label.
6407        let pad = "x".repeat(MAX_PROXY_ERROR_CLASSIFY_BYTES * 4);
6408        let body = format!(
6409            "{{\"error\":{{\"message\":\"ECHOED {HOSTILE_SENTINEL} {pad}\",\"type\":\"access_terminated_error\"}}}}"
6410        );
6411        let msg = stream_error_message(axum::http::StatusCode::FORBIDDEN, body).await;
6412        assert!(
6413            msg.ends_with("provider request failed: 403 Forbidden"),
6414            "no label expected for oversized body: {msg}"
6415        );
6416        assert!(!msg.contains('['), "unexpected label: {msg}");
6417        assert!(
6418            msg.len() < 200,
6419            "oversized body leaked: {} bytes",
6420            msg.len()
6421        );
6422        assert_no_hostile_bytes(&msg);
6423    }
6424
6425    /// The truncating reader itself: overflow returns the first `cap` bytes
6426    /// (never an error), and a multibyte char split at the cap yields `None`
6427    /// rather than panicking.
6428    #[tokio::test]
6429    async fn read_error_body_for_classification_truncates_and_never_fails() {
6430        use axum::routing::get;
6431        let app = axum::Router::new()
6432            .route("/big", get(|| async { "a".repeat(100) }))
6433            .route("/utf8", get(|| async { format!("{}é", "a".repeat(9)) }));
6434        let url = spawn_upstream(app).await;
6435        let client = reqwest::Client::new();
6436
6437        let resp = client.get(format!("{url}/big")).send().await.unwrap();
6438        let got = read_error_body_for_classification(resp, 10).await;
6439        assert_eq!(got.as_deref(), Some("aaaaaaaaaa"));
6440
6441        // 'é' is 2 bytes at offset 9..11; cap 10 splits it.
6442        let resp = client.get(format!("{url}/utf8")).send().await.unwrap();
6443        let got = read_error_body_for_classification(resp, 10).await;
6444        assert_eq!(got, None);
6445
6446        // Under the cap: whole body.
6447        let resp = client.get(format!("{url}/utf8")).send().await.unwrap();
6448        let got = read_error_body_for_classification(resp, 1024).await;
6449        assert_eq!(got.as_deref(), Some("aaaaaaaaaé"));
6450    }
6451
6452    /// LocalBroker::anthropic_usage: a non-2xx usage-endpoint body must never
6453    /// enter the error; operation + status survive. Serial: mutates
6454    /// `SYNAPS_BASE_DIR`, which other `#[serial]` tests also depend on.
6455    #[tokio::test]
6456    #[serial_test::serial]
6457    async fn local_broker_usage_error_omits_upstream_body() {
6458        use axum::routing::get;
6459        let app = axum::Router::new().route(
6460            "/usage",
6461            get(|| async {
6462                (
6463                    axum::http::StatusCode::INTERNAL_SERVER_ERROR,
6464                    hostile_body(),
6465                )
6466            }),
6467        );
6468        let url = spawn_upstream(app).await;
6469
6470        // Provide fresh Anthropic OAuth creds behind an isolated base dir so
6471        // token resolution succeeds without touching the real auth store.
6472        // Redirect via HOME (not SYNAPS_BASE_DIR): the non-serial
6473        // `config::tests::test_base_dir` asserts only the `.synaps-cli`
6474        // suffix, which every HOME-derived base dir preserves.
6475        let dir = tempfile::tempdir().unwrap();
6476        let base = dir.path().join(".synaps-cli");
6477        std::fs::create_dir_all(&base).unwrap();
6478        let creds = crate::auth::OAuthCredentials {
6479            auth_type: "oauth".into(),
6480            refresh: String::new(),
6481            access: "test-access-token".into(),
6482            expires: crate::epoch_millis() + 3_600_000,
6483            account_id: None,
6484        };
6485        super::super::storage::save_provider_auth_at_test_hook(
6486            &base.join("auth.json"),
6487            "anthropic",
6488            &creds,
6489        )
6490        .unwrap();
6491        let old_base = std::env::var("SYNAPS_BASE_DIR").ok();
6492        let old_home = std::env::var("HOME").ok();
6493        std::env::remove_var("SYNAPS_BASE_DIR");
6494        std::env::set_var("HOME", dir.path());
6495
6496        let broker = LocalBroker::new(reqwest::Client::new())
6497            .with_anthropic_usage_url(format!("{url}/usage"));
6498        let result = broker.anthropic_usage().await;
6499
6500        match old_home {
6501            Some(v) => std::env::set_var("HOME", v),
6502            None => std::env::remove_var("HOME"),
6503        }
6504        if let Some(v) = old_base {
6505            std::env::set_var("SYNAPS_BASE_DIR", v);
6506        }
6507
6508        let err = result.expect_err("non-2xx usage response must fail");
6509        assert!(matches!(err, BrokerError::Transport(_)));
6510        let msg = format!("{err}");
6511        assert!(
6512            msg.contains("usage request failed") && msg.contains("500"),
6513            "operation + status must survive: {msg}"
6514        );
6515        assert_no_hostile_bytes(&msg);
6516    }
6517
6518    async fn spawn_remote_broker_error(status: u16) -> RemoteBroker {
6519        use axum::routing::{get, post};
6520        let code = axum::http::StatusCode::from_u16(status).unwrap();
6521        let app = axum::Router::new()
6522            .route(
6523                "/proxy",
6524                post(move || async move { (code, hostile_broker_json()) }),
6525            )
6526            .route(
6527                "/usage",
6528                get(move || async move { (code, hostile_broker_json()) }),
6529            );
6530        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6531        let addr = listener.local_addr().unwrap();
6532        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6533        RemoteBroker::new(
6534            format!("http://{addr}"),
6535            "opaque-machine-token",
6536            reqwest::Client::new(),
6537            super::super::TokenCache::new(),
6538        )
6539    }
6540
6541    fn groq_request(stream: bool) -> ProxyRequest {
6542        ProxyRequest {
6543            provider: "groq".into(),
6544            method: ProxyMethod::Post,
6545            path: "/chat/completions".into(),
6546            body: None,
6547            stream,
6548            body_bytes: None,
6549        }
6550    }
6551
6552    /// RemoteBroker 400/403: a compromised broker's `error` field must not
6553    /// reach the Denied Display; the typed Denied category survives.
6554    #[tokio::test]
6555    async fn remote_broker_denied_omits_broker_error_body() {
6556        for status in [400u16, 403] {
6557            let broker = spawn_remote_broker_error(status).await;
6558            let err = broker
6559                .proxy(groq_request(false))
6560                .await
6561                .expect_err("4xx broker response must fail");
6562            assert!(
6563                matches!(err, BrokerError::Denied(_)),
6564                "typed category must survive, got {err:?}"
6565            );
6566            let msg = format!("{err}");
6567            assert!(msg.contains(&status.to_string()), "status lost: {msg}");
6568            assert_no_hostile_bytes(&msg);
6569        }
6570    }
6571
6572    /// R2-I1: Gemini can think silently for minutes before its first data
6573    /// chunk, so its no-redirect client idles longer than everyone else's.
6574    #[test]
6575    fn gemini_gets_the_longer_stream_idle_timeout() {
6576        let broker = LocalBroker::new(reqwest::Client::new());
6577        assert_eq!(broker.stream_idle_timeout_for("google-gemini"), Duration::from_secs(300));
6578        for key in ["muse", "kimi-code", "openai-codex", "anthropic"] {
6579            assert_eq!(broker.stream_idle_timeout_for(key), Duration::from_secs(90), "{key}");
6580        }
6581        let short = LocalBroker::new(reqwest::Client::new())
6582            .with_stream_idle_timeout_for_tests(Duration::from_millis(10));
6583        assert_eq!(short.stream_idle_timeout_for("muse"), Duration::from_millis(10));
6584        assert!(short.stream_idle_timeout_for("google-gemini") >= Duration::from_secs(300));
6585    }
6586
6587    async fn spawn_remote_broker_body(status: u16, body: String) -> RemoteBroker {
6588        use axum::routing::post;
6589        let code = axum::http::StatusCode::from_u16(status).unwrap();
6590        let app = axum::Router::new().route(
6591            "/proxy",
6592            post(move || {
6593                let body = body.clone();
6594                async move { (code, body) }
6595            }),
6596        );
6597        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6598        let addr = listener.local_addr().unwrap();
6599        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6600        RemoteBroker::new(
6601            format!("http://{addr}"),
6602            "opaque-machine-token",
6603            reqwest::Client::new(),
6604            super::super::TokenCache::new(),
6605        )
6606    }
6607
6608    /// Round-2 item 4: the daemon's 502 for a retryable refresh failure
6609    /// (a Muse re-mint that hit the network) reaches a remote client as the
6610    /// typed retryable refresh error, so the "retry shortly" hint survives.
6611    /// Only the kind is taken from the body; the text is static, so a
6612    /// hostile broker still can't put its bytes into the Display.
6613    #[tokio::test]
6614    async fn remote_broker_502_keeps_a_retryable_refresh_failure() {
6615        use super::super::token::{RefreshFailure, RefreshFailureKind as K};
6616        for kind in [K::Transient, K::LockBusy, K::Other] {
6617            let daemon_text = BrokerError::Refresh(RefreshFailure::new(
6618                kind,
6619                format!("HTTP error 503 {HOSTILE_SENTINEL} ECHOED input_schema"),
6620            ))
6621            .to_string();
6622            let body = serde_json::json!({ "error": daemon_text }).to_string();
6623            let broker = spawn_remote_broker_body(502, body.clone()).await;
6624            let stream_err = broker.proxy_stream(groq_request(true)).await.err().unwrap();
6625            let buffered_err = broker.proxy(groq_request(false)).await.unwrap_err();
6626            for err in [stream_err, buffered_err] {
6627                let BrokerError::Refresh(f) = &err else {
6628                    panic!("{kind:?}: expected Refresh, got {err:?}");
6629                };
6630                assert_eq!(f.kind, kind);
6631                let msg = err.to_string();
6632                assert!(msg.contains("retry shortly"), "{msg}");
6633                assert_no_hostile_bytes(&msg);
6634            }
6635        }
6636        // Anything else in a 502 stays an opaque transport error.
6637        let broker = spawn_remote_broker_body(502, hostile_broker_json()).await;
6638        let err = broker.proxy_stream(groq_request(true)).await.err().unwrap();
6639        assert!(matches!(err, BrokerError::Transport(_)), "{err:?}");
6640        assert_no_hostile_bytes(&err.to_string());
6641    }
6642
6643    /// RemoteBroker::proxy_stream: a non-2xx broker body must never enter the
6644    /// error; operation + status survive.
6645    #[tokio::test]
6646    async fn remote_broker_stream_error_omits_broker_body() {
6647        let broker = spawn_remote_broker_error(500).await;
6648        let err = broker
6649            .proxy_stream(groq_request(true))
6650            .await
6651            .err()
6652            .expect("non-2xx broker response must not yield a stream");
6653        assert!(matches!(err, BrokerError::Transport(_)));
6654        let msg = format!("{err}");
6655        assert!(
6656            msg.contains("broker proxy stream failed") && msg.contains("500"),
6657            "operation + status must survive: {msg}"
6658        );
6659        assert_no_hostile_bytes(&msg);
6660    }
6661
6662    /// RemoteBroker::anthropic_usage: a non-2xx broker body must never enter
6663    /// the error; operation + status survive.
6664    #[tokio::test]
6665    async fn remote_broker_usage_error_omits_broker_body() {
6666        let broker = spawn_remote_broker_error(502).await;
6667        let err = broker
6668            .anthropic_usage()
6669            .await
6670            .expect_err("non-2xx broker response must fail");
6671        assert!(matches!(err, BrokerError::Transport(_)));
6672        let msg = format!("{err}");
6673        assert!(
6674            msg.contains("broker usage returned HTTP 502"),
6675            "operation + status must survive: {msg}"
6676        );
6677        assert_no_hostile_bytes(&msg);
6678    }
6679
6680    /// Capability rows carry configured-ness only — the serialized form can
6681    /// never contain key material because there is no field for it.
6682    #[tokio::test]
6683    async fn capabilities_expose_no_secret_fields() {
6684        let broker = LocalBroker::new(reqwest::Client::new());
6685        let caps = broker.capabilities().await.unwrap();
6686        assert!(caps
6687            .iter()
6688            .any(|c| c.key == "anthropic" && c.kind == CredentialKind::OAuth));
6689        assert!(caps
6690            .iter()
6691            .any(|c| c.key == "groq" && c.kind == CredentialKind::StaticKey));
6692        let json = serde_json::to_string(&caps).unwrap();
6693        for field in ["key\":", "name\":", "kind\":", "configured\":"] {
6694            assert!(json.contains(field));
6695        }
6696        assert!(!json.contains("refresh"));
6697        assert!(!json.contains("access"));
6698    }
6699
6700    /// No Anthropic refresh seam exists, so the refresh-failure → kind map
6701    /// is pinned here: only a rejected login is a login problem; transient,
6702    /// lock-busy, local and cooldown failures are "retry", action-required
6703    /// is neither.
6704    #[test]
6705    fn refresh_failure_kinds_map_to_observation_kinds() {
6706        use super::super::quota_policy::{AccountFailureKind as K, QuotaObservation};
6707        use super::super::token::{RefreshFailure, RefreshFailureKind as R};
6708        for (kind, expected, transient, login) in [
6709            (R::Transient, K::RefreshTransient, true, false),
6710            (R::LockBusy, K::RefreshLockBusy, true, false),
6711            (R::Other, K::RefreshOther, true, false),
6712            (
6713                R::Cooldown { retry_in_secs: 42 },
6714                K::RefreshOther,
6715                true,
6716                false,
6717            ),
6718            (R::Credential, K::RefreshRejected, false, true),
6719            (R::ActionRequired, K::RefreshActionRequired, false, false),
6720        ] {
6721            let read = UsageReadError::refresh(RefreshFailure::new(kind, "why"));
6722            assert_eq!(read.failure.kind, expected, "{kind:?}");
6723            assert_eq!(read.failure.kind.is_transient(), transient, "{kind:?}");
6724            assert_eq!(read.failure.kind.needs_login(), login, "{kind:?}");
6725            // `usage()` keeps its historical text for every refresh failure.
6726            assert_eq!(
6727                read.error,
6728                BrokerError::Credential("why".into()),
6729                "{kind:?}"
6730            );
6731            let observation = read.failure.observation();
6732            if login {
6733                assert_eq!(observation, QuotaObservation::AuthError);
6734            } else {
6735                assert_eq!(observation, QuotaObservation::Unknown);
6736            }
6737        }
6738        let cooldown =
6739            refresh_failure(&RefreshFailure::new(R::Cooldown { retry_in_secs: 42 }, "x"));
6740        assert_eq!(cooldown.retry_after_s, Some(42));
6741        // Slot errors: vanished vs unreadable store.
6742        let gone = UsageReadError::slot(BrokerError::UnknownAccount {
6743            provider: "anthropic".into(),
6744            label: "x".into(),
6745        });
6746        assert_eq!(gone.failure.kind, K::CredentialMissing);
6747        assert!(gone.failure.kind.needs_login());
6748        let store = UsageReadError::slot(BrokerError::Credential("unreadable".into()));
6749        assert_eq!(store.failure.kind, K::CredentialStore);
6750        assert!(!store.failure.kind.needs_login() && !store.failure.kind.is_transient());
6751    }
6752
6753    /// Every usage error keeps a kind; 429 keeps its Retry-After and the
6754    /// status codes survive.
6755    #[test]
6756    fn usage_errors_map_to_observation_kinds_with_status() {
6757        use super::super::quota_policy::AccountFailureKind as K;
6758        use super::super::usage::UsageError as U;
6759        for (err, kind, status) in [
6760            (
6761                U::RateLimited {
6762                    retry_after_s: Some(9),
6763                },
6764                K::UsageRateLimited,
6765                Some(429),
6766            ),
6767            (
6768                U::Unauthorized { status: 403 },
6769                K::UsageUnauthorized,
6770                Some(403),
6771            ),
6772            (
6773                U::UpstreamStatus {
6774                    status: 503,
6775                    retry_after_s: None,
6776                },
6777                K::UsageUpstream,
6778                Some(503),
6779            ),
6780            (U::Redirected { status: 302 }, K::UsageUpstream, Some(302)),
6781            (U::Timeout, K::UsageTimeout, None),
6782            (
6783                U::Transport {
6784                    detail: "connect".into(),
6785                },
6786                K::UsageTransport,
6787                None,
6788            ),
6789            (U::Malformed { detail: "x".into() }, K::UsageMalformed, None),
6790            (U::BodyTooLarge { cap: 1 }, K::UsageMalformed, None),
6791            (U::AccountIdUnavailable, K::UsageUnauthorized, None),
6792            (
6793                U::UnsupportedProvider {
6794                    provider: "p".into(),
6795                },
6796                K::Unsupported,
6797                None,
6798            ),
6799        ] {
6800            let read = UsageReadError::usage(err.clone());
6801            assert_eq!(read.failure.kind, kind, "{err:?}");
6802            assert_eq!(read.failure.http_status, status, "{err:?}");
6803            assert_eq!(read.error, err.clone().into_broker_error(), "{err:?}");
6804        }
6805        assert_eq!(
6806            UsageReadError::usage(U::RateLimited {
6807                retry_after_s: Some(9)
6808            })
6809            .failure
6810            .retry_after_s,
6811            Some(9)
6812        );
6813    }
6814}