1use std::collections::BTreeMap;
23use std::pin::Pin;
24use std::sync::Arc;
25use std::time::Duration;
26
27use async_trait::async_trait;
28use futures::Stream;
29use serde::{Deserialize, Serialize};
30
31use super::account::{
32 seat_fingerprint, Account, AccountPolicy, AccountSelector, AccountSummary, CredentialRef,
33 MaterialDigest, SeatIdentity,
34};
35use super::cloud::{CloudProviderId, InvokeRequest};
36use super::provider::OAuthProviderId;
37use super::static_providers::{
38 allowed_proxy_paths, static_provider, StaticProviderSpec, LOCAL_DEFAULT_BASE_URL,
39 LOCAL_PROVIDER_KEY, STATIC_PROVIDERS,
40};
41use super::storage;
42
43pub const MAX_PROXY_REQUEST_BYTES: usize = 2 * 1024 * 1024;
52pub const MAX_PROXY_RESPONSE_BYTES: usize = 8 * 1024 * 1024;
54pub const MAX_UPSTREAM_ERROR_BYTES: usize = 2 * 1024;
58pub const PROXY_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
60
61const ANTHROPIC_USAGE_URL: &str = "https://api.anthropic.com/api/oauth/usage";
64const ANTHROPIC_OAUTH_BETA: &str = "oauth-2025-04-20";
65
66pub const GOOGLE_GEMINI_CODE_ASSIST_BASE_URL: &str = "https://cloudcode-pa.googleapis.com";
70
71pub(crate) fn is_allowed_google_gemini_path(path: &str) -> bool {
75 matches!(
76 path,
77 "/v1internal:loadCodeAssist"
78 | "/v1internal:onboardUser"
79 | "/v1internal:streamGenerateContent"
80 | "/v1internal:countTokens"
81 ) || path.starts_with("/v1internal/operations/")
82}
83
84pub const OPENAI_CODEX_BACKEND_BASE_URL: &str = "https://chatgpt.com/backend-api";
86
87pub(crate) const OPENAI_CODEX_MODELS_PATH: &str = "/codex/models";
96
97pub(crate) fn is_allowed_openai_codex_path(path: &str) -> bool {
98 let Some((base, query)) = path.split_once('?') else {
99 return false;
100 };
101 if base != OPENAI_CODEX_MODELS_PATH {
102 return false;
103 }
104 let mut client_version: Option<&str> = None;
105 for pair in query.split('&') {
106 let Some((k, v)) = pair.split_once('=') else {
107 return false;
108 };
109 if k != "client_version" || client_version.is_some() {
110 return false;
111 }
112 if v.trim().is_empty() {
113 return false;
114 }
115 client_version = Some(v);
116 }
117 client_version.is_some()
118}
119
120pub(crate) fn is_allowed_anthropic_path(path: &str) -> bool {
125 if path == "/v1/models" {
126 return true;
127 }
128 let Some((base, query)) = path.split_once('?') else {
129 return false;
130 };
131 if base != "/v1/models" {
132 return false;
133 }
134 if query.is_empty() {
135 return false;
136 }
137 let mut saw_limit = false;
138 let mut saw_after_id = false;
139 for pair in query.split('&') {
140 let Some((k, v)) = pair.split_once('=') else {
141 return false;
142 };
143 if v.is_empty() {
144 return false;
145 }
146 match k {
147 "limit" if !saw_limit => saw_limit = true,
148 "after_id" if !saw_after_id => saw_after_id = true,
149 _ => return false,
150 }
151 }
152 true
155}
156
157pub(crate) fn anthropic_oauth_catalog_request_headers() -> &'static [(&'static str, &'static str)] {
161 &[
162 ("anthropic-version", "2023-06-01"),
163 ("anthropic-beta", ANTHROPIC_OAUTH_BETA),
164 ("accept", "application/json"),
165 ]
166}
167
168#[derive(Debug, Clone, PartialEq, Eq)]
173#[non_exhaustive]
174pub enum BrokerError {
175 UnknownProvider(String),
177 RegistrationRequired {
179 provider: String,
180 remediation: String,
181 },
182 NotConfigured(String),
184 Unauthorized,
186 Denied(String),
188 UnsupportedCapability {
191 provider: String,
192 capability: String,
193 },
194 Transport(String),
196 Credential(String),
198 UnknownAccount { provider: String, label: String },
200 UnsupportedAccount { provider: String, label: String },
202 InvalidAccount(String),
204 NoAccountAvailable {
208 provider: String,
209 reason: String,
210 failure: Option<super::quota_policy::SelectionFailure>,
211 },
212 BrokerTooOld(String),
215 Refresh(super::token::RefreshFailure),
219}
220
221impl std::fmt::Display for BrokerError {
222 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
223 match self {
224 Self::UnknownProvider(p) => write!(f, "unknown provider: {p}"),
225 Self::RegistrationRequired {
226 provider,
227 remediation,
228 } => {
229 write!(f, "registration required for '{provider}': {remediation}")
230 }
231 Self::NotConfigured(p) => write!(
232 f,
233 "no credential configured for '{p}'. Run `synaps login` to add one."
234 ),
235 Self::Unauthorized => f.write_str(&super::broker_token::unauthorized_hint()),
236 Self::Denied(msg) => write!(f, "broker denied request: {msg}"),
237 Self::UnsupportedCapability {
238 provider,
239 capability,
240 } => write!(
241 f,
242 "cloud provider '{provider}' is text-only: {capability} are not supported yet \
243 (no credential was used and no network request was made)"
244 ),
245 Self::Transport(msg) => write!(f, "broker transport error: {msg}"),
246 Self::Credential(msg) => write!(f, "credential error: {msg}"),
247 Self::Refresh(failure) => {
248 use super::token::RefreshFailureKind as K;
249 let hint = match failure.kind {
250 K::Transient => " (network or provider error, not a login problem; retry shortly)",
251 K::LockBusy => " (another refresh of this account is still running; retry shortly)",
252 K::Other => " (retry shortly)",
253 K::Credential | K::Cooldown { .. } | K::ActionRequired => "",
254 };
255 write!(f, "credential refresh failed{hint}: {failure}")
256 }
257 Self::UnknownAccount { provider, label } => write!(
258 f,
259 "unknown account '{label}' for provider '{provider}' (no fallback; run \
260 `synaps auth list` or `synaps login --provider {provider} --account {label}`)"
261 ),
262 Self::UnsupportedAccount { provider, label } => write!(
263 f,
264 "this credential broker cannot address account '{label}' for provider '{provider}'"
265 ),
266 Self::InvalidAccount(msg) => write!(f, "invalid account: {msg}"),
267 Self::NoAccountAvailable {
268 provider,
269 reason,
270 failure: Some(failure),
271 } => write!(
272 f,
273 "no account with proven capacity for '{provider}' ({}): {reason}",
274 failure.kind
275 ),
276 Self::NoAccountAvailable {
277 provider, reason, ..
278 } => {
279 write!(
280 f,
281 "no account with proven capacity for '{provider}': {reason}"
282 )
283 }
284 Self::BrokerTooOld(what) => write!(
285 f,
286 "broker is older than this client; {what} unavailable (upgrade the broker host)"
287 ),
288 }
289 }
290}
291
292impl std::error::Error for BrokerError {}
293
294#[cfg(test)]
295#[path = "broker_remote_compat_tests.rs"]
296mod remote_compat_tests;
297
298#[derive(Debug, Clone, Serialize, Deserialize)]
305pub struct AccessToken {
306 pub token: String,
307 pub expires: u64,
309}
310
311#[derive(Debug, Clone)]
316pub struct PinnedToken {
317 pub credential: CredentialRef,
318 pub token: AccessToken,
319}
320
321impl PinnedToken {
322 pub fn seat_fingerprint(&self) -> Option<String> {
328 match self.credential.provider {
329 OAuthProviderId::OpenAiCodex => super::extract_codex_account_id(&self.token.token)
330 .and_then(|id| seat_fingerprint(self.credential.provider, &id)),
331 _ => None,
332 }
333 }
334}
335
336#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
338#[serde(rename_all = "lowercase")]
339pub enum ProxyMethod {
340 Get,
341 Post,
342}
343
344#[derive(Debug, Clone, Serialize, Deserialize)]
352pub struct ProxyRequest {
353 pub provider: String,
359 pub method: ProxyMethod,
360 pub path: String,
362 #[serde(default, skip_serializing_if = "Option::is_none")]
364 pub body: Option<serde_json::Value>,
365 #[serde(default)]
367 pub stream: bool,
368 #[serde(skip)]
377 pub body_bytes: Option<bytes::Bytes>,
378}
379
380impl ProxyRequest {
381 pub fn validate(&self) -> Result<(), BrokerError> {
384 let (provider, _account) = self.split_provider()?;
385 let provider = provider.as_str();
386 if provider != LOCAL_PROVIDER_KEY
387 && provider != "xai-auth"
388 && provider != "github-copilot"
389 && provider != "google-gemini"
390 && provider != "openai-codex"
391 && provider != "anthropic"
392 && provider != "kimi-code"
393 && provider != "muse"
394 && static_provider(provider).is_none()
395 {
396 return Err(BrokerError::UnknownProvider(self.provider.clone()));
397 }
398 if !self.path.starts_with('/') {
399 return Err(BrokerError::Denied("proxy path must be relative".into()));
400 }
401 if self.path.contains("://") || self.path.contains("..") {
402 return Err(BrokerError::Denied(
403 "proxy path is not a plain relative path".into(),
404 ));
405 }
406 let oauth_path_allowed = provider == "xai-auth" && self.path == "/responses"
410 || provider == "github-copilot"
411 && matches!(
412 self.path.as_str(),
413 "/models" | "/chat/completions" | "/responses"
414 )
415 || provider == "google-gemini" && is_allowed_google_gemini_path(&self.path)
416 || provider == "kimi-code"
419 && (self.path == "/chat/completions"
420 || self.method == ProxyMethod::Get
421 && matches!(self.path.as_str(), "/models" | "/me" | "/usages"))
422 || provider == "muse"
425 && (self.method == ProxyMethod::Post && self.path == "/v1/responses"
426 || self.method == ProxyMethod::Get && self.path == "/muse-code/models")
427 || provider == "openai-codex"
428 && self.method == ProxyMethod::Get
429 && is_allowed_openai_codex_path(&self.path)
430 || provider == "anthropic"
431 && self.method == ProxyMethod::Get
432 && is_allowed_anthropic_path(&self.path);
433 if !oauth_path_allowed && !allowed_proxy_paths(provider).contains(&self.path.as_str()) {
434 return Err(BrokerError::Denied(format!(
435 "proxy path '{}' is not in the provider's endpoint allowlist",
436 self.path
437 )));
438 }
439 let size = match (&self.body_bytes, &self.body) {
444 (Some(bytes), _) => Some(bytes.len()),
446 (None, Some(body)) => Some(
447 serde_json::to_vec(body)
448 .map(|v| v.len())
449 .unwrap_or(usize::MAX),
450 ),
451 (None, None) => None,
452 };
453 if let Some(size) = size {
454 if size > MAX_PROXY_REQUEST_BYTES {
455 return Err(BrokerError::Denied(format!(
456 "request body exceeds the {MAX_PROXY_REQUEST_BYTES}-byte broker limit"
457 )));
458 }
459 }
460 #[cfg(any(test, debug_assertions))]
466 if let (Some(bytes), Some(body)) = (&self.body_bytes, &self.body) {
467 let parsed = serde_json::from_slice::<serde_json::Value>(bytes).ok();
468 if parsed.as_ref() != Some(body) {
469 return Err(BrokerError::Denied(
470 "body_bytes does not match the JSON body (exact-byte handoff incoherent)"
471 .into(),
472 ));
473 }
474 }
475 Ok(())
476 }
477
478 pub fn post_json_exact(
487 provider: impl Into<String>,
488 path: impl Into<String>,
489 body: serde_json::Value,
490 stream: bool,
491 ) -> Result<(Self, bytes::Bytes), BrokerError> {
492 let bytes =
493 bytes::Bytes::from(serde_json::to_vec(&body).map_err(|e| {
494 BrokerError::Denied(format!("request body failed to serialize: {e}"))
495 })?);
496 let request = Self {
497 provider: provider.into(),
498 method: ProxyMethod::Post,
499 path: path.into(),
500 body: Some(body),
501 stream,
502 body_bytes: Some(bytes.clone()),
503 };
504 Ok((request, bytes))
505 }
506
507 pub fn with_account(mut self, account: &Account) -> Self {
510 let base = self
511 .provider
512 .split_once('@')
513 .map(|(p, _)| p.to_string())
514 .unwrap_or_else(|| self.provider.clone());
515 self.provider = format!("{base}@{}", account.label_str());
516 self
517 }
518
519 pub fn split_provider(&self) -> Result<(String, Option<Account>), BrokerError> {
522 match self.provider.split_once('@') {
523 None => Ok((self.provider.clone(), None)),
524 Some((base, account)) => {
525 if base.parse::<OAuthProviderId>().is_err() {
526 return Err(BrokerError::UnknownProvider(self.provider.clone()));
527 }
528 let account = Account::parse(account).map_err(BrokerError::InvalidAccount)?;
529 Ok((base.to_string(), Some(account)))
530 }
531 }
532 }
533
534 pub fn base_provider(&self) -> Result<String, BrokerError> {
536 Ok(self.split_provider()?.0)
537 }
538
539 pub fn explicit_account(&self) -> Result<Option<Account>, BrokerError> {
541 Ok(self.split_provider()?.1)
542 }
543}
544
545#[derive(Debug, Clone, Serialize, Deserialize)]
547pub struct ProxyResponse {
548 pub status: u16,
549 pub body: String,
550}
551
552pub type ProxyByteStream = Pin<Box<dyn Stream<Item = Result<bytes::Bytes, BrokerError>> + Send>>;
554
555#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
557#[serde(rename_all = "snake_case")]
558pub enum CredentialKind {
559 OAuth,
560 StaticKey,
561 LocalEndpoint,
562}
563
564#[derive(Debug, Clone, Serialize, Deserialize)]
566pub struct ProviderStatus {
567 pub key: String,
568 pub name: String,
569 pub kind: CredentialKind,
570 pub configured: bool,
573 #[serde(default, skip_serializing_if = "Vec::is_empty")]
575 pub accounts: Vec<AccountSummary>,
576}
577
578#[derive(Debug, Clone, PartialEq, Eq)]
580pub enum StaticKeyStatus {
581 NotSet,
582 Configured {
585 masked: String,
586 },
587 FromEnv,
589}
590
591impl StaticKeyStatus {
592 pub fn is_configured(&self) -> bool {
593 !matches!(self, StaticKeyStatus::NotSet)
594 }
595}
596
597#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
601pub struct CloudCatalogEntry {
602 pub provider: CloudProviderId,
603 pub id: String,
604 pub display_name: String,
605 pub context_ref: String,
607 pub context_label: String,
608 pub stale: bool,
609 pub fetched_at: u64,
611}
612
613#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
615#[serde(tag = "type", rename_all = "snake_case")]
616pub enum CloudEvent {
617 TextDelta {
618 delta: String,
619 },
620 ToolArguments {
621 id: String,
622 name: Option<String>,
623 delta: String,
624 },
625 Usage {
626 input_tokens: u64,
627 output_tokens: u64,
628 },
629 Done,
630}
631pub type CloudEventStream = Pin<Box<dyn Stream<Item = Result<CloudEvent, BrokerError>> + Send>>;
632
633#[async_trait]
637pub trait CredentialBroker: Send + Sync {
638 async fn access_token(&self, provider: OAuthProviderId) -> Result<AccessToken, BrokerError>;
641
642 fn account_selector(&self, provider: OAuthProviderId) -> AccountSelector {
645 let _ = provider;
646 AccountSelector::Account(Account::Default)
647 }
648
649 async fn access_token_for(&self, cred: &CredentialRef) -> Result<AccessToken, BrokerError> {
655 match &cred.account {
656 Account::Default => self.access_token(cred.provider).await,
657 Account::Named(label) => Err(BrokerError::UnsupportedAccount {
658 provider: cred.provider.as_str().to_string(),
659 label: label.as_str().to_string(),
660 }),
661 }
662 }
663
664 async fn recover_rejected_token(
668 &self,
669 cred: &CredentialRef,
670 _rejected: &str,
671 ) -> Result<AccessToken, BrokerError> {
672 self.access_token_for(cred).await
673 }
674
675 async fn access_token_pinned(
678 &self,
679 provider: OAuthProviderId,
680 ) -> Result<PinnedToken, BrokerError> {
681 let token = self.access_token(provider).await?;
682 Ok(PinnedToken {
683 credential: CredentialRef::default_for(provider),
684 token,
685 })
686 }
687
688 async fn access_token_pinned_for(
694 &self,
695 provider: OAuthProviderId,
696 model: Option<&str>,
697 ) -> Result<PinnedToken, BrokerError> {
698 let _ = model;
699 self.access_token_pinned(provider).await
700 }
701
702 async fn access_token_pinned_excluding(
710 &self,
711 provider: OAuthProviderId,
712 model: Option<&str>,
713 exclude: &[CredentialRef],
714 ) -> Result<PinnedToken, BrokerError> {
715 let pinned = self.access_token_pinned_for(provider, model).await?;
716 if exclude.contains(&pinned.credential) {
717 return Err(BrokerError::NoAccountAvailable {
718 provider: provider.as_str().to_string(),
719 reason: format!(
720 "the broker chose {}, which this request already used",
721 pinned.credential.account
722 ),
723 failure: None,
724 });
725 }
726 Ok(pinned)
727 }
728
729 async fn accounts(&self, provider: OAuthProviderId) -> Result<Vec<AccountSummary>, BrokerError> {
731 Err(BrokerError::UnsupportedCapability {
732 provider: provider.as_str().to_string(),
733 capability: "accounts".into(),
734 })
735 }
736
737 async fn usage(&self, cred: &CredentialRef) -> Result<super::usage::UsageSnapshot, BrokerError> {
741 Err(BrokerError::UnsupportedCapability {
742 provider: cred.provider.as_str().to_string(),
743 capability: "usage".into(),
744 })
745 }
746
747 async fn report_cooldown(
751 &self,
752 cred: &CredentialRef,
753 until_ms: Option<u64>,
754 reason: &str,
755 ) -> Result<(), BrokerError> {
756 let _ = (cred, until_ms, reason);
757 Ok(())
758 }
759
760 fn active_cooldown(&self, cred: &CredentialRef) -> Option<u64> {
763 let _ = cred;
764 None
765 }
766
767 fn clear_cooldown(&self, cred: &CredentialRef, sent_at_ms: u64) {
774 let _ = (cred, sent_at_ms);
775 }
776
777 async fn access_token_for_auto(&self, cred: &CredentialRef) -> Result<AccessToken, BrokerError> {
785 self.access_token_for(cred).await
786 }
787
788 async fn proxy(&self, request: ProxyRequest) -> Result<ProxyResponse, BrokerError>;
790
791 async fn proxy_stream(&self, request: ProxyRequest) -> Result<ProxyByteStream, BrokerError>;
795
796 async fn anthropic_usage(&self) -> Result<serde_json::Value, BrokerError>;
803
804 async fn cloud_catalog(
806 &self,
807 provider: CloudProviderId,
808 context_ref: &str,
809 allow_stale: bool,
810 ) -> Result<Vec<CloudCatalogEntry>, BrokerError> {
811 let _ = (context_ref, allow_stale);
812 Err(BrokerError::NotConfigured(provider.to_string()))
813 }
814
815 async fn cloud_invoke(
818 &self,
819 provider: CloudProviderId,
820 context_ref: &str,
821 model_id: &str,
822 request: InvokeRequest,
823 ) -> Result<CloudEventStream, BrokerError> {
824 let _ = (context_ref, model_id, request);
825 Err(BrokerError::NotConfigured(provider.to_string()))
826 }
827
828 async fn capabilities(&self) -> Result<Vec<ProviderStatus>, BrokerError>;
830}
831
832#[async_trait]
835pub trait CloudBackend: Send + Sync {
836 async fn catalog(
837 &self,
838 provider: CloudProviderId,
839 context_ref: &str,
840 allow_stale: bool,
841 ) -> Result<Vec<CloudCatalogEntry>, BrokerError>;
842 async fn invoke(
843 &self,
844 provider: CloudProviderId,
845 context_ref: &str,
846 model_id: &str,
847 request: InvokeRequest,
848 ) -> Result<CloudEventStream, BrokerError>;
849}
850
851#[derive(Clone)]
852struct ProductionCloudBackend {
853 http: reqwest::Client,
856 refresh_lock: Arc<tokio::sync::Mutex<()>>,
857}
858impl ProductionCloudBackend {
859 fn new(_http: reqwest::Client) -> Self {
860 let http = reqwest::Client::builder()
861 .redirect(reqwest::redirect::Policy::none())
862 .connect_timeout(Duration::from_secs(10))
863 .timeout(Duration::from_secs(90))
864 .build()
865 .expect("cloud HTTPS client configuration is valid");
866 Self {
867 http,
868 refresh_lock: Arc::new(tokio::sync::Mutex::new(())),
869 }
870 }
871
872 fn state(&self, provider: CloudProviderId) -> Result<serde_json::Value, BrokerError> {
873 storage::load_cloud_state(provider.as_str())
874 .map_err(BrokerError::Credential)?
875 .ok_or_else(|| BrokerError::NotConfigured(provider.to_string()))
876 }
877
878 fn context_ref(&self, provider: CloudProviderId) -> Result<String, BrokerError> {
879 use sha2::{Digest, Sha256};
880 let state = self.state(provider)?;
881 let public = serde_json::to_vec(&state["config"])
882 .map_err(|_| BrokerError::Credential("invalid cloud context".into()))?;
883 let digest = Sha256::digest([provider.as_str().as_bytes(), &public].concat());
884 Ok(format!(
885 "ctx-{}",
886 digest[..16]
887 .iter()
888 .map(|b| format!("{b:02x}"))
889 .collect::<String>()
890 ))
891 }
892
893 fn validate_context(
894 &self,
895 provider: CloudProviderId,
896 supplied: &str,
897 ) -> Result<String, BrokerError> {
898 let opaque = self.context_ref(provider)?;
899 if supplied == provider.as_str() || supplied == opaque {
902 Ok(opaque)
903 } else {
904 Err(BrokerError::Denied(
905 "cloud context does not match stored provider".into(),
906 ))
907 }
908 }
909
910 async fn aws(
911 &self,
912 ) -> Result<super::aws_bedrock::AwsBedrockBroker<super::aws_bedrock::AwsHttpApi>, BrokerError>
913 {
914 #[derive(Deserialize)]
915 struct State {
916 config: super::cloud::AwsBedrockConfig,
917 access_key: String,
918 secret_key: String,
919 session_token: String,
920 expires_at: u64,
921 registered_client: serde_json::Value,
922 sso_access_token: String,
923 sso_refresh_token: Option<String>,
924 sso_expires_at: u64,
925 }
926 let decode = |value| {
927 serde_json::from_value::<State>(value)
928 .map_err(|_| BrokerError::Credential("invalid AWS broker state".into()))
929 };
930 let mut raw = self.state(CloudProviderId::AwsBedrock)?;
931 let now = std::time::SystemTime::now()
932 .duration_since(std::time::UNIX_EPOCH)
933 .unwrap_or_default()
934 .as_millis() as u64;
935 let mut s = decode(raw.clone())?;
936 if s.expires_at <= now + 60_000 {
937 let _guard = self.refresh_lock.lock().await;
940 raw = self.state(CloudProviderId::AwsBedrock)?;
941 s = decode(raw.clone())?;
942 if s.expires_at <= now + 60_000 {
943 let api =
944 super::aws_bedrock::AwsHttpApi::new(self.http.clone(), &s.config.sso_region);
945 if s.sso_expires_at <= now + 60_000 {
946 let refresh = s
947 .sso_refresh_token
948 .as_deref()
949 .filter(|v| !v.is_empty())
950 .ok_or_else(|| {
951 BrokerError::Credential("AWS SSO session expired; login again".into())
952 })?;
953 let client = super::aws_bedrock::RegisteredClient::new(
954 s.registered_client["id"].as_str().ok_or_else(|| {
955 BrokerError::Credential("invalid AWS client registration".into())
956 })?,
957 s.registered_client["secret"].as_str().ok_or_else(|| {
958 BrokerError::Credential("invalid AWS client registration".into())
959 })?,
960 s.registered_client["expires_at"].as_u64().ok_or_else(|| {
961 BrokerError::Credential("invalid AWS client registration".into())
962 })?,
963 );
964 if client.expires_at <= now / 1000 {
965 return Err(BrokerError::Credential(
966 "AWS client registration expired; login again".into(),
967 ));
968 }
969 use super::aws_bedrock::AwsApi;
970 let token = api
971 .create_token(
972 &client,
973 &s.config.sso_region,
974 super::aws_bedrock::TokenGrant::RefreshToken(refresh),
975 )
976 .await
977 .map_err(|_| {
978 BrokerError::Credential("AWS SSO refresh rejected; login again".into())
979 })?;
980 raw["sso_access_token"] = token.access().into();
981 if let Some(rotated) = token.refresh() {
982 raw["sso_refresh_token"] = rotated.into();
983 }
984 raw["sso_expires_at"] = (now + token.expires_in * 1000).into();
985 storage::save_cloud_state("aws-bedrock", &raw)
990 .map_err(BrokerError::Credential)?;
991 s = decode(raw.clone())?;
992 }
993 use super::aws_bedrock::AwsApi;
994 let credentials = api
995 .get_role_credentials(
996 &s.config.sso_region,
997 &s.sso_access_token,
998 &s.config.account_id,
999 &s.config.role_name,
1000 )
1001 .await
1002 .map_err(|_| {
1003 BrokerError::Credential("AWS role refresh rejected; login again".into())
1004 })?;
1005 raw["access_key"] = credentials.access_key().into();
1006 raw["secret_key"] = credentials.secret_key().into();
1007 raw["session_token"] = credentials.session_token().into();
1008 raw["expires_at"] = credentials.expires_at.into();
1009 storage::save_cloud_state("aws-bedrock", &raw).map_err(BrokerError::Credential)?;
1010 s = decode(raw)?;
1011 }
1012 }
1013 let api = super::aws_bedrock::AwsHttpApi::new(self.http.clone(), &s.config.sso_region);
1014 Ok(super::aws_bedrock::AwsBedrockBroker::from_credentials(
1015 api,
1016 s.config,
1017 super::aws_bedrock::RoleCredentials::new(
1018 s.access_key,
1019 s.secret_key,
1020 s.session_token,
1021 s.expires_at,
1022 ),
1023 ))
1024 }
1025 async fn azure_request(
1026 &self,
1027 state: &mut serde_json::Value,
1028 audience: super::azure_openai::AzureAudience,
1029 ) -> Result<String, BrokerError> {
1030 let now = std::time::SystemTime::now()
1031 .duration_since(std::time::UNIX_EPOCH)
1032 .unwrap_or_default()
1033 .as_millis() as u64;
1034 let key = match audience {
1035 super::azure_openai::AzureAudience::Arm => "arm",
1036 super::azure_openai::AzureAudience::Inference => "inference",
1037 };
1038 if state[key]["expires_at"].as_u64().unwrap_or(0) > now + 60_000 {
1039 return state[key]["access_token"]
1040 .as_str()
1041 .map(str::to_owned)
1042 .ok_or_else(|| BrokerError::Credential("invalid Azure token state".into()));
1043 }
1044 let _refresh = self.refresh_lock.lock().await;
1047 *state = self.state(CloudProviderId::AzureOpenAi)?;
1048 if state[key]["expires_at"].as_u64().unwrap_or(0) > now + 60_000 {
1049 return state[key]["access_token"]
1050 .as_str()
1051 .map(str::to_owned)
1052 .ok_or_else(|| BrokerError::Credential("invalid Azure token state".into()));
1053 }
1054 let config: super::cloud::AzureOpenAiConfig =
1055 serde_json::from_value(state["config"].clone())
1056 .map_err(|_| BrokerError::Credential("invalid Azure broker state".into()))?;
1057 let client_id = state["client_id"]
1058 .as_str()
1059 .ok_or_else(|| BrokerError::Credential("invalid Azure client registration".into()))?;
1060 let refresh = state["refresh_token"]
1061 .as_str()
1062 .ok_or_else(|| BrokerError::Credential("invalid Azure refresh state".into()))?;
1063 let reg = super::azure_openai::AzureRegistration::production(Some(client_id.into()))
1064 .map_err(|e| BrokerError::Credential(e.to_string()))?;
1065 let r = super::azure_openai::refresh_request(&config, ®, audience, refresh)
1066 .map_err(|e| BrokerError::Credential(e.to_string()))?;
1067 let response = self
1068 .http
1069 .post(r.url)
1070 .form(&r.form)
1071 .send()
1072 .await
1073 .map_err(|_| BrokerError::Transport("Azure refresh failed".into()))?;
1074 if !response.status().is_success() {
1075 return Err(BrokerError::Credential(
1076 "Azure refresh rejected; login again".into(),
1077 ));
1078 }
1079 let wire: serde_json::Value = response
1080 .json()
1081 .await
1082 .map_err(|_| BrokerError::Transport("invalid Azure token response".into()))?;
1083 let access = wire["access_token"]
1084 .as_str()
1085 .filter(|v| !v.is_empty())
1086 .ok_or_else(|| BrokerError::Transport("invalid Azure token response".into()))?
1087 .to_owned();
1088 if let Some(r) = wire["refresh_token"].as_str().filter(|v| !v.is_empty()) {
1089 state["refresh_token"] = r.into();
1090 }
1091 state[key] = serde_json::json!({"access_token": access, "expires_at": now + wire["expires_in"].as_u64().unwrap_or(3600) * 1000});
1092 storage::save_cloud_state("azure-openai", state).map_err(BrokerError::Credential)?;
1093 Ok(access)
1094 }
1095
1096 async fn vertex_request(&self, state: &mut serde_json::Value) -> Result<String, BrokerError> {
1097 let now = std::time::SystemTime::now()
1098 .duration_since(std::time::UNIX_EPOCH)
1099 .unwrap_or_default()
1100 .as_millis() as u64;
1101 if state["expires_at"].as_u64().unwrap_or(0) > now + 60_000 {
1102 return state["access_token"]
1103 .as_str()
1104 .map(str::to_owned)
1105 .ok_or_else(|| BrokerError::Credential("invalid Vertex token state".into()));
1106 }
1107 let _refresh = self.refresh_lock.lock().await;
1108 *state = self.state(CloudProviderId::GoogleVertex)?;
1109 if state["expires_at"].as_u64().unwrap_or(0) > now + 60_000 {
1110 return state["access_token"]
1111 .as_str()
1112 .map(str::to_owned)
1113 .ok_or_else(|| BrokerError::Credential("invalid Vertex token state".into()));
1114 }
1115 let client_id = state["client_id"]
1116 .as_str()
1117 .ok_or_else(|| BrokerError::Credential("invalid Vertex registration".into()))?;
1118 let refresh = state["refresh_token"]
1119 .as_str()
1120 .ok_or_else(|| BrokerError::Credential("invalid Vertex refresh state".into()))?;
1121 let response = self
1122 .http
1123 .post(super::google_vertex::TOKEN_URL)
1124 .form(&[
1125 ("client_id", client_id),
1126 ("grant_type", "refresh_token"),
1127 ("refresh_token", refresh),
1128 ])
1129 .send()
1130 .await
1131 .map_err(|_| BrokerError::Transport("Vertex refresh failed".into()))?;
1132 if !response.status().is_success() {
1133 return Err(BrokerError::Credential(
1134 "Vertex refresh rejected; login again".into(),
1135 ));
1136 }
1137 let wire: serde_json::Value = response
1138 .json()
1139 .await
1140 .map_err(|_| BrokerError::Transport("invalid Vertex token response".into()))?;
1141 let access = wire["access_token"]
1142 .as_str()
1143 .filter(|v| !v.is_empty())
1144 .ok_or_else(|| BrokerError::Transport("invalid Vertex token response".into()))?
1145 .to_owned();
1146 if let Some(r) = wire["refresh_token"].as_str().filter(|v| !v.is_empty()) {
1147 state["refresh_token"] = r.into();
1148 }
1149 state["access_token"] = access.clone().into();
1150 state["expires_at"] = (now + wire["expires_in"].as_u64().unwrap_or(3600) * 1000).into();
1151 storage::save_cloud_state("google-vertex", state).map_err(BrokerError::Credential)?;
1152 Ok(access)
1153 }
1154}
1155
1156#[async_trait]
1157impl CloudBackend for ProductionCloudBackend {
1158 async fn catalog(
1159 &self,
1160 provider: CloudProviderId,
1161 context_ref: &str,
1162 _allow_stale: bool,
1163 ) -> Result<Vec<CloudCatalogEntry>, BrokerError> {
1164 let opaque_context = self.validate_context(provider, context_ref)?;
1165 let fetched_at = std::time::SystemTime::now()
1166 .duration_since(std::time::UNIX_EPOCH)
1167 .unwrap_or_default()
1168 .as_millis() as u64;
1169 match provider {
1170 CloudProviderId::AwsBedrock => {
1171 let broker = self.aws().await?;
1172 let label = broker.public_context().bedrock_region;
1173 broker
1174 .catalog()
1175 .await
1176 .map_err(|_| {
1177 BrokerError::Transport("AWS catalog failed (details redacted)".into())
1178 })
1179 .map(|xs| {
1180 xs.into_iter()
1181 .map(|x| CloudCatalogEntry {
1182 provider,
1183 id: x.id,
1184 display_name: x.display_name,
1185 context_ref: opaque_context.clone(),
1186 context_label: label.clone(),
1187 stale: false,
1188 fetched_at,
1189 })
1190 .collect()
1191 })
1192 }
1193 CloudProviderId::AzureOpenAi => {
1194 let mut state = self.state(provider)?;
1195 let config: super::cloud::AzureOpenAiConfig =
1196 serde_json::from_value(state["config"].clone()).map_err(|_| {
1197 BrokerError::Credential("invalid Azure broker state".into())
1198 })?;
1199 let token = self
1200 .azure_request(&mut state, super::azure_openai::AzureAudience::Arm)
1201 .await?;
1202 let mut discovery =
1203 super::azure_openai::DeploymentDiscovery::new(config.clone(), 20, 1000);
1204 let mut url = Some(discovery.initial_url());
1205 while let Some(next) = url {
1206 let r = self
1207 .http
1208 .get(next)
1209 .bearer_auth(&token)
1210 .send()
1211 .await
1212 .map_err(|_| BrokerError::Transport("Azure catalog failed".into()))?;
1213 if !r.status().is_success() {
1214 return Err(BrokerError::Transport("Azure catalog rejected".into()));
1215 }
1216 let body = read_body_capped(r, MAX_CLOUD_CATALOG_BODY_BYTES)
1217 .await
1218 .map_err(|_| {
1219 BrokerError::Transport("Azure catalog exceeded body limit".into())
1220 })?;
1221 url = discovery
1222 .accept_page(&body)
1223 .map_err(|_| BrokerError::Transport("invalid Azure catalog".into()))?;
1224 }
1225 discovery
1226 .finish()
1227 .map_err(|_| BrokerError::Transport("Azure catalog unavailable".into()))
1228 .map(|xs| {
1229 xs.into_iter()
1230 .map(|x| CloudCatalogEntry {
1231 provider,
1232 id: x.id,
1233 display_name: x.display_name,
1234 context_ref: opaque_context.clone(),
1235 context_label: format!(
1236 "{}/{}",
1237 config.resource_name, config.resource_group
1238 ),
1239 stale: false,
1240 fetched_at,
1241 })
1242 .collect()
1243 })
1244 }
1245 CloudProviderId::GoogleVertex => {
1246 let mut state = self.state(provider)?;
1247 let token = self.vertex_request(&mut state).await?;
1248 let config: super::cloud::GoogleVertexConfig =
1249 serde_json::from_value(state["config"].clone()).map_err(|_| {
1250 BrokerError::Credential("invalid Vertex broker state".into())
1251 })?;
1252 let mut page: Option<String> = None;
1253 let mut out = Vec::new();
1254 let mut seen = std::collections::HashSet::new();
1255 for _ in 0..20 {
1256 let mut url=format!("https://{}-aiplatform.googleapis.com/v1/projects/{}/locations/{}/publishers/google/models",config.location,config.project_id,config.location);
1257 if let Some(p) = &page {
1258 let mut parsed = url::Url::parse(&url).map_err(|_| {
1259 BrokerError::Transport("invalid Vertex catalog URL".into())
1260 })?;
1261 parsed.query_pairs_mut().append_pair("pageToken", p);
1262 url = parsed.into();
1263 }
1264 let r = self
1265 .http
1266 .get(url)
1267 .bearer_auth(&token)
1268 .send()
1269 .await
1270 .map_err(|_| BrokerError::Transport("Vertex catalog failed".into()))?;
1271 if !r.status().is_success() {
1272 return Err(BrokerError::Transport("Vertex catalog rejected".into()));
1273 }
1274 let v = cloud_catalog_json(r).await?;
1275 for m in v["publisherModels"].as_array().into_iter().flatten() {
1276 if let Some(name) = m["name"]
1277 .as_str()
1278 .filter(|n| n.starts_with("publishers/google/models/"))
1279 {
1280 if out.len() >= MAX_CLOUD_CATALOG_ENTRIES {
1281 return Err(BrokerError::Transport(
1282 "Vertex catalog exceeded entry limit".into(),
1283 ));
1284 }
1285 if !out.iter().any(|entry: &CloudCatalogEntry| {
1286 entry.id == format!("google-vertex/{name}")
1287 }) {
1288 out.push(CloudCatalogEntry {
1289 provider,
1290 id: format!("google-vertex/{name}"),
1291 display_name: m["displayName"].as_str().unwrap_or(name).into(),
1292 context_ref: opaque_context.clone(),
1293 context_label: format!(
1294 "{}/{}",
1295 config.project_id, config.location
1296 ),
1297 stale: false,
1298 fetched_at,
1299 });
1300 }
1301 }
1302 }
1303 page = v["nextPageToken"].as_str().map(str::to_owned);
1304 match &page {
1305 Some(p) if seen.insert(p.clone()) => {}
1306 Some(_) => {
1307 return Err(BrokerError::Transport("Vertex pagination loop".into()))
1308 }
1309 None => break,
1310 }
1311 }
1312 if out.is_empty() {
1313 Err(BrokerError::Transport("Vertex catalog is empty".into()))
1314 } else {
1315 Ok(out)
1316 }
1317 }
1318 }
1319 }
1320 async fn invoke(
1321 &self,
1322 provider: CloudProviderId,
1323 context_ref: &str,
1324 model_id: &str,
1325 request: InvokeRequest,
1326 ) -> Result<CloudEventStream, BrokerError> {
1327 self.validate_context(provider, context_ref)?;
1328 if !request.tools.is_empty() {
1329 return Err(BrokerError::Denied(
1330 "tools are not yet supported by cloud providers".into(),
1331 ));
1332 }
1333 let authorized = self.catalog(provider, context_ref, false).await?;
1334 if !authorized.iter().any(|entry| entry.id == model_id) {
1335 return Err(BrokerError::Denied(
1336 "model is not present in the current provider catalog".into(),
1337 ));
1338 }
1339 match provider {
1340 CloudProviderId::AwsBedrock => {
1341 let broker = self.aws().await?;
1342 if request.stream {
1343 use futures::StreamExt;
1344 let stream = broker
1345 .converse_stream(model_id, request)
1346 .await
1347 .map_err(|_| {
1348 BrokerError::Transport(
1349 "AWS invocation failed (details redacted)".into(),
1350 )
1351 })?
1352 .map(|event| {
1353 event
1354 .map_err(|_| {
1355 BrokerError::Transport(
1356 "AWS stream failed (details redacted)".into(),
1357 )
1358 })
1359 .map(|e| match e {
1360 super::aws_bedrock::ConverseEvent::TextDelta(delta) => {
1361 CloudEvent::TextDelta { delta }
1362 }
1363 super::aws_bedrock::ConverseEvent::ToolArguments {
1364 id,
1365 delta,
1366 } => CloudEvent::ToolArguments {
1367 id,
1368 name: None,
1369 delta,
1370 },
1371 super::aws_bedrock::ConverseEvent::Usage(u) => {
1372 CloudEvent::Usage {
1373 input_tokens: u.input_tokens,
1374 output_tokens: u.output_tokens,
1375 }
1376 }
1377 super::aws_bedrock::ConverseEvent::Done => CloudEvent::Done,
1378 })
1379 });
1380 Ok(Box::pin(stream))
1381 } else {
1382 let o = broker.converse(model_id, request).await.map_err(|_| {
1383 BrokerError::Transport("AWS invocation failed (details redacted)".into())
1384 })?;
1385 Ok(Box::pin(futures::stream::iter(vec![
1386 Ok(CloudEvent::TextDelta { delta: o.text }),
1387 Ok(CloudEvent::Usage {
1388 input_tokens: o.usage.input_tokens,
1389 output_tokens: o.usage.output_tokens,
1390 }),
1391 Ok(CloudEvent::Done),
1392 ])))
1393 }
1394 }
1395 CloudProviderId::AzureOpenAi => {
1396 let mut state = self.state(provider)?;
1397 let token = self
1398 .azure_request(&mut state, super::azure_openai::AzureAudience::Inference)
1399 .await?;
1400 let endpoint = super::azure_openai::AzureEndpoint::parse(
1401 state["endpoint"]
1402 .as_str()
1403 .ok_or_else(|| BrokerError::Credential("invalid Azure endpoint".into()))?,
1404 )
1405 .map_err(|_| BrokerError::Credential("invalid Azure endpoint".into()))?;
1406 let deployment = model_id
1407 .strip_prefix("azure-openai/")
1408 .ok_or_else(|| BrokerError::Denied("invalid Azure model".into()))?;
1409 let body = serde_json::json!({"input":request.messages.into_iter().map(|m|serde_json::json!({"role":match m.role{super::cloud::MessageRole::Assistant=>"assistant",super::cloud::MessageRole::System=>"system",_=>"user"},"content":m.content})).collect::<Vec<_>>(),"stream":request.stream});
1410 let rr = super::azure_openai::responses_request(
1411 &endpoint,
1412 deployment,
1413 &serde_json::to_vec(&body).unwrap(),
1414 )
1415 .map_err(|_| BrokerError::Denied("invalid Azure request".into()))?;
1416 let r = self
1417 .http
1418 .post(rr.url)
1419 .bearer_auth(token)
1420 .header("content-type", "application/json")
1421 .body(rr.body)
1422 .send()
1423 .await
1424 .map_err(|_| BrokerError::Transport("Azure invocation failed".into()))?;
1425 if !r.status().is_success() {
1426 return Err(BrokerError::Transport("Azure invocation rejected".into()));
1427 }
1428 if request.stream {
1429 use futures::StreamExt;
1430 use std::sync::{
1431 atomic::{AtomicBool, Ordering},
1432 Arc,
1433 };
1434 let terminal = Arc::new(AtomicBool::new(false));
1435 let seen = terminal.clone();
1436 let events = sse_json_stream(r).filter_map(move |item| {
1437 let seen = seen.clone();
1438 async move {
1439 match item {
1440 Ok(Some(v)) => v["delta"]
1441 .as_str()
1442 .or_else(|| v["text"].as_str())
1443 .map(|delta| {
1444 Ok(CloudEvent::TextDelta {
1445 delta: delta.into(),
1446 })
1447 }),
1448 Ok(None) => {
1449 seen.store(true, Ordering::SeqCst);
1450 Some(Ok(CloudEvent::Done))
1451 }
1452 Err(error) => Some(Err(error)),
1453 }
1454 }
1455 });
1456 let stream = events.chain(futures::stream::unfold(
1457 (terminal, false),
1458 |(terminal, emitted)| async move {
1459 if terminal.load(Ordering::SeqCst) || emitted {
1460 None
1461 } else {
1462 Some((
1463 Err(BrokerError::Transport(
1464 "Azure stream ended without [DONE]".into(),
1465 )),
1466 (terminal, true),
1467 ))
1468 }
1469 },
1470 ));
1471 Ok(Box::pin(stream))
1472 } else {
1473 let text = read_body_capped(r, MAX_CLOUD_STREAM_EVENT_BYTES).await?;
1474 let v: serde_json::Value = serde_json::from_str(&text)
1475 .map_err(|_| BrokerError::Transport("invalid Azure response".into()))?;
1476 let delta = v["output_text"]
1477 .as_str()
1478 .ok_or_else(|| {
1479 BrokerError::Transport("Azure response omitted output".into())
1480 })?
1481 .to_owned();
1482 Ok(Box::pin(futures::stream::iter(vec![
1483 Ok(CloudEvent::TextDelta { delta }),
1484 Ok(CloudEvent::Done),
1485 ])))
1486 }
1487 }
1488 CloudProviderId::GoogleVertex => {
1489 let mut state = self.state(provider)?;
1490 let token = self.vertex_request(&mut state).await?;
1491 let config: super::cloud::GoogleVertexConfig =
1492 serde_json::from_value(state["config"].clone())
1493 .map_err(|_| BrokerError::Credential("invalid Vertex state".into()))?;
1494 let model = model_id
1495 .strip_prefix("google-vertex/publishers/google/models/")
1496 .filter(|m| {
1497 m.bytes()
1498 .all(|b| b.is_ascii_alphanumeric() || b"-._".contains(&b))
1499 })
1500 .ok_or_else(|| BrokerError::Denied("invalid Vertex model".into()))?;
1501 let action = if request.stream {
1502 "streamGenerateContent?alt=sse"
1503 } else {
1504 "generateContent"
1505 };
1506 let url=format!("https://{}-aiplatform.googleapis.com/v1/projects/{}/locations/{}/publishers/google/models/{}:{}",config.location,config.project_id,config.location,model,action);
1507 let body = serde_json::json!({"contents":request.messages.into_iter().map(|m|serde_json::json!({"role":if matches!(m.role,super::cloud::MessageRole::Assistant){"model"}else{"user"},"parts":[{"text":m.content}]})).collect::<Vec<_>>()});
1508 let r = self
1509 .http
1510 .post(url)
1511 .bearer_auth(token)
1512 .json(&body)
1513 .send()
1514 .await
1515 .map_err(|_| BrokerError::Transport("Vertex invocation failed".into()))?;
1516 if !r.status().is_success() {
1517 return Err(BrokerError::Transport("Vertex invocation rejected".into()));
1518 }
1519 if request.stream {
1520 use futures::StreamExt;
1521 let events = sse_json_stream(r).map(|item| {
1522 item.and_then(|value| {
1523 value.ok_or_else(|| {
1524 BrokerError::Transport(
1525 "Vertex stream terminated without a provider event".into(),
1526 )
1527 })
1528 })
1529 });
1530 use std::sync::{
1531 atomic::{AtomicBool, Ordering},
1532 Arc,
1533 };
1534 let terminal = Arc::new(AtomicBool::new(false));
1535 let terminal_in_events = terminal.clone();
1536 let stream = events
1537 .flat_map(move |item| {
1538 let mut out = Vec::new();
1539 match item {
1540 Ok(v) => {
1541 if terminal_in_events.load(Ordering::SeqCst) {
1542 out.push(Err(BrokerError::Transport(
1543 "Vertex emitted data after terminal metadata".into(),
1544 )));
1545 return futures::stream::iter(out);
1546 }
1547 for c in v["candidates"].as_array().into_iter().flatten() {
1548 for p in
1549 c["content"]["parts"].as_array().into_iter().flatten()
1550 {
1551 if let Some(delta) = p["text"].as_str() {
1552 out.push(Ok(CloudEvent::TextDelta {
1553 delta: delta.into(),
1554 }));
1555 }
1556 }
1557 if let Some(reason) = c["finishReason"].as_str() {
1558 const VALID: &[&str] = &[
1559 "STOP",
1560 "MAX_TOKENS",
1561 "SAFETY",
1562 "RECITATION",
1563 "OTHER",
1564 "BLOCKLIST",
1565 "PROHIBITED_CONTENT",
1566 "SPII",
1567 "MALFORMED_FUNCTION_CALL",
1568 ];
1569 if !VALID.contains(&reason) {
1570 out.push(Err(BrokerError::Transport(
1571 "Vertex returned an invalid finish reason"
1572 .into(),
1573 )));
1574 } else if terminal_in_events
1575 .swap(true, Ordering::SeqCst)
1576 {
1577 out.push(Err(BrokerError::Transport(
1578 "Vertex returned duplicate terminal metadata"
1579 .into(),
1580 )));
1581 } else {
1582 out.push(Ok(CloudEvent::Done));
1583 }
1584 }
1585 }
1586 if let Some(u) = v.get("usageMetadata") {
1587 out.push(Ok(CloudEvent::Usage {
1588 input_tokens: u["promptTokenCount"]
1589 .as_u64()
1590 .unwrap_or(0),
1591 output_tokens: u["candidatesTokenCount"]
1592 .as_u64()
1593 .unwrap_or(0),
1594 }));
1595 }
1596 }
1597 Err(error) => out.push(Err(error)),
1598 }
1599 futures::stream::iter(out)
1600 })
1601 .chain(futures::stream::unfold(
1602 (terminal, false),
1603 |(terminal, emitted)| async move {
1604 if terminal.load(Ordering::SeqCst) || emitted {
1605 None
1606 } else {
1607 Some((
1608 Err(BrokerError::Transport(
1609 "Vertex stream ended without finish metadata".into(),
1610 )),
1611 (terminal, true),
1612 ))
1613 }
1614 },
1615 ));
1616 Ok(Box::pin(stream))
1617 } else {
1618 let text = read_body_capped(r, MAX_CLOUD_STREAM_EVENT_BYTES).await?;
1619 let v: serde_json::Value = serde_json::from_str(&text)
1620 .map_err(|_| BrokerError::Transport("invalid Vertex response".into()))?;
1621 let mut out = Vec::new();
1622 for c in v["candidates"].as_array().into_iter().flatten() {
1623 for p in c["content"]["parts"].as_array().into_iter().flatten() {
1624 if let Some(delta) = p["text"].as_str() {
1625 out.push(Ok(CloudEvent::TextDelta {
1626 delta: delta.into(),
1627 }));
1628 }
1629 }
1630 }
1631 out.push(Ok(CloudEvent::Done));
1632 Ok(Box::pin(futures::stream::iter(out)))
1633 }
1634 }
1635 }
1636 }
1637}
1638
1639const MAX_CLOUD_CATALOG_BODY_BYTES: usize = 2 * 1024 * 1024;
1640const MAX_CLOUD_CATALOG_ENTRIES: usize = 10_000;
1641const MAX_CLOUD_STREAM_EVENT_BYTES: usize = 1024 * 1024;
1642
1643fn sse_json_stream(
1647 resp: reqwest::Response,
1648) -> Pin<Box<dyn Stream<Item = Result<Option<serde_json::Value>, BrokerError>> + Send>> {
1649 use futures::StreamExt;
1650 let chunks = Box::pin(resp.bytes_stream());
1651 Box::pin(futures::stream::unfold(
1652 (chunks, Vec::<u8>::new(), false),
1653 |(mut chunks, mut buf, done)| async move {
1654 if done {
1655 return None;
1656 }
1657 loop {
1658 if let Some(end) = buf.windows(2).position(|w| w == b"\n\n") {
1659 let frame: Vec<u8> = buf.drain(..end + 2).collect();
1660 let text = match std::str::from_utf8(&frame[..end]) {
1661 Ok(v) => v,
1662 Err(_) => {
1663 return Some((
1664 Err(BrokerError::Transport("invalid SSE encoding".into())),
1665 (chunks, buf, true),
1666 ))
1667 }
1668 };
1669 let data = text
1670 .lines()
1671 .filter_map(|line| line.strip_prefix("data:"))
1672 .map(str::trim_start)
1673 .collect::<Vec<_>>()
1674 .join("\n");
1675 if data.is_empty() {
1676 continue;
1677 }
1678 if data == "[DONE]" {
1679 return Some((Ok(None), (chunks, buf, true)));
1680 }
1681 let value = serde_json::from_str(&data)
1682 .map_err(|_| BrokerError::Transport("invalid SSE event".into()));
1683 let terminal = value.is_err();
1684 return Some((value.map(Some), (chunks, buf, terminal)));
1685 }
1686 match chunks.next().await {
1687 Some(Ok(chunk)) if buf.len() + chunk.len() <= MAX_CLOUD_STREAM_EVENT_BYTES => {
1688 buf.extend_from_slice(&chunk)
1689 }
1690 Some(_) => {
1691 return Some((
1692 Err(BrokerError::Transport(
1693 "cloud stream event exceeded limit".into(),
1694 )),
1695 (chunks, buf, true),
1696 ))
1697 }
1698 None if buf.is_empty() => return None,
1699 None => {
1700 return Some((
1701 Err(BrokerError::Transport("truncated SSE event".into())),
1702 (chunks, buf, true),
1703 ))
1704 }
1705 }
1706 }
1707 },
1708 ))
1709}
1710
1711async fn cloud_catalog_json(resp: reqwest::Response) -> Result<serde_json::Value, BrokerError> {
1713 let body = read_body_capped(resp, MAX_CLOUD_CATALOG_BODY_BYTES).await?;
1714 serde_json::from_str(&body)
1715 .map_err(|_| BrokerError::Transport("invalid cloud catalog response".into()))
1716}
1717
1718pub struct LocalBroker {
1722 http: reqwest::Client,
1723 local_base_url: Option<String>,
1725 anthropic_usage_url: Option<String>,
1727 google_gemini_base_url: Option<String>,
1729 openai_codex_base_url: Option<String>,
1731 muse_base_url: Option<String>,
1733 refresh_endpoints: super::provider::RefreshEndpoints,
1735 stream_idle_timeout: Duration,
1737 request_timeout: Duration,
1739 max_response_bytes: usize,
1741 cloud_backend: Option<Arc<dyn CloudBackend>>,
1742 account_policy: Option<AccountPolicy>,
1745 auto_policy: Option<super::AutoSelectionPolicy>,
1746 auto_policy_cache: std::sync::Mutex<
1747 Option<(
1748 std::path::PathBuf,
1749 Option<std::time::SystemTime>,
1750 super::AutoSelectionPolicy,
1751 )>,
1752 >,
1753 current_accounts: SharedMap<(CredentialRef, SeatIdentity)>,
1755 cooldowns: Arc<std::sync::Mutex<BTreeMap<String, Cooldown>>>,
1757 usage_endpoint_override: Option<String>,
1759 max_snapshot_age: Duration,
1761 snapshots: SharedMap<super::usage::UsageSnapshot>,
1768 usage_reads: Arc<super::usage_reads::UsageReads>,
1771 instance: u64,
1773 policy_cache: Arc<std::sync::Mutex<Option<CachedPolicy>>>,
1776 observations: SharedMap<SlotObservations>,
1780 selections: SharedMap<super::auth_state::SelectionState>,
1782 refresh_holds: SharedMap<RefreshHold>,
1785 seat_cache: Arc<std::sync::Mutex<Option<SeatCache>>>,
1788 seat_loads: Arc<std::sync::atomic::AtomicU64>,
1790}
1791
1792type FileStamp = (std::time::SystemTime, u64, u64);
1796
1797fn file_stamp(path: &std::path::Path) -> Option<FileStamp> {
1799 let meta = std::fs::metadata(path).ok()?;
1800 #[cfg(unix)]
1801 let inode = std::os::unix::fs::MetadataExt::ino(&meta);
1802 #[cfg(not(unix))]
1803 let inode = 0;
1804 Some((meta.modified().ok()?, meta.len(), inode))
1805}
1806
1807struct SeatCache {
1810 path: std::path::PathBuf,
1811 stamp: FileStamp,
1812 seats: BTreeMap<String, Option<SeatIdentity>>,
1813}
1814
1815#[derive(Debug, Clone, Default)]
1817struct SlotObservations {
1818 last_usage: Option<super::auth_state::ObservedOutcome>,
1819 last_usage_ok_ms: Option<u64>,
1820}
1821
1822type SharedMap<T> = Arc<std::sync::Mutex<BTreeMap<String, T>>>;
1824
1825type FailedThisSelection = BTreeMap<String, super::quota_policy::ObservationFailure>;
1829
1830type CachedPolicy = (Option<std::time::SystemTime>, AccountPolicy);
1832
1833#[derive(Debug, Clone, PartialEq, Eq)]
1835struct Cooldown {
1836 until_ms: u64,
1837 reason: String,
1838 reported_at_ms: u64,
1841}
1842
1843struct CooldownKeys {
1847 seat: Option<String>,
1849 slot: String,
1851 identified: bool,
1853}
1854
1855impl CooldownKeys {
1856 fn written(&self) -> impl Iterator<Item = &str> + '_ {
1859 let slot = (!self.identified).then_some(self.slot.as_str());
1860 self.seat.as_deref().into_iter().chain(slot)
1861 }
1862
1863 fn read(&self) -> impl Iterator<Item = &str> + '_ {
1865 self.seat.as_deref().into_iter().chain([self.slot.as_str()])
1866 }
1867}
1868
1869const DEFAULT_COOLDOWN: Duration = Duration::from_secs(15 * 60);
1871
1872struct RefreshHold {
1875 material: MaterialDigest,
1877 failure: super::token::RefreshFailure,
1878 at_ms: u64,
1879}
1880
1881pub const REFUSED_REFRESH_HOLD: Duration = Duration::from_secs(15 * 60);
1893
1894struct SlotHold {
1902 holds: SharedMap<RefreshHold>,
1903 key: String,
1905 provider: OAuthProviderId,
1906 answered: std::sync::atomic::AtomicBool,
1908}
1909
1910impl SlotHold {
1911 fn answered(&self) -> bool {
1912 self.answered.load(std::sync::atomic::Ordering::SeqCst)
1913 }
1914}
1915
1916impl super::token::RefusalMemory for SlotHold {
1917 fn held(&self, creds: &super::OAuthCredentials) -> Option<super::token::RefreshFailure> {
1920 let material = MaterialDigest::of(self.provider, creds);
1921 let window_ms = REFUSED_REFRESH_HOLD.as_millis() as u64;
1922 let mut holds = self.holds.lock().unwrap_or_else(|e| e.into_inner());
1923 let hold = holds.get(&self.key)?;
1924 if crate::epoch_millis() < hold.at_ms.saturating_add(window_ms)
1925 && hold.material == material
1926 {
1927 self.answered.store(true, std::sync::atomic::Ordering::SeqCst);
1928 return Some(hold.failure.clone());
1929 }
1930 holds.remove(&self.key);
1931 None
1932 }
1933
1934 fn refused(&self, creds: &super::OAuthCredentials, failure: &super::token::RefreshFailure) {
1937 use super::token::RefreshFailureKind as R;
1938 if !matches!(failure.kind, R::Credential | R::ActionRequired) {
1939 return;
1940 }
1941 let hold = RefreshHold {
1942 material: MaterialDigest::of(self.provider, creds),
1943 failure: failure.clone(),
1944 at_ms: crate::epoch_millis(),
1945 };
1946 self.holds
1947 .lock()
1948 .unwrap_or_else(|e| e.into_inner())
1949 .insert(self.key.clone(), hold);
1950 }
1951}
1952
1953const MAX_COOLDOWN: Duration = Duration::from_secs(8 * 24 * 60 * 60);
1956pub const DEFAULT_MAX_SNAPSHOT_AGE: Duration = Duration::from_secs(60);
1959
1960pub const BROKER_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(90);
1964
1965pub const GEMINI_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
1969
1970fn next_broker_instance() -> u64 {
1972 static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
1973 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
1974}
1975
1976impl LocalBroker {
1977 pub fn new(http: reqwest::Client) -> Self {
1978 let cloud_backend = Arc::new(ProductionCloudBackend::new(http.clone()));
1979 Self {
1980 http,
1981 local_base_url: None,
1982 anthropic_usage_url: None,
1983 google_gemini_base_url: None,
1984 openai_codex_base_url: None,
1985 muse_base_url: None,
1986 refresh_endpoints: Default::default(),
1987 stream_idle_timeout: BROKER_STREAM_IDLE_TIMEOUT,
1988 request_timeout: PROXY_REQUEST_TIMEOUT,
1989 max_response_bytes: MAX_PROXY_RESPONSE_BYTES,
1990 cloud_backend: Some(cloud_backend),
1991 account_policy: None,
1992 auto_policy: None,
1993 auto_policy_cache: std::sync::Mutex::new(None),
1994 current_accounts: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
1995 cooldowns: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
1996 usage_endpoint_override: None,
1997 max_snapshot_age: DEFAULT_MAX_SNAPSHOT_AGE,
1998 snapshots: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
1999 usage_reads: super::usage_reads::process_table(),
2000 instance: next_broker_instance(),
2001 policy_cache: Arc::new(std::sync::Mutex::new(None)),
2002 observations: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
2003 selections: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
2004 refresh_holds: Arc::new(std::sync::Mutex::new(BTreeMap::new())),
2005 seat_cache: Arc::new(std::sync::Mutex::new(None)),
2006 seat_loads: Arc::new(std::sync::atomic::AtomicU64::new(0)),
2007 }
2008 }
2009
2010 pub fn with_cloud_backend(mut self, backend: Arc<dyn CloudBackend>) -> Self {
2011 self.cloud_backend = Some(backend);
2012 self
2013 }
2014
2015 pub fn with_account_policy(mut self, policy: AccountPolicy) -> Self {
2017 self.account_policy = Some(policy);
2018 self
2019 }
2020
2021 pub fn with_auto_policy(mut self, policy: super::AutoSelectionPolicy) -> Self {
2023 self.auto_policy = Some(policy);
2024 self
2025 }
2026
2027 fn auto_policy(&self) -> super::AutoSelectionPolicy {
2028 if let Some(policy) = &self.auto_policy {
2029 return policy.clone();
2030 }
2031 let path = crate::config::resolve_read_path("config");
2032 let mtime = std::fs::metadata(&path).and_then(|m| m.modified()).ok();
2033 let mut cache = self
2034 .auto_policy_cache
2035 .lock()
2036 .unwrap_or_else(|e| e.into_inner());
2037 if let Some((cached_path, cached_time, policy)) = cache.as_ref() {
2038 if cached_path == &path && *cached_time == mtime {
2039 return policy.clone();
2040 }
2041 }
2042 let (policy, warnings) =
2043 super::AutoSelectionPolicy::from_config_map(&crate::config::load_config().auth.auto);
2044 for warning in warnings {
2045 tracing::warn!(%warning, "automatic account policy");
2046 }
2047 *cache = Some((path, mtime, policy.clone()));
2048 policy
2049 }
2050
2051 fn current_key(provider: OAuthProviderId, model: Option<&str>) -> String {
2052 format!(
2053 "{}|{}|{}",
2054 storage::auth_file_path().display(),
2055 provider,
2056 model.unwrap_or("")
2057 )
2058 }
2059
2060 #[doc(hidden)]
2064 pub fn with_openai_codex_base_url_for_tests(mut self, base_url: impl Into<String>) -> Self {
2065 self.openai_codex_base_url = Some(base_url.into().trim_end_matches('/').to_string());
2066 self
2067 }
2068
2069 #[doc(hidden)]
2073 pub fn with_muse_base_url_for_tests(mut self, base_url: impl Into<String>) -> Self {
2074 self.muse_base_url = Some(base_url.into().trim_end_matches('/').to_string());
2075 self
2076 }
2077
2078 fn stream_idle_timeout_for(&self, provider_key: &str) -> Duration {
2080 if provider_key == "google-gemini" {
2081 GEMINI_STREAM_IDLE_TIMEOUT.max(self.stream_idle_timeout)
2082 } else {
2083 self.stream_idle_timeout
2084 }
2085 }
2086
2087 #[doc(hidden)]
2089 pub fn with_stream_idle_timeout_for_tests(mut self, idle: Duration) -> Self {
2090 self.stream_idle_timeout = idle;
2091 self
2092 }
2093
2094 #[doc(hidden)]
2099 pub fn with_muse_mint_url_for_tests(mut self, mint_url: impl Into<String>) -> Self {
2100 self.refresh_endpoints =
2101 std::mem::take(&mut self.refresh_endpoints).with_muse_mint_url_for_tests(mint_url);
2102 self
2103 }
2104
2105 #[doc(hidden)]
2109 pub fn with_anthropic_token_url_for_tests(mut self, url: impl Into<String>) -> Self {
2110 self.refresh_endpoints =
2111 std::mem::take(&mut self.refresh_endpoints).with_anthropic_token_url_for_tests(url);
2112 self
2113 }
2114
2115 pub async fn access_token_auto(
2121 &self,
2122 provider: OAuthProviderId,
2123 model: Option<&str>,
2124 ) -> Result<PinnedToken, BrokerError> {
2125 self.vend_auto(provider, model, &[]).await
2126 }
2127
2128 #[doc(hidden)]
2130 pub fn with_usage_endpoint_override(mut self, url: impl Into<String>) -> Self {
2131 self.usage_endpoint_override = Some(url.into());
2132 self
2133 }
2134
2135 pub fn with_max_snapshot_age(mut self, age: Duration) -> Self {
2137 self.max_snapshot_age = age;
2138 self
2139 }
2140
2141 fn policy(&self) -> AccountPolicy {
2145 if let Some(policy) = &self.account_policy {
2146 return policy.clone();
2147 }
2148 let mtime = std::fs::metadata(crate::config::resolve_read_path("config"))
2149 .and_then(|m| m.modified())
2150 .ok();
2151 let mut cache = self.policy_cache.lock().unwrap_or_else(|e| e.into_inner());
2152 let config_policy = match cache.as_ref() {
2153 Some((cached_mtime, policy)) if *cached_mtime == mtime => policy.clone(),
2154 _ => {
2155 let (policy, _warnings) = AccountPolicy::from_config_map(
2156 &crate::config::load_config().auth.accounts,
2157 );
2158 *cache = Some((mtime, policy.clone()));
2159 policy
2160 }
2161 };
2162 config_policy.with_env_overlay()
2163 }
2164
2165 fn cooldown_keys(&self, cred: &CredentialRef) -> CooldownKeys {
2175 let store = storage::auth_file_path().display().to_string();
2176 let slot = format!("{store}|{}|{}", cred.provider, cred.storage_key());
2177 let seat = self.cooldown_seat(cred);
2178 CooldownKeys {
2179 identified: seat.as_ref().is_some_and(|s| s.seat_fingerprint().is_some()),
2180 seat: seat.map(|s| format!("{store}|{}|{}", cred.provider, s.as_str())),
2181 slot,
2182 }
2183 }
2184
2185 fn cooldown_seat(&self, cred: &CredentialRef) -> Option<SeatIdentity> {
2194 let path = storage::auth_file_path();
2195 let stamp = file_stamp(&path);
2196 let key = cred.storage_key();
2197 let mut cache = self.seat_cache.lock().unwrap_or_else(|e| e.into_inner());
2198 if let (Some(stamp), Some(c)) = (stamp, cache.as_ref()) {
2199 if c.path == path && c.stamp == stamp {
2200 if let Some(seat) = c.seats.get(&key) {
2201 return seat.clone();
2202 }
2203 }
2204 }
2205 self.seat_loads
2206 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2207 let seat = self.stored_seat(cred).ok();
2208 match stamp {
2209 Some(stamp) => {
2210 let fresh = cache
2211 .as_ref()
2212 .is_some_and(|c| c.path == path && c.stamp == stamp);
2213 if !fresh {
2214 *cache = Some(SeatCache {
2215 path,
2216 stamp,
2217 seats: BTreeMap::new(),
2218 });
2219 }
2220 if let Some(c) = cache.as_mut() {
2221 c.seats.insert(key, seat.clone());
2222 }
2223 }
2224 None => *cache = None,
2225 }
2226 seat
2227 }
2228
2229 #[doc(hidden)]
2231 pub fn cooldown_seat_loads_for_tests(&self) -> u64 {
2232 self.seat_loads.load(std::sync::atomic::Ordering::Relaxed)
2233 }
2234
2235 fn any_cooldown_active(&self, now_ms: u64) -> bool {
2237 self.cooldowns
2238 .lock()
2239 .unwrap_or_else(|e| e.into_inner())
2240 .values()
2241 .any(|c| c.until_ms > now_ms)
2242 }
2243
2244 fn active_cooldown_at(&self, cred: &CredentialRef, now_ms: u64) -> Option<Cooldown> {
2246 if !self.any_cooldown_active(now_ms) {
2249 return None;
2250 }
2251 self.cooldown_for_keys(&self.cooldown_keys(cred), now_ms)
2252 }
2253
2254 fn cooldown_for_keys(&self, keys: &CooldownKeys, now_ms: u64) -> Option<Cooldown> {
2256 let cooldowns = self.cooldowns.lock().unwrap_or_else(|e| e.into_inner());
2257 keys.read()
2258 .filter_map(|key| cooldowns.get(key))
2259 .filter(|c| c.until_ms > now_ms)
2260 .max_by_key(|c| c.until_ms)
2261 .cloned()
2262 }
2263
2264 fn cooldown_until(&self, storage_key: &str, now_ms: u64) -> Option<u64> {
2266 let cred = CredentialRef::parse_storage_key(storage_key)?;
2267 self.active_cooldown_at(&cred, now_ms).map(|c| c.until_ms)
2268 }
2269
2270 pub fn auth_state(
2276 &self,
2277 query: &super::auth_state::AuthStateQuery,
2278 ) -> Result<super::auth_state::AuthStateReport, BrokerError> {
2279 auth_state_report(Some(self), query)
2280 }
2281
2282 fn cooldown_state(
2284 &self,
2285 cred: &CredentialRef,
2286 now_ms: u64,
2287 ) -> Option<super::auth_state::CooldownState> {
2288 self.active_cooldown_at(cred, now_ms)
2289 .map(|c| super::auth_state::CooldownState {
2290 until_ms: c.until_ms,
2291 reason: super::auth_state::CooldownReason::from_reported(&c.reason),
2292 })
2293 }
2294
2295 fn clear_state(&self, creds: &[CredentialRef], whole_providers: &[OAuthProviderId]) {
2303 let store = storage::auth_file_path().display().to_string();
2304 let keys: Vec<CooldownKeys> = creds.iter().map(|c| self.cooldown_keys(c)).collect();
2305 {
2306 let mut cooldowns = self.cooldowns.lock().unwrap_or_else(|e| e.into_inner());
2307 for key in keys.iter().flat_map(|k| k.read()) {
2308 cooldowns.remove(key);
2309 }
2310 for provider in whole_providers {
2311 let prefix = format!("{store}|{provider}|");
2312 cooldowns.retain(|k, _| !k.starts_with(&prefix));
2313 }
2314 }
2315 for cred in creds {
2316 self.forget_snapshots(cred);
2317 self.usage_reads.forget(&self.read_key(cred));
2318 }
2319 {
2320 let mut observations = self.observations.lock().unwrap_or_else(|e| e.into_inner());
2321 for cred in creds {
2322 observations.remove(&Self::state_key(cred));
2323 }
2324 }
2325 {
2326 let mut holds = self.refresh_holds.lock().unwrap_or_else(|e| e.into_inner());
2327 for cred in creds {
2328 holds.remove(&Self::state_key(cred));
2329 }
2330 for provider in whole_providers {
2331 let slot = format!("{store}|{}", provider.as_str());
2332 let named = format!("{slot}@");
2333 holds.retain(|k, _| *k != slot && !k.starts_with(&named));
2334 }
2335 }
2336 self.current_accounts
2337 .lock()
2338 .unwrap_or_else(|e| e.into_inner())
2339 .retain(|_, (current, _)| !creds.contains(current));
2340 let mut selections = self.selections.lock().unwrap_or_else(|e| e.into_inner());
2341 for provider in whole_providers {
2342 let prefix = format!("{store}|{provider}|");
2343 selections.retain(|k, _| !k.starts_with(&prefix));
2344 }
2345 }
2346
2347 fn check_slot_exists(&self, cred: &CredentialRef) -> Result<(), BrokerError> {
2352 if let Account::Named(label) = &cred.account {
2353 match storage::load_credential(cred) {
2354 Ok(Some(_)) => {}
2355 Ok(None) => {
2356 return Err(BrokerError::UnknownAccount {
2357 provider: cred.provider.as_str().to_string(),
2358 label: label.as_str().to_string(),
2359 })
2360 }
2361 Err(e) => return Err(BrokerError::Credential(e)),
2362 }
2363 }
2364 Ok(())
2365 }
2366
2367 async fn resolve_and_vend(
2373 &self,
2374 provider: OAuthProviderId,
2375 explicit: Option<Account>,
2376 model: Option<&str>,
2377 exclude: &[CredentialRef],
2378 ) -> Result<PinnedToken, BrokerError> {
2379 let selector = match explicit {
2380 Some(account) => AccountSelector::Account(account),
2381 None => self.policy().selector(provider),
2382 };
2383 match selector {
2384 AccountSelector::Account(account) => {
2385 let credential = CredentialRef::new(provider, account);
2386 if exclude.contains(&credential) {
2387 return Err(BrokerError::NoAccountAvailable {
2388 provider: provider.as_str().to_string(),
2389 reason: format!(
2390 "the selected account {} was already used by this request",
2391 credential.account
2392 ),
2393 failure: None,
2394 });
2395 }
2396 let token = self.access_token_for(&credential).await?;
2397 Ok(PinnedToken { credential, token })
2398 }
2399 AccountSelector::Auto => self.vend_auto(provider, model, exclude).await,
2400 AccountSelector::Invalid { source, reason } => {
2401 Err(BrokerError::InvalidAccount(format!("{source}: {reason}")))
2402 }
2403 }
2404 }
2405
2406 const MAX_AUTO_VEND_ROUNDS: usize = 2;
2409
2410 async fn vend_auto(
2433 &self,
2434 provider: OAuthProviderId,
2435 model: Option<&str>,
2436 exclude: &[CredentialRef],
2437 ) -> Result<PinnedToken, BrokerError> {
2438 use super::token::{ensure_fresh_credential_for_seat, SeatVendError};
2439 let mut failed = FailedThisSelection::new();
2440 let mut seat_changes = 0;
2441 loop {
2442 let (credential, seat) = match self
2443 .select_auto(provider, model, exclude, &mut failed)
2444 .await
2445 {
2446 Ok(choice) => choice,
2447 Err(e) => {
2448 let failure = match &e {
2449 BrokerError::NoAccountAvailable { failure, .. } => failure.clone(),
2450 _ => Some(super::quota_policy::SelectionFailure::new(
2453 super::quota_policy::SelectionFailureKind::CredentialStore,
2454 )),
2455 };
2456 self.record_selection(provider, model, None, failure, None);
2457 return Err(e);
2458 }
2459 };
2460 if let Some(held) = self.held_refresh(&credential) {
2462 let cause = refresh_failure(&held);
2463 tracing::info!(
2464 credential = %credential,
2465 kind = %cause.kind,
2466 "automatic choice not vended: its refused token refresh is held until the stored login changes; selecting again"
2467 );
2468 failed.insert(credential.storage_key(), cause);
2469 continue;
2470 }
2471 let hold = self.slot_hold(&credential);
2474 let memory: Arc<dyn super::token::RefusalMemory> = hold.clone();
2475 match ensure_fresh_credential_for_seat(
2476 &self.http,
2477 &credential,
2478 &seat,
2479 &self.refresh_endpoints,
2480 Some(memory),
2481 )
2482 .await
2483 {
2484 Ok(creds) => {
2485 self.current_accounts
2486 .lock()
2487 .unwrap_or_else(|e| e.into_inner())
2488 .insert(
2489 Self::current_key(provider, model),
2490 (credential.clone(), seat),
2491 );
2492 self.record_selection(provider, model, Some(&credential), None, None);
2493 return Ok(PinnedToken {
2494 credential,
2495 token: AccessToken {
2496 token: creds.access,
2497 expires: creds.expires,
2498 },
2499 });
2500 }
2501 Err(SeatVendError::SeatChanged) => {
2502 tracing::info!(credential = %credential, "slot changed after selection; re-selecting");
2503 self.forget_snapshots(&credential);
2504 seat_changes += 1;
2505 if seat_changes >= Self::MAX_AUTO_VEND_ROUNDS {
2506 break;
2507 }
2508 }
2509 Err(SeatVendError::Other(f)) => {
2512 let cause = refresh_failure(&f);
2513 if !matches!(
2521 f.kind,
2522 super::token::RefreshFailureKind::Credential
2523 | super::token::RefreshFailureKind::ActionRequired
2524 ) {
2525 tracing::warn!(
2526 credential = %credential,
2527 kind = %cause.kind,
2528 "automatic choice could not be vended: token refresh failed"
2529 );
2530 self.record_selection(provider, model, Some(&credential), None, Some(cause));
2531 return Err(BrokerError::Refresh(auto_vend_failure(&credential, f)));
2532 }
2533 if hold.answered() {
2534 tracing::info!(
2536 credential = %credential,
2537 kind = %cause.kind,
2538 "automatic choice not vended: its refused token refresh is held until the stored login changes; selecting again"
2539 );
2540 } else {
2541 tracing::warn!(
2542 credential = %credential,
2543 kind = %cause.kind,
2544 "automatic choice could not be vended: token refresh failed; selecting again"
2545 );
2546 }
2547 failed.insert(credential.storage_key(), cause);
2548 }
2549 }
2550 }
2551 let failure = super::quota_policy::SelectionFailure::new(
2552 super::quota_policy::SelectionFailureKind::SeatChurn,
2553 );
2554 self.record_selection(provider, model, None, Some(failure.clone()), None);
2555 Err(selection_failed(
2556 provider,
2557 model,
2558 "selected credential kept changing while its capacity was being proven".into(),
2559 failure,
2560 ))
2561 }
2562
2563 fn record_selection(
2568 &self,
2569 provider: OAuthProviderId,
2570 model: Option<&str>,
2571 chosen: Option<&CredentialRef>,
2572 failure: Option<super::quota_policy::SelectionFailure>,
2573 vend_failure: Option<super::quota_policy::ObservationFailure>,
2574 ) {
2575 let state = super::auth_state::SelectionState {
2576 provider: provider.as_str().to_string(),
2577 model: model.map(str::to_string),
2578 at_ms: crate::epoch_millis(),
2579 chosen: chosen.map(|c| c.account.label_str().to_string()),
2580 failure,
2581 vend_failure,
2582 };
2583 self.selections
2584 .lock()
2585 .unwrap_or_else(|e| e.into_inner())
2586 .insert(Self::current_key(provider, model), state);
2587 }
2588
2589 fn seat_key(cred: &CredentialRef, seat: &SeatIdentity) -> String {
2592 format!(
2593 "{}|{}|{}",
2594 storage::auth_file_path().display(),
2595 cred.storage_key(),
2596 seat.as_str()
2597 )
2598 }
2599
2600 fn stored_seat(&self, cred: &CredentialRef) -> Result<SeatIdentity, BrokerError> {
2602 match storage::load_credential(cred).map_err(BrokerError::Credential)? {
2603 Some(creds) => Ok(SeatIdentity::of(cred.provider, &creds)),
2604 None => Err(BrokerError::UnknownAccount {
2605 provider: cred.provider.as_str().to_string(),
2606 label: cred.account.label_str().to_string(),
2607 }),
2608 }
2609 }
2610
2611 fn cached_snapshot(&self, key: &str) -> Option<super::usage::UsageSnapshot> {
2612 let max_age_ms = self.max_snapshot_age.as_millis() as u64;
2613 let now_ms = crate::epoch_millis();
2614 self.snapshots
2615 .lock()
2616 .unwrap_or_else(|e| e.into_inner())
2617 .get(key)
2618 .filter(|s| !s.is_stale(now_ms, max_age_ms))
2619 .cloned()
2620 }
2621
2622 fn read_key(&self, cred: &CredentialRef) -> String {
2625 format!(
2626 "{}|{}|{}",
2627 storage::auth_file_path().display(),
2628 cred.storage_key(),
2629 self.usage_endpoint_override.as_deref().unwrap_or("")
2630 )
2631 }
2632
2633 fn forget_snapshots(&self, cred: &CredentialRef) {
2637 let prefix = format!("|{}|", cred.storage_key());
2638 self.snapshots
2639 .lock()
2640 .unwrap_or_else(|e| e.into_inner())
2641 .retain(|k, _| !k.contains(&prefix));
2642 }
2643
2644 async fn fresh_usage(
2653 &self,
2654 cred: &CredentialRef,
2655 ) -> Result<(super::usage::UsageSnapshot, SeatIdentity), super::usage_reads::ReadError> {
2656 if let Some(held) = self.held_refresh(cred) {
2657 return Err(Self::held_read(cred, held).into_read_error());
2658 }
2659 let seat = self
2660 .stored_seat(cred)
2661 .map_err(|e| UsageReadError::slot(e).into_read_error())?;
2662 let key = Self::seat_key(cred, &seat);
2663 if let Some(cached) = self.cached_snapshot(&key) {
2664 return Ok((cached, seat));
2665 }
2666 let (snapshot, used) = self.shared_read(cred).await?;
2667 self.forget_snapshots(cred);
2670 let used_key = Self::seat_key(cred, &used);
2671 self.snapshots
2672 .lock()
2673 .unwrap_or_else(|e| e.into_inner())
2674 .insert(used_key, snapshot.clone());
2675 Ok((snapshot, used))
2676 }
2677
2678 async fn usage_with_seat(
2682 &self,
2683 cred: &CredentialRef,
2684 ) -> Result<(super::usage::UsageSnapshot, SeatIdentity), UsageReadError> {
2685 if let Some(held) = self.held_refresh(cred) {
2686 return Err(Self::held_read(cred, held));
2687 }
2688 self.shared_read(cred)
2689 .await
2690 .map_err(|e| UsageReadError::from_read(&e))
2691 }
2692
2693 fn state_key(cred: &CredentialRef) -> String {
2696 format!(
2697 "{}|{}",
2698 storage::auth_file_path().display(),
2699 cred.storage_key()
2700 )
2701 }
2702
2703 fn stored_material(cred: &CredentialRef) -> Option<MaterialDigest> {
2706 storage::load_credential(cred)
2707 .ok()
2708 .flatten()
2709 .map(|creds| MaterialDigest::of(cred.provider, &creds))
2710 }
2711
2712 fn held_refresh(&self, cred: &CredentialRef) -> Option<super::token::RefreshFailure> {
2718 let key = Self::state_key(cred);
2719 let (material, failure, at_ms) = {
2720 let holds = self.refresh_holds.lock().unwrap_or_else(|e| e.into_inner());
2721 let hold = holds.get(&key)?;
2722 (hold.material.clone(), hold.failure.clone(), hold.at_ms)
2723 };
2724 let window_ms = REFUSED_REFRESH_HOLD.as_millis() as u64;
2725 let in_window = crate::epoch_millis() < at_ms.saturating_add(window_ms);
2726 if in_window && Self::stored_material(cred).as_ref() == Some(&material) {
2727 return Some(failure);
2728 }
2729 let mut holds = self.refresh_holds.lock().unwrap_or_else(|e| e.into_inner());
2730 if holds.get(&key).is_some_and(|h| h.at_ms == at_ms) {
2731 holds.remove(&key);
2732 }
2733 None
2734 }
2735
2736 fn slot_hold(&self, cred: &CredentialRef) -> Arc<SlotHold> {
2739 Arc::new(SlotHold {
2740 holds: self.refresh_holds.clone(),
2741 key: Self::state_key(cred),
2742 provider: cred.provider,
2743 answered: std::sync::atomic::AtomicBool::new(false),
2744 })
2745 }
2746
2747 fn held_read(cred: &CredentialRef, held: super::token::RefreshFailure) -> UsageReadError {
2751 let err = UsageReadError::refresh(held);
2752 tracing::debug!(
2753 credential = %cred,
2754 kind = %err.failure.kind,
2755 "usage read not sent: the refused token refresh is held until the stored login changes"
2756 );
2757 err
2758 }
2759
2760 fn refresh_hold_state(
2763 &self,
2764 cred: &CredentialRef,
2765 now_ms: u64,
2766 ) -> Option<super::auth_state::RefreshHoldState> {
2767 let (material, failure, at_ms) = {
2768 let holds = self.refresh_holds.lock().unwrap_or_else(|e| e.into_inner());
2769 let hold = holds.get(&Self::state_key(cred))?;
2770 (hold.material.clone(), hold.failure.clone(), hold.at_ms)
2771 };
2772 let until_ms = at_ms.saturating_add(REFUSED_REFRESH_HOLD.as_millis() as u64);
2773 (now_ms < until_ms && Self::stored_material(cred).as_ref() == Some(&material)).then(|| {
2774 super::auth_state::RefreshHoldState {
2775 failure: refresh_failure(&failure),
2776 refused_at_ms: at_ms,
2777 until_ms,
2778 }
2779 })
2780 }
2781
2782 #[doc(hidden)]
2785 pub fn age_refresh_holds_for_tests(&self, by: Duration) {
2786 let by_ms = by.as_millis() as u64;
2787 for hold in self
2788 .refresh_holds
2789 .lock()
2790 .unwrap_or_else(|e| e.into_inner())
2791 .values_mut()
2792 {
2793 hold.at_ms = hold.at_ms.saturating_sub(by_ms);
2794 }
2795 }
2796
2797 #[doc(hidden)]
2800 pub fn refresh_hold_digest_for_tests(&self, cred: &CredentialRef) -> Option<String> {
2801 self.refresh_holds
2802 .lock()
2803 .unwrap_or_else(|e| e.into_inner())
2804 .get(&Self::state_key(cred))
2805 .map(|h| h.material.expose_for_tests().to_string())
2806 }
2807
2808 fn record_usage(
2809 &self,
2810 cred: &CredentialRef,
2811 failure: Option<super::quota_policy::ObservationFailure>,
2812 ) {
2813 let at_ms = crate::epoch_millis();
2814 let mut observations = self.observations.lock().unwrap_or_else(|e| e.into_inner());
2815 let slot = observations.entry(Self::state_key(cred)).or_default();
2816 if failure.is_none() {
2817 slot.last_usage_ok_ms = Some(at_ms);
2818 }
2819 slot.last_usage = Some(super::auth_state::ObservedOutcome { at_ms, failure });
2820 }
2821
2822 async fn shared_read(&self, cred: &CredentialRef) -> super::usage_reads::ReadResult {
2828 use super::usage_reads::{ReadError, ReadPath, USAGE_READ_TARGET};
2829 let key = self.read_key(cred);
2830 let (result, path) = self
2831 .usage_reads
2832 .read(&key, || self.network_read(cred))
2833 .await;
2834 if path != ReadPath::NegativeCacheHit {
2835 self.record_usage(cred, result.as_ref().err().map(ReadError::cause));
2836 }
2837 let outcome = match (path, &result) {
2838 (ReadPath::Network, Ok(_)) => "ok",
2839 (ReadPath::Network, Err(_)) => "failed",
2840 (ReadPath::Joined, _) => "joined",
2841 (ReadPath::NegativeCacheHit, _) => "negative_cache_hit",
2842 };
2843 let now_ms = crate::epoch_millis();
2844 let (kind, status, retry_in_s) = match &result {
2845 Ok(_) => ("none", String::new(), 0),
2846 Err(ReadError::Transient {
2847 failure,
2848 retry_at_ms,
2849 }) => (
2850 failure.kind,
2851 failure.status.map(|s| s.to_string()).unwrap_or_default(),
2852 retry_at_ms.saturating_sub(now_ms) / 1000,
2853 ),
2854 Err(ReadError::Other { cause, .. }) => (
2855 cause.kind.as_str(),
2856 cause.http_status.map(|s| s.to_string()).unwrap_or_default(),
2857 0,
2858 ),
2859 };
2860 tracing::info!(
2861 target: USAGE_READ_TARGET,
2862 broker = self.instance,
2863 account = %cred.account.label_str(),
2864 outcome = %outcome,
2865 kind = %kind,
2866 status = %status,
2867 retry_in_s,
2868 "usage read"
2869 );
2870 result
2871 }
2872
2873 async fn network_read(&self, cred: &CredentialRef) -> super::usage_reads::Fetched {
2878 use super::usage::{fetch_usage, supports_usage, UsageClient, UsageError, UsageFetchOptions};
2879 use super::usage_reads::Fetched;
2880 if !supports_usage(cred.provider) {
2881 return Fetched::Usage(UsageError::UnsupportedProvider {
2882 provider: cred.provider.as_str().to_string(),
2883 });
2884 }
2885 if let Err(e) = self.check_slot_exists(cred) {
2886 return UsageReadError::slot(e).into_fetched();
2887 }
2888 let memory: Arc<dyn super::token::RefusalMemory> = self.slot_hold(cred);
2891 let creds = match super::token::ensure_fresh_credential_via(
2892 &self.http,
2893 cred,
2894 &self.refresh_endpoints,
2895 Some(memory),
2896 )
2897 .await
2898 {
2899 Ok(creds) => creds,
2900 Err(f) => return UsageReadError::refresh(f).into_fetched(),
2901 };
2902 let seat = SeatIdentity::of(cred.provider, &creds);
2903 let client = match UsageClient::new() {
2904 Ok(client) => client,
2905 Err(e) => return Fetched::Usage(e),
2906 };
2907 let opts = UsageFetchOptions {
2908 endpoint_override: self.usage_endpoint_override.clone(),
2909 ..UsageFetchOptions::default()
2910 };
2911 match fetch_usage(
2912 &client,
2913 cred.provider,
2914 cred.account.label_str(),
2915 &creds.access,
2916 &opts,
2917 )
2918 .await
2919 {
2920 Ok(snapshot) => Fetched::Ok(snapshot, seat),
2921 Err(e) => Fetched::Usage(e),
2922 }
2923 }
2924
2925 async fn capacity_for(&self, cred: &CredentialRef) -> CandidateRead {
2933 use super::quota_policy::AccountCapacity;
2934 use super::usage_reads::ReadError;
2935 match self.fresh_usage(cred).await {
2936 Ok((snapshot, seat)) => CandidateRead {
2937 capacity: capacity_from_snapshot(cred, &snapshot),
2938 seat: Some(seat),
2939 transient: None,
2940 grace: None,
2941 },
2942 Err(read_err) => {
2943 let err = UsageReadError::from_read(&read_err);
2944 if err.failure.kind.needs_action() {
2945 tracing::warn!(
2949 credential = %cred,
2950 kind = %err.failure.kind,
2951 detail = %action_detail(cred, &err),
2952 "usage read needs action"
2953 );
2954 } else {
2955 tracing::debug!(credential = %cred, cause = %err.failure, "usage read failed");
2956 }
2957 let capacity = AccountCapacity::failed(cred.clone(), err.failure);
2958 let ReadError::Transient {
2959 failure,
2960 retry_at_ms,
2961 } = read_err
2962 else {
2963 return CandidateRead {
2964 capacity,
2965 seat: None,
2966 transient: None,
2967 grace: None,
2968 };
2969 };
2970 let graced = self.stored_seat(cred).ok().and_then(|seat| {
2973 let good = self
2974 .usage_reads
2975 .last_good_for(&self.read_key(cred), &seat)?;
2976 Some((good, seat))
2977 });
2978 let (grace, seat) = match graced {
2979 Some((good, seat)) => (Some(capacity_from_snapshot(cred, &good)), Some(seat)),
2980 None => (None, None),
2981 };
2982 CandidateRead {
2983 capacity,
2984 seat,
2985 transient: Some((failure, retry_at_ms)),
2986 grace,
2987 }
2988 }
2989 }
2990 }
2991
2992 pub async fn plan(
2996 &self,
2997 provider: OAuthProviderId,
2998 model: Option<&str>,
2999 ) -> Result<super::SelectionPlan, BrokerError> {
3000 self.selection_plan(provider, model, &[], &mut FailedThisSelection::new())
3001 .await
3002 .map(|(plan, ..)| plan)
3003 }
3004
3005 #[allow(clippy::type_complexity)]
3012 async fn selection_plan(
3013 &self,
3014 provider: OAuthProviderId,
3015 model: Option<&str>,
3016 exclude: &[CredentialRef],
3017 failed: &mut FailedThisSelection,
3018 ) -> Result<
3019 (
3020 super::SelectionPlan,
3021 BTreeMap<String, SeatIdentity>,
3022 BTreeMap<String, (super::usage_reads::TransientReadFailure, u64)>,
3023 ),
3024 BrokerError,
3025 > {
3026 use super::quota_policy::{AccountCapacity, SelectionRequest};
3027 let summaries = storage::list_accounts(provider).map_err(BrokerError::Credential)?;
3028 let mut candidates = Vec::with_capacity(summaries.len());
3029 let mut seats = BTreeMap::new();
3030 let mut transient = BTreeMap::new();
3031 let mut grace = Vec::new();
3032 let mut seen = std::collections::BTreeSet::new();
3033 let mut exclude = exclude.to_vec();
3034 let cooling_any = self.any_cooldown_active(crate::epoch_millis());
3038 let mut keys = BTreeMap::new();
3039 for summary in &summaries {
3040 let Some(cred) = summary.credential_ref() else {
3041 continue;
3042 };
3043 if cooling_any {
3044 keys.insert(cred.storage_key(), self.cooldown_keys(&cred));
3045 }
3046 if let Some(failure) = failed.get(&cred.storage_key()) {
3047 candidates.push(AccountCapacity::failed(cred, failure.clone()));
3048 continue;
3049 }
3050 let read = if keys
3055 .get(&cred.storage_key())
3056 .and_then(|k| self.cooldown_for_keys(k, crate::epoch_millis()))
3057 .is_some()
3058 {
3059 CandidateRead {
3060 capacity: AccountCapacity::unobserved(cred.clone()),
3061 seat: self.stored_seat(&cred).ok(),
3062 transient: None,
3063 grace: None,
3064 }
3065 } else {
3066 self.capacity_for(&cred).await
3067 };
3068 let (mut capacity, seat) = (read.capacity, read.seat);
3069 match (read.transient, &capacity.failure) {
3070 (Some(held), _) => {
3071 transient.insert(cred.storage_key(), held);
3072 }
3073 (None, Some(failure)) => {
3074 failed.insert(cred.storage_key(), failure.clone());
3075 }
3076 (None, None) => {}
3077 }
3078 require_requested_model(provider, model, &mut capacity);
3081 grace.extend(read.grace.map(|mut good| {
3082 require_requested_model(provider, model, &mut good);
3083 good
3084 }));
3085 if let Some(seat) = seat {
3086 if !seen.insert(seat.as_str().to_string()) {
3087 exclude.push(cred.clone());
3088 }
3089 seats.insert(cred.storage_key(), seat);
3090 }
3091 candidates.push(capacity);
3092 }
3093 let now_ms = crate::epoch_millis();
3094 if !cooling_any && self.any_cooldown_active(now_ms) {
3096 for capacity in &candidates {
3097 let cred = &capacity.credential;
3098 keys.insert(cred.storage_key(), self.cooldown_keys(cred));
3099 }
3100 }
3101 for capacity in &mut candidates {
3102 capacity.cooldown_until_ms = keys
3103 .get(&capacity.credential.storage_key())
3104 .and_then(|k| self.cooldown_for_keys(k, now_ms))
3105 .map(|c| c.until_ms);
3106 }
3107 let previous = self
3108 .current_accounts
3109 .lock()
3110 .unwrap_or_else(|e| e.into_inner())
3111 .get(&Self::current_key(provider, model))
3112 .cloned();
3113 let current = previous
3114 .filter(|(cred, seat)| seats.get(&cred.storage_key()) == Some(seat))
3115 .map(|(cred, _)| cred);
3116 let policy = self.auto_policy();
3117 let preference: Vec<_> = summaries.iter().map(|s| s.label.clone()).collect();
3118 let request = SelectionRequest {
3119 model,
3120 preference: &preference,
3121 exclude: &exclude,
3122 strategy: policy.strategy_for(provider),
3123 current: current.as_ref(),
3124 urgent_horizon_ms: policy.urgent_horizon_ms,
3125 sticky: policy.sticky,
3126 grace: &grace,
3127 ..SelectionRequest::new(provider, now_ms, self.max_snapshot_age.as_millis() as u64)
3128 };
3129 Ok((
3130 super::SelectionPlan::from_candidates(&request, &candidates),
3131 seats,
3132 transient,
3133 ))
3134 }
3135
3136 async fn select_auto(
3137 &self,
3138 provider: OAuthProviderId,
3139 model: Option<&str>,
3140 exclude: &[CredentialRef],
3141 failed: &mut FailedThisSelection,
3142 ) -> Result<(CredentialRef, SeatIdentity), BrokerError> {
3143 use super::quota_policy::{Selection, SelectionFailure, SelectionFailureKind};
3144 use super::usage_reads::USAGE_READ_TARGET;
3145 let (plan, mut seats, transient) = match self
3146 .selection_plan(provider, model, exclude, failed)
3147 .await
3148 {
3149 Ok(planned) => planned,
3150 Err(e) => {
3154 log_selection_failure(
3155 provider,
3156 model,
3157 &SelectionFailure::new(SelectionFailureKind::CredentialStore),
3158 );
3159 return Err(e);
3160 }
3161 };
3162 match plan.selection {
3163 Selection::Selected {
3164 credential,
3165 tier,
3166 budget_reset_ms,
3167 utilization,
3168 ..
3169 } => {
3170 let seat = seats.remove(&credential.storage_key());
3173 debug_assert!(
3174 seat.is_some(),
3175 "selected account has no paired seat identity"
3176 );
3177 let Some(seat) = seat else {
3178 return Err(selection_failed(
3179 provider,
3180 model,
3181 "selected account has no paired seat identity".into(),
3182 SelectionFailure::new(SelectionFailureKind::SeatChurn),
3183 ));
3184 };
3185 if let Some((failure, retry_at_ms)) = transient.get(&credential.storage_key()) {
3188 tracing::info!(
3189 target: USAGE_READ_TARGET,
3190 broker = self.instance,
3191 account = %credential.account.label_str(),
3192 outcome = %"grace",
3193 kind = %failure.kind,
3194 status = %failure.status.map(|s| s.to_string()).unwrap_or_default(),
3195 retry_in_s = retry_at_ms.saturating_sub(crate::epoch_millis()) / 1000,
3196 "usage read"
3197 );
3198 }
3199 tracing::info!(%credential, ?plan.strategy, tier, ?budget_reset_ms, ?utilization, "automatic subscription choice");
3200 Ok((credential, seat))
3201 }
3202 Selection::NoCapacity {
3203 rejections,
3204 earliest_reset_ms,
3205 } => {
3206 let detail: Vec<_> = rejections
3207 .iter()
3208 .map(|r| format!("{}: {}", r.credential.account, r.describe()))
3209 .collect();
3210 let reset = earliest_reset_ms
3211 .map(|t| format!("; earliest reset at {t}"))
3212 .unwrap_or_default();
3213 let mut failure = super::quota_policy::SelectionFailure::from_rejections(
3214 &rejections,
3215 plan.now_ms,
3216 );
3217 if failure.kind == SelectionFailureKind::UsageReadFailed {
3220 failure.usage_read_retry_ms = rejections
3221 .iter()
3222 .filter_map(|r| transient.get(&r.credential.storage_key()))
3223 .map(|(_, at)| *at)
3224 .min();
3225 }
3226 let reason = if detail.is_empty() {
3227 format!("no {provider} account is logged in")
3228 } else {
3229 format!("{}{reset}", detail.join(", "))
3230 };
3231 Err(selection_failed(provider, model, reason, failure))
3232 }
3233 }
3234 }
3235
3236 pub fn with_local_base_url(http: reqwest::Client, base_url: impl Into<String>) -> Self {
3238 Self {
3239 local_base_url: Some(base_url.into()),
3240 ..Self::new(http)
3241 }
3242 }
3243
3244 #[doc(hidden)]
3248 pub fn with_google_gemini_base_url_for_tests(
3249 http: reqwest::Client,
3250 base_url: impl Into<String>,
3251 ) -> Self {
3252 Self {
3253 google_gemini_base_url: Some(base_url.into().trim_end_matches('/').to_string()),
3254 ..Self::new(http)
3255 }
3256 }
3257
3258 #[doc(hidden)]
3260 pub fn with_anthropic_usage_url(mut self, url: impl Into<String>) -> Self {
3261 self.anthropic_usage_url = Some(url.into());
3262 self
3263 }
3264
3265 #[doc(hidden)]
3267 pub fn with_request_timeout(mut self, timeout: Duration) -> Self {
3268 self.request_timeout = timeout;
3269 self
3270 }
3271
3272 #[doc(hidden)]
3274 pub fn with_max_response_bytes(mut self, cap: usize) -> Self {
3275 self.max_response_bytes = cap;
3276 self
3277 }
3278
3279 fn resolve_static_key(&self, provider: &str) -> Result<String, BrokerError> {
3283 if provider == LOCAL_PROVIDER_KEY {
3284 return Ok(self.resolve_local_key());
3285 }
3286 let spec = static_provider(provider)
3287 .ok_or_else(|| BrokerError::UnknownProvider(provider.to_string()))?;
3288 if let Ok(Some(key)) = storage::load_static_key(provider) {
3289 return Ok(key);
3290 }
3291 if let Some(key) = discover_legacy_static_key(spec) {
3292 if let Err(e) = storage::save_static_key(provider, &key) {
3295 tracing::warn!(provider, "static key migration failed: {e}");
3296 }
3297 return Ok(key);
3298 }
3299 Err(BrokerError::NotConfigured(provider.to_string()))
3300 }
3301
3302 fn resolve_local_key(&self) -> String {
3303 storage::load_static_key(LOCAL_PROVIDER_KEY)
3304 .ok()
3305 .flatten()
3306 .or_else(|| {
3307 crate::config::get_provider_keys()
3308 .get(LOCAL_PROVIDER_KEY)
3309 .filter(|s| !s.is_empty())
3310 .cloned()
3311 })
3312 .or_else(|| {
3313 std::env::var("LOCAL_API_KEY")
3314 .ok()
3315 .filter(|s| !s.is_empty())
3316 })
3317 .unwrap_or_else(|| "local".to_string())
3318 }
3319
3320 fn base_url_for(&self, provider: &str) -> Result<String, BrokerError> {
3321 if provider == LOCAL_PROVIDER_KEY {
3322 if let Some(url) = &self.local_base_url {
3323 return Ok(url.trim_end_matches('/').to_string());
3324 }
3325 return Ok(local_endpoint_url());
3326 }
3327 static_provider(provider)
3328 .map(|s| s.base_url.trim_end_matches('/').to_string())
3329 .ok_or_else(|| BrokerError::UnknownProvider(provider.to_string()))
3330 }
3331
3332 async fn send(&self, request: &ProxyRequest) -> Result<reqwest::Response, BrokerError> {
3333 request.validate()?;
3334 let (provider_key, explicit_account) = request.split_provider()?;
3335 let provider_key = provider_key.as_str();
3336 let oauth_provider: Option<OAuthProviderId> = provider_key.parse().ok();
3340 let mut pinned: Option<PinnedToken> = None;
3341 let muse_env_key = (provider_key == "muse" && explicit_account.is_none())
3345 .then(super::muse::api_key_from_env)
3346 .flatten();
3347 if let Some(provider) = oauth_provider.filter(|_| muse_env_key.is_none()) {
3348 if provider_key != LOCAL_PROVIDER_KEY && static_provider(provider_key).is_none() {
3349 pinned = Some(
3350 self.resolve_and_vend(provider, explicit_account, None, &[])
3351 .await?,
3352 );
3353 }
3354 }
3355 let bearer = |pinned: &Option<PinnedToken>| -> Result<String, BrokerError> {
3356 pinned
3357 .as_ref()
3358 .map(|p| p.token.token.clone())
3359 .ok_or_else(|| BrokerError::UnknownProvider(request.provider.clone()))
3360 };
3361 let (key, base) = if provider_key == "xai-auth" {
3362 (bearer(&pinned)?, "https://api.x.ai/v1".to_string())
3363 } else if provider_key == "github-copilot" {
3364 (
3367 bearer(&pinned)?,
3368 super::github_copilot_models_base_url().to_string(),
3369 )
3370 } else if provider_key == "google-gemini" {
3371 let base = self
3374 .google_gemini_base_url
3375 .clone()
3376 .unwrap_or_else(|| GOOGLE_GEMINI_CODE_ASSIST_BASE_URL.to_string());
3377 (bearer(&pinned)?, base)
3378 } else if provider_key == "openai-codex" {
3379 let base = self
3382 .openai_codex_base_url
3383 .clone()
3384 .unwrap_or_else(|| OPENAI_CODEX_BACKEND_BASE_URL.to_string());
3385 (bearer(&pinned)?, base)
3386 } else if provider_key == "kimi-code" {
3387 (bearer(&pinned)?, super::kimi_code::API_BASE_URL.to_string())
3391 } else if provider_key == "muse" {
3392 let key = match &muse_env_key {
3397 Some(key) => key.clone(),
3398 None => bearer(&pinned)?,
3399 };
3400 let base = self
3401 .muse_base_url
3402 .clone()
3403 .unwrap_or_else(|| super::muse::API_ORIGIN.to_string());
3404 (key, base)
3405 } else if provider_key == "anthropic" {
3406 (bearer(&pinned)?, "https://api.anthropic.com".to_string())
3409 } else {
3410 (
3411 self.resolve_static_key(provider_key)?,
3412 self.base_url_for(provider_key)?,
3413 )
3414 };
3415 let response = self
3416 .dispatch(request, provider_key, &key, &base, &pinned)
3417 .await?;
3418 if provider_key == "muse" && response.status() == reqwest::StatusCode::UNAUTHORIZED {
3432 if let Some(rejected) = pinned.as_ref() {
3433 tracing::info!(credential = %rejected.credential, "muse key rejected (401); re-minting once");
3434 match self.recover_rejected_token(&rejected.credential, &key).await {
3435 Ok(fresh) => {
3436 let pinned = Some(PinnedToken {
3437 credential: rejected.credential.clone(),
3438 token: fresh,
3439 });
3440 let key = bearer(&pinned)?;
3441 return self
3442 .dispatch(request, provider_key, &key, &base, &pinned)
3443 .await;
3444 }
3445 Err(BrokerError::Refresh(f))
3451 if f.kind.is_retryable()
3452 || f.kind == super::token::RefreshFailureKind::ActionRequired =>
3453 {
3454 tracing::warn!(
3455 credential = %rejected.credential,
3456 reason = %f,
3457 "muse key re-mint failed; returning a retryable error"
3458 );
3459 return Err(BrokerError::Refresh(f));
3460 }
3461 Err(e) => {
3462 tracing::warn!(
3465 credential = %rejected.credential,
3466 reason = %e,
3467 "muse key re-mint refused or rejected; returning the upstream 401"
3468 );
3469 }
3470 }
3471 }
3472 }
3473 Ok(response)
3474 }
3475
3476 async fn dispatch(
3478 &self,
3479 request: &ProxyRequest,
3480 provider_key: &str,
3481 key: &str,
3482 base: &str,
3483 pinned: &Option<PinnedToken>,
3484 ) -> Result<reqwest::Response, BrokerError> {
3485 let url = format!("{base}{}", request.path);
3486 let mut builder = if provider_key == "google-gemini"
3489 || provider_key == "openai-codex"
3490 || provider_key == "anthropic"
3491 || provider_key == "kimi-code"
3492 || provider_key == "muse"
3493 {
3494 let no_redirect = reqwest::Client::builder()
3498 .redirect(reqwest::redirect::Policy::none())
3499 .connect_timeout(Duration::from_secs(10))
3500 .read_timeout(self.stream_idle_timeout_for(provider_key))
3501 .build()
3502 .map_err(|e| BrokerError::Transport(format!("client build failed: {e}")))?;
3503 match request.method {
3504 ProxyMethod::Get => no_redirect.get(&url),
3505 ProxyMethod::Post => no_redirect.post(&url),
3506 }
3507 } else {
3508 match request.method {
3509 ProxyMethod::Get => self.http.get(&url),
3510 ProxyMethod::Post => self.http.post(&url),
3511 }
3512 };
3513 builder = builder.bearer_auth(key);
3514 if provider_key == "openai-codex" {
3515 let cred = pinned
3519 .as_ref()
3520 .map(|p| p.credential.clone())
3521 .unwrap_or_else(|| CredentialRef::default_for(OAuthProviderId::OpenAiCodex));
3522 let account_id = super::extract_codex_account_id(key)
3523 .or_else(|| {
3524 storage::load_credential(&cred)
3525 .ok()
3526 .flatten()
3527 .and_then(|c| c.account_id)
3528 .filter(|s| !s.trim().is_empty())
3529 })
3530 .ok_or_else(|| {
3531 BrokerError::Credential(
3532 "openai-codex credential is missing chatgpt account id".into(),
3533 )
3534 })?;
3535 builder = builder
3536 .header("chatgpt-account-id", account_id)
3537 .header("originator", "synaps")
3538 .header("OpenAI-Beta", "responses=experimental")
3539 .header("accept", "application/json");
3540 }
3541 if provider_key == "anthropic" {
3542 for (name, value) in anthropic_oauth_catalog_request_headers() {
3545 builder = builder.header(*name, *value);
3546 }
3547 }
3548 if provider_key == "github-copilot" {
3549 for (name, value) in super::github_copilot_models_request_headers() {
3550 builder = builder.header(*name, *value);
3551 }
3552 if request.path != "/models" {
3553 builder = builder
3554 .header("Openai-Intent", "conversation-edits")
3555 .header("X-Initiator", "agent");
3556 }
3557 }
3558 if provider_key == "google-gemini" {
3559 if request.stream && request.path == "/v1internal:streamGenerateContent" {
3563 builder = builder.query(&[("alt", "sse")]);
3564 }
3565 builder = builder.header("user-agent", "SynapsCLI/0.6.0 (google-gemini)");
3566 }
3567 if provider_key == "muse" {
3568 builder = builder.header("user-agent", super::muse::USER_AGENT);
3571 }
3572 if provider_key == "kimi-code" {
3573 let device_id = super::kimi_code::load_or_create_device_id();
3577 for (name, value) in super::kimi_code::identity_request_headers(&device_id) {
3578 builder = builder.header(name, value);
3579 }
3580 }
3581 if let Some(bytes) = &request.body_bytes {
3582 builder = builder
3586 .header("content-type", "application/json")
3587 .body(bytes.clone());
3588 } else if let Some(body) = &request.body {
3589 builder = builder.json(body);
3590 }
3591 if request.stream {
3592 builder = builder.header("accept", "text/event-stream");
3593 } else {
3594 builder = builder.timeout(self.request_timeout);
3598 }
3599 builder.send().await.map_err(|e| {
3600 if e.is_connect() && provider_key == LOCAL_PROVIDER_KEY {
3601 BrokerError::Transport(format!(
3602 "can't reach local endpoint at {url} — is Ollama/LM Studio running?"
3603 ))
3604 } else {
3605 BrokerError::Transport(format!("request to {} failed: {e}", request.provider))
3607 }
3608 })
3609 }
3610}
3611
3612pub fn capacity_from_snapshot(
3618 cred: &CredentialRef,
3619 snapshot: &super::usage::UsageSnapshot,
3620) -> super::quota_policy::AccountCapacity {
3621 use super::quota_policy::{
3622 AccountCapacity, ModelAvailability, ModelState, QuotaObservation, WindowLimit,
3623 };
3624 use super::usage::{Availability, WindowScope};
3625 let mut windows: Vec<WindowLimit> = snapshot
3626 .windows
3627 .iter()
3628 .filter_map(|w| {
3629 let models = match &w.scope {
3630 WindowScope::Account => None,
3631 WindowScope::Model { model } => Some(vec![model.clone()]),
3632 WindowScope::Feature { .. } => return None,
3633 };
3634 Some(WindowLimit {
3635 id: w.id.clone(),
3636 duration_ms: w.duration_secs.map(|s| s.saturating_mul(1000)),
3637 used_percent: w.used_percent.valid(),
3638 limit_reached: w.limit_reached,
3639 resets_at_ms: w.reset_at,
3640 models,
3641 })
3642 })
3643 .collect();
3644 if snapshot.limit_reached == Some(true) {
3645 windows.push(WindowLimit {
3647 id: "limit_reached".into(),
3648 duration_ms: None,
3649 used_percent: None,
3650 limit_reached: Some(true),
3651 resets_at_ms: snapshot.earliest_reset_at(),
3652 models: None,
3653 });
3654 }
3655 let models = if snapshot.model_availability.is_empty() {
3656 None
3657 } else {
3658 Some(
3659 snapshot
3660 .model_availability
3661 .iter()
3662 .map(|m| ModelAvailability {
3663 model: m.model.clone(),
3664 state: match m.availability {
3665 Availability::Available => ModelState::Available,
3666 Availability::Exhausted => ModelState::Exhausted,
3667 Availability::Unknown => ModelState::Unknown,
3668 },
3669 })
3670 .collect(),
3671 )
3672 };
3673 AccountCapacity {
3674 credential: cred.clone(),
3675 observed_at_ms: Some(snapshot.observed_at),
3676 observation: QuotaObservation::Ok { windows, models },
3677 cooldown_until_ms: None,
3678 failure: None,
3679 }
3680}
3681
3682#[derive(Debug)]
3685struct UsageReadError {
3686 error: BrokerError,
3687 failure: super::quota_policy::ObservationFailure,
3688}
3689
3690struct CandidateRead {
3692 capacity: super::quota_policy::AccountCapacity,
3693 seat: Option<SeatIdentity>,
3695 transient: Option<(super::usage_reads::TransientReadFailure, u64)>,
3697 grace: Option<super::quota_policy::AccountCapacity>,
3699}
3700
3701impl UsageReadError {
3702 fn from_read(e: &super::usage_reads::ReadError) -> Self {
3705 Self {
3706 error: e.to_broker_error(),
3707 failure: e.cause(),
3708 }
3709 }
3710
3711 fn into_read_error(self) -> super::usage_reads::ReadError {
3714 super::usage_reads::ReadError::Other {
3715 error: self.error,
3716 cause: self.failure,
3717 }
3718 }
3719
3720 fn into_fetched(self) -> super::usage_reads::Fetched {
3722 super::usage_reads::Fetched::Failed {
3723 error: self.error,
3724 cause: self.failure,
3725 }
3726 }
3727
3728 fn refresh(f: super::token::RefreshFailure) -> Self {
3731 Self {
3732 failure: refresh_failure(&f),
3733 error: BrokerError::Credential(f.into()),
3734 }
3735 }
3736
3737 fn slot(error: BrokerError) -> Self {
3739 use super::quota_policy::{AccountFailureKind as K, ObservationFailure};
3740 let kind = match &error {
3741 BrokerError::UnknownAccount { .. } => K::CredentialMissing,
3742 _ => K::CredentialStore,
3743 };
3744 Self {
3745 failure: ObservationFailure::new(kind),
3746 error,
3747 }
3748 }
3749
3750 #[cfg(test)]
3753 fn usage(e: super::usage::UsageError) -> Self {
3754 let table = super::usage_reads::UsageReads::default();
3755 match table.record_at("slot", super::usage_reads::Fetched::Usage(e), 0) {
3756 Err(read) => Self::from_read(&read),
3757 Ok(_) => unreachable!("a usage error is never a reading"),
3758 }
3759 }
3760}
3761
3762fn require_requested_model(
3767 provider: OAuthProviderId,
3768 model: Option<&str>,
3769 capacity: &mut super::quota_policy::AccountCapacity,
3770) {
3771 use super::quota_policy::{model_matches, ModelAvailability, ModelState, QuotaObservation};
3772 if provider != OAuthProviderId::OpenAiCodex {
3773 return;
3774 }
3775 if let (Some(model), QuotaObservation::Ok { models, .. }) = (model, &mut capacity.observation) {
3776 let availability = models.get_or_insert_with(Vec::new);
3777 if !availability.iter().any(|m| model_matches(&m.model, model)) {
3778 availability.push(ModelAvailability {
3779 model: model.to_string(),
3780 state: ModelState::Unknown,
3781 });
3782 }
3783 }
3784}
3785
3786fn action_detail(cred: &CredentialRef, err: &UsageReadError) -> String {
3791 use super::quota_policy::AccountFailureKind as K;
3792 match err.failure.kind {
3793 K::CredentialStore => format!(
3794 "the stored login of '{}' could not be read from {}; repair or restore the \
3795 credential store",
3796 cred.account.label_str(),
3797 storage::auth_file_path().display()
3798 ),
3799 _ => err.error.to_string(),
3800 }
3801}
3802
3803fn refresh_failure(f: &super::token::RefreshFailure) -> super::quota_policy::ObservationFailure {
3806 let failure = refresh_kind_failure(f.kind);
3807 match f.http_status() {
3808 Some(status) => failure.with_status(status),
3809 None => failure,
3810 }
3811}
3812
3813fn refresh_kind_failure(
3815 kind: super::token::RefreshFailureKind,
3816) -> super::quota_policy::ObservationFailure {
3817 use super::quota_policy::{AccountFailureKind as K, ObservationFailure};
3818 use super::token::RefreshFailureKind as R;
3819 match kind {
3820 R::Transient => ObservationFailure::new(K::RefreshTransient),
3821 R::LockBusy => ObservationFailure::new(K::RefreshLockBusy),
3822 R::Other => ObservationFailure::new(K::RefreshOther),
3823 R::Cooldown { retry_in_secs } => {
3824 ObservationFailure::new(K::RefreshOther).with_retry_after(Some(retry_in_secs))
3825 }
3826 R::Credential => ObservationFailure::new(K::RefreshRejected),
3827 R::ActionRequired => ObservationFailure::new(K::RefreshActionRequired),
3828 }
3829}
3830
3831fn auto_vend_failure(
3834 credential: &CredentialRef,
3835 f: super::token::RefreshFailure,
3836) -> super::token::RefreshFailure {
3837 let label = credential.account.label_str();
3838 let mut message = format!("account '{label}': {}", f.message());
3839 if f.kind == super::token::RefreshFailureKind::Credential && !message.contains("synaps login") {
3840 let account = if credential.account.is_default() {
3841 String::new()
3842 } else {
3843 format!(" --account {label}")
3844 };
3845 message.push_str(&format!(
3846 "; log it in again with `synaps login --provider {}{account}`",
3847 credential.provider
3848 ));
3849 }
3850 super::token::RefreshFailure::new(f.kind, message).with_http_status(f.http_status())
3851}
3852
3853fn selection_failed(
3856 provider: OAuthProviderId,
3857 model: Option<&str>,
3858 reason: String,
3859 failure: super::quota_policy::SelectionFailure,
3860) -> BrokerError {
3861 log_selection_failure(provider, model, &failure);
3862 BrokerError::NoAccountAvailable {
3863 provider: provider.to_string(),
3864 reason,
3865 failure: Some(failure),
3866 }
3867}
3868
3869fn log_selection_failure(
3873 provider: OAuthProviderId,
3874 model: Option<&str>,
3875 failure: &super::quota_policy::SelectionFailure,
3876) {
3877 tracing::warn!(
3878 provider = %provider,
3879 model = model.unwrap_or("any"),
3880 kind = %failure.kind,
3881 accounts = %failure.accounts_field(),
3882 earliest_reset_ms = ?failure.earliest_reset_ms,
3883 "auth_selection_failed"
3884 );
3885}
3886
3887async fn read_body_capped(resp: reqwest::Response, cap: usize) -> Result<String, BrokerError> {
3892 use futures::StreamExt;
3893 let mut buf: Vec<u8> = Vec::new();
3894 let mut stream = resp.bytes_stream();
3895 while let Some(chunk) = stream.next().await {
3896 let chunk =
3897 chunk.map_err(|e| BrokerError::Transport(format!("failed to read response: {e}")))?;
3898 if buf.len() + chunk.len() > cap {
3899 return Err(BrokerError::Transport(format!(
3900 "response body exceeded the {cap}-byte broker buffering limit"
3901 )));
3902 }
3903 buf.extend_from_slice(&chunk);
3904 }
3905 String::from_utf8(buf)
3906 .map_err(|_| BrokerError::Transport("response body was not valid UTF-8".into()))
3907}
3908
3909const MAX_PROXY_ERROR_CLASSIFY_BYTES: usize = 8 * 1024;
3913
3914async fn read_error_body_for_classification(resp: reqwest::Response, cap: usize) -> Option<String> {
3919 use futures::StreamExt;
3920 let mut buf: Vec<u8> = Vec::with_capacity(cap.min(1024));
3921 let mut stream = resp.bytes_stream();
3922 while let Some(chunk) = stream.next().await {
3923 let chunk = chunk.ok()?;
3924 let room = cap.saturating_sub(buf.len());
3925 if chunk.len() >= room {
3926 buf.extend_from_slice(&chunk[..room]);
3927 return String::from_utf8(buf).ok();
3930 }
3931 buf.extend_from_slice(&chunk);
3932 }
3933 String::from_utf8(buf).ok()
3934}
3935
3936fn discover_legacy_static_key(spec: &StaticProviderSpec) -> Option<String> {
3939 if let Some(v) = crate::config::get_provider_keys().get(spec.key) {
3940 if !v.is_empty() {
3941 return Some(v.clone());
3942 }
3943 }
3944 spec.env_vars
3945 .iter()
3946 .find_map(|var| std::env::var(var).ok().filter(|v| !v.is_empty()))
3947}
3948
3949pub fn static_key_configured(provider: &str) -> bool {
3952 if provider == LOCAL_PROVIDER_KEY {
3953 return true; }
3955 let Some(spec) = static_provider(provider) else {
3956 return false;
3957 };
3958 storage::load_static_key(provider).ok().flatten().is_some()
3959 || discover_legacy_static_key(spec).is_some()
3960}
3961
3962pub fn static_key_status(provider: &str) -> StaticKeyStatus {
3966 let stored = storage::load_static_key(provider)
3967 .ok()
3968 .flatten()
3969 .or_else(|| {
3970 crate::config::get_provider_keys()
3971 .get(provider)
3972 .filter(|s| !s.is_empty())
3973 .cloned()
3974 });
3975 if let Some(key) = stored {
3976 return StaticKeyStatus::Configured {
3977 masked: mask_key(&key),
3978 };
3979 }
3980 if let Some(spec) = static_provider(provider) {
3981 if spec
3982 .env_vars
3983 .iter()
3984 .any(|var| std::env::var(var).is_ok_and(|v| !v.is_empty()))
3985 {
3986 return StaticKeyStatus::FromEnv;
3987 }
3988 }
3989 StaticKeyStatus::NotSet
3990}
3991
3992fn mask_key(key: &str) -> String {
3994 let chars: Vec<char> = key.chars().collect();
3995 if chars.len() <= 8 {
3996 return "…".to_string();
3997 }
3998 let head: String = chars[..4].iter().collect();
3999 let tail: String = chars[chars.len() - 4..].iter().collect();
4000 format!("{head}…{tail}")
4001}
4002
4003pub fn configured_static_provider_keys() -> std::collections::BTreeSet<String> {
4005 STATIC_PROVIDERS
4006 .iter()
4007 .filter(|s| static_key_configured(s.key))
4008 .map(|s| s.key.to_string())
4009 .collect()
4010}
4011
4012pub fn local_endpoint_url() -> String {
4015 local_endpoint_config()
4016 .unwrap_or_else(|| LOCAL_DEFAULT_BASE_URL.to_string())
4017 .trim_end_matches('/')
4018 .to_string()
4019}
4020
4021pub fn local_endpoint_config() -> Option<String> {
4023 crate::config::get_provider_keys()
4024 .get("local.url")
4025 .filter(|s| !s.is_empty())
4026 .cloned()
4027 .or_else(|| {
4028 std::env::var("LOCAL_ENDPOINT")
4029 .ok()
4030 .filter(|s| !s.is_empty())
4031 })
4032}
4033
4034pub fn local_model_ids() -> Vec<String> {
4036 crate::config::get_provider_keys()
4037 .get("local.models")
4038 .map(|value| {
4039 value
4040 .split(',')
4041 .map(|s| s.trim().to_string())
4042 .filter(|s| !s.is_empty())
4043 .collect()
4044 })
4045 .unwrap_or_default()
4046}
4047
4048pub fn anthropic_credential_available() -> bool {
4052 oauth_provider_logged_in(OAuthProviderId::Anthropic)
4053 || std::env::var("ANTHROPIC_API_KEY").is_ok_and(|v| !v.is_empty())
4054}
4055
4056pub fn oauth_provider_logged_in(provider: OAuthProviderId) -> bool {
4067 oauth_provider_logged_in_at(
4068 &storage::auth_file_path(),
4069 &AccountPolicy::from_environment(),
4070 provider,
4071 )
4072}
4073
4074pub(crate) fn oauth_provider_logged_in_at(
4076 path: &std::path::Path,
4077 policy: &AccountPolicy,
4078 provider: OAuthProviderId,
4079) -> bool {
4080 let Ok(inventory) = storage::list_accounts_detailed_at(path, Some(provider)) else {
4081 return false;
4082 };
4083 match policy.selector(provider) {
4084 AccountSelector::Account(Account::Named(label)) => inventory
4085 .accounts
4086 .iter()
4087 .any(|row| row.label == label.as_str()),
4088 AccountSelector::Account(Account::Default)
4089 | AccountSelector::Auto
4090 | AccountSelector::Invalid { .. } => !inventory.accounts.is_empty(),
4091 }
4092}
4093
4094#[async_trait]
4095impl CredentialBroker for LocalBroker {
4096 async fn recover_rejected_token(
4097 &self,
4098 cred: &CredentialRef,
4099 rejected: &str,
4100 ) -> Result<AccessToken, BrokerError> {
4101 self.check_slot_exists(cred)?;
4102 let creds = super::token::recover_rejected_credential(
4103 &self.http,
4104 cred,
4105 rejected,
4106 &self.refresh_endpoints,
4107 )
4108 .await
4109 .map_err(BrokerError::Refresh)?;
4110 Ok(AccessToken {
4111 token: creds.access,
4112 expires: creds.expires,
4113 })
4114 }
4115
4116 async fn access_token(&self, provider: OAuthProviderId) -> Result<AccessToken, BrokerError> {
4117 Ok(self.access_token_pinned(provider).await?.token)
4118 }
4119
4120 fn account_selector(&self, provider: OAuthProviderId) -> AccountSelector {
4121 self.policy().selector(provider)
4122 }
4123
4124 async fn access_token_for(&self, cred: &CredentialRef) -> Result<AccessToken, BrokerError> {
4125 self.check_slot_exists(cred)?;
4126 let creds = super::token::ensure_fresh_credential_via(
4128 &self.http,
4129 cred,
4130 &self.refresh_endpoints,
4131 None,
4132 )
4133 .await
4134 .map_err(|f| match f.kind {
4135 super::token::RefreshFailureKind::ActionRequired => BrokerError::Refresh(f),
4138 _ => BrokerError::Credential(f.into()),
4139 })?;
4140 Ok(AccessToken {
4142 token: creds.access,
4143 expires: creds.expires,
4144 })
4145 }
4146
4147 async fn access_token_pinned(
4148 &self,
4149 provider: OAuthProviderId,
4150 ) -> Result<PinnedToken, BrokerError> {
4151 self.access_token_pinned_for(provider, None).await
4152 }
4153
4154 async fn access_token_pinned_for(
4155 &self,
4156 provider: OAuthProviderId,
4157 model: Option<&str>,
4158 ) -> Result<PinnedToken, BrokerError> {
4159 self.resolve_and_vend(provider, None, model, &[]).await
4160 }
4161
4162 async fn access_token_pinned_excluding(
4166 &self,
4167 provider: OAuthProviderId,
4168 model: Option<&str>,
4169 exclude: &[CredentialRef],
4170 ) -> Result<PinnedToken, BrokerError> {
4171 self.resolve_and_vend(provider, None, model, exclude).await
4172 }
4173
4174 async fn accounts(&self, provider: OAuthProviderId) -> Result<Vec<AccountSummary>, BrokerError> {
4175 let mut rows = storage::list_accounts(provider).map_err(BrokerError::Credential)?;
4176 let selector = self.policy().selector(provider);
4177 let now_ms = crate::epoch_millis();
4178 for row in &mut rows {
4179 row.selected = matches!(&selector, AccountSelector::Account(a) if a.label_str() == row.label);
4180 if let Some(cred) = row.credential_ref() {
4181 row.cooldown_until = self.cooldown_until(&cred.storage_key(), now_ms);
4182 }
4183 }
4184 Ok(rows)
4185 }
4186
4187 async fn usage(&self, cred: &CredentialRef) -> Result<super::usage::UsageSnapshot, BrokerError> {
4188 self.usage_with_seat(cred)
4189 .await
4190 .map(|(snapshot, _seat)| snapshot)
4191 .map_err(|e| e.error)
4192 }
4193
4194 fn active_cooldown(&self, cred: &CredentialRef) -> Option<u64> {
4195 self.active_cooldown_at(cred, crate::epoch_millis())
4196 .map(|c| c.until_ms)
4197 }
4198
4199 fn clear_cooldown(&self, cred: &CredentialRef, sent_at_ms: u64) {
4200 let now_ms = crate::epoch_millis();
4201 if !self.any_cooldown_active(now_ms) {
4202 return;
4203 }
4204 let keys = self.cooldown_keys(cred);
4205 let mut cleared = None;
4206 {
4207 let mut cooldowns = self.cooldowns.lock().unwrap_or_else(|e| e.into_inner());
4208 for key in keys.read() {
4209 if cooldowns
4210 .get(key)
4211 .is_some_and(|c| c.until_ms > now_ms && c.reported_at_ms <= sent_at_ms)
4212 {
4213 cleared = cooldowns.remove(key).or(cleared);
4214 }
4215 }
4216 }
4217 if let Some(c) = cleared {
4218 tracing::info!(
4219 credential = %cred,
4220 until_ms = c.until_ms,
4221 reason = %c.reason,
4222 "account cooldown cleared: a request sent after it was reported was accepted"
4223 );
4224 }
4225 }
4226
4227 async fn access_token_for_auto(&self, cred: &CredentialRef) -> Result<AccessToken, BrokerError> {
4228 self.check_slot_exists(cred)?;
4229 if let Some(held) = self.held_refresh(cred) {
4231 return Err(BrokerError::Refresh(auto_vend_failure(cred, held)));
4232 }
4233 let memory: Arc<dyn super::token::RefusalMemory> = self.slot_hold(cred);
4234 let creds = super::token::ensure_fresh_credential_via(
4235 &self.http,
4236 cred,
4237 &self.refresh_endpoints,
4238 Some(memory),
4239 )
4240 .await
4241 .map_err(|f| BrokerError::Refresh(auto_vend_failure(cred, f)))?;
4242 Ok(AccessToken {
4243 token: creds.access,
4244 expires: creds.expires,
4245 })
4246 }
4247
4248 async fn report_cooldown(
4249 &self,
4250 cred: &CredentialRef,
4251 until_ms: Option<u64>,
4252 reason: &str,
4253 ) -> Result<(), BrokerError> {
4254 let now_ms = crate::epoch_millis();
4255 let max_until = now_ms.saturating_add(MAX_COOLDOWN.as_millis() as u64);
4256 let until_ms = until_ms
4257 .filter(|t| *t > now_ms)
4258 .unwrap_or(now_ms + DEFAULT_COOLDOWN.as_millis() as u64)
4259 .min(max_until);
4260 let reason = crate::truncate_str(reason, 64).to_string();
4261 tracing::info!(credential = %cred, until_ms, reason = %reason, "account cooldown reported");
4262 let keys = self.cooldown_keys(cred);
4263 {
4264 let mut cooldowns = self.cooldowns.lock().unwrap_or_else(|e| e.into_inner());
4265 for key in keys.written() {
4266 cooldowns.insert(
4267 key.to_string(),
4268 Cooldown {
4269 until_ms,
4270 reason: reason.clone(),
4271 reported_at_ms: now_ms,
4272 },
4273 );
4274 }
4275 }
4276 self.forget_snapshots(cred);
4280 self.usage_reads.forget_last_good(&self.read_key(cred));
4281 Ok(())
4282 }
4283
4284 async fn proxy(&self, request: ProxyRequest) -> Result<ProxyResponse, BrokerError> {
4285 let resp = self.send(&request).await?;
4286 let status = resp.status().as_u16();
4287 let body = read_body_capped(resp, self.max_response_bytes).await?;
4288 Ok(ProxyResponse { status, body })
4289 }
4290
4291 async fn proxy_stream(&self, request: ProxyRequest) -> Result<ProxyByteStream, BrokerError> {
4292 let mut request = request;
4293 request.stream = true;
4294 let resp = self.send(&request).await?;
4295 let status = resp.status();
4296 if !status.is_success() {
4297 let label = read_error_body_for_classification(resp, MAX_PROXY_ERROR_CLASSIFY_BYTES)
4307 .await
4308 .as_deref()
4309 .and_then(crate::error::vetted_proxy_error_label);
4310 tracing::warn!(
4311 provider = %request.provider,
4312 status = status.as_u16(),
4313 label = label.unwrap_or("-"),
4314 "provider request failed"
4315 );
4316 let label_suffix = label.map(|l| format!(" [{l}]")).unwrap_or_default();
4317 return Err(BrokerError::Transport(format!(
4318 "provider request failed: {status}{label_suffix}"
4319 )));
4320 }
4321 use futures::StreamExt;
4322 let stream = resp
4323 .bytes_stream()
4324 .map(|chunk| chunk.map_err(|e| BrokerError::Transport(format!("stream error: {e}"))));
4325 Ok(Box::pin(stream))
4326 }
4327
4328 async fn anthropic_usage(&self) -> Result<serde_json::Value, BrokerError> {
4329 let token = self.access_token(OAuthProviderId::Anthropic).await?;
4332 let url = self
4333 .anthropic_usage_url
4334 .as_deref()
4335 .unwrap_or(ANTHROPIC_USAGE_URL);
4336 let resp = self
4337 .http
4338 .get(url)
4339 .timeout(self.request_timeout)
4340 .bearer_auth(&token.token)
4341 .header("anthropic-beta", ANTHROPIC_OAUTH_BETA)
4342 .send()
4343 .await
4344 .map_err(|e| BrokerError::Transport(format!("usage request failed: {e}")))?;
4345 let status = resp.status();
4346 if !status.is_success() {
4347 drop(resp);
4350 return Err(BrokerError::Transport(format!(
4351 "usage request failed: {status}"
4352 )));
4353 }
4354 let body = read_body_capped(resp, self.max_response_bytes).await?;
4355 serde_json::from_str(&body)
4356 .map_err(|e| BrokerError::Transport(format!("invalid usage response: {e}")))
4357 }
4358
4359 async fn cloud_catalog(
4360 &self,
4361 provider: CloudProviderId,
4362 context_ref: &str,
4363 allow_stale: bool,
4364 ) -> Result<Vec<CloudCatalogEntry>, BrokerError> {
4365 self.cloud_backend
4366 .as_ref()
4367 .ok_or_else(|| BrokerError::NotConfigured(provider.to_string()))?
4368 .catalog(provider, context_ref, allow_stale)
4369 .await
4370 }
4371
4372 async fn cloud_invoke(
4373 &self,
4374 provider: CloudProviderId,
4375 context_ref: &str,
4376 model_id: &str,
4377 request: InvokeRequest,
4378 ) -> Result<CloudEventStream, BrokerError> {
4379 self.cloud_backend
4380 .as_ref()
4381 .ok_or_else(|| BrokerError::NotConfigured(provider.to_string()))?
4382 .invoke(provider, context_ref, model_id, request)
4383 .await
4384 }
4385
4386 async fn capabilities(&self) -> Result<Vec<ProviderStatus>, BrokerError> {
4387 let mut out = Vec::new();
4388 let policy = self.policy();
4389 for descriptor in super::provider::registry().iter() {
4390 let accounts = self.accounts(descriptor.id).await.unwrap_or_default();
4391 let configured = match policy.selector(descriptor.id) {
4392 AccountSelector::Account(account) => accounts
4393 .iter()
4394 .any(|a| a.label == account.label_str()),
4395 AccountSelector::Auto => !accounts.is_empty(),
4396 AccountSelector::Invalid { .. } => false,
4397 };
4398 out.push(ProviderStatus {
4399 key: descriptor.id.as_str().to_string(),
4400 name: descriptor.display_name.to_string(),
4401 kind: CredentialKind::OAuth,
4402 configured,
4403 accounts,
4404 });
4405 }
4406 for spec in STATIC_PROVIDERS {
4407 out.push(ProviderStatus {
4408 key: spec.key.to_string(),
4409 name: spec.name.to_string(),
4410 kind: CredentialKind::StaticKey,
4411 configured: static_key_configured(spec.key),
4412 accounts: Vec::new(),
4413 });
4414 }
4415 out.push(ProviderStatus {
4416 key: LOCAL_PROVIDER_KEY.to_string(),
4417 name: "Local endpoint".to_string(),
4418 kind: CredentialKind::LocalEndpoint,
4419 configured: true,
4420 accounts: Vec::new(),
4421 });
4422 Ok(out)
4423 }
4424}
4425
4426pub struct RemoteBroker {
4432 http: reqwest::Client,
4433 endpoint: String,
4434 machine_token: super::MachineToken,
4437 cache: super::TokenCache,
4438 account_policy: Option<AccountPolicy>,
4440}
4441
4442async fn route_missing(resp: reqwest::Response) -> bool {
4446 let json = resp
4447 .headers()
4448 .get(reqwest::header::CONTENT_TYPE)
4449 .and_then(|v| v.to_str().ok())
4450 .is_some_and(|v| v.contains("json"));
4451 !json
4452 && read_body_capped(resp, 1024)
4453 .await
4454 .is_ok_and(|body| body.trim().is_empty())
4455}
4456
4457impl RemoteBroker {
4458 async fn predates_usage_snapshot(&self) -> Result<bool, BrokerError> {
4462 let resp = self
4463 .machine_token
4464 .send(|t| {
4465 self.http
4466 .get(format!("{}/usage/snapshot", self.endpoint))
4467 .query(&[("provider", "anthropic"), ("account", "")])
4468 .bearer_auth(t)
4469 })
4470 .await
4471 .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4472 match resp.status().as_u16() {
4473 401 => Err(BrokerError::Unauthorized),
4474 404 => Ok(route_missing(resp).await),
4475 _ => Ok(false),
4476 }
4477 }
4478
4479 pub fn new(
4480 endpoint: impl Into<String>,
4481 machine_token: impl Into<String>,
4482 http: reqwest::Client,
4483 cache: super::TokenCache,
4484 ) -> Self {
4485 let endpoint = endpoint.into().trim_end_matches('/').to_string();
4486 let http = reqwest::Client::builder()
4489 .redirect(reqwest::redirect::Policy::none())
4490 .connect_timeout(Duration::from_secs(10))
4491 .timeout(Duration::from_secs(90))
4492 .build()
4493 .unwrap_or(http);
4494 let machine_token = super::MachineToken::for_endpoint(&endpoint, machine_token);
4495 Self {
4496 http,
4497 endpoint,
4498 machine_token,
4499 cache,
4500 account_policy: None,
4501 }
4502 }
4503
4504 pub fn with_account_policy(mut self, policy: AccountPolicy) -> Self {
4506 self.account_policy = Some(policy);
4507 self
4508 }
4509
4510 fn policy(&self) -> AccountPolicy {
4511 self.account_policy
4512 .clone()
4513 .unwrap_or_else(AccountPolicy::from_environment)
4514 }
4515
4516 fn fetcher(&self) -> super::BrokerClient {
4517 super::BrokerClient::with_machine_token(
4518 self.endpoint.clone(),
4519 self.machine_token.clone(),
4520 self.http.clone(),
4521 )
4522 }
4523
4524 fn scope(&self) -> String {
4526 super::credential_source::source_scope(&self.endpoint, &self.machine_token.get())
4527 }
4528
4529 async fn resolve_pinned(
4534 &self,
4535 provider: OAuthProviderId,
4536 model: Option<&str>,
4537 ) -> Result<PinnedToken, BrokerError> {
4538 match self.policy().selector(provider) {
4539 AccountSelector::Account(account) => {
4540 let credential = CredentialRef::new(provider, account);
4541 let token = self.access_token_for(&credential).await?;
4542 Ok(PinnedToken { credential, token })
4543 }
4544 AccountSelector::Auto => self.fetch_auto(provider, model).await,
4545 AccountSelector::Invalid { source, reason } => {
4546 Err(BrokerError::InvalidAccount(format!("{source}: {reason}")))
4547 }
4548 }
4549 }
4550
4551 async fn fetch_auto(
4555 &self,
4556 provider: OAuthProviderId,
4557 model: Option<&str>,
4558 ) -> Result<PinnedToken, BrokerError> {
4559 let mut query: Vec<(&str, &str)> = vec![
4560 ("provider", provider.as_str()),
4561 ("account", super::account::AUTO_ACCOUNT_NAME),
4562 ];
4563 if let Some(model) = model {
4564 query.push(("model", model));
4565 }
4566 let resp = self
4567 .machine_token
4568 .send(|t| {
4569 self.http
4570 .get(format!("{}/token", self.endpoint))
4571 .query(&query)
4572 .bearer_auth(t)
4573 })
4574 .await
4575 .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4576 match resp.status().as_u16() {
4577 401 => return Err(BrokerError::Unauthorized),
4578 503 => {
4579 drop(resp);
4580 return Err(BrokerError::NoAccountAvailable {
4581 provider: provider.as_str().to_string(),
4582 reason: "broker reported no account with proven capacity".into(),
4583 failure: Some(super::quota_policy::SelectionFailure::new(
4585 super::quota_policy::SelectionFailureKind::RemoteNoCapacity,
4586 )),
4587 });
4588 }
4589 s if !(200..300).contains(&s) => {
4590 drop(resp);
4591 return Err(BrokerError::Transport(format!("broker returned HTTP {s}")));
4592 }
4593 _ => {}
4594 }
4595 #[derive(Deserialize)]
4596 struct AutoToken {
4597 access_token: String,
4598 expires: u64,
4599 #[serde(default)]
4600 ttl_ms: Option<u64>,
4601 account: Option<String>,
4602 }
4603 let body = read_body_capped(resp, MAX_PROXY_RESPONSE_BYTES).await?;
4604 let tok: AutoToken = serde_json::from_str(&body)
4605 .map_err(|e| BrokerError::Transport(format!("invalid broker token response: {e}")))?;
4606 let account = tok
4607 .account
4608 .as_deref()
4609 .ok_or_else(|| BrokerError::Transport("broker did not report the selected account".into()))
4610 .and_then(|a| Account::parse(a).map_err(BrokerError::InvalidAccount))?;
4611 if tok.access_token.is_empty() {
4612 return Err(BrokerError::Transport("broker returned an empty access_token".into()));
4613 }
4614 let expires = match tok.ttl_ms {
4615 Some(ttl) => crate::epoch_millis().saturating_add(ttl),
4616 None => tok.expires,
4617 };
4618 Ok(PinnedToken {
4619 credential: CredentialRef::new(provider, account),
4620 token: AccessToken {
4621 token: tok.access_token,
4622 expires,
4623 },
4624 })
4625 }
4626
4627 async fn proxy_failure(resp: reqwest::Response, fallback: String) -> BrokerError {
4634 if resp.status() != reqwest::StatusCode::BAD_GATEWAY {
4635 return BrokerError::Transport(fallback);
4636 }
4637 let Ok(body) = read_body_capped(resp, 4096).await else {
4638 return BrokerError::Transport(fallback);
4639 };
4640 let text = serde_json::from_str::<serde_json::Value>(&body)
4641 .ok()
4642 .and_then(|v| v.get("error")?.as_str().map(str::to_string))
4643 .unwrap_or_default();
4644 use super::token::{RefreshFailure, RefreshFailureKind as K};
4645 for kind in [K::Transient, K::LockBusy, K::Other] {
4646 let prefix = BrokerError::Refresh(RefreshFailure::new(kind, "")).to_string();
4647 if text.starts_with(&prefix) {
4648 return BrokerError::Refresh(RefreshFailure::new(
4649 kind,
4650 "the remote credential broker could not refresh this credential",
4651 ));
4652 }
4653 }
4654 BrokerError::Transport(fallback)
4655 }
4656
4657 async fn post_proxy(&self, request: &ProxyRequest) -> Result<reqwest::Response, BrokerError> {
4658 request.validate()?;
4659 let resp = self
4660 .machine_token
4661 .send(|t| {
4662 self.http
4663 .post(format!("{}/proxy", self.endpoint))
4664 .bearer_auth(t)
4665 .json(request)
4666 })
4667 .await
4668 .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4669 match resp.status().as_u16() {
4670 401 => Err(BrokerError::Unauthorized),
4671 s @ (400 | 403) => {
4672 drop(resp);
4676 Err(BrokerError::Denied(format!(
4677 "broker rejected the request (HTTP {s})"
4678 )))
4679 }
4680 _ => Ok(resp),
4681 }
4682 }
4683}
4684
4685#[async_trait]
4686impl CredentialBroker for RemoteBroker {
4687 async fn access_token(&self, provider: OAuthProviderId) -> Result<AccessToken, BrokerError> {
4688 Ok(self.access_token_pinned(provider).await?.token)
4689 }
4690
4691 fn account_selector(&self, provider: OAuthProviderId) -> AccountSelector {
4692 self.policy().selector(provider)
4693 }
4694
4695 async fn access_token_for(&self, cred: &CredentialRef) -> Result<AccessToken, BrokerError> {
4696 let fetcher = self.fetcher();
4697 let tok = super::credential_source::resolve_remote_credential(
4698 &fetcher,
4699 &self.cache,
4700 &self.scope(),
4701 cred,
4702 super::DEFAULT_MARGIN_MS,
4703 )
4704 .await
4705 .map_err(|e| e.into_broker_error(cred))?;
4706 Ok(AccessToken {
4707 token: tok.access_token,
4708 expires: tok.expires,
4709 })
4710 }
4711
4712 async fn access_token_pinned(
4713 &self,
4714 provider: OAuthProviderId,
4715 ) -> Result<PinnedToken, BrokerError> {
4716 self.resolve_pinned(provider, None).await
4717 }
4718
4719 async fn access_token_pinned_for(
4720 &self,
4721 provider: OAuthProviderId,
4722 model: Option<&str>,
4723 ) -> Result<PinnedToken, BrokerError> {
4724 self.resolve_pinned(provider, model).await
4725 }
4726
4727 async fn accounts(&self, provider: OAuthProviderId) -> Result<Vec<AccountSummary>, BrokerError> {
4728 let caps = self.capabilities().await?;
4729 let accounts = caps
4730 .into_iter()
4731 .find(|c| c.key == provider.as_str())
4732 .map(|c| c.accounts)
4733 .unwrap_or_default();
4734 if accounts.is_empty() && self.predates_usage_snapshot().await? {
4737 return Err(BrokerError::BrokerTooOld("account listing".into()));
4738 }
4739 Ok(accounts)
4740 }
4741
4742 async fn usage(&self, cred: &CredentialRef) -> Result<super::usage::UsageSnapshot, BrokerError> {
4743 let resp = self
4744 .machine_token
4745 .send(|t| {
4746 self.http
4747 .get(format!("{}/usage/snapshot", self.endpoint))
4748 .query(&[
4749 ("provider", cred.provider.as_str()),
4750 ("account", cred.account.label_str()),
4751 ])
4752 .bearer_auth(t)
4753 })
4754 .await
4755 .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4756 match resp.status().as_u16() {
4757 401 => Err(BrokerError::Unauthorized),
4758 404 => {
4759 if !route_missing(resp).await {
4760 return Err(BrokerError::UnknownAccount {
4761 provider: cred.provider.as_str().to_string(),
4762 label: cred.account.label_str().to_string(),
4763 });
4764 }
4765 if cred.provider != OAuthProviderId::Anthropic || !cred.account.is_default() {
4766 return Err(BrokerError::BrokerTooOld("per-account usage".into()));
4767 }
4768 let body = self.anthropic_usage().await?.to_string();
4770 super::usage::parse_anthropic_usage(
4771 &body,
4772 cred.account.label_str(),
4773 crate::epoch_millis(),
4774 )
4775 .map_err(super::usage::UsageError::into_broker_error)
4776 }
4777 s if !(200..300).contains(&s) => {
4778 drop(resp);
4780 Err(BrokerError::Transport(format!(
4781 "broker usage snapshot returned HTTP {s}"
4782 )))
4783 }
4784 _ => {
4785 let body = read_body_capped(resp, MAX_PROXY_RESPONSE_BYTES).await?;
4786 let snapshot: super::usage::UsageSnapshot = serde_json::from_str(&body)
4787 .map_err(|e| BrokerError::Transport(format!("invalid usage snapshot: {e}")))?;
4788 if snapshot.provider != cred.provider.as_str()
4789 || snapshot.account != cred.account.label_str()
4790 {
4791 return Err(BrokerError::Transport(
4792 "broker returned a usage snapshot for a different account".into(),
4793 ));
4794 }
4795 Ok(snapshot)
4796 }
4797 }
4798 }
4799
4800 async fn report_cooldown(
4801 &self,
4802 cred: &CredentialRef,
4803 until_ms: Option<u64>,
4804 reason: &str,
4805 ) -> Result<(), BrokerError> {
4806 let resp = self
4807 .machine_token
4808 .send(|t| {
4809 self.http
4810 .post(format!("{}/accounts/cooldown", self.endpoint))
4811 .bearer_auth(t)
4812 .json(&serde_json::json!({
4813 "provider": cred.provider.as_str(),
4814 "account": cred.account.label_str(),
4815 "until_ms": until_ms,
4816 "reason": crate::truncate_str(reason, 64),
4817 }))
4818 })
4819 .await
4820 .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4821 match resp.status().as_u16() {
4822 401 => Err(BrokerError::Unauthorized),
4823 s if !(200..300).contains(&s) => {
4824 Err(BrokerError::Transport(format!("broker returned HTTP {s}")))
4825 }
4826 _ => Ok(()),
4827 }
4828 }
4829
4830 async fn proxy(&self, request: ProxyRequest) -> Result<ProxyResponse, BrokerError> {
4831 let mut request = request;
4832 request.stream = false;
4833 let resp = self.post_proxy(&request).await?;
4834 if !resp.status().is_success() {
4835 let fallback = format!("broker proxy returned HTTP {}", resp.status());
4836 return Err(Self::proxy_failure(resp, fallback).await);
4837 }
4838 let body = read_body_capped(resp, MAX_PROXY_RESPONSE_BYTES).await?;
4839 serde_json::from_str::<ProxyResponse>(&body)
4840 .map_err(|e| BrokerError::Transport(format!("invalid broker proxy response: {e}")))
4841 }
4842
4843 async fn proxy_stream(&self, request: ProxyRequest) -> Result<ProxyByteStream, BrokerError> {
4844 let mut request = request;
4845 request.stream = true;
4846 let resp = self.post_proxy(&request).await?;
4847 let status = resp.status();
4848 if !status.is_success() {
4849 let fallback = format!("broker proxy stream failed: {status}");
4852 return Err(Self::proxy_failure(resp, fallback).await);
4853 }
4854 use futures::StreamExt;
4855 let stream = resp
4856 .bytes_stream()
4857 .map(|chunk| chunk.map_err(|e| BrokerError::Transport(format!("stream error: {e}"))));
4858 Ok(Box::pin(stream))
4859 }
4860
4861 async fn anthropic_usage(&self) -> Result<serde_json::Value, BrokerError> {
4862 let resp = self
4865 .machine_token
4866 .send(|t| {
4867 self.http
4868 .get(format!("{}/usage", self.endpoint))
4869 .bearer_auth(t)
4870 })
4871 .await
4872 .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
4873 match resp.status().as_u16() {
4874 401 => Err(BrokerError::Unauthorized),
4875 s if !(200..300).contains(&s) => {
4876 drop(resp);
4878 Err(BrokerError::Transport(format!(
4879 "broker usage returned HTTP {s}"
4880 )))
4881 }
4882 _ => {
4883 let body = read_body_capped(resp, MAX_PROXY_RESPONSE_BYTES).await?;
4884 serde_json::from_str(&body)
4885 .map_err(|e| BrokerError::Transport(format!("invalid usage response: {e}")))
4886 }
4887 }
4888 }
4889
4890 async fn cloud_catalog(
4891 &self,
4892 provider: CloudProviderId,
4893 context_ref: &str,
4894 allow_stale: bool,
4895 ) -> Result<Vec<CloudCatalogEntry>, BrokerError> {
4896 let resp = self.machine_token.send(|t| self.http.post(format!("{}/cloud/catalog", self.endpoint)).bearer_auth(t).json(&serde_json::json!({"provider":provider,"context_ref":context_ref,"allow_stale":allow_stale}))).await.map_err(|e| BrokerError::Transport(e.to_string()))?;
4897 if resp.status() == reqwest::StatusCode::UNAUTHORIZED {
4898 return Err(BrokerError::Unauthorized);
4899 }
4900 if !resp.status().is_success() {
4901 return Err(BrokerError::Transport(format!(
4902 "cloud catalog returned HTTP {}",
4903 resp.status()
4904 )));
4905 }
4906 let body = read_body_capped(resp, MAX_CLOUD_CATALOG_BODY_BYTES).await?;
4907 let entries: Vec<CloudCatalogEntry> = serde_json::from_str(&body)
4908 .map_err(|e| BrokerError::Transport(format!("invalid cloud catalog: {e}")))?;
4909 if entries.len() > MAX_CLOUD_CATALOG_ENTRIES
4910 || entries.iter().any(|e| {
4911 e.provider != provider || !e.context_ref.starts_with("ctx-") || e.id.len() > 512
4912 })
4913 {
4914 return Err(BrokerError::Transport(
4915 "broker returned an invalid cloud catalog".into(),
4916 ));
4917 }
4918 Ok(entries)
4919 }
4920
4921 async fn cloud_invoke(
4922 &self,
4923 provider: CloudProviderId,
4924 context_ref: &str,
4925 model_id: &str,
4926 request: InvokeRequest,
4927 ) -> Result<CloudEventStream, BrokerError> {
4928 let resp = self.machine_token.send(|t| self.http.post(format!("{}/cloud/invoke", self.endpoint)).bearer_auth(t).json(&serde_json::json!({"provider":provider,"context_ref":context_ref,"model_id":model_id,"request":request}))).await.map_err(|e| BrokerError::Transport(e.to_string()))?;
4929 if !resp.status().is_success() {
4930 return Err(if resp.status() == reqwest::StatusCode::UNAUTHORIZED {
4931 BrokerError::Unauthorized
4932 } else {
4933 BrokerError::Transport(format!("cloud invoke returned HTTP {}", resp.status()))
4934 });
4935 }
4936 use futures::StreamExt;
4937 let chunks = Box::pin(resp.bytes_stream());
4938 let stream = futures::stream::unfold(
4939 (chunks, Vec::<u8>::new(), false, false),
4940 |(mut chunks, mut buffer, done, failed)| async move {
4941 if failed {
4942 return None;
4943 }
4944 loop {
4945 if let Some(end) = buffer.iter().position(|b| *b == b'\n') {
4946 let line: Vec<u8> = buffer.drain(..=end).collect();
4947 let parsed = serde_json::from_slice::<CloudEvent>(&line[..end])
4948 .map_err(|_| BrokerError::Transport("invalid cloud event".into()));
4949 let event = match parsed {
4950 Ok(CloudEvent::Done) if done => Err(BrokerError::Transport(
4951 "duplicate cloud terminal event".into(),
4952 )),
4953 Ok(CloudEvent::Done) => Ok(CloudEvent::Done),
4954 Ok(_) if done => Err(BrokerError::Transport(
4955 "cloud data followed terminal event".into(),
4956 )),
4957 other => other,
4958 };
4959 let next_done = done || matches!(event, Ok(CloudEvent::Done));
4960 let next_failed = event.is_err();
4961 return Some((event, (chunks, buffer, next_done, next_failed)));
4962 }
4963 match chunks.next().await {
4964 Some(Ok(chunk))
4965 if buffer.len() + chunk.len() <= MAX_CLOUD_STREAM_EVENT_BYTES =>
4966 {
4967 buffer.extend_from_slice(&chunk)
4968 }
4969 Some(Ok(_)) => {
4970 return Some((
4971 Err(BrokerError::Transport("cloud event exceeded limit".into())),
4972 (chunks, Vec::new(), done, true),
4973 ))
4974 }
4975 Some(Err(_)) => {
4976 return Some((
4977 Err(BrokerError::Transport("broker stream failed".into())),
4978 (chunks, Vec::new(), done, true),
4979 ))
4980 }
4981 None if buffer.is_empty() && done => return None,
4982 None if buffer.is_empty() => {
4983 return Some((
4984 Err(BrokerError::Transport(
4985 "cloud stream ended without terminal event".into(),
4986 )),
4987 (chunks, Vec::new(), done, true),
4988 ))
4989 }
4990 None => {
4991 return Some((
4992 Err(BrokerError::Transport("truncated cloud event".into())),
4993 (chunks, Vec::new(), done, true),
4994 ))
4995 }
4996 }
4997 }
4998 },
4999 );
5000 Ok(Box::pin(stream))
5001 }
5002
5003 async fn capabilities(&self) -> Result<Vec<ProviderStatus>, BrokerError> {
5004 let resp = self
5005 .machine_token
5006 .send(|t| {
5007 self.http
5008 .get(format!("{}/capabilities", self.endpoint))
5009 .bearer_auth(t)
5010 })
5011 .await
5012 .map_err(|e| BrokerError::Transport(format!("broker request failed: {e}")))?;
5013 match resp.status().as_u16() {
5014 401 => Err(BrokerError::Unauthorized),
5015 s if !(200..300).contains(&s) => {
5016 Err(BrokerError::Transport(format!("broker returned HTTP {s}")))
5017 }
5018 _ => {
5019 let body = read_body_capped(resp, MAX_PROXY_RESPONSE_BYTES).await?;
5020 serde_json::from_str::<Vec<ProviderStatus>>(&body).map_err(|e| {
5021 BrokerError::Transport(format!("invalid capabilities response: {e}"))
5022 })
5023 }
5024 }
5025 }
5026}
5027
5028pub fn preflight_cloud_capability(
5037 provider: CloudProviderId,
5038 needs_tools: bool,
5039) -> Result<(), BrokerError> {
5040 if needs_tools && !provider.supports_tools() {
5041 return Err(BrokerError::UnsupportedCapability {
5042 provider: provider.to_string(),
5043 capability: "tools".into(),
5044 });
5045 }
5046 Ok(())
5047}
5048
5049pub fn local_auth_state(
5055 cache: &super::TokenCache,
5056 query: &super::auth_state::AuthStateQuery,
5057 live: bool,
5058) -> Result<super::auth_state::AuthStateReport, BrokerError> {
5059 let key = (
5060 storage::auth_file_path(),
5061 crate::config::resolve_read_path("config"),
5062 );
5063 let broker = cache
5064 .local_brokers
5065 .lock()
5066 .unwrap_or_else(|e| e.into_inner())
5067 .get(&key)
5068 .cloned();
5069 let mut report = auth_state_report(broker.as_deref(), query)?;
5070 report.live = live;
5071 Ok(report)
5072}
5073
5074fn auth_state_report(
5075 broker: Option<&LocalBroker>,
5076 query: &super::auth_state::AuthStateQuery,
5077) -> Result<super::auth_state::AuthStateReport, BrokerError> {
5078 use super::auth_state::{
5079 AccountState, AuthStateReport, IdentityKind, ObservedOutcome, AUTH_STATE_SCHEMA_VERSION,
5080 };
5081 let provider = match query.provider.as_deref() {
5082 Some(p) => Some(
5083 p.parse::<OAuthProviderId>()
5084 .map_err(|_| BrokerError::UnknownProvider(p.to_string()))?,
5085 ),
5086 None => None,
5087 };
5088 let mut summaries = match provider {
5089 Some(p) => storage::list_accounts(p),
5090 None => storage::list_all_accounts(),
5091 }
5092 .map_err(BrokerError::Credential)?;
5093 if let Some(label) = query.account.as_deref() {
5094 summaries.retain(|s| s.label == label);
5095 if summaries.is_empty() {
5096 return Err(BrokerError::UnknownAccount {
5097 provider: provider.map_or("any provider", |p| p.as_str()).to_string(),
5098 label: label.to_string(),
5099 });
5100 }
5101 }
5102 let creds: Vec<CredentialRef> = summaries
5103 .iter()
5104 .filter_map(AccountSummary::credential_ref)
5105 .collect();
5106 let mut cleared = Vec::new();
5107 if query.clear {
5108 if let Some(broker) = broker {
5109 let whole: Vec<OAuthProviderId> = match (&query.account, provider) {
5110 (Some(_), _) => Vec::new(),
5111 (None, Some(p)) => vec![p],
5112 (None, None) => super::provider::DESCRIPTORS.iter().map(|d| d.id).collect(),
5113 };
5114 broker.clear_state(&creds, &whole);
5115 }
5116 cleared = creds
5117 .iter()
5118 .map(|c| format!("{}:{}", c.provider, c.account.label_str()))
5119 .collect();
5120 tracing::info!(
5121 accounts = %cleared.join(","),
5122 broker_active = broker.is_some(),
5123 "auth_state_cleared"
5124 );
5125 }
5126 let store = storage::auth_file_path();
5127 let now_ms = crate::epoch_millis();
5128 let mut accounts = Vec::with_capacity(summaries.len());
5129 for summary in &summaries {
5130 let Some(cred) = summary.credential_ref() else {
5131 continue;
5132 };
5133 let observed = broker
5134 .and_then(|b| {
5135 b.observations
5136 .lock()
5137 .unwrap_or_else(|e| e.into_inner())
5138 .get(&LocalBroker::state_key(&cred))
5139 .cloned()
5140 })
5141 .unwrap_or_default();
5142 accounts.push(AccountState {
5143 provider: summary.provider.clone(),
5144 label: summary.label.clone(),
5145 identity: if summary.seat_fingerprint.is_some() {
5146 IdentityKind::Stable
5147 } else {
5148 IdentityKind::Rotating
5149 },
5150 access_expires_ms: (summary.expires != 0).then_some(summary.expires),
5151 cooldown: broker.and_then(|b| b.cooldown_state(&cred, now_ms)),
5152 last_usage: observed.last_usage,
5153 last_usage_ok_ms: observed.last_usage_ok_ms,
5154 last_refresh: super::token::last_refresh_exchange(&store, &cred.storage_key()).map(
5155 |x| ObservedOutcome {
5156 at_ms: x.at_ms,
5157 failure: x.failure.map(|kind| {
5158 let failure = refresh_kind_failure(kind);
5159 match x.http_status {
5160 Some(status) => failure.with_status(status),
5161 None => failure,
5162 }
5163 }),
5164 },
5165 ),
5166 refresh_hold: broker.and_then(|b| b.refresh_hold_state(&cred, now_ms)),
5167 });
5168 }
5169 let store_prefix = format!("{}|", store.display());
5170 let mut selections: Vec<_> = broker
5171 .map(|b| {
5172 b.selections
5173 .lock()
5174 .unwrap_or_else(|e| e.into_inner())
5175 .iter()
5176 .filter(|(k, s)| {
5177 k.starts_with(&store_prefix)
5178 && provider.is_none_or(|p| s.provider == p.as_str())
5179 && query
5180 .account
5181 .as_deref()
5182 .is_none_or(|label| selection_involves(s, label))
5183 })
5184 .map(|(_, s)| s.clone())
5185 .collect()
5186 })
5187 .unwrap_or_default();
5188 selections.sort_by_key(|s| std::cmp::Reverse(s.at_ms));
5189 Ok(AuthStateReport {
5190 schema_version: AUTH_STATE_SCHEMA_VERSION,
5191 live: false,
5192 broker_active: broker.is_some(),
5193 generated_at_ms: now_ms,
5194 store: store.display().to_string(),
5195 accounts,
5196 selections,
5197 cleared,
5198 })
5199}
5200
5201fn selection_involves(s: &super::auth_state::SelectionState, label: &str) -> bool {
5204 s.chosen.as_deref() == Some(label)
5205 || s.failure
5206 .as_ref()
5207 .is_some_and(|f| f.accounts.iter().any(|a| a.account == label))
5208}
5209
5210pub fn broker_from_source(
5212 source: &super::CredentialSource,
5213 cache: &super::TokenCache,
5214 http: reqwest::Client,
5215) -> Arc<dyn CredentialBroker> {
5216 match source {
5217 super::CredentialSource::Local => {
5218 let key = (
5222 storage::auth_file_path(),
5223 crate::config::resolve_read_path("config"),
5224 );
5225 let mut brokers = cache
5226 .local_brokers
5227 .lock()
5228 .unwrap_or_else(|e| e.into_inner());
5229 brokers
5230 .entry(key)
5231 .or_insert_with(|| Arc::new(LocalBroker::new(http)))
5232 .clone()
5233 }
5234 super::CredentialSource::Remote {
5235 endpoint,
5236 machine_token,
5237 } => Arc::new(RemoteBroker::new(
5238 endpoint.clone(),
5239 machine_token.clone(),
5240 http,
5241 cache.clone(),
5242 )),
5243 }
5244}
5245
5246static GLOBAL_BROKER: std::sync::RwLock<Option<Arc<dyn CredentialBroker>>> =
5247 std::sync::RwLock::new(None);
5248
5249static GLOBAL_BROKER_INSTALLS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
5250
5251pub fn set_global_broker(broker: Arc<dyn CredentialBroker>) {
5256 *GLOBAL_BROKER.write().expect("broker registry poisoned") = Some(broker);
5257 let n = GLOBAL_BROKER_INSTALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
5258 if crate::core::memstat::mem_trace_enabled() {
5259 tracing::info!(target: "agent_core::memstat", installs = n, "global broker installed");
5260 }
5261}
5262
5263pub fn global_broker_install_count() -> u64 {
5265 GLOBAL_BROKER_INSTALLS.load(std::sync::atomic::Ordering::Relaxed)
5266}
5267
5268pub fn global_broker() -> Arc<dyn CredentialBroker> {
5271 if let Some(b) = GLOBAL_BROKER
5272 .read()
5273 .expect("broker registry poisoned")
5274 .clone()
5275 {
5276 return b;
5277 }
5278 let default: Arc<dyn CredentialBroker> = Arc::new(LocalBroker::new(
5279 reqwest::Client::builder()
5280 .connect_timeout(std::time::Duration::from_secs(10))
5281 .timeout(std::time::Duration::from_secs(300))
5282 .build()
5283 .unwrap_or_default(),
5284 ));
5285 let mut guard = GLOBAL_BROKER.write().expect("broker registry poisoned");
5286 if let Some(b) = guard.clone() {
5287 return b;
5288 }
5289 *guard = Some(default.clone());
5290 default
5291}
5292
5293pub async fn broker_access_token(
5296 provider: OAuthProviderId,
5297 source: &super::CredentialSource,
5298 cache: &super::TokenCache,
5299 http: &reqwest::Client,
5300) -> Result<AccessToken, BrokerError> {
5301 broker_from_source(source, cache, http.clone())
5302 .access_token(provider)
5303 .await
5304}
5305
5306pub fn static_key_status_map() -> BTreeMap<String, StaticKeyStatus> {
5309 STATIC_PROVIDERS
5310 .iter()
5311 .map(|s| (s.key.to_string(), static_key_status(s.key)))
5312 .collect()
5313}
5314
5315#[cfg(test)]
5316mod tests {
5317 use super::*;
5318
5319 #[test]
5320 fn login_availability_handles_named_only_and_explicit_policy() {
5321 let dir = tempfile::tempdir().unwrap();
5322 let path = dir.path().join("auth.json");
5323 let provider = OAuthProviderId::Anthropic;
5324 let empty = AccountPolicy::new();
5325 assert!(!oauth_provider_logged_in_at(&path, &empty, provider));
5326 std::fs::write(&path, serde_json::json!({
5327 "anthropic@work": {"type":"oauth", "access":"synthetic-access",
5328 "refresh":"synthetic-refresh", "expires":1}
5329 }).to_string()).unwrap();
5330 assert!(oauth_provider_logged_in_at(&path, &empty, provider));
5332 for selector in [AccountSelector::Auto,
5333 AccountSelector::parse_or_invalid("../bad", "test"),
5334 AccountSelector::Account(Account::named("work").unwrap())] {
5335 let policy = AccountPolicy::new().with(provider, selector);
5336 assert!(oauth_provider_logged_in_at(&path, &policy, provider));
5337 }
5338 let missing = AccountPolicy::new().with(provider,
5339 AccountSelector::Account(Account::named("missing").unwrap()));
5340 assert!(!oauth_provider_logged_in_at(&path, &missing, provider));
5341 assert!(!oauth_provider_logged_in_at(&path, &empty, OAuthProviderId::OpenAiCodex));
5342 std::fs::write(&path, "{broken").unwrap();
5343 assert!(!oauth_provider_logged_in_at(&path, &empty, provider));
5344 }
5345
5346 #[test]
5351 fn preflight_rejects_tool_requiring_cloud_routes_with_typed_error() {
5352 for provider in [
5353 CloudProviderId::AzureOpenAi,
5354 CloudProviderId::AwsBedrock,
5355 CloudProviderId::GoogleVertex,
5356 ] {
5357 let err = preflight_cloud_capability(provider, true)
5358 .expect_err("tool-requiring cloud route must fail pre-flight");
5359 assert_eq!(
5360 err,
5361 BrokerError::UnsupportedCapability {
5362 provider: provider.to_string(),
5363 capability: "tools".into(),
5364 }
5365 );
5366 assert!(err.to_string().contains("text-only"));
5367 assert_eq!(preflight_cloud_capability(provider, false), Ok(()));
5368 }
5369 }
5370
5371 async fn spawn_ndjson(body: &'static str) -> String {
5372 use axum::{body::Body, routing::post, Router};
5373 use bytes::Bytes;
5374 let app = Router::new().route(
5375 "/cloud/invoke",
5376 post(move || async move {
5377 let chunks = body
5378 .as_bytes()
5379 .chunks(3)
5380 .map(|chunk| Ok::<_, std::convert::Infallible>(Bytes::copy_from_slice(chunk)));
5381 axum::response::Response::builder()
5382 .header("content-type", "application/x-ndjson")
5383 .body(Body::from_stream(futures::stream::iter(chunks)))
5384 .unwrap()
5385 }),
5386 );
5387 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5388 let addr = listener.local_addr().unwrap();
5389 tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
5390 format!("http://{addr}")
5391 }
5392
5393 async fn remote_events(body: &'static str) -> Vec<Result<CloudEvent, BrokerError>> {
5394 use futures::StreamExt;
5395 let broker = RemoteBroker::new(
5396 spawn_ndjson(body).await,
5397 "opaque-machine-token",
5398 reqwest::Client::new(),
5399 super::super::TokenCache::new(),
5400 );
5401 broker
5402 .cloud_invoke(
5403 CloudProviderId::GoogleVertex,
5404 "ctx-opaque",
5405 "google-vertex/publishers/google/models/test",
5406 InvokeRequest {
5407 messages: vec![],
5408 tools: vec![],
5409 stream: true,
5410 options: Default::default(),
5411 },
5412 )
5413 .await
5414 .unwrap()
5415 .collect()
5416 .await
5417 }
5418
5419 #[tokio::test]
5420 async fn remote_ndjson_requires_exactly_one_terminal_and_stops_after_error() {
5421 let valid =
5422 remote_events("{\"type\":\"text_delta\",\"delta\":\"hi\"}\n{\"type\":\"done\"}\n")
5423 .await;
5424 assert_eq!(valid.len(), 2);
5425 assert!(matches!(valid[1], Ok(CloudEvent::Done)));
5426
5427 for body in [
5428 "{\"type\":\"text_delta\",\"delta\":\"hi\"}\n",
5429 "{\"type\":\"done\"}\n{\"type\":\"done\"}\n",
5430 "{\"type\":\"done\"}\n{\"type\":\"text_delta\",\"delta\":\"late\"}\n",
5431 ] {
5432 let events = remote_events(body).await;
5433 assert_eq!(events.iter().filter(|event| event.is_err()).count(), 1);
5434 assert!(events.last().unwrap().is_err());
5435 }
5436 }
5437
5438 #[test]
5441 fn proxy_rejects_unlisted_same_host_paths() {
5442 for (provider, path) in [
5443 ("groq", "/v1/keys"),
5444 ("groq", "/admin"),
5445 ("openrouter", "/api/v1/auth/keys"),
5446 ("local", "/audio/speech"),
5447 ] {
5448 let req = ProxyRequest {
5449 provider: provider.into(),
5450 method: ProxyMethod::Get,
5451 path: path.into(),
5452 body: None,
5453 stream: false,
5454 body_bytes: None,
5455 };
5456 match req.validate() {
5457 Err(BrokerError::Denied(msg)) => {
5458 assert!(msg.contains("allowlist"), "got: {msg}")
5459 }
5460 other => panic!("{provider} {path} must be denied, got {other:?}"),
5461 }
5462 }
5463 for path in ["/models", "/chat/completions"] {
5465 let req = ProxyRequest {
5466 provider: "groq".into(),
5467 method: ProxyMethod::Get,
5468 path: path.into(),
5469 body: None,
5470 stream: false,
5471 body_bytes: None,
5472 };
5473 assert!(req.validate().is_ok(), "{path} must be allowed");
5474 }
5475 }
5476
5477 #[test]
5480 fn proxy_rejects_oversize_request_body() {
5481 let req = ProxyRequest {
5482 provider: "groq".into(),
5483 method: ProxyMethod::Post,
5484 path: "/chat/completions".into(),
5485 body: Some(serde_json::json!({
5486 "blob": "x".repeat(MAX_PROXY_REQUEST_BYTES + 1)
5487 })),
5488 stream: false,
5489 body_bytes: None,
5490 };
5491 match req.validate() {
5492 Err(BrokerError::Denied(msg)) => assert!(msg.contains("byte"), "got: {msg}"),
5493 other => panic!("oversize body must be denied, got {other:?}"),
5494 }
5495 }
5496
5497 #[test]
5501 fn proxy_rejects_oversize_body_bytes_without_json_body() {
5502 let req = ProxyRequest {
5503 provider: "groq".into(),
5504 method: ProxyMethod::Post,
5505 path: "/chat/completions".into(),
5506 body: None,
5507 stream: false,
5508 body_bytes: Some(bytes::Bytes::from(vec![b'x'; MAX_PROXY_REQUEST_BYTES + 1])),
5509 };
5510 match req.validate() {
5511 Err(BrokerError::Denied(msg)) => assert!(msg.contains("byte"), "got: {msg}"),
5512 other => panic!("oversize body_bytes must be denied, got {other:?}"),
5513 }
5514 }
5515
5516 #[test]
5520 fn post_json_exact_sets_coherent_body_and_bytes() {
5521 let value = serde_json::json!({"model": "m", "stream": true});
5522 let (req, digest_bytes) =
5523 ProxyRequest::post_json_exact("groq", "/chat/completions", value.clone(), true)
5524 .expect("serializable body must construct");
5525 assert_eq!(req.method, ProxyMethod::Post);
5526 assert!(req.stream);
5527 assert_eq!(req.body.as_ref(), Some(&value));
5528 let stored = req.body_bytes.as_ref().expect("bytes must be set");
5529 assert_eq!(stored, &digest_bytes, "caller digest bytes == wire bytes");
5530 assert_eq!(
5531 serde_json::from_slice::<serde_json::Value>(stored).unwrap(),
5532 value
5533 );
5534 req.validate().expect("coherent request must validate");
5535 }
5536
5537 #[test]
5541 fn mismatched_body_bytes_rejected_by_debug_validation() {
5542 let req = ProxyRequest {
5543 provider: "groq".into(),
5544 method: ProxyMethod::Post,
5545 path: "/chat/completions".into(),
5546 body: Some(serde_json::json!({"model": "claimed"})),
5547 stream: true,
5548 body_bytes: Some(bytes::Bytes::from_static(b"{\"model\":\"actually-sent\"}")),
5549 };
5550 match req.validate() {
5551 Err(BrokerError::Denied(msg)) => {
5552 assert!(msg.contains("body_bytes"), "got: {msg}");
5553 }
5554 other => panic!("incoherent handoff must be denied, got {other:?}"),
5555 }
5556 }
5557
5558 #[test]
5559 fn proxy_request_validation_fails_closed() {
5560 let ok = ProxyRequest {
5561 provider: "groq".into(),
5562 method: ProxyMethod::Post,
5563 path: "/chat/completions".into(),
5564 body: None,
5565 stream: false,
5566 body_bytes: None,
5567 };
5568 assert!(ok.validate().is_ok());
5569
5570 let unknown = ProxyRequest {
5571 provider: "evil".into(),
5572 ..ok.clone()
5573 };
5574 assert!(matches!(
5575 unknown.validate(),
5576 Err(BrokerError::UnknownProvider(_))
5577 ));
5578
5579 let absolute = ProxyRequest {
5580 path: "https://evil.example/x".into(),
5581 ..ok.clone()
5582 };
5583 assert!(matches!(absolute.validate(), Err(BrokerError::Denied(_))));
5584
5585 let traversal = ProxyRequest {
5586 path: "/../secrets".into(),
5587 ..ok.clone()
5588 };
5589 assert!(matches!(traversal.validate(), Err(BrokerError::Denied(_))));
5590
5591 let relative = ProxyRequest {
5592 path: "chat/completions".into(),
5593 ..ok
5594 };
5595 assert!(matches!(relative.validate(), Err(BrokerError::Denied(_))));
5596 }
5597
5598 #[test]
5601 fn proxy_allows_only_pinned_kimi_code_paths() {
5602 let chat = ProxyRequest {
5603 provider: "kimi-code".into(),
5604 method: ProxyMethod::Post,
5605 path: "/chat/completions".into(),
5606 body: None,
5607 stream: true,
5608 body_bytes: None,
5609 };
5610 assert!(chat.validate().is_ok(), "kimi-code chat must be allowed");
5611
5612 for path in ["/models", "/me", "/usages"] {
5613 let get = ProxyRequest {
5614 provider: "kimi-code".into(),
5615 method: ProxyMethod::Get,
5616 path: path.into(),
5617 body: None,
5618 stream: false,
5619 body_bytes: None,
5620 };
5621 assert!(get.validate().is_ok(), "kimi-code GET {path} must pass");
5622 let post = ProxyRequest {
5624 method: ProxyMethod::Post,
5625 ..get
5626 };
5627 assert!(
5628 matches!(post.validate(), Err(BrokerError::Denied(_))),
5629 "kimi-code POST {path} must be denied"
5630 );
5631 }
5632
5633 for path in ["/feedback", "/search", "/fetch", "/completions"] {
5635 let denied = ProxyRequest {
5636 provider: "kimi-code".into(),
5637 method: ProxyMethod::Post,
5638 path: path.into(),
5639 body: None,
5640 stream: false,
5641 body_bytes: None,
5642 };
5643 assert!(
5644 matches!(denied.validate(), Err(BrokerError::Denied(_))),
5645 "kimi-code {path} must be denied"
5646 );
5647 }
5648
5649 let static_kimi = ProxyRequest {
5652 provider: "kimi".into(),
5653 method: ProxyMethod::Post,
5654 path: "/chat/completions".into(),
5655 body: None,
5656 stream: false,
5657 body_bytes: None,
5658 };
5659 assert!(static_kimi.validate().is_ok());
5660 }
5661
5662 #[test]
5666 fn proxy_rejects_oauth_providers_except_codex_catalog() {
5667 let claude = ProxyRequest {
5669 provider: "claude".into(),
5670 method: ProxyMethod::Get,
5671 path: "/models".into(),
5672 body: None,
5673 stream: false,
5674 body_bytes: None,
5675 };
5676 assert!(
5677 matches!(claude.validate(), Err(BrokerError::UnknownProvider(_))),
5678 "claude alias must not be proxyable"
5679 );
5680 let anth = ProxyRequest {
5682 provider: "anthropic".into(),
5683 method: ProxyMethod::Get,
5684 path: "/models".into(),
5685 body: None,
5686 stream: false,
5687 body_bytes: None,
5688 };
5689 assert!(
5690 matches!(anth.validate(), Err(BrokerError::Denied(_))),
5691 "anthropic /models must be denied"
5692 );
5693 let bare = ProxyRequest {
5695 provider: "openai-codex".into(),
5696 method: ProxyMethod::Get,
5697 path: "/models".into(),
5698 body: None,
5699 stream: false,
5700 body_bytes: None,
5701 };
5702 assert!(
5703 matches!(bare.validate(), Err(BrokerError::Denied(_))),
5704 "openai-codex /models without client_version must be denied"
5705 );
5706 }
5707
5708 #[test]
5712 fn proxy_allows_only_pinned_openai_codex_models_path() {
5713 let allowed = ProxyRequest {
5714 provider: "openai-codex".into(),
5715 method: ProxyMethod::Get,
5716 path: "/codex/models?client_version=0.153.3".into(),
5717 body: None,
5718 stream: false,
5719 body_bytes: None,
5720 };
5721 assert!(
5722 allowed.validate().is_ok(),
5723 "exact codex models path must be allowed"
5724 );
5725
5726 for path in [
5727 "/codex/models",
5728 "/codex/models?client_version=",
5729 "/codex/models?foo=1",
5730 "/codex/models?client_version=0.153.3&extra=1",
5731 "/codex/responses",
5732 "/models",
5733 "/models?client_version=0.153.3",
5734 "/backend-api/codex/models?client_version=0.153.3",
5735 "/v1/models?client_version=0.153.3",
5736 "/chat/completions",
5737 ] {
5738 let req = ProxyRequest {
5739 provider: "openai-codex".into(),
5740 method: ProxyMethod::Get,
5741 path: path.into(),
5742 body: None,
5743 stream: false,
5744 body_bytes: None,
5745 };
5746 assert!(
5747 matches!(req.validate(), Err(BrokerError::Denied(_))),
5748 "openai-codex path {path} must be denied"
5749 );
5750 }
5751
5752 let post = ProxyRequest {
5754 provider: "openai-codex".into(),
5755 method: ProxyMethod::Post,
5756 path: "/codex/models?client_version=0.153.3".into(),
5757 body: None,
5758 stream: false,
5759 body_bytes: None,
5760 };
5761 assert!(
5762 matches!(post.validate(), Err(BrokerError::Denied(_))),
5763 "POST openai-codex models must be denied"
5764 );
5765 }
5766
5767 #[test]
5769 fn proxy_allows_only_pinned_github_copilot_paths() {
5770 let models = ProxyRequest {
5771 provider: "github-copilot".into(),
5772 method: ProxyMethod::Get,
5773 path: "/models".into(),
5774 body: None,
5775 stream: false,
5776 body_bytes: None,
5777 };
5778 assert!(models.validate().is_ok());
5779
5780 let chat = ProxyRequest {
5781 provider: "github-copilot".into(),
5782 method: ProxyMethod::Post,
5783 path: "/chat/completions".into(),
5784 body: None,
5785 stream: false,
5786 body_bytes: None,
5787 };
5788 assert!(chat.validate().is_ok());
5789 let responses = ProxyRequest {
5790 provider: "github-copilot".into(),
5791 method: ProxyMethod::Post,
5792 path: "/responses".into(),
5793 body: None,
5794 stream: false,
5795 body_bytes: None,
5796 };
5797 assert!(responses.validate().is_ok());
5798 for path in ["/v1/messages", "/models?x=1", "/embeddings"] {
5799 let request = ProxyRequest {
5800 provider: "github-copilot".into(),
5801 method: ProxyMethod::Post,
5802 path: path.into(),
5803 body: None,
5804 stream: false,
5805 body_bytes: None,
5806 };
5807 assert!(matches!(request.validate(), Err(BrokerError::Denied(_))));
5808 }
5809 }
5810
5811 #[test]
5813 fn proxy_allows_only_pinned_anthropic_models_path() {
5814 for path in [
5815 "/v1/models",
5816 "/v1/models?limit=100",
5817 "/v1/models?limit=100&after_id=claude-opus-4-7",
5818 ] {
5819 let req = ProxyRequest {
5820 provider: "anthropic".into(),
5821 method: ProxyMethod::Get,
5822 path: path.into(),
5823 body: None,
5824 stream: false,
5825 body_bytes: None,
5826 };
5827 assert!(req.validate().is_ok(), "{path} must be allowed");
5828 }
5829 for path in [
5830 "/v1/messages",
5831 "/models",
5832 "/v1/models?foo=1",
5833 "/v1/models?limit=",
5834 "/api/oauth/usage",
5835 "/v1/models?limit=100&evil=1",
5836 "/v1/models?limit=100&limit=50",
5837 "/v1/models?after_id=a&after_id=b",
5838 "/v1/models?limit=100&after_id=a&limit=50",
5839 ] {
5840 let req = ProxyRequest {
5841 provider: "anthropic".into(),
5842 method: ProxyMethod::Get,
5843 path: path.into(),
5844 body: None,
5845 stream: false,
5846 body_bytes: None,
5847 };
5848 assert!(
5849 matches!(req.validate(), Err(BrokerError::Denied(_))),
5850 "anthropic path {path} must be denied"
5851 );
5852 }
5853 }
5854
5855 #[test]
5856 fn anthropic_oauth_catalog_headers_are_bearer_version_beta_not_x_api_key() {
5857 let headers = anthropic_oauth_catalog_request_headers();
5858 let names: Vec<_> = headers.iter().map(|(n, _)| *n).collect();
5859 assert!(names.contains(&"anthropic-version"));
5860 assert!(names.contains(&"anthropic-beta"));
5861 assert!(names.contains(&"accept"));
5862 assert!(!names.iter().any(|n| n.eq_ignore_ascii_case("x-api-key")));
5863 assert!(!names
5864 .iter()
5865 .any(|n| n.eq_ignore_ascii_case("authorization")));
5866 let beta = headers
5867 .iter()
5868 .find(|(n, _)| *n == "anthropic-beta")
5869 .map(|(_, v)| *v)
5870 .expect("beta");
5871 assert_eq!(beta, "oauth-2025-04-20");
5872 }
5873
5874 #[test]
5875 fn anthropic_models_path_round_trip_and_rejects_duplicates() {
5876 assert!(is_allowed_anthropic_path("/v1/models"));
5877 assert!(is_allowed_anthropic_path("/v1/models?limit=100"));
5878 assert!(is_allowed_anthropic_path(
5879 "/v1/models?limit=100&after_id=claude-opus-4-7"
5880 ));
5881 assert!(!is_allowed_anthropic_path("/v1/models?limit=100&limit=50"));
5882 assert!(!is_allowed_anthropic_path(
5883 "/v1/models?after_id=a&after_id=b"
5884 ));
5885 let page0 = "/v1/models?limit=100";
5887 let page1 = "/v1/models?limit=100&after_id=model-1";
5888 assert!(is_allowed_anthropic_path(page0));
5889 assert!(is_allowed_anthropic_path(page1));
5890 }
5891
5892 #[test]
5894 fn proxy_allows_only_pinned_google_gemini_paths() {
5895 for path in [
5896 "/v1internal:loadCodeAssist",
5897 "/v1internal:onboardUser",
5898 "/v1internal:streamGenerateContent",
5899 "/v1internal:countTokens",
5900 "/v1internal/operations/op-12345",
5901 ] {
5902 let req = ProxyRequest {
5903 provider: "google-gemini".into(),
5904 method: ProxyMethod::Post,
5905 path: path.into(),
5906 body: None,
5907 stream: false,
5908 body_bytes: None,
5909 };
5910 assert!(req.validate().is_ok(), "{path} must be allowed");
5911 }
5912 for path in [
5913 "/v1internal:listExperiments",
5915 "/v1internal:fetchAdminControls",
5916 "/v1internal:setCodeAssistGlobalUserSetting",
5917 "/v1internal:generateContent",
5918 "/v2/models",
5920 "/v1beta/models",
5921 "/",
5923 "/v1internal:",
5924 ] {
5925 let req = ProxyRequest {
5926 provider: "google-gemini".into(),
5927 method: ProxyMethod::Post,
5928 path: path.into(),
5929 body: None,
5930 stream: false,
5931 body_bytes: None,
5932 };
5933 assert!(
5934 matches!(req.validate(), Err(BrokerError::Denied(_))),
5935 "{path} must be denied"
5936 );
5937 }
5938 }
5939
5940 #[test]
5941 fn access_token_type_has_no_refresh_field() {
5942 let t: AccessToken =
5945 serde_json::from_str(r#"{"token":"sk-x","expires":123,"refresh":"MUST-NOT-EXIST"}"#)
5946 .unwrap();
5947 let round = serde_json::to_value(&t).unwrap();
5948 assert_eq!(round.get("refresh"), None);
5949 assert_eq!(round["token"], "sk-x");
5950 }
5951
5952 #[test]
5953 fn broker_error_display_never_echoes_values() {
5954 let e = BrokerError::NotConfigured("groq".into());
5956 assert!(!format!("{e}").contains("sk-"));
5957 let e = BrokerError::Unauthorized;
5958 let shown = format!("{e}");
5959 assert!(shown.starts_with("broker rejected machine auth"), "{shown}");
5960 assert!(shown.contains("auth.machine_token") && shown.contains("SYNAPS_MACHINE_TOKEN"), "{shown}");
5962 assert!(shown.contains("auth-broker.token") && shown.contains("restart"), "{shown}");
5963 }
5964
5965 #[test]
5966 fn mask_key_is_short_and_lossy() {
5967 assert_eq!(mask_key("gsk-live-1234567890abcdef"), "gsk-…cdef");
5968 assert_eq!(mask_key("short"), "…");
5969 }
5970
5971 #[test]
5972 fn proxy_request_serde_roundtrip() {
5973 let req = ProxyRequest {
5974 provider: "openrouter".into(),
5975 method: ProxyMethod::Get,
5976 path: "/models".into(),
5977 body: Some(serde_json::json!({"a": 1})),
5978 stream: true,
5979 body_bytes: None,
5980 };
5981 let json = serde_json::to_string(&req).unwrap();
5982 let back: ProxyRequest = serde_json::from_str(&json).unwrap();
5983 assert_eq!(back.provider, "openrouter");
5984 assert!(back.stream);
5985 assert!(matches!(back.method, ProxyMethod::Get));
5986 }
5987
5988 async fn spawn_upstream(app: axum::Router) -> String {
5991 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5992 let addr = listener.local_addr().unwrap();
5993 tokio::spawn(async move {
5994 axum::serve(listener, app).await.unwrap();
5995 });
5996 format!("http://{addr}")
5997 }
5998
5999 #[tokio::test]
6003 async fn local_broker_streams_local_endpoint_and_applies_key() {
6004 use axum::routing::post;
6005 let seen_auth = Arc::new(std::sync::Mutex::new(String::new()));
6006 let seen = seen_auth.clone();
6007 let app = axum::Router::new().route(
6008 "/chat/completions",
6009 post(move |headers: axum::http::HeaderMap| {
6010 let seen = seen.clone();
6011 async move {
6012 *seen.lock().unwrap() = headers
6013 .get("authorization")
6014 .and_then(|v| v.to_str().ok())
6015 .unwrap_or("")
6016 .to_string();
6017 (
6018 [("content-type", "text/event-stream")],
6019 "data: {\"x\":1}\n\ndata: [DONE]\n\n",
6020 )
6021 }
6022 }),
6023 );
6024 let url = spawn_upstream(app).await;
6025 let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6026 let mut stream = broker
6027 .proxy_stream(ProxyRequest {
6028 provider: LOCAL_PROVIDER_KEY.into(),
6029 method: ProxyMethod::Post,
6030 path: "/chat/completions".into(),
6031 body: Some(serde_json::json!({"model": "m"})),
6032 stream: true,
6033 body_bytes: None,
6034 })
6035 .await
6036 .expect("stream must open");
6037 use futures::StreamExt;
6038 let mut collected = Vec::new();
6039 while let Some(chunk) = stream.next().await {
6040 collected.extend_from_slice(&chunk.unwrap());
6041 }
6042 let text = String::from_utf8(collected).unwrap();
6043 assert!(text.contains("data: {\"x\":1}"));
6044 assert!(text.contains("[DONE]"));
6045 assert_eq!(&*seen_auth.lock().unwrap(), "Bearer local");
6047 }
6048
6049 #[tokio::test]
6052 async fn local_broker_stream_error_is_typed_and_keyless() {
6053 use axum::routing::post;
6054 let app = axum::Router::new().route(
6055 "/chat/completions",
6056 post(|| async {
6057 (
6058 axum::http::StatusCode::UNAUTHORIZED,
6059 "{\"error\":\"bad key\"}",
6060 )
6061 }),
6062 );
6063 let url = spawn_upstream(app).await;
6064 let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6065 let err = match broker
6066 .proxy_stream(ProxyRequest {
6067 provider: LOCAL_PROVIDER_KEY.into(),
6068 method: ProxyMethod::Post,
6069 path: "/chat/completions".into(),
6070 body: None,
6071 stream: true,
6072 body_bytes: None,
6073 })
6074 .await
6075 {
6076 Err(e) => e,
6077 Ok(_) => panic!("non-2xx upstream must not yield a stream"),
6078 };
6079 let msg = format!("{err}");
6080 assert!(msg.contains("401"), "got: {msg}");
6081 assert!(
6082 !msg.to_lowercase().contains("bearer"),
6083 "no auth material in errors"
6084 );
6085 }
6086
6087 #[tokio::test]
6089 async fn local_broker_proxy_returns_status_and_body() {
6090 use axum::routing::get;
6091 let app =
6092 axum::Router::new().route("/models", get(|| async { "{\"data\":[{\"id\":\"m1\"}]}" }));
6093 let url = spawn_upstream(app).await;
6094 let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6095 let resp = broker
6096 .proxy(ProxyRequest {
6097 provider: LOCAL_PROVIDER_KEY.into(),
6098 method: ProxyMethod::Get,
6099 path: "/models".into(),
6100 body: None,
6101 stream: false,
6102 body_bytes: None,
6103 })
6104 .await
6105 .unwrap();
6106 assert_eq!(resp.status, 200);
6107 assert!(resp.body.contains("m1"));
6108 }
6109
6110 #[tokio::test]
6113 async fn local_broker_proxy_rejects_oversize_response() {
6114 use axum::routing::get;
6115 let app = axum::Router::new().route("/models", get(|| async { "x".repeat(4096) }));
6116 let url = spawn_upstream(app).await;
6117 let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url)
6118 .with_max_response_bytes(64);
6119 let err = broker
6120 .proxy(ProxyRequest {
6121 provider: LOCAL_PROVIDER_KEY.into(),
6122 method: ProxyMethod::Get,
6123 path: "/models".into(),
6124 body: None,
6125 stream: false,
6126 body_bytes: None,
6127 })
6128 .await
6129 .expect_err("oversize body must be rejected");
6130 let msg = format!("{err}");
6131 assert!(msg.contains("limit"), "got: {msg}");
6132 }
6133
6134 #[tokio::test]
6137 async fn local_broker_stream_error_is_status_only() {
6138 use axum::routing::post;
6139 let app = axum::Router::new().route(
6140 "/chat/completions",
6141 post(|| async {
6142 (
6143 axum::http::StatusCode::INTERNAL_SERVER_ERROR,
6144 format!("\x1b[2J{}", "E".repeat(64 * 1024)),
6145 )
6146 }),
6147 );
6148 let url = spawn_upstream(app).await;
6149 let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6150 let err = broker
6151 .proxy_stream(ProxyRequest {
6152 provider: LOCAL_PROVIDER_KEY.into(),
6153 method: ProxyMethod::Post,
6154 path: "/chat/completions".into(),
6155 body: None,
6156 stream: true,
6157 body_bytes: None,
6158 })
6159 .await
6160 .err()
6161 .expect("non-2xx upstream must not yield a stream");
6162 let msg = format!("{err}");
6163 assert!(
6164 msg.len() < 700,
6165 "error must be bounded, got {} bytes",
6166 msg.len()
6167 );
6168 assert!(!msg.contains('\x1b'), "control chars must be stripped");
6169 assert!(msg.contains("500"));
6170 }
6171
6172 #[tokio::test]
6175 async fn local_broker_buffered_request_times_out() {
6176 use axum::routing::get;
6177 let app = axum::Router::new().route(
6178 "/models",
6179 get(|| async {
6180 tokio::time::sleep(Duration::from_secs(30)).await;
6181 "too late"
6182 }),
6183 );
6184 let url = spawn_upstream(app).await;
6185 let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url)
6186 .with_request_timeout(Duration::from_millis(200));
6187 let started = std::time::Instant::now();
6188 let err = broker
6189 .proxy(ProxyRequest {
6190 provider: LOCAL_PROVIDER_KEY.into(),
6191 method: ProxyMethod::Get,
6192 path: "/models".into(),
6193 body: None,
6194 stream: false,
6195 body_bytes: None,
6196 })
6197 .await
6198 .expect_err("hung upstream must time out");
6199 assert!(matches!(err, BrokerError::Transport(_)));
6200 assert!(
6201 started.elapsed() < Duration::from_secs(5),
6202 "timeout must be enforced promptly"
6203 );
6204 }
6205
6206 #[tokio::test]
6209 async fn global_broker_defaults_to_in_process_local() {
6210 let broker = global_broker();
6211 let caps = broker
6212 .capabilities()
6213 .await
6214 .expect("in-process broker needs no daemon");
6215 assert!(caps.iter().any(|c| c.key == LOCAL_PROVIDER_KEY));
6216 let again = global_broker();
6218 assert!(again.capabilities().await.is_ok());
6219 }
6220
6221 const HOSTILE_SENTINEL: &str = "ZX9-HOSTILE-SENTINEL-7Q";
6231
6232 fn hostile_body() -> String {
6235 format!(
6236 "{{\"error\":{{\"message\":\"ECHOED {HOSTILE_SENTINEL} \
6237 {{\\\"messages\\\":[{{\\\"content\\\":\\\"my secret prompt\\\"}}],\
6238 \\\"tools\\\":[{{\\\"input_schema\\\":{{}}}}]}}\"}}}}"
6239 )
6240 }
6241
6242 fn hostile_broker_json() -> String {
6245 format!(
6246 "{{\"error\":\"ECHOED {HOSTILE_SENTINEL} input_schema \
6247 my secret prompt\"}}"
6248 )
6249 }
6250
6251 fn assert_no_hostile_bytes(msg: &str) {
6252 assert!(
6253 !msg.contains(HOSTILE_SENTINEL),
6254 "sentinel leaked into error: {msg}"
6255 );
6256 assert!(!msg.contains("ECHOED"), "echoed body leaked: {msg}");
6257 assert!(
6258 !msg.contains("input_schema"),
6259 "request-shaped data leaked: {msg}"
6260 );
6261 assert!(
6262 !msg.contains("secret prompt"),
6263 "prompt content leaked: {msg}"
6264 );
6265 }
6266
6267 #[tokio::test]
6272 async fn local_broker_stream_error_omits_upstream_body() {
6273 use axum::routing::post;
6274 let app = axum::Router::new().route(
6275 "/chat/completions",
6276 post(|| async {
6277 (
6278 axum::http::StatusCode::TOO_MANY_REQUESTS,
6279 format!("\x1b[2J{}", hostile_body()),
6280 )
6281 }),
6282 );
6283 let url = spawn_upstream(app).await;
6284 let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6285 let err = broker
6286 .proxy_stream(ProxyRequest {
6287 provider: LOCAL_PROVIDER_KEY.into(),
6288 method: ProxyMethod::Post,
6289 path: "/chat/completions".into(),
6290 body: None,
6291 stream: true,
6292 body_bytes: None,
6293 })
6294 .await
6295 .err()
6296 .expect("non-2xx upstream must not yield a stream");
6297 assert!(matches!(err, BrokerError::Transport(_)));
6298 let msg = format!("{err}");
6299 assert!(
6300 msg.contains("provider request failed: 429 Too Many Requests"),
6301 "status prefix must survive for engine classification: {msg}"
6302 );
6303 assert!(
6304 !msg.contains('\x1b'),
6305 "escapes must not reach errors: {msg}"
6306 );
6307 assert_no_hostile_bytes(&msg);
6308 }
6309
6310 async fn stream_error_message(status: axum::http::StatusCode, body: String) -> String {
6313 use axum::routing::post;
6314 let app = axum::Router::new().route(
6315 "/chat/completions",
6316 post(move || async move { (status, body) }),
6317 );
6318 let url = spawn_upstream(app).await;
6319 let broker = LocalBroker::with_local_base_url(reqwest::Client::new(), url);
6320 let err = broker
6321 .proxy_stream(ProxyRequest {
6322 provider: LOCAL_PROVIDER_KEY.into(),
6323 method: ProxyMethod::Post,
6324 path: "/chat/completions".into(),
6325 body: None,
6326 stream: true,
6327 body_bytes: None,
6328 })
6329 .await
6330 .err()
6331 .expect("non-2xx upstream must not yield a stream");
6332 assert!(matches!(err, BrokerError::Transport(_)));
6333 format!("{err}")
6334 }
6335
6336 fn after_marker(msg: &str) -> &str {
6338 let marker = "provider request failed: ";
6339 let idx = msg.find(marker).expect("marker present");
6340 &msg[idx + marker.len()..]
6341 }
6342
6343 #[tokio::test]
6349 async fn local_broker_stream_403_kimi_quota_surfaces_vetted_label_only() {
6350 let kimi_body = "{\"error\":{\"type\":\"access_terminated_error\",\"message\":\
6351 \"You've reached your weekly (7-day) usage limit. Your quota will \
6352 reset when the current 7-day window ends. \"}}";
6353 let msg = stream_error_message(axum::http::StatusCode::FORBIDDEN, kimi_body.into()).await;
6354 assert!(
6355 msg.ends_with("provider request failed: 403 Forbidden [access_terminated_error]"),
6356 "expected exact labelled prefix: {msg}"
6357 );
6358 assert!(!msg.contains("weekly"), "body text leaked: {msg}");
6359 assert!(!msg.contains("quota"), "body text leaked: {msg}");
6360 assert!(!msg.contains('{'), "raw JSON leaked: {msg}");
6361 assert!(
6362 !after_marker(&msg).contains(':'),
6363 "suffix must not contain ':' (engine truncates there): {msg}"
6364 );
6365 }
6366
6367 #[tokio::test]
6370 async fn local_broker_stream_429_unvetted_type_yields_no_label() {
6371 let body = format!(
6372 "{{\"error\":{{\"type\":\"<script>ECHO:secret\",\"message\":\"ECHOED {HOSTILE_SENTINEL}\"}}}}"
6373 );
6374 let msg = stream_error_message(axum::http::StatusCode::TOO_MANY_REQUESTS, body).await;
6375 assert!(
6376 msg.ends_with("provider request failed: 429 Too Many Requests"),
6377 "no label expected: {msg}"
6378 );
6379 assert!(!msg.contains('['), "unexpected label: {msg}");
6380 assert!(!msg.contains("script"), "hostile type leaked: {msg}");
6381 assert!(!msg.contains("secret"), "hostile type leaked: {msg}");
6382 assert!(!after_marker(&msg).contains(':'), "got: {msg}");
6383 assert_no_hostile_bytes(&msg);
6384 }
6385
6386 #[tokio::test]
6388 async fn local_broker_stream_html_body_yields_no_label() {
6389 let body =
6390 format!("<html><body>ECHOED {HOSTILE_SENTINEL} access_terminated_error</body></html>");
6391 let msg = stream_error_message(axum::http::StatusCode::BAD_GATEWAY, body).await;
6392 assert!(
6393 msg.ends_with("provider request failed: 502 Bad Gateway"),
6394 "no label expected: {msg}"
6395 );
6396 assert!(!msg.contains("html"), "body leaked: {msg}");
6397 assert!(!msg.contains('['), "unexpected label: {msg}");
6398 assert_no_hostile_bytes(&msg);
6399 }
6400
6401 #[tokio::test]
6404 async fn local_broker_stream_oversized_body_yields_no_label_without_panic() {
6405 let pad = "x".repeat(MAX_PROXY_ERROR_CLASSIFY_BYTES * 4);
6408 let body = format!(
6409 "{{\"error\":{{\"message\":\"ECHOED {HOSTILE_SENTINEL} {pad}\",\"type\":\"access_terminated_error\"}}}}"
6410 );
6411 let msg = stream_error_message(axum::http::StatusCode::FORBIDDEN, body).await;
6412 assert!(
6413 msg.ends_with("provider request failed: 403 Forbidden"),
6414 "no label expected for oversized body: {msg}"
6415 );
6416 assert!(!msg.contains('['), "unexpected label: {msg}");
6417 assert!(
6418 msg.len() < 200,
6419 "oversized body leaked: {} bytes",
6420 msg.len()
6421 );
6422 assert_no_hostile_bytes(&msg);
6423 }
6424
6425 #[tokio::test]
6429 async fn read_error_body_for_classification_truncates_and_never_fails() {
6430 use axum::routing::get;
6431 let app = axum::Router::new()
6432 .route("/big", get(|| async { "a".repeat(100) }))
6433 .route("/utf8", get(|| async { format!("{}é", "a".repeat(9)) }));
6434 let url = spawn_upstream(app).await;
6435 let client = reqwest::Client::new();
6436
6437 let resp = client.get(format!("{url}/big")).send().await.unwrap();
6438 let got = read_error_body_for_classification(resp, 10).await;
6439 assert_eq!(got.as_deref(), Some("aaaaaaaaaa"));
6440
6441 let resp = client.get(format!("{url}/utf8")).send().await.unwrap();
6443 let got = read_error_body_for_classification(resp, 10).await;
6444 assert_eq!(got, None);
6445
6446 let resp = client.get(format!("{url}/utf8")).send().await.unwrap();
6448 let got = read_error_body_for_classification(resp, 1024).await;
6449 assert_eq!(got.as_deref(), Some("aaaaaaaaaé"));
6450 }
6451
6452 #[tokio::test]
6456 #[serial_test::serial]
6457 async fn local_broker_usage_error_omits_upstream_body() {
6458 use axum::routing::get;
6459 let app = axum::Router::new().route(
6460 "/usage",
6461 get(|| async {
6462 (
6463 axum::http::StatusCode::INTERNAL_SERVER_ERROR,
6464 hostile_body(),
6465 )
6466 }),
6467 );
6468 let url = spawn_upstream(app).await;
6469
6470 let dir = tempfile::tempdir().unwrap();
6476 let base = dir.path().join(".synaps-cli");
6477 std::fs::create_dir_all(&base).unwrap();
6478 let creds = crate::auth::OAuthCredentials {
6479 auth_type: "oauth".into(),
6480 refresh: String::new(),
6481 access: "test-access-token".into(),
6482 expires: crate::epoch_millis() + 3_600_000,
6483 account_id: None,
6484 };
6485 super::super::storage::save_provider_auth_at_test_hook(
6486 &base.join("auth.json"),
6487 "anthropic",
6488 &creds,
6489 )
6490 .unwrap();
6491 let old_base = std::env::var("SYNAPS_BASE_DIR").ok();
6492 let old_home = std::env::var("HOME").ok();
6493 std::env::remove_var("SYNAPS_BASE_DIR");
6494 std::env::set_var("HOME", dir.path());
6495
6496 let broker = LocalBroker::new(reqwest::Client::new())
6497 .with_anthropic_usage_url(format!("{url}/usage"));
6498 let result = broker.anthropic_usage().await;
6499
6500 match old_home {
6501 Some(v) => std::env::set_var("HOME", v),
6502 None => std::env::remove_var("HOME"),
6503 }
6504 if let Some(v) = old_base {
6505 std::env::set_var("SYNAPS_BASE_DIR", v);
6506 }
6507
6508 let err = result.expect_err("non-2xx usage response must fail");
6509 assert!(matches!(err, BrokerError::Transport(_)));
6510 let msg = format!("{err}");
6511 assert!(
6512 msg.contains("usage request failed") && msg.contains("500"),
6513 "operation + status must survive: {msg}"
6514 );
6515 assert_no_hostile_bytes(&msg);
6516 }
6517
6518 async fn spawn_remote_broker_error(status: u16) -> RemoteBroker {
6519 use axum::routing::{get, post};
6520 let code = axum::http::StatusCode::from_u16(status).unwrap();
6521 let app = axum::Router::new()
6522 .route(
6523 "/proxy",
6524 post(move || async move { (code, hostile_broker_json()) }),
6525 )
6526 .route(
6527 "/usage",
6528 get(move || async move { (code, hostile_broker_json()) }),
6529 );
6530 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6531 let addr = listener.local_addr().unwrap();
6532 tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6533 RemoteBroker::new(
6534 format!("http://{addr}"),
6535 "opaque-machine-token",
6536 reqwest::Client::new(),
6537 super::super::TokenCache::new(),
6538 )
6539 }
6540
6541 fn groq_request(stream: bool) -> ProxyRequest {
6542 ProxyRequest {
6543 provider: "groq".into(),
6544 method: ProxyMethod::Post,
6545 path: "/chat/completions".into(),
6546 body: None,
6547 stream,
6548 body_bytes: None,
6549 }
6550 }
6551
6552 #[tokio::test]
6555 async fn remote_broker_denied_omits_broker_error_body() {
6556 for status in [400u16, 403] {
6557 let broker = spawn_remote_broker_error(status).await;
6558 let err = broker
6559 .proxy(groq_request(false))
6560 .await
6561 .expect_err("4xx broker response must fail");
6562 assert!(
6563 matches!(err, BrokerError::Denied(_)),
6564 "typed category must survive, got {err:?}"
6565 );
6566 let msg = format!("{err}");
6567 assert!(msg.contains(&status.to_string()), "status lost: {msg}");
6568 assert_no_hostile_bytes(&msg);
6569 }
6570 }
6571
6572 #[test]
6575 fn gemini_gets_the_longer_stream_idle_timeout() {
6576 let broker = LocalBroker::new(reqwest::Client::new());
6577 assert_eq!(broker.stream_idle_timeout_for("google-gemini"), Duration::from_secs(300));
6578 for key in ["muse", "kimi-code", "openai-codex", "anthropic"] {
6579 assert_eq!(broker.stream_idle_timeout_for(key), Duration::from_secs(90), "{key}");
6580 }
6581 let short = LocalBroker::new(reqwest::Client::new())
6582 .with_stream_idle_timeout_for_tests(Duration::from_millis(10));
6583 assert_eq!(short.stream_idle_timeout_for("muse"), Duration::from_millis(10));
6584 assert!(short.stream_idle_timeout_for("google-gemini") >= Duration::from_secs(300));
6585 }
6586
6587 async fn spawn_remote_broker_body(status: u16, body: String) -> RemoteBroker {
6588 use axum::routing::post;
6589 let code = axum::http::StatusCode::from_u16(status).unwrap();
6590 let app = axum::Router::new().route(
6591 "/proxy",
6592 post(move || {
6593 let body = body.clone();
6594 async move { (code, body) }
6595 }),
6596 );
6597 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6598 let addr = listener.local_addr().unwrap();
6599 tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
6600 RemoteBroker::new(
6601 format!("http://{addr}"),
6602 "opaque-machine-token",
6603 reqwest::Client::new(),
6604 super::super::TokenCache::new(),
6605 )
6606 }
6607
6608 #[tokio::test]
6614 async fn remote_broker_502_keeps_a_retryable_refresh_failure() {
6615 use super::super::token::{RefreshFailure, RefreshFailureKind as K};
6616 for kind in [K::Transient, K::LockBusy, K::Other] {
6617 let daemon_text = BrokerError::Refresh(RefreshFailure::new(
6618 kind,
6619 format!("HTTP error 503 {HOSTILE_SENTINEL} ECHOED input_schema"),
6620 ))
6621 .to_string();
6622 let body = serde_json::json!({ "error": daemon_text }).to_string();
6623 let broker = spawn_remote_broker_body(502, body.clone()).await;
6624 let stream_err = broker.proxy_stream(groq_request(true)).await.err().unwrap();
6625 let buffered_err = broker.proxy(groq_request(false)).await.unwrap_err();
6626 for err in [stream_err, buffered_err] {
6627 let BrokerError::Refresh(f) = &err else {
6628 panic!("{kind:?}: expected Refresh, got {err:?}");
6629 };
6630 assert_eq!(f.kind, kind);
6631 let msg = err.to_string();
6632 assert!(msg.contains("retry shortly"), "{msg}");
6633 assert_no_hostile_bytes(&msg);
6634 }
6635 }
6636 let broker = spawn_remote_broker_body(502, hostile_broker_json()).await;
6638 let err = broker.proxy_stream(groq_request(true)).await.err().unwrap();
6639 assert!(matches!(err, BrokerError::Transport(_)), "{err:?}");
6640 assert_no_hostile_bytes(&err.to_string());
6641 }
6642
6643 #[tokio::test]
6646 async fn remote_broker_stream_error_omits_broker_body() {
6647 let broker = spawn_remote_broker_error(500).await;
6648 let err = broker
6649 .proxy_stream(groq_request(true))
6650 .await
6651 .err()
6652 .expect("non-2xx broker response must not yield a stream");
6653 assert!(matches!(err, BrokerError::Transport(_)));
6654 let msg = format!("{err}");
6655 assert!(
6656 msg.contains("broker proxy stream failed") && msg.contains("500"),
6657 "operation + status must survive: {msg}"
6658 );
6659 assert_no_hostile_bytes(&msg);
6660 }
6661
6662 #[tokio::test]
6665 async fn remote_broker_usage_error_omits_broker_body() {
6666 let broker = spawn_remote_broker_error(502).await;
6667 let err = broker
6668 .anthropic_usage()
6669 .await
6670 .expect_err("non-2xx broker response must fail");
6671 assert!(matches!(err, BrokerError::Transport(_)));
6672 let msg = format!("{err}");
6673 assert!(
6674 msg.contains("broker usage returned HTTP 502"),
6675 "operation + status must survive: {msg}"
6676 );
6677 assert_no_hostile_bytes(&msg);
6678 }
6679
6680 #[tokio::test]
6683 async fn capabilities_expose_no_secret_fields() {
6684 let broker = LocalBroker::new(reqwest::Client::new());
6685 let caps = broker.capabilities().await.unwrap();
6686 assert!(caps
6687 .iter()
6688 .any(|c| c.key == "anthropic" && c.kind == CredentialKind::OAuth));
6689 assert!(caps
6690 .iter()
6691 .any(|c| c.key == "groq" && c.kind == CredentialKind::StaticKey));
6692 let json = serde_json::to_string(&caps).unwrap();
6693 for field in ["key\":", "name\":", "kind\":", "configured\":"] {
6694 assert!(json.contains(field));
6695 }
6696 assert!(!json.contains("refresh"));
6697 assert!(!json.contains("access"));
6698 }
6699
6700 #[test]
6705 fn refresh_failure_kinds_map_to_observation_kinds() {
6706 use super::super::quota_policy::{AccountFailureKind as K, QuotaObservation};
6707 use super::super::token::{RefreshFailure, RefreshFailureKind as R};
6708 for (kind, expected, transient, login) in [
6709 (R::Transient, K::RefreshTransient, true, false),
6710 (R::LockBusy, K::RefreshLockBusy, true, false),
6711 (R::Other, K::RefreshOther, true, false),
6712 (
6713 R::Cooldown { retry_in_secs: 42 },
6714 K::RefreshOther,
6715 true,
6716 false,
6717 ),
6718 (R::Credential, K::RefreshRejected, false, true),
6719 (R::ActionRequired, K::RefreshActionRequired, false, false),
6720 ] {
6721 let read = UsageReadError::refresh(RefreshFailure::new(kind, "why"));
6722 assert_eq!(read.failure.kind, expected, "{kind:?}");
6723 assert_eq!(read.failure.kind.is_transient(), transient, "{kind:?}");
6724 assert_eq!(read.failure.kind.needs_login(), login, "{kind:?}");
6725 assert_eq!(
6727 read.error,
6728 BrokerError::Credential("why".into()),
6729 "{kind:?}"
6730 );
6731 let observation = read.failure.observation();
6732 if login {
6733 assert_eq!(observation, QuotaObservation::AuthError);
6734 } else {
6735 assert_eq!(observation, QuotaObservation::Unknown);
6736 }
6737 }
6738 let cooldown =
6739 refresh_failure(&RefreshFailure::new(R::Cooldown { retry_in_secs: 42 }, "x"));
6740 assert_eq!(cooldown.retry_after_s, Some(42));
6741 let gone = UsageReadError::slot(BrokerError::UnknownAccount {
6743 provider: "anthropic".into(),
6744 label: "x".into(),
6745 });
6746 assert_eq!(gone.failure.kind, K::CredentialMissing);
6747 assert!(gone.failure.kind.needs_login());
6748 let store = UsageReadError::slot(BrokerError::Credential("unreadable".into()));
6749 assert_eq!(store.failure.kind, K::CredentialStore);
6750 assert!(!store.failure.kind.needs_login() && !store.failure.kind.is_transient());
6751 }
6752
6753 #[test]
6756 fn usage_errors_map_to_observation_kinds_with_status() {
6757 use super::super::quota_policy::AccountFailureKind as K;
6758 use super::super::usage::UsageError as U;
6759 for (err, kind, status) in [
6760 (
6761 U::RateLimited {
6762 retry_after_s: Some(9),
6763 },
6764 K::UsageRateLimited,
6765 Some(429),
6766 ),
6767 (
6768 U::Unauthorized { status: 403 },
6769 K::UsageUnauthorized,
6770 Some(403),
6771 ),
6772 (
6773 U::UpstreamStatus {
6774 status: 503,
6775 retry_after_s: None,
6776 },
6777 K::UsageUpstream,
6778 Some(503),
6779 ),
6780 (U::Redirected { status: 302 }, K::UsageUpstream, Some(302)),
6781 (U::Timeout, K::UsageTimeout, None),
6782 (
6783 U::Transport {
6784 detail: "connect".into(),
6785 },
6786 K::UsageTransport,
6787 None,
6788 ),
6789 (U::Malformed { detail: "x".into() }, K::UsageMalformed, None),
6790 (U::BodyTooLarge { cap: 1 }, K::UsageMalformed, None),
6791 (U::AccountIdUnavailable, K::UsageUnauthorized, None),
6792 (
6793 U::UnsupportedProvider {
6794 provider: "p".into(),
6795 },
6796 K::Unsupported,
6797 None,
6798 ),
6799 ] {
6800 let read = UsageReadError::usage(err.clone());
6801 assert_eq!(read.failure.kind, kind, "{err:?}");
6802 assert_eq!(read.failure.http_status, status, "{err:?}");
6803 assert_eq!(read.error, err.clone().into_broker_error(), "{err:?}");
6804 }
6805 assert_eq!(
6806 UsageReadError::usage(U::RateLimited {
6807 retry_after_s: Some(9)
6808 })
6809 .failure
6810 .retry_after_s,
6811 Some(9)
6812 );
6813 }
6814}