Skip to main content

rtdb_admin/
lib.rs

1//! Safe Google service-account authentication and token lifecycle management.
2//! This crate owns credentials and OAuth; `rtdb-rs` owns database transport.
3
4use async_trait::async_trait;
5use jsonwebtoken::{encode, Algorithm, EncodingKey, Header};
6use reqwest::{Client, StatusCode};
7use serde::{Deserialize, Serialize};
8use sha2::{Digest, Sha256};
9use std::{
10    fmt,
11    path::Path,
12    sync::{
13        atomic::{AtomicU64, Ordering},
14        Arc,
15    },
16    time::{Duration, SystemTime, UNIX_EPOCH},
17};
18use thiserror::Error;
19use tokio::sync::{Mutex, Notify};
20
21pub const GOOGLE_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token";
22pub const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform";
23
24#[derive(Debug, Clone, PartialEq, Eq)]
25pub struct DatabaseBinding {
26    pub project_id: String,
27    pub database_url: String,
28}
29
30#[derive(Debug, Clone, PartialEq, Eq)]
31pub struct IdentityHealth {
32    pub identity: String,
33    pub state: TokenState,
34    pub fingerprint: Option<String>,
35    pub refreshes: u64,
36    pub failures: u64,
37}
38
39#[derive(Debug, Error)]
40pub enum Error {
41    #[error("credential JSON is invalid")]
42    Json(#[from] serde_json::Error),
43    #[error("credential is missing required field `{0}`")]
44    MissingField(&'static str),
45    #[error("credential field `{0}` is invalid")]
46    InvalidField(&'static str),
47    #[error("could not read credential file")]
48    Io(#[from] std::io::Error),
49    #[error("JWT signing failed")]
50    Jwt(#[from] jsonwebtoken::errors::Error),
51    #[error("token endpoint request failed")]
52    Http(#[from] reqwest::Error),
53    #[error("token endpoint returned HTTP {status}")]
54    HttpStatus { status: StatusCode },
55    #[error("token endpoint response is malformed")]
56    MalformedResponse,
57    #[error("token endpoint retries exhausted")]
58    RetriesExhausted(Box<Error>),
59    #[error("credential source failed: {0}")]
60    Source(String),
61    #[error("manager is shutting down")]
62    Shutdown,
63}
64
65#[derive(Clone)]
66pub struct ServiceAccount {
67    pub project_id: Option<String>,
68    pub client_email: String,
69    pub private_key: String,
70    pub token_uri: String,
71}
72
73impl fmt::Debug for ServiceAccount {
74    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
75        f.debug_struct("ServiceAccount")
76            .field("project_id", &self.project_id)
77            .field("client_email", &self.client_email)
78            .field("private_key", &"[REDACTED]")
79            .field("token_uri", &self.token_uri)
80            .finish()
81    }
82}
83
84#[derive(Deserialize)]
85struct RawAccount {
86    project_id: Option<String>,
87    client_email: Option<String>,
88    private_key: Option<String>,
89    token_uri: Option<String>,
90    #[serde(rename = "type")]
91    kind: Option<String>,
92}
93
94impl ServiceAccount {
95    pub fn from_json(input: &str) -> Result<Self, Error> {
96        let r: RawAccount = serde_json::from_str(input)?;
97        if let Some(k) = r.kind.as_deref() {
98            if k != "service_account" {
99                return Err(Error::InvalidField("type"));
100            }
101        }
102        let client_email = r
103            .client_email
104            .filter(|v| v.contains('@'))
105            .ok_or(Error::MissingField("client_email"))?;
106        let private_key = r
107            .private_key
108            .filter(|v| v.contains("BEGIN PRIVATE KEY"))
109            .ok_or(Error::MissingField("private_key"))?;
110        let token_uri = r.token_uri.unwrap_or_else(|| GOOGLE_TOKEN_ENDPOINT.into());
111        if !token_uri.starts_with("https://")
112            && !token_uri.starts_with("http://127.0.0.1")
113            && !token_uri.starts_with("http://localhost")
114        {
115            return Err(Error::InvalidField("token_uri"));
116        }
117        Ok(Self {
118            project_id: r.project_id,
119            client_email,
120            private_key,
121            token_uri,
122        })
123    }
124    pub fn from_reader<R: std::io::Read>(mut reader: R) -> Result<Self, Error> {
125        let mut s = String::new();
126        reader.read_to_string(&mut s)?;
127        Self::from_json(&s)
128    }
129    pub fn from_file(path: impl AsRef<Path>) -> Result<Self, Error> {
130        Self::from_reader(std::fs::File::open(path)?)
131    }
132    pub fn fingerprint(&self) -> String {
133        let mut h = Sha256::new();
134        h.update(self.client_email.as_bytes());
135        h.update(self.project_id.as_deref().unwrap_or("").as_bytes());
136        format!("sha256:{:x}", h.finalize())
137    }
138}
139
140#[async_trait]
141pub trait CredentialSource: Send + Sync {
142    async fn load(&self) -> Result<ServiceAccount, Error>;
143}
144
145#[derive(Clone)]
146pub struct StaticCredentials(Arc<ServiceAccount>);
147impl StaticCredentials {
148    pub fn new(credentials: ServiceAccount) -> Self {
149        Self(Arc::new(credentials))
150    }
151}
152#[async_trait]
153impl CredentialSource for StaticCredentials {
154    async fn load(&self) -> Result<ServiceAccount, Error> {
155        Ok((*self.0).clone())
156    }
157}
158
159#[derive(Clone)]
160pub struct EnvironmentCredentials {
161    variable: String,
162}
163impl EnvironmentCredentials {
164    pub fn new(variable: impl Into<String>) -> Self {
165        Self {
166            variable: variable.into(),
167        }
168    }
169}
170#[async_trait]
171impl CredentialSource for EnvironmentCredentials {
172    async fn load(&self) -> Result<ServiceAccount, Error> {
173        let p = std::env::var(&self.variable).map_err(|_| {
174            Error::Source(format!("environment variable {} is not set", self.variable))
175        })?;
176        ServiceAccount::from_file(p)
177    }
178}
179
180#[derive(Clone)]
181pub struct FileCredentials {
182    path: Arc<std::path::PathBuf>,
183}
184impl FileCredentials {
185    pub fn new(path: impl Into<std::path::PathBuf>) -> Self {
186        Self {
187            path: Arc::new(path.into()),
188        }
189    }
190}
191#[async_trait]
192impl CredentialSource for FileCredentials {
193    async fn load(&self) -> Result<ServiceAccount, Error> {
194        ServiceAccount::from_file(&*self.path)
195    }
196}
197
198pub trait Clock: Send + Sync {
199    fn now(&self) -> SystemTime;
200}
201#[derive(Clone, Default)]
202pub struct SystemClock;
203impl Clock for SystemClock {
204    fn now(&self) -> SystemTime {
205        SystemTime::now()
206    }
207}
208
209#[derive(Clone, PartialEq, Eq)]
210pub struct AccessToken {
211    pub value: String,
212    pub expires_at: SystemTime,
213    pub generation: u64,
214}
215impl fmt::Debug for AccessToken {
216    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
217        f.debug_struct("AccessToken")
218            .field("value", &"[REDACTED]")
219            .field("expires_at", &self.expires_at)
220            .field("generation", &self.generation)
221            .finish()
222    }
223}
224
225pub trait RefreshObserver: Send + Sync {
226    fn refreshed(&self, fingerprint: &str, generation: u64, lifetime: Duration);
227    fn failed(&self, fingerprint: Option<&str>);
228}
229
230#[derive(Debug, Clone, Copy, PartialEq, Eq)]
231pub enum TokenState {
232    Empty,
233    Refreshing,
234    Ready,
235    Failed,
236}
237#[derive(Debug, Clone)]
238pub struct RetryPolicy {
239    pub max_attempts: usize,
240    pub initial_backoff: Duration,
241    pub max_backoff: Duration,
242}
243impl Default for RetryPolicy {
244    fn default() -> Self {
245        Self {
246            max_attempts: 3,
247            initial_backoff: Duration::from_millis(25),
248            max_backoff: Duration::from_secs(2),
249        }
250    }
251}
252#[derive(Debug, Clone)]
253pub struct ExchangePolicy {
254    pub endpoint: Option<String>,
255    pub timeout: Duration,
256    pub refresh_skew: Duration,
257    pub retry: RetryPolicy,
258}
259impl Default for ExchangePolicy {
260    fn default() -> Self {
261        Self {
262            endpoint: None,
263            timeout: Duration::from_secs(10),
264            refresh_skew: Duration::from_secs(60),
265            retry: RetryPolicy::default(),
266        }
267    }
268}
269
270#[derive(Serialize)]
271struct Claims {
272    iss: String,
273    scope: String,
274    aud: String,
275    iat: u64,
276    exp: u64,
277}
278pub fn signed_assertion(
279    account: &ServiceAccount,
280    scope: &str,
281    lifetime: Duration,
282    now: SystemTime,
283) -> Result<String, Error> {
284    let iat = now
285        .duration_since(UNIX_EPOCH)
286        .map_err(|_| Error::InvalidField("clock"))?
287        .as_secs();
288    let exp = iat.saturating_add(lifetime.as_secs().min(3600));
289    let claims = Claims {
290        iss: account.client_email.clone(),
291        scope: scope.into(),
292        aud: account.token_uri.clone(),
293        iat,
294        exp,
295    };
296    Ok(encode(
297        &Header::new(Algorithm::RS256),
298        &claims,
299        &EncodingKey::from_rsa_pem(account.private_key.as_bytes())?,
300    )?)
301}
302
303#[derive(Deserialize)]
304struct TokenResponse {
305    access_token: Option<String>,
306    expires_in: Option<u64>,
307}
308#[derive(Clone)]
309pub struct OAuthClient {
310    http: Client,
311    policy: ExchangePolicy,
312}
313#[async_trait]
314pub trait TokenExchanger: Send + Sync {
315    async fn exchange(
316        &self,
317        account: &ServiceAccount,
318        scope: &str,
319        now: SystemTime,
320    ) -> Result<(String, Duration), Error>;
321}
322#[async_trait]
323impl TokenExchanger for OAuthClient {
324    async fn exchange(
325        &self,
326        account: &ServiceAccount,
327        scope: &str,
328        now: SystemTime,
329    ) -> Result<(String, Duration), Error> {
330        OAuthClient::exchange(self, account, scope, now).await
331    }
332}
333impl OAuthClient {
334    pub fn new(policy: ExchangePolicy) -> Result<Self, Error> {
335        Ok(Self {
336            http: Client::builder().timeout(policy.timeout).build()?,
337            policy,
338        })
339    }
340    pub async fn exchange(
341        &self,
342        account: &ServiceAccount,
343        scope: &str,
344        now: SystemTime,
345    ) -> Result<(String, Duration), Error> {
346        let assertion = signed_assertion(account, scope, Duration::from_secs(3600), now)?;
347        self.exchange_assertion(&account.token_uri, &assertion)
348            .await
349    }
350    pub async fn exchange_assertion(
351        &self,
352        endpoint: &str,
353        assertion: &str,
354    ) -> Result<(String, Duration), Error> {
355        let mut delay = self.policy.retry.initial_backoff;
356        let attempts = self.policy.retry.max_attempts.max(1);
357        let mut last = None;
358        for n in 0..attempts {
359            let r = self
360                .http
361                .post(self.policy.endpoint.as_deref().unwrap_or(endpoint))
362                .form(&[
363                    ("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"),
364                    ("assertion", assertion),
365                ])
366                .send()
367                .await;
368            match r {
369                Ok(resp) if resp.status().is_success() => {
370                    let body: TokenResponse =
371                        resp.json().await.map_err(|_| Error::MalformedResponse)?;
372                    let token = body
373                        .access_token
374                        .filter(|x| !x.is_empty())
375                        .ok_or(Error::MalformedResponse)?;
376                    let secs = body
377                        .expires_in
378                        .filter(|x| *x > 0)
379                        .ok_or(Error::MalformedResponse)?;
380                    return Ok((token, Duration::from_secs(secs)));
381                }
382                Ok(resp) => {
383                    let status = resp.status();
384                    let _ = resp.bytes().await;
385                    let e = Error::HttpStatus { status };
386                    if !status.is_server_error() && status != StatusCode::TOO_MANY_REQUESTS {
387                        return Err(e);
388                    }
389                    last = Some(e);
390                }
391                Err(e) => last = Some(Error::Http(e)),
392            }
393            if n + 1 < attempts {
394                tokio::time::sleep(delay).await;
395                delay = (delay * 2).min(self.policy.retry.max_backoff);
396            }
397        }
398        Err(Error::RetriesExhausted(Box::new(
399            last.unwrap_or(Error::MalformedResponse),
400        )))
401    }
402}
403
404struct Inner {
405    token: Option<AccessToken>,
406    state: TokenState,
407    generation: u64,
408    credential_fingerprint: Option<String>,
409    pending_fingerprint: Option<String>,
410    closed: bool,
411}
412pub struct TokenManager<
413    S: CredentialSource,
414    C: Clock = SystemClock,
415    E: TokenExchanger = OAuthClient,
416> {
417    source: S,
418    exchanger: E,
419    scope: String,
420    skew: Duration,
421    clock: C,
422    inner: Mutex<Inner>,
423    notify: Notify,
424    refreshes: AtomicU64,
425    failures: AtomicU64,
426    observer: Option<Arc<dyn RefreshObserver>>,
427}
428impl<S: CredentialSource> TokenManager<S> {
429    pub fn new(source: S, policy: ExchangePolicy) -> Result<Self, Error> {
430        Self::with_clock(source, policy, SystemClock)
431    }
432}
433impl<S: CredentialSource, C: Clock> TokenManager<S, C, OAuthClient> {
434    pub fn with_clock(source: S, policy: ExchangePolicy, clock: C) -> Result<Self, Error> {
435        Ok(Self {
436            source,
437            exchanger: OAuthClient::new(policy.clone())?,
438            scope: CLOUD_PLATFORM_SCOPE.into(),
439            skew: policy.refresh_skew,
440            clock,
441            inner: Mutex::new(Inner {
442                token: None,
443                state: TokenState::Empty,
444                generation: 0,
445                credential_fingerprint: None,
446                pending_fingerprint: None,
447                closed: false,
448            }),
449            notify: Notify::new(),
450            refreshes: AtomicU64::new(0),
451            failures: AtomicU64::new(0),
452            observer: None,
453        })
454    }
455}
456impl<S: CredentialSource, C: Clock, E: TokenExchanger + 'static> TokenManager<S, C, E> {
457    pub fn with_exchanger(source: S, exchanger: E, refresh_skew: Duration, clock: C) -> Self {
458        Self {
459            source,
460            exchanger,
461            scope: CLOUD_PLATFORM_SCOPE.into(),
462            skew: refresh_skew,
463            clock,
464            inner: Mutex::new(Inner {
465                token: None,
466                state: TokenState::Empty,
467                generation: 0,
468                credential_fingerprint: None,
469                pending_fingerprint: None,
470                closed: false,
471            }),
472            notify: Notify::new(),
473            refreshes: AtomicU64::new(0),
474            failures: AtomicU64::new(0),
475            observer: None,
476        }
477    }
478    pub fn with_scope(mut self, scope: impl Into<String>) -> Self {
479        self.scope = scope.into();
480        self
481    }
482    pub fn with_observer(mut self, observer: Arc<dyn RefreshObserver>) -> Self {
483        self.observer = Some(observer);
484        self
485    }
486    pub async fn token(&self) -> Result<AccessToken, Error> {
487        loop {
488            let mut g = self.inner.lock().await;
489            let now = self.clock.now();
490            if let Some(t) = &g.token {
491                if t.expires_at
492                    .duration_since(now)
493                    .map(|remaining| remaining > self.skew)
494                    .unwrap_or(false)
495                {
496                    return Ok(t.clone());
497                }
498            }
499            if g.closed {
500                return Err(Error::Shutdown);
501            }
502            if g.state == TokenState::Refreshing {
503                let wait = self.notify.notified();
504                drop(g);
505                wait.await;
506                continue;
507            }
508            g.state = TokenState::Refreshing;
509            let old_fp = g.credential_fingerprint.clone();
510            drop(g);
511            let result = self.refresh(old_fp).await;
512            let mut g = self.inner.lock().await;
513            match result {
514                Ok((token, fp)) => {
515                    self.refreshes.fetch_add(1, Ordering::Relaxed);
516                    g.generation += 1;
517                    let t = AccessToken {
518                        value: token.0,
519                        expires_at: now + token.1,
520                        generation: g.generation,
521                    };
522                    g.token = Some(t.clone());
523                    g.credential_fingerprint = Some(fp);
524                    g.pending_fingerprint = None;
525                    g.state = TokenState::Ready;
526                    if let Some(observer) = &self.observer {
527                        observer.refreshed(
528                            g.credential_fingerprint.as_deref().unwrap_or("unknown"),
529                            g.generation,
530                            token.1,
531                        );
532                    }
533                    self.notify.notify_waiters();
534                    return Ok(t);
535                }
536                Err(e) => {
537                    self.failures.fetch_add(1, Ordering::Relaxed);
538                    g.state = TokenState::Failed;
539                    g.pending_fingerprint = None;
540                    if let Some(observer) = &self.observer {
541                        observer.failed(g.credential_fingerprint.as_deref());
542                    }
543                    self.notify.notify_waiters();
544                    return Err(e);
545                }
546            }
547        }
548    }
549    async fn refresh(&self, old_fp: Option<String>) -> Result<((String, Duration), String), Error> {
550        let account = self.source.load().await?;
551        let fp = account.fingerprint();
552        let _rotated = old_fp.as_deref() != Some(fp.as_str());
553        Ok((
554            self.exchanger
555                .exchange(&account, &self.scope, self.clock.now())
556                .await?,
557            fp,
558        ))
559    }
560    pub async fn refresh_now(&self) -> Result<AccessToken, Error> {
561        self.invalidate().await;
562        self.token().await
563    }
564    pub async fn refresh_if_changed(&self) -> Result<AccessToken, Error> {
565        let account = self.source.load().await?;
566        let fingerprint = account.fingerprint();
567        loop {
568            let mut state = self.inner.lock().await;
569            if state.credential_fingerprint.as_deref() == Some(fingerprint.as_str()) {
570                drop(state);
571                return self.token().await;
572            }
573            if state.pending_fingerprint.as_deref() == Some(fingerprint.as_str())
574                || state.state == TokenState::Refreshing
575            {
576                let wait = self.notify.notified();
577                drop(state);
578                wait.await;
579                continue;
580            }
581            state.pending_fingerprint = Some(fingerprint.clone());
582            state.token = None;
583            state.state = TokenState::Empty;
584            drop(state);
585            return self.token().await;
586        }
587    }
588    pub async fn invalidate(&self) {
589        let mut g = self.inner.lock().await;
590        g.token = None;
591        g.state = TokenState::Empty;
592        self.notify.notify_waiters();
593    }
594    pub async fn state(&self) -> TokenState {
595        self.inner.lock().await.state
596    }
597    pub async fn fingerprint(&self) -> Option<String> {
598        self.inner.lock().await.credential_fingerprint.clone()
599    }
600    pub async fn shutdown(&self) {
601        let mut g = self.inner.lock().await;
602        g.closed = true;
603        self.notify.notify_waiters();
604    }
605    pub async fn ready(&self) -> bool {
606        self.state().await == TokenState::Ready
607    }
608    pub fn refresh_count(&self) -> u64 {
609        self.refreshes.load(Ordering::Relaxed)
610    }
611    pub fn failure_count(&self) -> u64 {
612        self.failures.load(Ordering::Relaxed)
613    }
614    pub fn spawn_proactive_refresh(
615        self: Arc<Self>,
616        interval: Duration,
617    ) -> tokio::task::JoinHandle<()>
618    where
619        S: 'static,
620        C: 'static,
621    {
622        tokio::spawn(async move {
623            let mut ticker = tokio::time::interval(interval.max(Duration::from_millis(10)));
624            loop {
625                ticker.tick().await;
626                if self.inner.lock().await.closed {
627                    break;
628                }
629                let _ = self.token().await;
630            }
631        })
632    }
633}
634
635#[cfg(feature = "rtdb-client")]
636impl<S: CredentialSource, C: Clock, E: TokenExchanger + 'static> TokenManager<S, C, E> {
637    pub async fn rtdb_client(
638        &self,
639        base_url: impl Into<String>,
640    ) -> Result<rtdb_rs::RtdbClient, Error> {
641        let t = self.token().await?;
642        Ok(rtdb_rs::RtdbClient::new(base_url, t.value))
643    }
644    pub async fn update_rtdb_client(
645        &self,
646        client: rtdb_rs::RtdbClient,
647    ) -> Result<rtdb_rs::RtdbClient, Error> {
648        let t = self.token().await?;
649        Ok(client.with_token(t.value))
650    }
651}
652
653pub struct CredentialManager<S: CredentialSource, C: Clock = SystemClock> {
654    entries: Mutex<std::collections::HashMap<String, Arc<TokenManager<S, C>>>>,
655    source_factory: Arc<dyn Fn(&str) -> S + Send + Sync>,
656    capacity: usize,
657}
658impl<S: CredentialSource + 'static, C: Clock + Default + 'static> CredentialManager<S, C> {
659    pub fn new(factory: impl Fn(&str) -> S + Send + Sync + 'static) -> Self {
660        Self::with_capacity(256, factory)
661    }
662    pub fn with_capacity(
663        capacity: usize,
664        factory: impl Fn(&str) -> S + Send + Sync + 'static,
665    ) -> Self {
666        Self {
667            entries: Mutex::new(std::collections::HashMap::new()),
668            source_factory: Arc::new(factory),
669            capacity: capacity.max(1),
670        }
671    }
672    pub async fn get(
673        &self,
674        identity: &str,
675        policy: ExchangePolicy,
676    ) -> Result<Arc<TokenManager<S, C>>, Error> {
677        let mut e = self.entries.lock().await;
678        if let Some(m) = e.get(identity) {
679            return Ok(m.clone());
680        }
681        let m = Arc::new(TokenManager::with_clock(
682            (self.source_factory)(identity),
683            policy,
684            C::default(),
685        )?);
686        if e.len() >= self.capacity {
687            if let Some(oldest) = e.keys().next().cloned() {
688                e.remove(&oldest);
689            }
690        }
691        e.insert(identity.into(), m.clone());
692        Ok(m)
693    }
694    pub async fn len(&self) -> usize {
695        self.entries.lock().await.len()
696    }
697    pub async fn is_empty(&self) -> bool {
698        self.entries.lock().await.is_empty()
699    }
700    pub async fn remove(&self, identity: &str) {
701        self.entries.lock().await.remove(identity);
702    }
703    pub async fn health(&self) -> Vec<IdentityHealth> {
704        let entries = self.entries.lock().await;
705        let mut result = Vec::with_capacity(entries.len());
706        for (identity, manager) in entries.iter() {
707            result.push(IdentityHealth {
708                identity: identity.clone(),
709                state: manager.state().await,
710                fingerprint: manager.fingerprint().await,
711                refreshes: manager.refresh_count(),
712                failures: manager.failure_count(),
713            });
714        }
715        result.sort_by(|left, right| left.identity.cmp(&right.identity));
716        result
717    }
718}
719
720#[cfg(test)]
721mod tests {
722    use super::*;
723    use std::sync::{
724        atomic::{AtomicBool, AtomicUsize, Ordering},
725        Mutex as StdMutex,
726    };
727    use tokio::io::{AsyncReadExt, AsyncWriteExt};
728    use tokio::net::TcpListener;
729
730    const JSON: &str = r#"{"type":"service_account","project_id":"test-project","client_email":"test@example.invalid","private_key":"-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----\n"}"#;
731
732    fn fake_account(id: &str) -> ServiceAccount {
733        ServiceAccount {
734            project_id: Some(id.into()),
735            client_email: format!("{id}@example.invalid"),
736            private_key: "not-used-by-test-exchanger".into(),
737            token_uri: "http://127.0.0.1/token".into(),
738        }
739    }
740
741    #[derive(Clone)]
742    struct MockExchange {
743        calls: Arc<AtomicUsize>,
744        fail: Arc<AtomicBool>,
745        delay: Duration,
746    }
747    #[async_trait]
748    impl TokenExchanger for MockExchange {
749        async fn exchange(
750            &self,
751            account: &ServiceAccount,
752            _scope: &str,
753            _now: SystemTime,
754        ) -> Result<(String, Duration), Error> {
755            self.calls.fetch_add(1, Ordering::SeqCst);
756            if self.delay > Duration::ZERO {
757                tokio::time::sleep(self.delay).await;
758            }
759            if self.fail.load(Ordering::SeqCst) {
760                return Err(Error::Source(format!(
761                    "exchange failed for {}",
762                    account.fingerprint()
763                )));
764            }
765            Ok((
766                format!(
767                    "token-{}",
768                    account.project_id.as_deref().unwrap_or("unknown")
769                ),
770                Duration::from_millis(100),
771            ))
772        }
773    }
774
775    #[derive(Clone)]
776    struct TestClock(Arc<AtomicU64>);
777    impl TestClock {
778        fn new() -> Self {
779            Self(Arc::new(AtomicU64::new(0)))
780        }
781        fn advance(&self, millis: u64) {
782            self.0.fetch_add(millis, Ordering::SeqCst);
783        }
784    }
785    impl Clock for TestClock {
786        fn now(&self) -> SystemTime {
787            UNIX_EPOCH + Duration::from_millis(self.0.load(Ordering::SeqCst))
788        }
789    }
790
791    #[derive(Clone)]
792    struct RotatingSource(Arc<StdMutex<ServiceAccount>>);
793    #[async_trait]
794    impl CredentialSource for RotatingSource {
795        async fn load(&self) -> Result<ServiceAccount, Error> {
796            Ok(self.0.lock().unwrap().clone())
797        }
798    }
799
800    struct CountingObserver(AtomicUsize);
801    impl RefreshObserver for CountingObserver {
802        fn refreshed(&self, _fingerprint: &str, _generation: u64, _lifetime: Duration) {
803            self.0.fetch_add(1, Ordering::SeqCst);
804        }
805        fn failed(&self, _fingerprint: Option<&str>) {}
806    }
807
808    #[test]
809    fn validates_and_redacts_credentials() {
810        let account = ServiceAccount::from_json(JSON).unwrap();
811        let debug = format!("{account:?}");
812        assert!(debug.contains("[REDACTED]"));
813        assert!(!debug.contains("fake"));
814        assert!(account.fingerprint().starts_with("sha256:"));
815        assert!(matches!(
816            ServiceAccount::from_json("{}"),
817            Err(Error::MissingField("client_email"))
818        ));
819    }
820
821    async fn mock_endpoint(status: &str, body: &str, requests: Arc<AtomicUsize>) -> String {
822        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
823        let addr = listener.local_addr().unwrap();
824        let status = status.to_string();
825        let body = body.to_string();
826        tokio::spawn(async move {
827            if let Ok((mut socket, _)) = listener.accept().await {
828                requests.fetch_add(1, Ordering::SeqCst);
829                let mut buf = [0u8; 1024];
830                let _ = socket.read(&mut buf).await;
831                let response = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{body}", body.len());
832                let _ = socket.write_all(response.as_bytes()).await;
833            }
834        });
835        format!("http://{addr}/token")
836    }
837
838    #[tokio::test]
839    async fn local_exchange_parses_success_without_google() {
840        let requests = Arc::new(AtomicUsize::new(0));
841        let endpoint = mock_endpoint(
842            "200 OK",
843            r#"{"access_token":"test-token","expires_in":120}"#,
844            requests.clone(),
845        )
846        .await;
847        let policy = ExchangePolicy {
848            endpoint: Some(endpoint.clone()),
849            retry: RetryPolicy {
850                max_attempts: 1,
851                ..Default::default()
852            },
853            ..Default::default()
854        };
855        let oauth = OAuthClient::new(policy).unwrap();
856        let (token, lifetime) = oauth
857            .exchange_assertion(&endpoint, "non-secret-test-assertion")
858            .await
859            .unwrap();
860        assert_eq!(token, "test-token");
861        assert_eq!(lifetime, Duration::from_secs(120));
862        assert_eq!(requests.load(Ordering::SeqCst), 1);
863    }
864
865    #[tokio::test]
866    async fn malformed_timeout_and_disconnect_are_deterministic() {
867        let requests = Arc::new(AtomicUsize::new(0));
868        let malformed = mock_endpoint("200 OK", "not-json", requests.clone()).await;
869        let policy = ExchangePolicy {
870            endpoint: Some(malformed.clone()),
871            retry: RetryPolicy {
872                max_attempts: 1,
873                ..Default::default()
874            },
875            ..Default::default()
876        };
877        let oauth = OAuthClient::new(policy).unwrap();
878        assert!(matches!(
879            oauth.exchange_assertion(&malformed, "assertion").await,
880            Err(Error::MalformedResponse)
881        ));
882
883        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
884        let timeout_endpoint = format!("http://{}/token", listener.local_addr().unwrap());
885        tokio::spawn(async move {
886            let Ok((mut socket, _)) = listener.accept().await else {
887                return;
888            };
889            let mut buf = [0u8; 256];
890            let _ = socket.read(&mut buf).await;
891            tokio::time::sleep(Duration::from_millis(100)).await;
892        });
893        let oauth = OAuthClient::new(ExchangePolicy {
894            endpoint: Some(timeout_endpoint.clone()),
895            timeout: Duration::from_millis(10),
896            retry: RetryPolicy {
897                max_attempts: 1,
898                ..Default::default()
899            },
900            ..Default::default()
901        })
902        .unwrap();
903        assert!(matches!(
904            oauth
905                .exchange_assertion(&timeout_endpoint, "assertion")
906                .await,
907            Err(Error::RetriesExhausted(_))
908        ));
909
910        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
911        let reset_endpoint = format!("http://{}/token", listener.local_addr().unwrap());
912        tokio::spawn(async move {
913            let _ = listener.accept().await;
914        });
915        let oauth = OAuthClient::new(ExchangePolicy {
916            endpoint: Some(reset_endpoint.clone()),
917            retry: RetryPolicy {
918                max_attempts: 1,
919                ..Default::default()
920            },
921            ..Default::default()
922        })
923        .unwrap();
924        assert!(matches!(
925            oauth.exchange_assertion(&reset_endpoint, "assertion").await,
926            Err(Error::RetriesExhausted(_))
927        ));
928    }
929
930    #[tokio::test]
931    async fn transient_server_error_retries_then_succeeds() {
932        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
933        let endpoint = format!("http://{}/token", listener.local_addr().unwrap());
934        let requests = Arc::new(AtomicUsize::new(0));
935        let seen = requests.clone();
936        tokio::spawn(async move {
937            for status in ["500 Internal Server Error", "200 OK"] {
938                let Ok((mut socket, _)) = listener.accept().await else {
939                    return;
940                };
941                seen.fetch_add(1, Ordering::SeqCst);
942                let mut buf = [0u8; 256];
943                let _ = socket.read(&mut buf).await;
944                let body = if status.starts_with("200") {
945                    r#"{"access_token":"recovered","expires_in":30}"#
946                } else {
947                    "{}"
948                };
949                let response = format!(
950                    "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
951                    body.len()
952                );
953                let _ = socket.write_all(response.as_bytes()).await;
954            }
955        });
956        let oauth = OAuthClient::new(ExchangePolicy {
957            endpoint: Some(endpoint.clone()),
958            retry: RetryPolicy {
959                max_attempts: 2,
960                initial_backoff: Duration::from_millis(1),
961                ..Default::default()
962            },
963            ..Default::default()
964        })
965        .unwrap();
966        assert_eq!(
967            oauth
968                .exchange_assertion(&endpoint, "assertion")
969                .await
970                .unwrap()
971                .0,
972            "recovered"
973        );
974        assert_eq!(requests.load(Ordering::SeqCst), 2);
975    }
976
977    #[tokio::test]
978    async fn permanent_http_errors_fail_without_leaking_body() {
979        let requests = Arc::new(AtomicUsize::new(0));
980        let endpoint = mock_endpoint(
981            "401 Unauthorized",
982            r#"{"error":"private-token-must-not-appear"}"#,
983            requests.clone(),
984        )
985        .await;
986        let mut policy = ExchangePolicy::default();
987        policy.retry.max_attempts = 4;
988        let oauth = OAuthClient::new(ExchangePolicy {
989            endpoint: Some(endpoint.clone()),
990            ..policy
991        })
992        .unwrap();
993        let error = oauth
994            .exchange_assertion(&endpoint, "assertion")
995            .await
996            .unwrap_err();
997        assert!(!error.to_string().contains("private-token"));
998        assert_eq!(requests.load(Ordering::SeqCst), 1);
999    }
1000
1001    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1002    async fn one_hundred_concurrent_callers_share_one_generation() {
1003        let calls = Arc::new(AtomicUsize::new(0));
1004        let manager = Arc::new(TokenManager::with_exchanger(
1005            StaticCredentials::new(fake_account("single-flight")),
1006            MockExchange {
1007                calls: calls.clone(),
1008                fail: Arc::new(AtomicBool::new(false)),
1009                delay: Duration::from_millis(20),
1010            },
1011            Duration::from_millis(5),
1012            SystemClock,
1013        ));
1014        let mut tasks = Vec::new();
1015        for _ in 0..100 {
1016            let manager = manager.clone();
1017            tasks.push(tokio::spawn(async move { manager.token().await.unwrap() }));
1018        }
1019        let tokens = futures_util::future::join_all(tasks)
1020            .await
1021            .into_iter()
1022            .map(|x| x.unwrap())
1023            .collect::<Vec<_>>();
1024        assert_eq!(calls.load(Ordering::SeqCst), 1);
1025        assert!(tokens
1026            .iter()
1027            .all(|t| t.generation == 1 && t.value == "token-single-flight"));
1028    }
1029
1030    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1031    async fn expiry_boundary_refreshes_once_for_sixty_four_callers() {
1032        let calls = Arc::new(AtomicUsize::new(0));
1033        let clock = TestClock::new();
1034        let manager = Arc::new(TokenManager::with_exchanger(
1035            StaticCredentials::new(fake_account("boundary")),
1036            MockExchange {
1037                calls: calls.clone(),
1038                fail: Arc::new(AtomicBool::new(false)),
1039                delay: Duration::from_millis(10),
1040            },
1041            Duration::from_millis(10),
1042            clock.clone(),
1043        ));
1044        assert_eq!(manager.token().await.unwrap().generation, 1);
1045        clock.advance(91);
1046        let mut tasks = Vec::new();
1047        for _ in 0..64 {
1048            let manager = manager.clone();
1049            tasks.push(tokio::spawn(async move { manager.token().await.unwrap() }));
1050        }
1051        let results = futures_util::future::join_all(tasks).await;
1052        assert!(results.iter().all(|r| r.as_ref().unwrap().generation == 2));
1053        assert_eq!(calls.load(Ordering::SeqCst), 2);
1054    }
1055
1056    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1057    async fn rotation_reauthenticates_and_preserves_new_identity() {
1058        let calls = Arc::new(AtomicUsize::new(0));
1059        let source = RotatingSource(Arc::new(StdMutex::new(fake_account("old"))));
1060        let manager = Arc::new(TokenManager::with_exchanger(
1061            source.clone(),
1062            MockExchange {
1063                calls: calls.clone(),
1064                fail: Arc::new(AtomicBool::new(false)),
1065                delay: Duration::from_millis(5),
1066            },
1067            Duration::ZERO,
1068            SystemClock,
1069        ));
1070        manager.token().await.unwrap();
1071        let old_fp = manager.fingerprint().await.unwrap();
1072        *source.0.lock().unwrap() = fake_account("new");
1073        let new_token = manager.refresh_if_changed().await.unwrap();
1074        assert_ne!(old_fp, manager.fingerprint().await.unwrap());
1075        assert_eq!(new_token.value, "token-new");
1076        assert_eq!(manager.refresh_count(), 2);
1077        *source.0.lock().unwrap() = fake_account("third");
1078        let mut consumers = Vec::new();
1079        for _ in 0..64 {
1080            let manager = manager.clone();
1081            consumers.push(tokio::spawn(async move {
1082                manager.refresh_if_changed().await.unwrap()
1083            }));
1084        }
1085        let results = futures_util::future::join_all(consumers).await;
1086        assert!(results
1087            .iter()
1088            .all(|result| result.as_ref().unwrap().value == "token-third"));
1089        assert_eq!(manager.refresh_count(), 3);
1090    }
1091
1092    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1093    async fn observer_receives_only_secret_free_refresh_events_and_task_stops() {
1094        let calls = Arc::new(AtomicUsize::new(0));
1095        let observer = Arc::new(CountingObserver(AtomicUsize::new(0)));
1096        let manager = Arc::new(
1097            TokenManager::with_exchanger(
1098                StaticCredentials::new(fake_account("observed")),
1099                MockExchange {
1100                    calls,
1101                    fail: Arc::new(AtomicBool::new(false)),
1102                    delay: Duration::ZERO,
1103                },
1104                Duration::ZERO,
1105                SystemClock,
1106            )
1107            .with_observer(observer.clone()),
1108        );
1109        manager.token().await.unwrap();
1110        assert_eq!(observer.0.load(Ordering::SeqCst), 1);
1111        let task = manager
1112            .clone()
1113            .spawn_proactive_refresh(Duration::from_millis(10));
1114        tokio::time::sleep(Duration::from_millis(25)).await;
1115        manager.shutdown().await;
1116        tokio::time::timeout(Duration::from_secs(1), task)
1117            .await
1118            .unwrap()
1119            .unwrap();
1120    }
1121
1122    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1123    async fn thousand_request_manual_profile_has_bounded_generations() {
1124        let calls = Arc::new(AtomicUsize::new(0));
1125        let manager = Arc::new(TokenManager::with_exchanger(
1126            StaticCredentials::new(fake_account("soak")),
1127            MockExchange {
1128                calls: calls.clone(),
1129                fail: Arc::new(AtomicBool::new(false)),
1130                delay: Duration::from_millis(1),
1131            },
1132            Duration::ZERO,
1133            SystemClock,
1134        ));
1135        for _generation in 0..10 {
1136            manager.invalidate().await;
1137            let mut tasks = Vec::new();
1138            for _ in 0..128 {
1139                let manager = manager.clone();
1140                tasks.push(tokio::spawn(async move {
1141                    let mut last = 0;
1142                    for _ in 0..8 {
1143                        last = manager.token().await.unwrap().generation;
1144                    }
1145                    last
1146                }));
1147            }
1148            let results = futures_util::future::join_all(tasks).await;
1149            assert!(results.iter().all(|r| r.is_ok()));
1150        }
1151        assert_eq!(manager.refresh_count(), 10);
1152        assert!(calls.load(Ordering::SeqCst) <= 10);
1153    }
1154
1155    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1156    async fn outage_fails_bounded_and_recovers_after_invalidate() {
1157        let calls = Arc::new(AtomicUsize::new(0));
1158        let fail = Arc::new(AtomicBool::new(true));
1159        let manager = Arc::new(TokenManager::with_exchanger(
1160            StaticCredentials::new(fake_account("outage")),
1161            MockExchange {
1162                calls: calls.clone(),
1163                fail: fail.clone(),
1164                delay: Duration::from_millis(5),
1165            },
1166            Duration::ZERO,
1167            SystemClock,
1168        ));
1169        let mut tasks = Vec::new();
1170        for _ in 0..128 {
1171            let manager = manager.clone();
1172            tasks.push(tokio::spawn(async move { manager.token().await.is_err() }));
1173        }
1174        assert!(futures_util::future::join_all(tasks)
1175            .await
1176            .into_iter()
1177            .all(|x| x.unwrap()));
1178        assert!(calls.load(Ordering::SeqCst) <= 128);
1179        fail.store(false, Ordering::SeqCst);
1180        manager.invalidate().await;
1181        assert_eq!(manager.token().await.unwrap().generation, 1);
1182    }
1183
1184    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1185    async fn multi_identity_managers_are_isolated_and_bounded() {
1186        let manager = CredentialManager::<StaticCredentials>::with_capacity(100, |id| {
1187            StaticCredentials::new(fake_account(id))
1188        });
1189        let mut entries = Vec::new();
1190        for i in 0..100 {
1191            entries.push(
1192                manager
1193                    .get(&format!("identity-{i}"), ExchangePolicy::default())
1194                    .await
1195                    .unwrap(),
1196            );
1197        }
1198        assert_eq!(manager.len().await, 100);
1199        assert!(!Arc::ptr_eq(&entries[0], &entries[1]));
1200        manager
1201            .get("identity-100", ExchangePolicy::default())
1202            .await
1203            .unwrap();
1204        assert_eq!(manager.len().await, 100);
1205    }
1206
1207    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
1208    async fn one_hundred_identities_with_ten_consumers_each_stay_isolated() {
1209        let mut all = Vec::new();
1210        for identity in 0..100 {
1211            let manager = Arc::new(TokenManager::with_exchanger(
1212                StaticCredentials::new(fake_account(&format!("identity-{identity}"))),
1213                MockExchange {
1214                    calls: Arc::new(AtomicUsize::new(0)),
1215                    fail: Arc::new(AtomicBool::new(false)),
1216                    delay: Duration::from_millis(1),
1217                },
1218                Duration::ZERO,
1219                SystemClock,
1220            ));
1221            let mut consumers = Vec::new();
1222            for _ in 0..10 {
1223                let manager = manager.clone();
1224                consumers.push(tokio::spawn(async move { manager.token().await.unwrap() }));
1225            }
1226            all.push((identity, manager, consumers));
1227        }
1228        for (identity, manager, consumers) in all {
1229            let results = futures_util::future::join_all(consumers).await;
1230            let expected = format!("token-identity-{identity}");
1231            assert!(results
1232                .iter()
1233                .all(|result| result.as_ref().unwrap().value == expected));
1234            assert_eq!(manager.refresh_count(), 1);
1235        }
1236    }
1237}