use async_trait::async_trait;
use jsonwebtoken::{encode, Algorithm, EncodingKey, Header};
use reqwest::{Client, StatusCode};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{
fmt,
path::Path,
sync::{
atomic::{AtomicU64, Ordering},
Arc,
},
time::{Duration, SystemTime, UNIX_EPOCH},
};
use thiserror::Error;
use tokio::sync::{Mutex, Notify};
pub const GOOGLE_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token";
pub const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform";
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DatabaseBinding {
pub project_id: String,
pub database_url: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct IdentityHealth {
pub identity: String,
pub state: TokenState,
pub fingerprint: Option<String>,
pub refreshes: u64,
pub failures: u64,
}
#[derive(Debug, Error)]
pub enum Error {
#[error("credential JSON is invalid")]
Json(#[from] serde_json::Error),
#[error("credential is missing required field `{0}`")]
MissingField(&'static str),
#[error("credential field `{0}` is invalid")]
InvalidField(&'static str),
#[error("could not read credential file")]
Io(#[from] std::io::Error),
#[error("JWT signing failed")]
Jwt(#[from] jsonwebtoken::errors::Error),
#[error("token endpoint request failed")]
Http(#[from] reqwest::Error),
#[error("token endpoint returned HTTP {status}")]
HttpStatus { status: StatusCode },
#[error("token endpoint response is malformed")]
MalformedResponse,
#[error("token endpoint retries exhausted")]
RetriesExhausted(Box<Error>),
#[error("credential source failed: {0}")]
Source(String),
#[error("manager is shutting down")]
Shutdown,
}
#[derive(Clone)]
pub struct ServiceAccount {
pub project_id: Option<String>,
pub client_email: String,
pub private_key: String,
pub token_uri: String,
}
impl fmt::Debug for ServiceAccount {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ServiceAccount")
.field("project_id", &self.project_id)
.field("client_email", &self.client_email)
.field("private_key", &"[REDACTED]")
.field("token_uri", &self.token_uri)
.finish()
}
}
#[derive(Deserialize)]
struct RawAccount {
project_id: Option<String>,
client_email: Option<String>,
private_key: Option<String>,
token_uri: Option<String>,
#[serde(rename = "type")]
kind: Option<String>,
}
impl ServiceAccount {
pub fn from_json(input: &str) -> Result<Self, Error> {
let r: RawAccount = serde_json::from_str(input)?;
if let Some(k) = r.kind.as_deref() {
if k != "service_account" {
return Err(Error::InvalidField("type"));
}
}
let client_email = r
.client_email
.filter(|v| v.contains('@'))
.ok_or(Error::MissingField("client_email"))?;
let private_key = r
.private_key
.filter(|v| v.contains("BEGIN PRIVATE KEY"))
.ok_or(Error::MissingField("private_key"))?;
let token_uri = r.token_uri.unwrap_or_else(|| GOOGLE_TOKEN_ENDPOINT.into());
if !token_uri.starts_with("https://")
&& !token_uri.starts_with("http://127.0.0.1")
&& !token_uri.starts_with("http://localhost")
{
return Err(Error::InvalidField("token_uri"));
}
Ok(Self {
project_id: r.project_id,
client_email,
private_key,
token_uri,
})
}
pub fn from_reader<R: std::io::Read>(mut reader: R) -> Result<Self, Error> {
let mut s = String::new();
reader.read_to_string(&mut s)?;
Self::from_json(&s)
}
pub fn from_file(path: impl AsRef<Path>) -> Result<Self, Error> {
Self::from_reader(std::fs::File::open(path)?)
}
pub fn fingerprint(&self) -> String {
let mut h = Sha256::new();
h.update(self.client_email.as_bytes());
h.update(self.project_id.as_deref().unwrap_or("").as_bytes());
format!("sha256:{:x}", h.finalize())
}
}
#[async_trait]
pub trait CredentialSource: Send + Sync {
async fn load(&self) -> Result<ServiceAccount, Error>;
}
#[derive(Clone)]
pub struct StaticCredentials(Arc<ServiceAccount>);
impl StaticCredentials {
pub fn new(credentials: ServiceAccount) -> Self {
Self(Arc::new(credentials))
}
}
#[async_trait]
impl CredentialSource for StaticCredentials {
async fn load(&self) -> Result<ServiceAccount, Error> {
Ok((*self.0).clone())
}
}
#[derive(Clone)]
pub struct EnvironmentCredentials {
variable: String,
}
impl EnvironmentCredentials {
pub fn new(variable: impl Into<String>) -> Self {
Self {
variable: variable.into(),
}
}
}
#[async_trait]
impl CredentialSource for EnvironmentCredentials {
async fn load(&self) -> Result<ServiceAccount, Error> {
let p = std::env::var(&self.variable).map_err(|_| {
Error::Source(format!("environment variable {} is not set", self.variable))
})?;
ServiceAccount::from_file(p)
}
}
#[derive(Clone)]
pub struct FileCredentials {
path: Arc<std::path::PathBuf>,
}
impl FileCredentials {
pub fn new(path: impl Into<std::path::PathBuf>) -> Self {
Self {
path: Arc::new(path.into()),
}
}
}
#[async_trait]
impl CredentialSource for FileCredentials {
async fn load(&self) -> Result<ServiceAccount, Error> {
ServiceAccount::from_file(&*self.path)
}
}
pub trait Clock: Send + Sync {
fn now(&self) -> SystemTime;
}
#[derive(Clone, Default)]
pub struct SystemClock;
impl Clock for SystemClock {
fn now(&self) -> SystemTime {
SystemTime::now()
}
}
#[derive(Clone, PartialEq, Eq)]
pub struct AccessToken {
pub value: String,
pub expires_at: SystemTime,
pub generation: u64,
}
impl fmt::Debug for AccessToken {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("AccessToken")
.field("value", &"[REDACTED]")
.field("expires_at", &self.expires_at)
.field("generation", &self.generation)
.finish()
}
}
pub trait RefreshObserver: Send + Sync {
fn refreshed(&self, fingerprint: &str, generation: u64, lifetime: Duration);
fn failed(&self, fingerprint: Option<&str>);
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenState {
Empty,
Refreshing,
Ready,
Failed,
}
#[derive(Debug, Clone)]
pub struct RetryPolicy {
pub max_attempts: usize,
pub initial_backoff: Duration,
pub max_backoff: Duration,
}
impl Default for RetryPolicy {
fn default() -> Self {
Self {
max_attempts: 3,
initial_backoff: Duration::from_millis(25),
max_backoff: Duration::from_secs(2),
}
}
}
#[derive(Debug, Clone)]
pub struct ExchangePolicy {
pub endpoint: Option<String>,
pub timeout: Duration,
pub refresh_skew: Duration,
pub retry: RetryPolicy,
}
impl Default for ExchangePolicy {
fn default() -> Self {
Self {
endpoint: None,
timeout: Duration::from_secs(10),
refresh_skew: Duration::from_secs(60),
retry: RetryPolicy::default(),
}
}
}
#[derive(Serialize)]
struct Claims {
iss: String,
scope: String,
aud: String,
iat: u64,
exp: u64,
}
pub fn signed_assertion(
account: &ServiceAccount,
scope: &str,
lifetime: Duration,
now: SystemTime,
) -> Result<String, Error> {
let iat = now
.duration_since(UNIX_EPOCH)
.map_err(|_| Error::InvalidField("clock"))?
.as_secs();
let exp = iat.saturating_add(lifetime.as_secs().min(3600));
let claims = Claims {
iss: account.client_email.clone(),
scope: scope.into(),
aud: account.token_uri.clone(),
iat,
exp,
};
Ok(encode(
&Header::new(Algorithm::RS256),
&claims,
&EncodingKey::from_rsa_pem(account.private_key.as_bytes())?,
)?)
}
#[derive(Deserialize)]
struct TokenResponse {
access_token: Option<String>,
expires_in: Option<u64>,
}
#[derive(Clone)]
pub struct OAuthClient {
http: Client,
policy: ExchangePolicy,
}
#[async_trait]
pub trait TokenExchanger: Send + Sync {
async fn exchange(
&self,
account: &ServiceAccount,
scope: &str,
now: SystemTime,
) -> Result<(String, Duration), Error>;
}
#[async_trait]
impl TokenExchanger for OAuthClient {
async fn exchange(
&self,
account: &ServiceAccount,
scope: &str,
now: SystemTime,
) -> Result<(String, Duration), Error> {
OAuthClient::exchange(self, account, scope, now).await
}
}
impl OAuthClient {
pub fn new(policy: ExchangePolicy) -> Result<Self, Error> {
Ok(Self {
http: Client::builder().timeout(policy.timeout).build()?,
policy,
})
}
pub async fn exchange(
&self,
account: &ServiceAccount,
scope: &str,
now: SystemTime,
) -> Result<(String, Duration), Error> {
let assertion = signed_assertion(account, scope, Duration::from_secs(3600), now)?;
self.exchange_assertion(&account.token_uri, &assertion)
.await
}
pub async fn exchange_assertion(
&self,
endpoint: &str,
assertion: &str,
) -> Result<(String, Duration), Error> {
let mut delay = self.policy.retry.initial_backoff;
let attempts = self.policy.retry.max_attempts.max(1);
let mut last = None;
for n in 0..attempts {
let r = self
.http
.post(self.policy.endpoint.as_deref().unwrap_or(endpoint))
.form(&[
("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"),
("assertion", assertion),
])
.send()
.await;
match r {
Ok(resp) if resp.status().is_success() => {
let body: TokenResponse =
resp.json().await.map_err(|_| Error::MalformedResponse)?;
let token = body
.access_token
.filter(|x| !x.is_empty())
.ok_or(Error::MalformedResponse)?;
let secs = body
.expires_in
.filter(|x| *x > 0)
.ok_or(Error::MalformedResponse)?;
return Ok((token, Duration::from_secs(secs)));
}
Ok(resp) => {
let status = resp.status();
let _ = resp.bytes().await;
let e = Error::HttpStatus { status };
if !status.is_server_error() && status != StatusCode::TOO_MANY_REQUESTS {
return Err(e);
}
last = Some(e);
}
Err(e) => last = Some(Error::Http(e)),
}
if n + 1 < attempts {
tokio::time::sleep(delay).await;
delay = (delay * 2).min(self.policy.retry.max_backoff);
}
}
Err(Error::RetriesExhausted(Box::new(
last.unwrap_or(Error::MalformedResponse),
)))
}
}
struct Inner {
token: Option<AccessToken>,
state: TokenState,
generation: u64,
credential_fingerprint: Option<String>,
pending_fingerprint: Option<String>,
closed: bool,
}
pub struct TokenManager<
S: CredentialSource,
C: Clock = SystemClock,
E: TokenExchanger = OAuthClient,
> {
source: S,
exchanger: E,
scope: String,
skew: Duration,
clock: C,
inner: Mutex<Inner>,
notify: Notify,
refreshes: AtomicU64,
failures: AtomicU64,
observer: Option<Arc<dyn RefreshObserver>>,
}
impl<S: CredentialSource> TokenManager<S> {
pub fn new(source: S, policy: ExchangePolicy) -> Result<Self, Error> {
Self::with_clock(source, policy, SystemClock)
}
}
impl<S: CredentialSource, C: Clock> TokenManager<S, C, OAuthClient> {
pub fn with_clock(source: S, policy: ExchangePolicy, clock: C) -> Result<Self, Error> {
Ok(Self {
source,
exchanger: OAuthClient::new(policy.clone())?,
scope: CLOUD_PLATFORM_SCOPE.into(),
skew: policy.refresh_skew,
clock,
inner: Mutex::new(Inner {
token: None,
state: TokenState::Empty,
generation: 0,
credential_fingerprint: None,
pending_fingerprint: None,
closed: false,
}),
notify: Notify::new(),
refreshes: AtomicU64::new(0),
failures: AtomicU64::new(0),
observer: None,
})
}
}
impl<S: CredentialSource, C: Clock, E: TokenExchanger + 'static> TokenManager<S, C, E> {
pub fn with_exchanger(source: S, exchanger: E, refresh_skew: Duration, clock: C) -> Self {
Self {
source,
exchanger,
scope: CLOUD_PLATFORM_SCOPE.into(),
skew: refresh_skew,
clock,
inner: Mutex::new(Inner {
token: None,
state: TokenState::Empty,
generation: 0,
credential_fingerprint: None,
pending_fingerprint: None,
closed: false,
}),
notify: Notify::new(),
refreshes: AtomicU64::new(0),
failures: AtomicU64::new(0),
observer: None,
}
}
pub fn with_scope(mut self, scope: impl Into<String>) -> Self {
self.scope = scope.into();
self
}
pub fn with_observer(mut self, observer: Arc<dyn RefreshObserver>) -> Self {
self.observer = Some(observer);
self
}
pub async fn token(&self) -> Result<AccessToken, Error> {
loop {
let mut g = self.inner.lock().await;
let now = self.clock.now();
if let Some(t) = &g.token {
if t.expires_at
.duration_since(now)
.map(|remaining| remaining > self.skew)
.unwrap_or(false)
{
return Ok(t.clone());
}
}
if g.closed {
return Err(Error::Shutdown);
}
if g.state == TokenState::Refreshing {
let wait = self.notify.notified();
drop(g);
wait.await;
continue;
}
g.state = TokenState::Refreshing;
let old_fp = g.credential_fingerprint.clone();
drop(g);
let result = self.refresh(old_fp).await;
let mut g = self.inner.lock().await;
match result {
Ok((token, fp)) => {
self.refreshes.fetch_add(1, Ordering::Relaxed);
g.generation += 1;
let t = AccessToken {
value: token.0,
expires_at: now + token.1,
generation: g.generation,
};
g.token = Some(t.clone());
g.credential_fingerprint = Some(fp);
g.pending_fingerprint = None;
g.state = TokenState::Ready;
if let Some(observer) = &self.observer {
observer.refreshed(
g.credential_fingerprint.as_deref().unwrap_or("unknown"),
g.generation,
token.1,
);
}
self.notify.notify_waiters();
return Ok(t);
}
Err(e) => {
self.failures.fetch_add(1, Ordering::Relaxed);
g.state = TokenState::Failed;
g.pending_fingerprint = None;
if let Some(observer) = &self.observer {
observer.failed(g.credential_fingerprint.as_deref());
}
self.notify.notify_waiters();
return Err(e);
}
}
}
}
async fn refresh(&self, old_fp: Option<String>) -> Result<((String, Duration), String), Error> {
let account = self.source.load().await?;
let fp = account.fingerprint();
let _rotated = old_fp.as_deref() != Some(fp.as_str());
Ok((
self.exchanger
.exchange(&account, &self.scope, self.clock.now())
.await?,
fp,
))
}
pub async fn refresh_now(&self) -> Result<AccessToken, Error> {
self.invalidate().await;
self.token().await
}
pub async fn refresh_if_changed(&self) -> Result<AccessToken, Error> {
let account = self.source.load().await?;
let fingerprint = account.fingerprint();
loop {
let mut state = self.inner.lock().await;
if state.credential_fingerprint.as_deref() == Some(fingerprint.as_str()) {
drop(state);
return self.token().await;
}
if state.pending_fingerprint.as_deref() == Some(fingerprint.as_str())
|| state.state == TokenState::Refreshing
{
let wait = self.notify.notified();
drop(state);
wait.await;
continue;
}
state.pending_fingerprint = Some(fingerprint.clone());
state.token = None;
state.state = TokenState::Empty;
drop(state);
return self.token().await;
}
}
pub async fn invalidate(&self) {
let mut g = self.inner.lock().await;
g.token = None;
g.state = TokenState::Empty;
self.notify.notify_waiters();
}
pub async fn state(&self) -> TokenState {
self.inner.lock().await.state
}
pub async fn fingerprint(&self) -> Option<String> {
self.inner.lock().await.credential_fingerprint.clone()
}
pub async fn shutdown(&self) {
let mut g = self.inner.lock().await;
g.closed = true;
self.notify.notify_waiters();
}
pub async fn ready(&self) -> bool {
self.state().await == TokenState::Ready
}
pub fn refresh_count(&self) -> u64 {
self.refreshes.load(Ordering::Relaxed)
}
pub fn failure_count(&self) -> u64 {
self.failures.load(Ordering::Relaxed)
}
pub fn spawn_proactive_refresh(
self: Arc<Self>,
interval: Duration,
) -> tokio::task::JoinHandle<()>
where
S: 'static,
C: 'static,
{
tokio::spawn(async move {
let mut ticker = tokio::time::interval(interval.max(Duration::from_millis(10)));
loop {
ticker.tick().await;
if self.inner.lock().await.closed {
break;
}
let _ = self.token().await;
}
})
}
}
#[cfg(feature = "rtdb-client")]
impl<S: CredentialSource, C: Clock, E: TokenExchanger + 'static> TokenManager<S, C, E> {
pub async fn rtdb_client(
&self,
base_url: impl Into<String>,
) -> Result<rtdb_rs::RtdbClient, Error> {
let t = self.token().await?;
Ok(rtdb_rs::RtdbClient::new(base_url, t.value))
}
pub async fn update_rtdb_client(
&self,
client: rtdb_rs::RtdbClient,
) -> Result<rtdb_rs::RtdbClient, Error> {
let t = self.token().await?;
Ok(client.with_token(t.value))
}
}
pub struct CredentialManager<S: CredentialSource, C: Clock = SystemClock> {
entries: Mutex<std::collections::HashMap<String, Arc<TokenManager<S, C>>>>,
source_factory: Arc<dyn Fn(&str) -> S + Send + Sync>,
capacity: usize,
}
impl<S: CredentialSource + 'static, C: Clock + Default + 'static> CredentialManager<S, C> {
pub fn new(factory: impl Fn(&str) -> S + Send + Sync + 'static) -> Self {
Self::with_capacity(256, factory)
}
pub fn with_capacity(
capacity: usize,
factory: impl Fn(&str) -> S + Send + Sync + 'static,
) -> Self {
Self {
entries: Mutex::new(std::collections::HashMap::new()),
source_factory: Arc::new(factory),
capacity: capacity.max(1),
}
}
pub async fn get(
&self,
identity: &str,
policy: ExchangePolicy,
) -> Result<Arc<TokenManager<S, C>>, Error> {
let mut e = self.entries.lock().await;
if let Some(m) = e.get(identity) {
return Ok(m.clone());
}
let m = Arc::new(TokenManager::with_clock(
(self.source_factory)(identity),
policy,
C::default(),
)?);
if e.len() >= self.capacity {
if let Some(oldest) = e.keys().next().cloned() {
e.remove(&oldest);
}
}
e.insert(identity.into(), m.clone());
Ok(m)
}
pub async fn len(&self) -> usize {
self.entries.lock().await.len()
}
pub async fn is_empty(&self) -> bool {
self.entries.lock().await.is_empty()
}
pub async fn remove(&self, identity: &str) {
self.entries.lock().await.remove(identity);
}
pub async fn health(&self) -> Vec<IdentityHealth> {
let entries = self.entries.lock().await;
let mut result = Vec::with_capacity(entries.len());
for (identity, manager) in entries.iter() {
result.push(IdentityHealth {
identity: identity.clone(),
state: manager.state().await,
fingerprint: manager.fingerprint().await,
refreshes: manager.refresh_count(),
failures: manager.failure_count(),
});
}
result.sort_by(|left, right| left.identity.cmp(&right.identity));
result
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{
atomic::{AtomicBool, AtomicUsize, Ordering},
Mutex as StdMutex,
};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
const JSON: &str = r#"{"type":"service_account","project_id":"test-project","client_email":"test@example.invalid","private_key":"-----BEGIN PRIVATE KEY-----\nfake\n-----END PRIVATE KEY-----\n"}"#;
fn fake_account(id: &str) -> ServiceAccount {
ServiceAccount {
project_id: Some(id.into()),
client_email: format!("{id}@example.invalid"),
private_key: "not-used-by-test-exchanger".into(),
token_uri: "http://127.0.0.1/token".into(),
}
}
#[derive(Clone)]
struct MockExchange {
calls: Arc<AtomicUsize>,
fail: Arc<AtomicBool>,
delay: Duration,
}
#[async_trait]
impl TokenExchanger for MockExchange {
async fn exchange(
&self,
account: &ServiceAccount,
_scope: &str,
_now: SystemTime,
) -> Result<(String, Duration), Error> {
self.calls.fetch_add(1, Ordering::SeqCst);
if self.delay > Duration::ZERO {
tokio::time::sleep(self.delay).await;
}
if self.fail.load(Ordering::SeqCst) {
return Err(Error::Source(format!(
"exchange failed for {}",
account.fingerprint()
)));
}
Ok((
format!(
"token-{}",
account.project_id.as_deref().unwrap_or("unknown")
),
Duration::from_millis(100),
))
}
}
#[derive(Clone)]
struct TestClock(Arc<AtomicU64>);
impl TestClock {
fn new() -> Self {
Self(Arc::new(AtomicU64::new(0)))
}
fn advance(&self, millis: u64) {
self.0.fetch_add(millis, Ordering::SeqCst);
}
}
impl Clock for TestClock {
fn now(&self) -> SystemTime {
UNIX_EPOCH + Duration::from_millis(self.0.load(Ordering::SeqCst))
}
}
#[derive(Clone)]
struct RotatingSource(Arc<StdMutex<ServiceAccount>>);
#[async_trait]
impl CredentialSource for RotatingSource {
async fn load(&self) -> Result<ServiceAccount, Error> {
Ok(self.0.lock().unwrap().clone())
}
}
struct CountingObserver(AtomicUsize);
impl RefreshObserver for CountingObserver {
fn refreshed(&self, _fingerprint: &str, _generation: u64, _lifetime: Duration) {
self.0.fetch_add(1, Ordering::SeqCst);
}
fn failed(&self, _fingerprint: Option<&str>) {}
}
#[test]
fn validates_and_redacts_credentials() {
let account = ServiceAccount::from_json(JSON).unwrap();
let debug = format!("{account:?}");
assert!(debug.contains("[REDACTED]"));
assert!(!debug.contains("fake"));
assert!(account.fingerprint().starts_with("sha256:"));
assert!(matches!(
ServiceAccount::from_json("{}"),
Err(Error::MissingField("client_email"))
));
}
async fn mock_endpoint(status: &str, body: &str, requests: Arc<AtomicUsize>) -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let status = status.to_string();
let body = body.to_string();
tokio::spawn(async move {
if let Ok((mut socket, _)) = listener.accept().await {
requests.fetch_add(1, Ordering::SeqCst);
let mut buf = [0u8; 1024];
let _ = socket.read(&mut buf).await;
let response = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{body}", body.len());
let _ = socket.write_all(response.as_bytes()).await;
}
});
format!("http://{addr}/token")
}
#[tokio::test]
async fn local_exchange_parses_success_without_google() {
let requests = Arc::new(AtomicUsize::new(0));
let endpoint = mock_endpoint(
"200 OK",
r#"{"access_token":"test-token","expires_in":120}"#,
requests.clone(),
)
.await;
let policy = ExchangePolicy {
endpoint: Some(endpoint.clone()),
retry: RetryPolicy {
max_attempts: 1,
..Default::default()
},
..Default::default()
};
let oauth = OAuthClient::new(policy).unwrap();
let (token, lifetime) = oauth
.exchange_assertion(&endpoint, "non-secret-test-assertion")
.await
.unwrap();
assert_eq!(token, "test-token");
assert_eq!(lifetime, Duration::from_secs(120));
assert_eq!(requests.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn malformed_timeout_and_disconnect_are_deterministic() {
let requests = Arc::new(AtomicUsize::new(0));
let malformed = mock_endpoint("200 OK", "not-json", requests.clone()).await;
let policy = ExchangePolicy {
endpoint: Some(malformed.clone()),
retry: RetryPolicy {
max_attempts: 1,
..Default::default()
},
..Default::default()
};
let oauth = OAuthClient::new(policy).unwrap();
assert!(matches!(
oauth.exchange_assertion(&malformed, "assertion").await,
Err(Error::MalformedResponse)
));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let timeout_endpoint = format!("http://{}/token", listener.local_addr().unwrap());
tokio::spawn(async move {
let Ok((mut socket, _)) = listener.accept().await else {
return;
};
let mut buf = [0u8; 256];
let _ = socket.read(&mut buf).await;
tokio::time::sleep(Duration::from_millis(100)).await;
});
let oauth = OAuthClient::new(ExchangePolicy {
endpoint: Some(timeout_endpoint.clone()),
timeout: Duration::from_millis(10),
retry: RetryPolicy {
max_attempts: 1,
..Default::default()
},
..Default::default()
})
.unwrap();
assert!(matches!(
oauth
.exchange_assertion(&timeout_endpoint, "assertion")
.await,
Err(Error::RetriesExhausted(_))
));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let reset_endpoint = format!("http://{}/token", listener.local_addr().unwrap());
tokio::spawn(async move {
let _ = listener.accept().await;
});
let oauth = OAuthClient::new(ExchangePolicy {
endpoint: Some(reset_endpoint.clone()),
retry: RetryPolicy {
max_attempts: 1,
..Default::default()
},
..Default::default()
})
.unwrap();
assert!(matches!(
oauth.exchange_assertion(&reset_endpoint, "assertion").await,
Err(Error::RetriesExhausted(_))
));
}
#[tokio::test]
async fn transient_server_error_retries_then_succeeds() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = format!("http://{}/token", listener.local_addr().unwrap());
let requests = Arc::new(AtomicUsize::new(0));
let seen = requests.clone();
tokio::spawn(async move {
for status in ["500 Internal Server Error", "200 OK"] {
let Ok((mut socket, _)) = listener.accept().await else {
return;
};
seen.fetch_add(1, Ordering::SeqCst);
let mut buf = [0u8; 256];
let _ = socket.read(&mut buf).await;
let body = if status.starts_with("200") {
r#"{"access_token":"recovered","expires_in":30}"#
} else {
"{}"
};
let response = format!(
"HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
let _ = socket.write_all(response.as_bytes()).await;
}
});
let oauth = OAuthClient::new(ExchangePolicy {
endpoint: Some(endpoint.clone()),
retry: RetryPolicy {
max_attempts: 2,
initial_backoff: Duration::from_millis(1),
..Default::default()
},
..Default::default()
})
.unwrap();
assert_eq!(
oauth
.exchange_assertion(&endpoint, "assertion")
.await
.unwrap()
.0,
"recovered"
);
assert_eq!(requests.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn permanent_http_errors_fail_without_leaking_body() {
let requests = Arc::new(AtomicUsize::new(0));
let endpoint = mock_endpoint(
"401 Unauthorized",
r#"{"error":"private-token-must-not-appear"}"#,
requests.clone(),
)
.await;
let mut policy = ExchangePolicy::default();
policy.retry.max_attempts = 4;
let oauth = OAuthClient::new(ExchangePolicy {
endpoint: Some(endpoint.clone()),
..policy
})
.unwrap();
let error = oauth
.exchange_assertion(&endpoint, "assertion")
.await
.unwrap_err();
assert!(!error.to_string().contains("private-token"));
assert_eq!(requests.load(Ordering::SeqCst), 1);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn one_hundred_concurrent_callers_share_one_generation() {
let calls = Arc::new(AtomicUsize::new(0));
let manager = Arc::new(TokenManager::with_exchanger(
StaticCredentials::new(fake_account("single-flight")),
MockExchange {
calls: calls.clone(),
fail: Arc::new(AtomicBool::new(false)),
delay: Duration::from_millis(20),
},
Duration::from_millis(5),
SystemClock,
));
let mut tasks = Vec::new();
for _ in 0..100 {
let manager = manager.clone();
tasks.push(tokio::spawn(async move { manager.token().await.unwrap() }));
}
let tokens = futures_util::future::join_all(tasks)
.await
.into_iter()
.map(|x| x.unwrap())
.collect::<Vec<_>>();
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert!(tokens
.iter()
.all(|t| t.generation == 1 && t.value == "token-single-flight"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn expiry_boundary_refreshes_once_for_sixty_four_callers() {
let calls = Arc::new(AtomicUsize::new(0));
let clock = TestClock::new();
let manager = Arc::new(TokenManager::with_exchanger(
StaticCredentials::new(fake_account("boundary")),
MockExchange {
calls: calls.clone(),
fail: Arc::new(AtomicBool::new(false)),
delay: Duration::from_millis(10),
},
Duration::from_millis(10),
clock.clone(),
));
assert_eq!(manager.token().await.unwrap().generation, 1);
clock.advance(91);
let mut tasks = Vec::new();
for _ in 0..64 {
let manager = manager.clone();
tasks.push(tokio::spawn(async move { manager.token().await.unwrap() }));
}
let results = futures_util::future::join_all(tasks).await;
assert!(results.iter().all(|r| r.as_ref().unwrap().generation == 2));
assert_eq!(calls.load(Ordering::SeqCst), 2);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn rotation_reauthenticates_and_preserves_new_identity() {
let calls = Arc::new(AtomicUsize::new(0));
let source = RotatingSource(Arc::new(StdMutex::new(fake_account("old"))));
let manager = Arc::new(TokenManager::with_exchanger(
source.clone(),
MockExchange {
calls: calls.clone(),
fail: Arc::new(AtomicBool::new(false)),
delay: Duration::from_millis(5),
},
Duration::ZERO,
SystemClock,
));
manager.token().await.unwrap();
let old_fp = manager.fingerprint().await.unwrap();
*source.0.lock().unwrap() = fake_account("new");
let new_token = manager.refresh_if_changed().await.unwrap();
assert_ne!(old_fp, manager.fingerprint().await.unwrap());
assert_eq!(new_token.value, "token-new");
assert_eq!(manager.refresh_count(), 2);
*source.0.lock().unwrap() = fake_account("third");
let mut consumers = Vec::new();
for _ in 0..64 {
let manager = manager.clone();
consumers.push(tokio::spawn(async move {
manager.refresh_if_changed().await.unwrap()
}));
}
let results = futures_util::future::join_all(consumers).await;
assert!(results
.iter()
.all(|result| result.as_ref().unwrap().value == "token-third"));
assert_eq!(manager.refresh_count(), 3);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn observer_receives_only_secret_free_refresh_events_and_task_stops() {
let calls = Arc::new(AtomicUsize::new(0));
let observer = Arc::new(CountingObserver(AtomicUsize::new(0)));
let manager = Arc::new(
TokenManager::with_exchanger(
StaticCredentials::new(fake_account("observed")),
MockExchange {
calls,
fail: Arc::new(AtomicBool::new(false)),
delay: Duration::ZERO,
},
Duration::ZERO,
SystemClock,
)
.with_observer(observer.clone()),
);
manager.token().await.unwrap();
assert_eq!(observer.0.load(Ordering::SeqCst), 1);
let task = manager
.clone()
.spawn_proactive_refresh(Duration::from_millis(10));
tokio::time::sleep(Duration::from_millis(25)).await;
manager.shutdown().await;
tokio::time::timeout(Duration::from_secs(1), task)
.await
.unwrap()
.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn thousand_request_manual_profile_has_bounded_generations() {
let calls = Arc::new(AtomicUsize::new(0));
let manager = Arc::new(TokenManager::with_exchanger(
StaticCredentials::new(fake_account("soak")),
MockExchange {
calls: calls.clone(),
fail: Arc::new(AtomicBool::new(false)),
delay: Duration::from_millis(1),
},
Duration::ZERO,
SystemClock,
));
for _generation in 0..10 {
manager.invalidate().await;
let mut tasks = Vec::new();
for _ in 0..128 {
let manager = manager.clone();
tasks.push(tokio::spawn(async move {
let mut last = 0;
for _ in 0..8 {
last = manager.token().await.unwrap().generation;
}
last
}));
}
let results = futures_util::future::join_all(tasks).await;
assert!(results.iter().all(|r| r.is_ok()));
}
assert_eq!(manager.refresh_count(), 10);
assert!(calls.load(Ordering::SeqCst) <= 10);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn outage_fails_bounded_and_recovers_after_invalidate() {
let calls = Arc::new(AtomicUsize::new(0));
let fail = Arc::new(AtomicBool::new(true));
let manager = Arc::new(TokenManager::with_exchanger(
StaticCredentials::new(fake_account("outage")),
MockExchange {
calls: calls.clone(),
fail: fail.clone(),
delay: Duration::from_millis(5),
},
Duration::ZERO,
SystemClock,
));
let mut tasks = Vec::new();
for _ in 0..128 {
let manager = manager.clone();
tasks.push(tokio::spawn(async move { manager.token().await.is_err() }));
}
assert!(futures_util::future::join_all(tasks)
.await
.into_iter()
.all(|x| x.unwrap()));
assert!(calls.load(Ordering::SeqCst) <= 128);
fail.store(false, Ordering::SeqCst);
manager.invalidate().await;
assert_eq!(manager.token().await.unwrap().generation, 1);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn multi_identity_managers_are_isolated_and_bounded() {
let manager = CredentialManager::<StaticCredentials>::with_capacity(100, |id| {
StaticCredentials::new(fake_account(id))
});
let mut entries = Vec::new();
for i in 0..100 {
entries.push(
manager
.get(&format!("identity-{i}"), ExchangePolicy::default())
.await
.unwrap(),
);
}
assert_eq!(manager.len().await, 100);
assert!(!Arc::ptr_eq(&entries[0], &entries[1]));
manager
.get("identity-100", ExchangePolicy::default())
.await
.unwrap();
assert_eq!(manager.len().await, 100);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn one_hundred_identities_with_ten_consumers_each_stay_isolated() {
let mut all = Vec::new();
for identity in 0..100 {
let manager = Arc::new(TokenManager::with_exchanger(
StaticCredentials::new(fake_account(&format!("identity-{identity}"))),
MockExchange {
calls: Arc::new(AtomicUsize::new(0)),
fail: Arc::new(AtomicBool::new(false)),
delay: Duration::from_millis(1),
},
Duration::ZERO,
SystemClock,
));
let mut consumers = Vec::new();
for _ in 0..10 {
let manager = manager.clone();
consumers.push(tokio::spawn(async move { manager.token().await.unwrap() }));
}
all.push((identity, manager, consumers));
}
for (identity, manager, consumers) in all {
let results = futures_util::future::join_all(consumers).await;
let expected = format!("token-identity-{identity}");
assert!(results
.iter()
.all(|result| result.as_ref().unwrap().value == expected));
assert_eq!(manager.refresh_count(), 1);
}
}
}