Skip to main content

udb_client/
auth.rs

1//! Login and the token lifecycle.
2//!
3//! The broker's refresh token is SINGLE-USE: every successful refresh mints a new
4//! one and invalidates the presented one atomically. A client that keeps sending
5//! its original refresh token authenticates, refreshes once, and then fails with
6//! `Unauthenticated: invalid credential` at the second refresh boundary — an hour
7//! or a day later, far from the code that caused it. [`TokenManager`] persists the
8//! rotated token, which is the whole reason it exists rather than leaving refresh
9//! to callers.
10//!
11//! # Recovery after a broker restart
12//!
13//! A client must outlive the broker it talks to. [`TokenManager::login`] retries
14//! a transient failure (broker unreachable or restarting, overloaded, reply
15//! lost), so a service that starts while the broker restarts still signs in.
16//! When the broker refuses the access token the manager issued before its local
17//! expiry (a restarted broker that lost the session, a revoked session),
18//! [`TokenManager::reject`] marks it, and the next [`TokenManager::access_token`]
19//! renews it: refresh, else sign in again. A [`crate::UdbClient`] built with
20//! [`crate::UdbClient::with_token_manager`] does this itself and repeats the
21//! refused call exactly once. A token renewed moments ago that is refused again
22//! is not renewed again, and while a refused token cannot be renewed, calls are
23//! refused locally with a backoff (1s doubling to 60s) instead of one sign-in
24//! per call.
25
26use std::collections::hash_map::RandomState;
27use std::hash::{BuildHasher, Hasher};
28use std::sync::Arc;
29use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
30
31use tokio::sync::Mutex;
32use tonic::transport::Channel;
33use tonic::{Code, Status};
34
35use crate::metadata::Metadata;
36use crate::proto::udb::core::authn::services::v1 as authn;
37use crate::proto::udb::core::authn::services::v1::authn_service_client::AuthnServiceClient;
38
39/// A stored credential set. `expires_at_unix` of 0 means "unknown"; the manager
40/// then treats the token as valid until an explicit refresh.
41#[derive(Clone, Debug, Default, PartialEq, Eq)]
42pub struct Token {
43    pub access_token: String,
44    pub refresh_token: String,
45    pub session_id: String,
46    pub expires_at_unix: u64,
47}
48
49impl Token {
50    /// Non-empty and not within `skew` of expiry.
51    pub fn is_valid(&self, now_unix: u64, skew: Duration) -> bool {
52        if self.access_token.is_empty() {
53            return false;
54        }
55        if self.expires_at_unix == 0 {
56            return true;
57        }
58        now_unix.saturating_add(skew.as_secs()) < self.expires_at_unix
59    }
60}
61
62/// Sign-in attempts [`TokenManager::login`] makes before it gives up on a
63/// transient failure.
64pub const LOGIN_ATTEMPTS: u32 = 3;
65
66/// First wait after a failed attempt; doubles per consecutive failure.
67const RENEW_BACKOFF_BASE: Duration = Duration::from_secs(1);
68/// Longest wait between attempts.
69const RENEW_BACKOFF_CAP: Duration = Duration::from_secs(60);
70/// How soon after renewing a refused token another refusal is taken as final
71/// rather than as a reason to renew again.
72const REJECTED_RENEW_INTERVAL: Duration = Duration::from_secs(5);
73
74/// Whether repeating a failed sign-in can help: the broker was unreachable,
75/// overloaded, or the reply was lost. A refused credential is final.
76fn transient(status: &Status) -> bool {
77    matches!(
78        status.code(),
79        Code::Unavailable | Code::DeadlineExceeded | Code::ResourceExhausted | Code::Aborted
80    )
81}
82
83/// The wait after the n-th consecutive failure: a random time in
84/// `[ceiling/2, ceiling]`, where `ceiling = base * 2^(n-1)`, capped at `limit`.
85fn jittered_backoff(failures: u32, base: Duration, limit: Duration) -> Duration {
86    let mut ceiling = base;
87    let mut i = 1;
88    while i < failures && ceiling < limit {
89        ceiling *= 2;
90        i += 1;
91    }
92    let ceiling = ceiling.min(limit);
93    let half = ceiling / 2;
94    let span = u64::try_from((ceiling - half).as_millis()).unwrap_or(u64::MAX);
95    // A freshly keyed SipHash of nothing: random enough for jitter, with no
96    // dependency on a random-number crate.
97    let random = RandomState::new().build_hasher().finish();
98    half + Duration::from_millis(random % span.saturating_add(1))
99}
100
101/// What the manager knows about tokens the broker refused before their expiry.
102#[derive(Default)]
103struct RefusalState {
104    /// The refused access token; a stored token carrying it is renewed as if
105    /// it were stale.
106    rejected: String,
107    /// When `rejected` was marked.
108    rejected_at: Option<Instant>,
109    /// Consecutive failed renewals of the refused token.
110    failures: u32,
111    /// Until when calls are refused locally after a failed renewal, and with
112    /// what.
113    retry_at: Option<Instant>,
114    last_error: Option<(Code, String)>,
115}
116
117impl RefusalState {
118    /// Whether the broker's refusal of `sent` should be answered by renewing
119    /// the token and repeating the call. `current` is the token the manager
120    /// holds now.
121    fn reject(&mut self, sent: &str, current: &str, now: Instant) -> bool {
122        if sent.is_empty() {
123            return false;
124        }
125        if sent == self.rejected {
126            // Already marked by a concurrent call; the next resolve renews it.
127            return true;
128        }
129        if sent != current {
130            // Not this manager's token, or one already replaced by an
131            // ordinary refresh: nothing to renew.
132            return false;
133        }
134        if let Some(at) = self.rejected_at {
135            if now.saturating_duration_since(at) < REJECTED_RENEW_INTERVAL {
136                // The token renewed moments ago is refused too: renewing
137                // cannot help.
138                return false;
139            }
140        }
141        self.rejected = sent.to_string();
142        self.rejected_at = Some(now);
143        self.renewed();
144        true
145    }
146
147    fn is_rejected(&self, access_token: &str) -> bool {
148        !self.rejected.is_empty() && self.rejected == access_token
149    }
150
151    /// The local refusal while a failed renewal backs off.
152    fn refused_until(&self, now: Instant) -> Option<Status> {
153        match (self.retry_at, &self.last_error) {
154            (Some(at), Some((code, message))) if now < at => Some(Status::new(
155                *code,
156                format!("credential refused by the broker and not yet renewed: {message}"),
157            )),
158            _ => None,
159        }
160    }
161
162    fn renew_failed(&mut self, err: &Status, now: Instant) {
163        self.failures = self.failures.saturating_add(1);
164        self.retry_at =
165            Some(now + jittered_backoff(self.failures, RENEW_BACKOFF_BASE, RENEW_BACKOFF_CAP));
166        self.last_error = Some((err.code(), err.message().to_string()));
167    }
168
169    fn renewed(&mut self) {
170        self.failures = 0;
171        self.retry_at = None;
172        self.last_error = None;
173    }
174}
175
176fn now_unix() -> u64 {
177    SystemTime::now()
178        .duration_since(UNIX_EPOCH)
179        .map(|d| d.as_secs())
180        .unwrap_or(0)
181}
182
183/// Logs in and keeps the token fresh.
184///
185/// Refresh is single-flighted: concurrent callers that all observe a stale token
186/// share one `RefreshToken` round-trip. Without that, N tasks would each present
187/// the same single-use refresh token, one would win, and the rest would be told
188/// their credential is invalid.
189#[derive(Clone)]
190pub struct TokenManager {
191    inner: Arc<Inner>,
192}
193
194struct Inner {
195    client: Mutex<AuthnServiceClient<Channel>>,
196    token: Mutex<Token>,
197    skew: Duration,
198    meta: Metadata,
199    /// The username and password of the last successful login, kept so a
200    /// session the broker lost (a restart) can sign in again when its refresh
201    /// token is refused too. Never printed.
202    sign_in: std::sync::Mutex<Option<(String, String)>>,
203    refusal: std::sync::Mutex<RefusalState>,
204}
205
206/// Hand-written rather than derived: a derived `Debug` would print the stored
207/// access and refresh tokens, and this type is exactly the sort of thing that
208/// ends up in a `tracing` field or a panic message.
209impl std::fmt::Debug for TokenManager {
210    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
211        f.debug_struct("TokenManager")
212            .field("tenant_id", &self.inner.meta.tenant_id)
213            .field("project_id", &self.inner.meta.project_id)
214            .field("skew", &self.inner.skew)
215            .field("token", &"<redacted>")
216            .finish()
217    }
218}
219
220impl TokenManager {
221    /// Default refresh skew: refresh this long before actual expiry.
222    pub const DEFAULT_SKEW: Duration = Duration::from_secs(30);
223
224    pub fn new(channel: Channel, meta: Metadata) -> Self {
225        Self::with_skew(channel, meta, Self::DEFAULT_SKEW)
226    }
227
228    pub fn with_skew(channel: Channel, meta: Metadata, skew: Duration) -> Self {
229        Self {
230            inner: Arc::new(Inner {
231                client: Mutex::new(AuthnServiceClient::new(channel)),
232                token: Mutex::new(Token::default()),
233                skew,
234                meta,
235                sign_in: std::sync::Mutex::new(None),
236                refusal: std::sync::Mutex::new(RefusalState::default()),
237            }),
238        }
239    }
240
241    // The two std mutexes below are never held across an await: every guard is
242    // a temporary dropped within its statement.
243    fn refusal(&self) -> std::sync::MutexGuard<'_, RefusalState> {
244        self.inner
245            .refusal
246            .lock()
247            .unwrap_or_else(std::sync::PoisonError::into_inner)
248    }
249
250    fn sign_in_credentials(&self) -> Option<(String, String)> {
251        self.inner
252            .sign_in
253            .lock()
254            .unwrap_or_else(std::sync::PoisonError::into_inner)
255            .clone()
256    }
257
258    fn remember_sign_in(&self, username: String, password: String) {
259        *self
260            .inner
261            .sign_in
262            .lock()
263            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some((username, password));
264    }
265
266    /// The currently stored token, without refreshing.
267    pub async fn peek(&self) -> Token {
268        self.inner.token.lock().await.clone()
269    }
270
271    /// Authenticate with username and password, storing the result.
272    ///
273    /// A transient failure (broker unreachable or restarting, overloaded, reply
274    /// lost) is retried, up to [`LOGIN_ATTEMPTS`] attempts in all with a
275    /// jittered, doubling wait between them. A refused credential is final.
276    /// The username and password are kept in memory so a session the broker
277    /// later loses can sign in again (see [`TokenManager::reject`]).
278    pub async fn login(
279        &self,
280        username: impl Into<String>,
281        password: impl Into<String>,
282    ) -> Result<Token, Status> {
283        let username = username.into();
284        let password = password.into();
285        let mut last = None;
286        for attempt in 0..LOGIN_ATTEMPTS {
287            if attempt > 0 {
288                let wait = jittered_backoff(attempt, RENEW_BACKOFF_BASE, RENEW_BACKOFF_CAP);
289                tokio::time::sleep(wait).await;
290            }
291            match self.login_rpc(username.clone(), password.clone()).await {
292                Ok(token) => {
293                    *self.inner.token.lock().await = token.clone();
294                    self.remember_sign_in(username, password);
295                    self.refusal().renewed();
296                    return Ok(token);
297                }
298                Err(err) if transient(&err) => last = Some(err),
299                Err(err) => return Err(err),
300            }
301        }
302        Err(last.unwrap_or_else(|| Status::unavailable("login was not attempted")))
303    }
304
305    /// One `Login` round-trip; the caller stores the result.
306    async fn login_rpc(&self, username: String, password: String) -> Result<Token, Status> {
307        let mut req = tonic::Request::new(authn::LoginRequest {
308            username,
309            password,
310            ..Default::default()
311        });
312        self.inner.meta.apply(&mut req)?;
313
314        let resp = {
315            let mut client = self.inner.client.lock().await;
316            client.login(req).await?.into_inner()
317        };
318
319        if resp.mfa_required {
320            return Err(Status::unauthenticated(
321                "login requires a second factor; re-call Login with the MFA credential",
322            ));
323        }
324
325        Ok(Token {
326            access_token: resp.access_token,
327            refresh_token: resp.refresh_token,
328            session_id: resp.session_id,
329            expires_at_unix: expiry_from(resp.access_token_expires_in),
330        })
331    }
332
333    /// The access token, refreshing first if it is stale or the broker refused it.
334    pub async fn access_token(&self) -> Result<String, Status> {
335        {
336            let token = self.inner.token.lock().await;
337            let rejected = self.refusal().is_rejected(&token.access_token);
338            if !rejected && token.is_valid(now_unix(), self.inner.skew) {
339                return Ok(token.access_token.clone());
340            }
341        }
342        self.refresh().await.map(|t| t.access_token)
343    }
344
345    /// Force a refresh. Holding the token lock across the RPC is what makes this
346    /// single-flighted: a second caller waits and then observes the new token
347    /// rather than presenting the spent one.
348    ///
349    /// A token the broker refused (see [`TokenManager::reject`]) is renewed even
350    /// though it has not expired: refresh, else sign in again with the stored
351    /// login. While such a renewal keeps failing it is retried at most once per
352    /// backoff interval (1s doubling to 60s); calls in between are refused
353    /// locally with the last failure.
354    pub async fn refresh(&self) -> Result<Token, Status> {
355        let mut stored = self.inner.token.lock().await;
356        let rejected = self.refusal().is_rejected(&stored.access_token);
357
358        // Someone refreshed while we waited for the lock.
359        if !rejected && stored.is_valid(now_unix(), self.inner.skew) {
360            return Ok(stored.clone());
361        }
362        if !rejected {
363            return self.refresh_rpc(&mut stored).await;
364        }
365
366        let refused = self.refusal().refused_until(Instant::now());
367        if let Some(status) = refused {
368            return Err(status);
369        }
370        let result = match self.refresh_rpc(&mut stored).await {
371            Err(err) if err.code() == Code::Unauthenticated => {
372                // The broker lost the session too (a restart): sign in again.
373                match self.sign_in_credentials() {
374                    Some((username, password)) => {
375                        self.login_rpc(username, password).await.inspect(|token| {
376                            *stored = token.clone();
377                        })
378                    }
379                    None => Err(err),
380                }
381            }
382            other => other,
383        };
384        match &result {
385            Ok(_) => self.refusal().renewed(),
386            Err(err) => self.refusal().renew_failed(err, Instant::now()),
387        }
388        result
389    }
390
391    /// One `RefreshToken` round-trip folded into `stored`.
392    async fn refresh_rpc(&self, stored: &mut Token) -> Result<Token, Status> {
393        if stored.refresh_token.is_empty() && stored.session_id.is_empty() {
394            return Err(Status::unauthenticated(
395                "no refresh token or session id stored; call login() first",
396            ));
397        }
398
399        let mut req = tonic::Request::new(refresh_request(stored));
400        self.inner.meta.apply(&mut req)?;
401
402        let resp = {
403            let mut client = self.inner.client.lock().await;
404            client.refresh_token(req).await?.into_inner()
405        };
406
407        apply_refresh(stored, resp);
408        Ok(stored.clone())
409    }
410
411    /// Report that the broker refused `access_token` (UNAUTHENTICATED) before
412    /// its local expiry. Returns true when a repeat of the refused call can
413    /// carry a different, renewed token: the token is this manager's current
414    /// one (or was already marked by a concurrent call), and it was not itself
415    /// renewed less than 5s ago. The next [`TokenManager::access_token`] then
416    /// renews it.
417    ///
418    /// Only for a call that carried this manager's token and is not an
419    /// `AuthnService` call (a refusal there is about the credential in the
420    /// request, not the bearer). [`crate::UdbClient::with_token_manager`] calls
421    /// it for you.
422    pub async fn reject(&self, access_token: &str) -> bool {
423        let current = self.inner.token.lock().await.access_token.clone();
424        self.refusal()
425            .reject(access_token, &current, Instant::now())
426    }
427
428    /// Metadata carrying the current access token, ready for a data-plane client.
429    pub async fn authenticated_metadata(&self) -> Result<Metadata, Status> {
430        let token = self.access_token().await?;
431        Ok(self.inner.meta.clone().with_bearer_token(token))
432    }
433}
434
435// ── Delegated job tokens ─────────────────────────────────────────────────────
436//
437// A durable async job (an export, a report, a nightly sync) often has to act for
438// the user who started it, long after that user's own short-lived access token
439// expired. `AuthnService.ExchangeTokenForJob` mints a delegated job token for
440// that: the service presents the user's current access token, and the broker
441// returns a token whose subject is the user, whose RFC 8693 actor is the
442// service, which carries the job's purpose and id, and whose scopes are those
443// both the user and the service's grant allow. It has no refresh token, cannot
444// be exchanged again, lives at most the broker's `UDB_DELEGATED_TOKEN_MAX_TTL`
445// (24 hours by default), and stops working as soon as the user, the token or
446// the service's `udb:token:delegate` grant is revoked
447// ([`crate::UdbError::is_token_expired`], [`crate::UdbError::is_token_revoked`]).
448
449/// What [`exchange_token_for_job`] asks for.
450#[derive(Clone, Default)]
451pub struct ExchangeTokenForJobRequest {
452    /// The user's current access token (`Bearer ` prefix optional).
453    pub subject_token: String,
454    /// Why the job acts for the user, e.g. `"export"`; signed into the token.
455    pub purpose: String,
456    /// The job the token is for; at most 200 bytes.
457    pub job_id: String,
458    /// The requested lifetime in seconds; 0 asks for the broker's maximum.
459    pub ttl_seconds: i64,
460}
461
462/// Hand-written rather than derived: a derived `Debug` would print the user's
463/// access token.
464impl std::fmt::Debug for ExchangeTokenForJobRequest {
465    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
466        f.debug_struct("ExchangeTokenForJobRequest")
467            .field("subject_token", &"<redacted>")
468            .field("purpose", &self.purpose)
469            .field("job_id", &self.job_id)
470            .field("ttl_seconds", &self.ttl_seconds)
471            .finish()
472    }
473}
474
475/// A token a job uses to act for `subject` (the user) as `actor` (the service).
476#[derive(Clone, Default, PartialEq, Eq)]
477pub struct DelegatedJobToken {
478    pub access_token: String,
479    /// The token's id; revoking it stops the token.
480    pub jti: String,
481    pub expires_at_unix: i64,
482    /// The user the token acts for.
483    pub subject: String,
484    /// The acting service's identity (the RFC 8693 `act` claim).
485    pub actor: String,
486    pub purpose: String,
487    pub job_id: String,
488    /// The scopes both the user's token and the service's grant allow.
489    pub scopes: Vec<String>,
490    pub tenant_id: String,
491    pub project_id: String,
492}
493
494/// Hand-written rather than derived: a derived `Debug` would print the token.
495impl std::fmt::Debug for DelegatedJobToken {
496    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
497        f.debug_struct("DelegatedJobToken")
498            .field("access_token", &"<redacted>")
499            .field("jti", &self.jti)
500            .field("expires_at_unix", &self.expires_at_unix)
501            .field("subject", &self.subject)
502            .field("actor", &self.actor)
503            .field("purpose", &self.purpose)
504            .field("job_id", &self.job_id)
505            .field("scopes", &self.scopes)
506            .field("tenant_id", &self.tenant_id)
507            .field("project_id", &self.project_id)
508            .finish()
509    }
510}
511
512/// Exchange a user's access token for a delegated job token.
513///
514/// `meta` must carry the SERVICE's own bearer (a service account whose grant
515/// holds `udb:token:delegate`, in the user's tenant); the broker refuses any
516/// other caller, and any subject token that is not a valid user access token of
517/// that tenant, with `PERMISSION_DENIED`. A blank subject token, purpose or job
518/// id, or a negative ttl, is refused locally with `INVALID_ARGUMENT` before any
519/// call. A refusal keeps the broker's code and typed reason
520/// ([`crate::UdbError::reason`]).
521pub async fn exchange_token_for_job(
522    client: &mut AuthnServiceClient<Channel>,
523    meta: &Metadata,
524    request: ExchangeTokenForJobRequest,
525) -> Result<DelegatedJobToken, crate::UdbError> {
526    let job_id = request.job_id.clone();
527    let mut req = tonic::Request::new(job_token_request(request)?);
528    meta.apply(&mut req)?;
529    let resp = client
530        .exchange_token_for_job(req)
531        .await
532        .map_err(crate::UdbError::from_status)?
533        .into_inner();
534    delegated_job_token(resp, &job_id)
535}
536
537fn job_token_request(
538    request: ExchangeTokenForJobRequest,
539) -> Result<authn::ExchangeTokenForJobRequest, Status> {
540    if request.subject_token.trim().is_empty() {
541        return Err(Status::invalid_argument(
542            "udb: exchange_token_for_job: subject_token is required",
543        ));
544    }
545    if request.job_id.trim().is_empty() {
546        return Err(Status::invalid_argument(
547            "udb: exchange_token_for_job: job_id is required",
548        ));
549    }
550    if request.purpose.trim().is_empty() {
551        return Err(Status::invalid_argument(
552            "udb: exchange_token_for_job: purpose is required",
553        ));
554    }
555    if request.ttl_seconds < 0 {
556        return Err(Status::invalid_argument(
557            "udb: exchange_token_for_job: ttl_seconds must not be negative",
558        ));
559    }
560    Ok(authn::ExchangeTokenForJobRequest {
561        subject_token: request.subject_token,
562        purpose: request.purpose,
563        job_id: request.job_id,
564        ttl_seconds: request.ttl_seconds,
565    })
566}
567
568fn delegated_job_token(
569    resp: authn::ExchangeTokenForJobResponse,
570    job_id: &str,
571) -> Result<DelegatedJobToken, crate::UdbError> {
572    if resp.access_token.is_empty() {
573        return Err(Status::internal(format!(
574            "udb: exchange the user's token for job {job_id}: the broker returned no token"
575        ))
576        .into());
577    }
578    Ok(DelegatedJobToken {
579        access_token: resp.access_token,
580        jti: resp.jti,
581        expires_at_unix: resp.expires_at_unix,
582        subject: resp.subject,
583        actor: resp.actor,
584        purpose: resp.purpose,
585        job_id: resp.job_id,
586        scopes: resp.scopes,
587        tenant_id: resp.tenant_id,
588        project_id: resp.project_id,
589    })
590}
591
592/// The `RefreshToken` request for the stored credential.
593fn refresh_request(stored: &Token) -> authn::RefreshTokenRequest {
594    authn::RefreshTokenRequest {
595        refresh_token: stored.refresh_token.clone(),
596        session_id: stored.session_id.clone(),
597    }
598}
599
600/// Fold a `RefreshToken` reply into the stored credential.
601fn apply_refresh(stored: &mut Token, resp: authn::RefreshTokenResponse) {
602    stored.access_token = resp.access_token;
603    stored.expires_at_unix = expiry_from(resp.access_token_expires_in);
604    // Persist the ROTATED refresh token. Guarded on non-empty because the
605    // response omits it when the caller refreshed with a legacy server-side
606    // session id rather than a token-family credential; assigning blindly
607    // would erase a working credential.
608    if !resp.refresh_token.is_empty() {
609        stored.refresh_token = resp.refresh_token;
610    }
611}
612
613fn expiry_from(expires_in_secs: i32) -> u64 {
614    if expires_in_secs <= 0 {
615        return 0;
616    }
617    now_unix().saturating_add(expires_in_secs as u64)
618}
619
620#[cfg(test)]
621mod tests {
622    use super::*;
623
624    #[test]
625    fn empty_access_token_is_never_valid() {
626        assert!(!Token::default().is_valid(1_000, Duration::from_secs(30)));
627    }
628
629    #[test]
630    fn unknown_expiry_is_treated_as_valid() {
631        let t = Token {
632            access_token: "a".into(),
633            expires_at_unix: 0,
634            ..Default::default()
635        };
636        assert!(t.is_valid(u64::MAX, Duration::from_secs(30)));
637    }
638
639    #[test]
640    fn skew_expires_the_token_early() {
641        let t = Token {
642            access_token: "a".into(),
643            expires_at_unix: 1_000,
644            ..Default::default()
645        };
646        assert!(t.is_valid(900, Duration::from_secs(30)), "930 < 1000");
647        assert!(
648            !t.is_valid(980, Duration::from_secs(30)),
649            "1010 >= 1000: inside the skew window, must refresh"
650        );
651    }
652
653    /// Stands in for the broker: it accepts only the refresh token it minted
654    /// last and mints a new one on every refresh, so each token works once.
655    /// `None` is the broker's "invalid credential".
656    struct SingleUseIssuer {
657        live: String,
658        minted: u32,
659        presented: Vec<String>,
660    }
661
662    impl SingleUseIssuer {
663        fn new(first: &str) -> Self {
664            Self {
665                live: first.to_string(),
666                minted: 1,
667                presented: Vec::new(),
668            }
669        }
670
671        fn refresh(
672            &mut self,
673            req: authn::RefreshTokenRequest,
674        ) -> Option<authn::RefreshTokenResponse> {
675            self.presented.push(req.refresh_token.clone());
676            if req.refresh_token != self.live {
677                return None;
678            }
679            self.minted += 1;
680            self.live = format!("refresh-{}", self.minted);
681            Some(authn::RefreshTokenResponse {
682                access_token: format!("access-{}", self.minted),
683                access_token_expires_in: 900,
684                refresh_token: self.live.clone(),
685                refresh_token_expires_in: 86_400,
686                session_public_id: "public-session".into(),
687            })
688        }
689    }
690
691    // The production failure: keeping the login refresh token refreshed once and
692    // then failed every later refresh. Each refresh must present the token the
693    // previous one returned.
694    #[test]
695    fn repeated_refreshes_present_each_rotated_token() {
696        let mut issuer = SingleUseIssuer::new("refresh-1");
697        let mut stored = Token {
698            access_token: "access-1".into(),
699            refresh_token: "refresh-1".into(),
700            session_id: "legacy-session".into(),
701            expires_at_unix: 0,
702        };
703
704        for _ in 0..4 {
705            let resp = issuer
706                .refresh(refresh_request(&stored))
707                .expect("the rotated refresh token must be accepted");
708            apply_refresh(&mut stored, resp);
709        }
710
711        assert_eq!(
712            issuer.presented,
713            ["refresh-1", "refresh-2", "refresh-3", "refresh-4"]
714        );
715        assert_eq!(stored.refresh_token, "refresh-5");
716        assert_eq!(stored.access_token, "access-5");
717        assert_eq!(
718            stored.session_id, "legacy-session",
719            "the legacy refresh session id is kept"
720        );
721        assert!(stored.expires_at_unix >= now_unix());
722    }
723
724    #[test]
725    fn legacy_session_refresh_without_a_rotated_token_keeps_the_stored_one() {
726        let mut stored = Token {
727            access_token: "access-1".into(),
728            refresh_token: "refresh-1".into(),
729            session_id: "legacy-session".into(),
730            expires_at_unix: 0,
731        };
732        apply_refresh(
733            &mut stored,
734            authn::RefreshTokenResponse {
735                access_token: "access-2".into(),
736                access_token_expires_in: 900,
737                ..Default::default()
738            },
739        );
740        assert_eq!(stored.refresh_token, "refresh-1");
741        assert_eq!(stored.access_token, "access-2");
742    }
743
744    // Guards the stand-in itself: without rotation the second refresh fails,
745    // which is exactly what the test above proves the manager avoids.
746    #[test]
747    fn a_spent_refresh_token_is_refused_by_the_issuer() {
748        let mut issuer = SingleUseIssuer::new("refresh-1");
749        let spent = Token {
750            refresh_token: "refresh-1".into(),
751            ..Default::default()
752        };
753        assert!(issuer.refresh(refresh_request(&spent)).is_some());
754        assert!(issuer.refresh(refresh_request(&spent)).is_none());
755    }
756
757    #[test]
758    fn a_refusal_of_the_current_token_marks_it_for_renewal() {
759        let now = Instant::now();
760        let mut state = RefusalState::default();
761        assert!(state.reject("access-1", "access-1", now));
762        assert!(state.is_rejected("access-1"));
763        assert!(!state.is_rejected("access-2"));
764        // A concurrent call that carried the same token is repeated too.
765        assert!(state.reject("access-1", "access-1", now));
766    }
767
768    // Negative: a token this manager does not hold now (a caller's own, or one
769    // already replaced by an ordinary refresh) is never renewed.
770    #[test]
771    fn a_refusal_of_a_token_the_manager_does_not_hold_is_final() {
772        let mut state = RefusalState::default();
773        assert!(!state.reject("someone-else", "access-1", Instant::now()));
774        assert!(!state.reject("", "", Instant::now()));
775        assert!(!state.is_rejected("access-1"));
776    }
777
778    // Negative: a token renewed moments ago that is refused again is not
779    // renewed again; renewing cannot help, and it must not become a storm.
780    #[test]
781    fn a_renewed_token_refused_within_five_seconds_is_final() {
782        let t0 = Instant::now();
783        let mut state = RefusalState::default();
784        assert!(state.reject("access-1", "access-1", t0));
785        state.renewed();
786        for offset in [0, 1, 4] {
787            assert!(
788                !state.reject("access-2", "access-2", t0 + Duration::from_secs(offset)),
789                "renewed {offset}s ago"
790            );
791        }
792        assert!(state.reject("access-2", "access-2", t0 + Duration::from_secs(6)));
793        assert!(state.is_rejected("access-2"));
794    }
795
796    #[test]
797    fn a_failed_renewal_refuses_locally_with_a_growing_backoff() {
798        let t0 = Instant::now();
799        let mut state = RefusalState::default();
800        assert!(state.reject("access-1", "access-1", t0));
801        assert!(state.refused_until(t0).is_none(), "the first renewal runs");
802
803        let refused = Status::unauthenticated("invalid credential");
804        for failure in 1..=8u32 {
805            state.renew_failed(&refused, t0);
806            let wait = state.retry_at.expect("a backoff") - t0;
807            let ceiling = (RENEW_BACKOFF_BASE * 2u32.pow(failure - 1)).min(RENEW_BACKOFF_CAP);
808            assert!(
809                wait >= ceiling / 2 && wait <= ceiling,
810                "failure {failure}: waited {wait:?}, ceiling {ceiling:?}"
811            );
812            let local = state.refused_until(t0).expect("refused locally");
813            assert_eq!(local.code(), Code::Unauthenticated);
814        }
815        assert!(state.refused_until(t0 + RENEW_BACKOFF_CAP).is_none());
816        state.renewed();
817        assert!(state.refused_until(t0).is_none());
818    }
819
820    #[test]
821    fn only_unreachable_overloaded_or_lost_reply_codes_are_transient() {
822        for code in [
823            Code::Unavailable,
824            Code::DeadlineExceeded,
825            Code::ResourceExhausted,
826            Code::Aborted,
827        ] {
828            assert!(transient(&Status::new(code, "")), "{code:?}");
829        }
830        for code in [
831            Code::Unauthenticated,
832            Code::PermissionDenied,
833            Code::InvalidArgument,
834            Code::Internal,
835        ] {
836            assert!(!transient(&Status::new(code, "")), "{code:?}");
837        }
838    }
839
840    #[test]
841    fn jittered_backoff_stays_within_its_ceiling() {
842        for _ in 0..50 {
843            let first = jittered_backoff(1, RENEW_BACKOFF_BASE, RENEW_BACKOFF_CAP);
844            assert!(first >= Duration::from_millis(500) && first <= RENEW_BACKOFF_BASE);
845            let second = jittered_backoff(2, RENEW_BACKOFF_BASE, RENEW_BACKOFF_CAP);
846            assert!(second >= RENEW_BACKOFF_BASE && second <= Duration::from_secs(2));
847            let capped = jittered_backoff(30, RENEW_BACKOFF_BASE, RENEW_BACKOFF_CAP);
848            assert!(capped >= RENEW_BACKOFF_CAP / 2 && capped <= RENEW_BACKOFF_CAP);
849        }
850    }
851
852    #[test]
853    fn job_token_request_carries_the_fields_and_refuses_blanks_locally() {
854        let ok = job_token_request(ExchangeTokenForJobRequest {
855            subject_token: "user.jwt".into(),
856            purpose: "export".into(),
857            job_id: "export-42".into(),
858            ttl_seconds: 600,
859        })
860        .expect("valid request");
861        assert_eq!(ok.subject_token, "user.jwt");
862        assert_eq!(ok.purpose, "export");
863        assert_eq!(ok.job_id, "export-42");
864        assert_eq!(ok.ttl_seconds, 600);
865
866        let valid = || ExchangeTokenForJobRequest {
867            subject_token: "t".into(),
868            purpose: "export".into(),
869            job_id: "j".into(),
870            ttl_seconds: 0,
871        };
872        for bad in [
873            ExchangeTokenForJobRequest {
874                subject_token: " ".into(),
875                ..valid()
876            },
877            ExchangeTokenForJobRequest {
878                purpose: String::new(),
879                ..valid()
880            },
881            ExchangeTokenForJobRequest {
882                job_id: String::new(),
883                ..valid()
884            },
885            ExchangeTokenForJobRequest {
886                ttl_seconds: -1,
887                ..valid()
888            },
889        ] {
890            let err = job_token_request(bad).expect_err("refused locally");
891            assert_eq!(err.code(), Code::InvalidArgument);
892        }
893    }
894
895    #[test]
896    fn delegated_job_token_maps_the_response_and_refuses_an_empty_token() {
897        let token = delegated_job_token(
898            authn::ExchangeTokenForJobResponse {
899                access_token: "delegated.jwt".into(),
900                jti: "dlg_1".into(),
901                expires_at_unix: 1_900_000_000,
902                subject: "user-1".into(),
903                actor: "spiffe://acme/exporter".into(),
904                purpose: "export".into(),
905                job_id: "export-42".into(),
906                scopes: vec!["udb:read".into()],
907                tenant_id: "tenant-1".into(),
908                project_id: "billing".into(),
909                ..Default::default()
910            },
911            "export-42",
912        )
913        .expect("token");
914        assert_eq!(
915            token,
916            DelegatedJobToken {
917                access_token: "delegated.jwt".into(),
918                jti: "dlg_1".into(),
919                expires_at_unix: 1_900_000_000,
920                subject: "user-1".into(),
921                actor: "spiffe://acme/exporter".into(),
922                purpose: "export".into(),
923                job_id: "export-42".into(),
924                scopes: vec!["udb:read".into()],
925                tenant_id: "tenant-1".into(),
926                project_id: "billing".into(),
927            }
928        );
929        assert!(!format!("{token:?}").contains("delegated.jwt"));
930        let err = delegated_job_token(authn::ExchangeTokenForJobResponse::default(), "export-42")
931            .expect_err("no token");
932        assert_eq!(err.code(), Code::Internal);
933    }
934
935    #[test]
936    fn expiry_from_ignores_non_positive() {
937        assert_eq!(expiry_from(0), 0);
938        assert_eq!(expiry_from(-5), 0);
939        assert!(expiry_from(60) >= now_unix());
940    }
941}