alien-bindings 3.3.0

Alien direct in-process resource bindings
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
//! Minting-backed client credential resolution.
//!
//! This is the *only* client-side credential resolver: every language SDK
//! inherits it through the napi addon, so it must stay runtime-agnostic (no
//! background timers, no spawned refresh loops — refresh happens lazily, on the
//! access path).
//!
//! Managed cloud workloads use their platform-projected service account or
//! machine-host metadata; they do not receive deployment bearer tokens. Minting is
//! an explicitly configured fallback for external/bootstrap integrations that
//! cannot use that native identity path. Such a client POSTs to
//! `{ALIEN_MANAGER_URL}/v1/credentials/mint` with a deployment token and
//! receives a short-lived [`ClientConfig`] plus a server-computed `expiresAt`
//! refresh hint. The minted config is cached and re-minted on access once it
//! passes the refresh threshold, with a single-flight guard so a burst of
//! concurrent binding loads triggers at most one mint.
//!
//! # Bootstrapping an external app
//!
//! An external/bootstrap integration may opt into minting by supplying the
//! following environment contract. The manager does not inject this contract
//! into managed Container/Daemon workloads. All values are required together
//! (see [`MintingCredentialSource::from_env`]):
//!
//! - `ALIEN_MANAGER_URL` — base URL of the deployment's manager; the mint
//!   endpoint is `{ALIEN_MANAGER_URL}/v1/credentials/mint`.
//! - `ALIEN_DEPLOYMENT_TOKEN` — bearer token scoped to this deployment (or its
//!   deployment group). Sent as `Authorization: Bearer …`; never logged.
//! - `ALIEN_DEPLOYMENT_SERVICE_ACCOUNT` — the service-account binding to mint
//!   credentials for (`bindingName` in the request body).
//! - `ALIEN_RESOURCE_ID` — the current app resource in the deployment stack
//!   (`resourceId` in the request body).
//!
//! `ALIEN_DEPLOYMENT_ID` (`deploymentId` in the request body) is reused from
//! the existing deployment-identity contract rather than duplicated.
//!
//! # Selection order
//!
//! [`crate::provider::LazyEnvBindingsProvider`] decides its strategy once, on
//! first binding use, in a fixed order:
//!
//! 1. **Native/projected identity** — `ClientConfig::from_env` succeeds (an
//!    IAM role, workload identity, metadata service, or explicit static
//!    credentials in the environment). Used as-is; minting is never attempted.
//! 2. **Mint** — native resolution failed and an external/bootstrap caller
//!    explicitly supplied the complete mint environment contract. Falls back
//!    to this module.
//! 3. **Original error** — native resolution failed and no mint gate is
//!    present. The original `from_env` error is surfaced unchanged; there is
//!    nothing left to fall back to.
//!
//! # Refresh semantics
//!
//! A minted [`ClientConfig`] is cached under its server-declared `expiresAt`
//! and treated as stale once `now >= expiresAt - 300s`
//! ([`REFRESH_SKEW_SECONDS`]) — the credential is re-minted *before* it would
//! actually expire, not after. Re-minting happens lazily, only on access
//! ([`MintingResolver::provider`]); there is no background timer.
//!
//! Two properties make this safe under concurrency and manager failures:
//!
//! - **Single-flight**: concurrent callers that observe a stale/empty cache
//!   contend on one lock; only the first re-mints, the rest observe the
//!   now-fresh cache after it releases. A burst of concurrent binding loads
//!   never produces a burst of mint calls.
//! - **Fail-closed**: if a mint call errors, that error propagates to the
//!   caller instead of serving the stale provider. Availability suffers (a
//!   manager blip can break every binding load), but no caller is ever handed
//!   credentials past their declared expiry.
//!
//! # Static/local mode
//!
//! Tests and local development don't need any of this: when
//! `ClientConfig::from_env` resolves directly (e.g. `ALIEN_DEPLOYMENT_TYPE=local`
//! with a `state_directory`), selection stops at step 1 above and this module
//! is never touched. The fake-mint-server tests in this file, and the
//! `mints_when_native_config_unavailable` / `native_config_wins_and_never_mints`
//! tests in `crate::provider`, exercise both sides of that boundary without
//! needing a real manager or real cloud credentials.

use std::collections::HashMap;
use std::fmt;
use std::sync::Arc;
use std::time::Duration;

use alien_core::{
    ClientConfig, ENV_ALIEN_DEPLOYMENT_ID, ENV_ALIEN_DEPLOYMENT_SERVICE_ACCOUNT,
    ENV_ALIEN_DEPLOYMENT_TOKEN, ENV_ALIEN_MANAGER_URL, ENV_ALIEN_RESOURCE_ID,
};
use alien_error::{AlienError, Context, IntoAlienError};
use chrono::{DateTime, Duration as ChronoDuration, Utc};
use serde::Deserialize;
use tokio::sync::{Mutex, RwLock};
use tracing::debug;

use crate::error::{ErrorData, Result};
use crate::provider::BindingsProvider;

/// Re-mint credentials once they are within this many seconds of their
/// server-declared expiry. Gives in-flight work a safety margin so it never
/// races a hard expiry, and absorbs modest client/server clock skew.
///
/// This margin is only safe from a per-call "mint storm" (re-minting on every
/// access instead of every refresh window) because the manager clamps minted
/// lifetimes to `[900, 3600]` seconds (`MIN_DURATION_SECONDS` /
/// `MAX_DURATION_SECONDS` in `crates/alien-manager/src/routes/credentials.rs`).
/// A 900s floor minus this 300s skew still leaves a 600s window of cache
/// hits between mints; if the server ever granted shorter-lived credentials
/// than the skew, every access would re-mint.
const REFRESH_SKEW_SECONDS: i64 = 300;

/// Timeout for a single mint HTTP request. The mint endpoint impersonates a
/// service account / calls STS, so it is not instant; this is generous enough
/// for that round-trip while still bounding a hung manager.
const MINT_TIMEOUT: Duration = Duration::from_secs(30);

/// The mint request inputs read from the process environment, plus the HTTP
/// client used to reach the manager.
///
/// External/bootstrap callers supply these vars explicitly. Managed workload
/// controllers do not inject them.
pub(crate) struct MintingCredentialSource {
    /// Base URL of the deployment's manager (`ALIEN_MANAGER_URL`).
    manager_url: String,
    /// Deployment bearer token (`ALIEN_DEPLOYMENT_TOKEN`). Secret material —
    /// see the manual [`fmt::Debug`] impl, which must never print it.
    token: String,
    /// Deployment id (`ALIEN_DEPLOYMENT_ID`).
    deployment_id: String,
    /// Service-account binding to mint credentials for
    /// (`ALIEN_DEPLOYMENT_SERVICE_ACCOUNT`).
    binding_name: String,
    /// Current app resource id (`ALIEN_RESOURCE_ID`).
    resource_id: String,
    http: reqwest::Client,
}

/// Manual `Debug`: `token` is a live bearer credential. Never let a `{:?}` of
/// this source (log line, panic message, test failure output) print it.
impl fmt::Debug for MintingCredentialSource {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("MintingCredentialSource")
            .field("manager_url", &self.manager_url)
            .field("token", &"<redacted>")
            .field("deployment_id", &self.deployment_id)
            .field("binding_name", &self.binding_name)
            .field("resource_id", &self.resource_id)
            .finish()
    }
}

impl MintingCredentialSource {
    /// Builds a source from the environment when the mint contract is present.
    ///
    /// Returns `Ok(None)` when the gate (`ALIEN_MANAGER_URL` +
    /// `ALIEN_DEPLOYMENT_TOKEN`) is absent — the caller then keeps the existing
    /// (non-minting) behaviour. When the gate *is* present but the rest of the
    /// request contract (`ALIEN_DEPLOYMENT_ID`, `ALIEN_DEPLOYMENT_SERVICE_ACCOUNT`,
    /// `ALIEN_RESOURCE_ID`) is missing, this fails fast: a half-injected mint
    /// environment is a manager bug, not a reason to silently fall back to
    /// static resolution.
    pub(crate) fn from_env(env: &HashMap<String, String>) -> Result<Option<Self>> {
        let (manager_url, token) = match (
            env.get(ENV_ALIEN_MANAGER_URL),
            env.get(ENV_ALIEN_DEPLOYMENT_TOKEN),
        ) {
            (None, None) => return Ok(None),
            (Some(manager_url), Some(token)) => (manager_url, token),
            (None, Some(_)) => {
                return Err(AlienError::new(ErrorData::EnvironmentVariableMissing {
                    variable_name: ENV_ALIEN_MANAGER_URL.to_string(),
                }));
            }
            (Some(_), None) => {
                return Err(AlienError::new(ErrorData::EnvironmentVariableMissing {
                    variable_name: ENV_ALIEN_DEPLOYMENT_TOKEN.to_string(),
                }));
            }
        };

        let deployment_id = env.get(ENV_ALIEN_DEPLOYMENT_ID).ok_or_else(|| {
            AlienError::new(ErrorData::EnvironmentVariableMissing {
                variable_name: ENV_ALIEN_DEPLOYMENT_ID.to_string(),
            })
        })?;
        let binding_name = env
            .get(ENV_ALIEN_DEPLOYMENT_SERVICE_ACCOUNT)
            .ok_or_else(|| {
                AlienError::new(ErrorData::EnvironmentVariableMissing {
                    variable_name: ENV_ALIEN_DEPLOYMENT_SERVICE_ACCOUNT.to_string(),
                })
            })?;
        let resource_id = env.get(ENV_ALIEN_RESOURCE_ID).ok_or_else(|| {
            AlienError::new(ErrorData::EnvironmentVariableMissing {
                variable_name: ENV_ALIEN_RESOURCE_ID.to_string(),
            })
        })?;

        let http = reqwest::Client::builder()
            .timeout(MINT_TIMEOUT)
            .build()
            .into_alien_error()
            .context(ErrorData::RemoteAccessFailed {
                operation: "build minting HTTP client".to_string(),
            })?;

        Ok(Some(Self {
            manager_url: manager_url.clone(),
            token: token.clone(),
            deployment_id: deployment_id.clone(),
            binding_name: binding_name.clone(),
            resource_id: resource_id.clone(),
            http,
        }))
    }

    /// POST the mint request and parse the response. Errors surface typed via
    /// `alien-error` with context; there are no panic paths.
    async fn mint(&self) -> Result<MintedConfig> {
        let url = format!(
            "{}/v1/credentials/mint",
            self.manager_url.trim_end_matches('/')
        );

        let response = self
            .http
            .post(&url)
            .bearer_auth(&self.token)
            .json(&serde_json::json!({
                "deploymentId": self.deployment_id,
                "resourceId": self.resource_id,
                "bindingName": self.binding_name,
            }))
            .send()
            .await
            .into_alien_error()
            .context(ErrorData::RemoteAccessFailed {
                operation: "mint credentials from manager".to_string(),
            })?;

        let response = response.error_for_status().into_alien_error().context(
            ErrorData::RemoteAccessFailed {
                operation: "mint credentials from manager (non-success status)".to_string(),
            },
        )?;

        let minted: MintResponse =
            response
                .json()
                .await
                .into_alien_error()
                .context(ErrorData::RemoteAccessFailed {
                    operation: "parse mint response".to_string(),
                })?;

        // Audit trail: never logs the client_config (credential material) — only
        // the non-secret principal and expiry.
        debug!(
            deployment_id = %self.deployment_id,
            resource_id = %self.resource_id,
            binding_name = %self.binding_name,
            principal = %minted.principal,
            expires_at = %minted.expires_at.to_rfc3339(),
            "Minted client credentials"
        );

        Ok(MintedConfig {
            client_config: minted.client_config,
            expires_at: minted.expires_at,
        })
    }
}

/// Deserialised mint response. Mirrors the manager's `MintCredentialsResponse`
/// (`crates/alien-manager/src/routes/credentials.rs`).
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct MintResponse {
    client_config: ClientConfig,
    /// Server-computed refresh hint (RFC3339). Treated as "re-mint at or before
    /// this instant", not as proof the credential is valid to the last second.
    expires_at: DateTime<Utc>,
    /// Human-readable identity the credentials act as. Non-secret; logged only.
    principal: String,
}

/// A minted config together with the server's expiry hint.
struct MintedConfig {
    client_config: ClientConfig,
    expires_at: DateTime<Utc>,
}

/// A cached [`BindingsProvider`] built from minted credentials, plus the expiry
/// that decides when it must be rebuilt.
struct Cached {
    provider: Arc<BindingsProvider>,
    expires_at: DateTime<Utc>,
}

/// Resolves a [`BindingsProvider`] from minted credentials, caching it until it
/// passes the refresh threshold and re-minting on access under a single-flight
/// guard.
pub(crate) struct MintingResolver {
    source: MintingCredentialSource,
    /// Binding JSON (`ALIEN_*_BINDING`), reused verbatim across re-mints — only
    /// the credentials change, not the binding topology.
    bindings: HashMap<String, serde_json::Value>,
    /// Cached provider + expiry. `None` until the first mint.
    cache: RwLock<Option<Cached>>,
    /// Single-flight guard: a burst of concurrent stale/first loads collapses to
    /// one mint. An async `Mutex` (not a timer) keeps this napi-runtime-agnostic.
    refresh_lock: Mutex<()>,
    /// Re-mint once within this many seconds of expiry. Field (not the bare
    /// const) so tests can pin it.
    refresh_skew_seconds: i64,
}

/// Manual `Debug`: the cached provider holds a live [`ClientConfig`] and the
/// source holds a bearer token. Redact both; delegating to the source's own
/// (already-redacting) `Debug` is safe.
impl fmt::Debug for MintingResolver {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("MintingResolver")
            .field("source", &self.source)
            .field("bindings", &self.bindings.keys().collect::<Vec<_>>())
            .field("cache", &"<redacted>")
            .field("refresh_skew_seconds", &self.refresh_skew_seconds)
            .finish()
    }
}

impl MintingResolver {
    pub(crate) fn new(
        source: MintingCredentialSource,
        bindings: HashMap<String, serde_json::Value>,
    ) -> Self {
        Self {
            source,
            bindings,
            cache: RwLock::new(None),
            refresh_lock: Mutex::new(()),
            refresh_skew_seconds: REFRESH_SKEW_SECONDS,
        }
    }

    /// Returns a provider backed by fresh-enough minted credentials, minting (or
    /// re-minting) only when the cache is empty or stale.
    pub(crate) async fn provider(&self) -> Result<Arc<BindingsProvider>> {
        // Fast path: a fresh cached provider needs no lock contention and no mint.
        if let Some(provider) = self.fresh_cached().await {
            return Ok(provider);
        }

        // Slow path: single-flight the mint. Only one task refreshes per
        // staleness window; the rest wait here and then observe the fresh cache.
        let _flight = self.refresh_lock.lock().await;

        // Double-check: a racing task may have refreshed while we waited for the
        // lock. This is what makes concurrent first-loads collapse to one mint.
        if let Some(provider) = self.fresh_cached().await {
            return Ok(provider);
        }

        // Fail closed: if the mint call errors, this returns the error to the
        // caller rather than serving a stale/expired provider. That trades
        // availability (a manager blip can break every binding load) for never
        // handing out credentials past their declared expiry.
        let minted = self.source.mint().await?;
        let provider = Arc::new(BindingsProvider::new(
            minted.client_config,
            self.bindings.clone(),
        )?);

        let mut cache = self.cache.write().await;
        *cache = Some(Cached {
            provider: provider.clone(),
            expires_at: minted.expires_at,
        });
        Ok(provider)
    }

    /// The cached provider if present and not yet within the refresh window.
    async fn fresh_cached(&self) -> Option<Arc<BindingsProvider>> {
        let cache = self.cache.read().await;
        cache.as_ref().and_then(|cached| {
            if self.is_stale(cached.expires_at) {
                None
            } else {
                Some(cached.provider.clone())
            }
        })
    }

    fn is_stale(&self, expires_at: DateTime<Utc>) -> bool {
        Utc::now() >= expires_at - ChronoDuration::seconds(self.refresh_skew_seconds)
    }

    /// Test-only: simulate elapsed time by pushing the cached expiry into the
    /// past, so the next access observes the entry as stale and re-mints. Keeps
    /// the re-mint test deterministic without a real sleep.
    #[cfg(test)]
    async fn force_stale(&self) {
        if let Some(cached) = self.cache.write().await.as_mut() {
            cached.expires_at = Utc::now() - ChronoDuration::seconds(1);
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::net::SocketAddr;
    use std::sync::atomic::{AtomicUsize, Ordering};

    use axum::{extract::State, routing::post, Json, Router};
    use serde_json::json;

    /// Shared state for the fake mint endpoint: counts requests and controls the
    /// expiry it hands back.
    #[derive(Clone)]
    struct MintServerState {
        calls: Arc<AtomicUsize>,
        /// Seconds from now used for each response's `expiresAt`.
        expiry_secs: i64,
        /// Optional artificial delay to widen the single-flight race window.
        delay: Option<Duration>,
    }

    async fn mint_handler(State(state): State<MintServerState>) -> Json<serde_json::Value> {
        state.calls.fetch_add(1, Ordering::SeqCst);
        if let Some(delay) = state.delay {
            tokio::time::sleep(delay).await;
        }
        let expires_at = (Utc::now() + ChronoDuration::seconds(state.expiry_secs)).to_rfc3339();
        // A `Local` client config deserialises without needing real cloud
        // credentials, so binding loads against the minted provider stay offline.
        Json(json!({
            "clientConfig": { "platform": "local", "state_directory": "/tmp/alien-mint-test" },
            "expiresAt": expires_at,
            "principal": "local:mint-test",
        }))
    }

    /// Spawn a fake mint server; returns its base URL and the call counter.
    async fn spawn_mint_server(
        expiry_secs: i64,
        delay: Option<Duration>,
    ) -> (String, Arc<AtomicUsize>) {
        let calls = Arc::new(AtomicUsize::new(0));
        let state = MintServerState {
            calls: calls.clone(),
            expiry_secs,
            delay,
        };
        let app = Router::new()
            .route("/v1/credentials/mint", post(mint_handler))
            .with_state(state);

        let listener = tokio::net::TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0)))
            .await
            .expect("bind fake mint server");
        let addr = listener.local_addr().expect("local addr");
        tokio::spawn(async move {
            axum::serve(listener, app).await.expect("serve");
        });
        (format!("http://{addr}"), calls)
    }

    fn source(manager_url: &str) -> MintingCredentialSource {
        let env = HashMap::from([
            (ENV_ALIEN_MANAGER_URL.to_string(), manager_url.to_string()),
            (
                ENV_ALIEN_DEPLOYMENT_TOKEN.to_string(),
                "ax_deploy_SECRET_TOKEN".to_string(),
            ),
            (ENV_ALIEN_DEPLOYMENT_ID.to_string(), "dep_123".to_string()),
            (
                ENV_ALIEN_DEPLOYMENT_SERVICE_ACCOUNT.to_string(),
                "management".to_string(),
            ),
            (ENV_ALIEN_RESOURCE_ID.to_string(), "api".to_string()),
        ]);
        MintingCredentialSource::from_env(&env)
            .expect("source builds")
            .expect("mint contract present")
    }

    #[tokio::test]
    async fn from_env_returns_none_without_gate() {
        // No manager URL / token -> not a minting environment.
        let env = HashMap::from([(ENV_ALIEN_DEPLOYMENT_ID.to_string(), "dep_1".to_string())]);
        assert!(MintingCredentialSource::from_env(&env)
            .expect("no error")
            .is_none());
    }

    #[tokio::test]
    async fn from_env_fails_fast_when_gate_present_but_contract_incomplete() {
        // Manager URL + token present, but deployment id / SA binding missing:
        // a half-injected mint environment must error, not silently fall back.
        let env = HashMap::from([
            (
                ENV_ALIEN_MANAGER_URL.to_string(),
                "http://localhost".to_string(),
            ),
            (ENV_ALIEN_DEPLOYMENT_TOKEN.to_string(), "tok".to_string()),
        ]);
        let error = MintingCredentialSource::from_env(&env)
            .expect_err("incomplete contract should fail fast");
        assert_eq!(error.code, "ENVIRONMENT_VARIABLE_MISSING");
    }

    #[test]
    fn from_env_fails_fast_when_only_one_gate_variable_is_present() {
        for (present, missing) in [
            (ENV_ALIEN_MANAGER_URL, ENV_ALIEN_DEPLOYMENT_TOKEN),
            (ENV_ALIEN_DEPLOYMENT_TOKEN, ENV_ALIEN_MANAGER_URL),
        ] {
            let env = HashMap::from([(present.to_string(), "configured".to_string())]);
            let error = MintingCredentialSource::from_env(&env)
                .expect_err("a partial mint gate must not silently disable minting");

            assert_eq!(error.code, "ENVIRONMENT_VARIABLE_MISSING");
            match error.error {
                Some(ErrorData::EnvironmentVariableMissing { variable_name }) => {
                    assert_eq!(variable_name, missing);
                }
                other => panic!("expected missing {missing}, got {other:?}"),
            }
        }
    }

    #[tokio::test]
    async fn first_use_mints_then_caches_then_re_mints_when_stale() {
        let (base_url, calls) = spawn_mint_server(3600, None).await;
        let resolver = MintingResolver::new(source(&base_url), HashMap::new());

        // First access mints.
        resolver.provider().await.expect("first mint");
        assert_eq!(calls.load(Ordering::SeqCst), 1, "first access mints once");

        // Second access is served from the fresh cache: no new mint.
        resolver.provider().await.expect("cached");
        assert_eq!(
            calls.load(Ordering::SeqCst),
            1,
            "fresh cache must not re-hit the manager"
        );

        // Once stale, the next access re-mints.
        resolver.force_stale().await;
        resolver.provider().await.expect("re-mint");
        assert_eq!(
            calls.load(Ordering::SeqCst),
            2,
            "stale credentials must trigger a re-mint on access"
        );
    }

    #[tokio::test]
    async fn near_expiry_config_is_treated_as_stale() {
        // Server hands back an expiry inside the refresh skew window, so the
        // very next access re-mints — the credential is never handed out cutting
        // it close to the hard expiry.
        let (base_url, calls) = spawn_mint_server(60, None).await;
        let resolver = MintingResolver::new(source(&base_url), HashMap::new());

        resolver.provider().await.expect("first mint");
        resolver.provider().await.expect("second mint");
        assert_eq!(
            calls.load(Ordering::SeqCst),
            2,
            "an expiry within the 300s skew window is stale on the next access"
        );
    }

    #[tokio::test]
    async fn concurrent_first_loads_mint_exactly_once() {
        // Two tasks race the empty cache; the single-flight guard must collapse
        // them to one mint. A server delay widens the window to make the race real.
        let (base_url, calls) = spawn_mint_server(3600, Some(Duration::from_millis(100))).await;
        let resolver = Arc::new(MintingResolver::new(source(&base_url), HashMap::new()));

        let a = {
            let resolver = resolver.clone();
            tokio::spawn(async move { resolver.provider().await.map(|_| ()) })
        };
        let b = {
            let resolver = resolver.clone();
            tokio::spawn(async move { resolver.provider().await.map(|_| ()) })
        };
        a.await.expect("join a").expect("mint a");
        b.await.expect("join b").expect("mint b");

        assert_eq!(
            calls.load(Ordering::SeqCst),
            1,
            "single-flight must collapse concurrent first-loads to one mint"
        );
    }

    #[tokio::test]
    async fn debug_never_leaks_token_or_credentials() {
        let (base_url, _calls) = spawn_mint_server(3600, None).await;
        let resolver = MintingResolver::new(source(&base_url), HashMap::new());
        resolver.provider().await.expect("mint");

        let rendered = format!("{resolver:?}");
        assert!(
            !rendered.contains("ax_deploy_SECRET_TOKEN"),
            "resolver Debug leaked the deployment token: {rendered}"
        );
        assert!(
            rendered.contains("<redacted>"),
            "resolver Debug should mark redacted fields: {rendered}"
        );

        // The source alone must also redact its token.
        let source_rendered = format!("{:?}", source(&base_url));
        assert!(!source_rendered.contains("ax_deploy_SECRET_TOKEN"));
    }
}