Skip to main content

alien_client_core/
runtime_credentials.rs

1//! Refresh bounded runtime credentials without retaining expired authority.
2
3#[cfg(unix)]
4use std::os::unix::fs::{MetadataExt, PermissionsExt};
5use std::{fmt, path::PathBuf, sync::Arc, time::Duration};
6
7use alien_error::AlienError;
8use async_trait::async_trait;
9use chrono::{DateTime, Utc};
10use reqwest::{redirect::Policy, Client, Url};
11use serde::{Deserialize, Serialize};
12use serde_json::{json, Value};
13use tokio::{io::AsyncReadExt, sync::Mutex};
14
15use crate::{ErrorData, Result};
16
17/// How far apart this process's clock and the manager's clock may be when we
18/// check the lifetime of a credential the manager issued. Without it, a
19/// manager clock one second ahead makes a one hour credential look too long
20/// and locks the caller out. AWS uses the same 5 minute tolerance for SigV4.
21pub const ALLOWED_CLOCK_SKEW: chrono::TimeDelta = chrono::TimeDelta::minutes(5);
22
23/// Longest lifetime of a registry pull credential. Pods pull images with it
24/// from a Kubernetes Secret that the operator refreshes, so it may last as
25/// long as the operator's daily key. Pods then still start while the operator
26/// is down. The manager caps it at the key's own expiry.
27pub const REGISTRY_PULL_LIFETIME: chrono::TimeDelta = chrono::TimeDelta::hours(24);
28
29/// Every purpose other than registry pull lasts at most one hour.
30fn max_lifetime(purpose: &Value) -> chrono::TimeDelta {
31    if purpose.get("type").and_then(Value::as_str) == Some("registryPull") {
32        REGISTRY_PULL_LIFETIME
33    } else {
34        chrono::TimeDelta::hours(1)
35    }
36}
37
38/// Supplies fresh identity evidence. Implementations must not return a runtime
39/// capability as proof or fall back to an unrelated stored credential.
40#[async_trait]
41pub trait RuntimeProofProvider: Send + Sync {
42    async fn proof(&self, audience: &str) -> Result<Value>;
43}
44
45/// A bounded credential suitable for an atomically replaced private file.
46/// It deliberately has no Debug implementation.
47#[derive(Clone, Deserialize, Serialize)]
48#[serde(rename_all = "camelCase", deny_unknown_fields)]
49pub struct RuntimeCapability {
50    pub token: String,
51    pub expires_at: DateTime<Utc>,
52}
53
54enum CredentialSource {
55    Exchange(Arc<dyn RuntimeProofProvider>),
56    File(PathBuf),
57}
58
59#[derive(Default)]
60struct CredentialState {
61    cached: Option<CachedCredential>,
62    rejected: Option<String>,
63}
64
65struct CachedCredential {
66    credential: RuntimeCapability,
67    refresh_at: DateTime<Utc>,
68}
69
70/// Nonsecret configuration supplied by deployment provisioning.
71pub struct RuntimeCredentialConfig {
72    pub manager_url: String,
73    /// Exact configured manager identity, independent of its transport URL.
74    pub audience: String,
75    pub deployment_id: String,
76    pub purpose: Value,
77}
78
79/// A single-purpose credential source shared by concurrent requests.
80/// Refresh errors propagate; expired credentials are never returned.
81pub struct RuntimeCredentials {
82    client: Client,
83    endpoint: Url,
84    receiver: Url,
85    audience: String,
86    deployment_id: String,
87    purpose: Value,
88    source: CredentialSource,
89    clock: Arc<dyn Fn() -> DateTime<Utc> + Send + Sync>,
90    state: Mutex<CredentialState>,
91}
92
93impl RuntimeCredentials {
94    /// The deployment these credentials are for.
95    pub fn deployment_id(&self) -> &str {
96        &self.deployment_id
97    }
98
99    /// The purpose these credentials are for, as sent to the manager.
100    pub fn purpose(&self) -> &Value {
101        &self.purpose
102    }
103}
104
105impl fmt::Debug for RuntimeCredentials {
106    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
107        formatter
108            .debug_struct("RuntimeCredentials")
109            .finish_non_exhaustive()
110    }
111}
112
113impl RuntimeCredentials {
114    /// Connect to a manager using a fixed deployment and purpose. The manager
115    /// independently verifies membership and never trusts these selectors.
116    pub fn new(
117        config: RuntimeCredentialConfig,
118        proof: Arc<dyn RuntimeProofProvider>,
119    ) -> Result<Self> {
120        Self::with_source(config, CredentialSource::Exchange(proof))
121    }
122
123    /// Exchange proof at a configured manager path for a credential restricted
124    /// to an independently configured receiver. Neither destination is selected
125    /// by the proof provider or an exchange response.
126    pub fn new_for_receiver(
127        config: RuntimeCredentialConfig,
128        proof: Arc<dyn RuntimeProofProvider>,
129        exchange_path: &str,
130        receiver_url: &str,
131    ) -> Result<Self> {
132        let mut credentials = Self::new(config, proof)?;
133        if !exchange_path.starts_with('/') || exchange_path.starts_with("//") {
134            return Err(invalid("exchange path must be an absolute manager path"));
135        }
136        let endpoint = credentials
137            .endpoint
138            .join(exchange_path)
139            .map_err(|_| invalid("invalid exchange path"))?;
140        if endpoint.origin() != credentials.endpoint.origin()
141            || endpoint.path() != exchange_path
142            || endpoint.query().is_some()
143            || endpoint.fragment().is_some()
144            || !endpoint.username().is_empty()
145            || endpoint.password().is_some()
146        {
147            return Err(invalid(
148                "exchange path must remain on the configured manager",
149            ));
150        }
151        credentials.receiver = secure_endpoint(receiver_url)?;
152        credentials.endpoint = endpoint;
153        Ok(credentials)
154    }
155
156    /// Read an externally refreshed capability on each request. This source
157    /// never exchanges or renews the supplied credential.
158    pub fn from_capability_file(config: RuntimeCredentialConfig, path: PathBuf) -> Result<Self> {
159        Self::with_source(config, CredentialSource::File(path))
160    }
161
162    fn with_source(config: RuntimeCredentialConfig, source: CredentialSource) -> Result<Self> {
163        let manager_url = &config.manager_url;
164        let base = secure_endpoint(manager_url)?;
165        let endpoint = Url::parse(&format!(
166            "{}/v1/identity/exchange",
167            manager_url.trim_end_matches('/')
168        ))
169        .map_err(|_| invalid("invalid exchange URL"))?;
170        let client = Client::builder()
171            .redirect(Policy::none())
172            .http1_only()
173            .timeout(Duration::from_secs(15))
174            .build()
175            .map_err(|_| invalid("cannot initialize credential transport"))?;
176        Ok(Self {
177            client,
178            endpoint,
179            receiver: base,
180            audience: config.audience,
181            deployment_id: config.deployment_id,
182            purpose: config.purpose,
183            source,
184            clock: Arc::new(Utc::now),
185            state: Mutex::new(CredentialState::default()),
186        })
187    }
188
189    /// Credentials may only be attached to their configured receiver origin.
190    pub fn allows_endpoint(&self, endpoint: &str) -> bool {
191        Url::parse(endpoint)
192            .map(|url| {
193                url.username().is_empty()
194                    && url.password().is_none()
195                    && url.origin() == self.receiver.origin()
196            })
197            .unwrap_or(false)
198    }
199
200    /// Inject a clock for precise expiry validation.
201    pub fn with_clock(mut self, clock: Arc<dyn Fn() -> DateTime<Utc> + Send + Sync>) -> Self {
202        self.clock = clock;
203        self
204    }
205
206    /// Return a current credential, refreshing halfway through its remaining
207    /// lifetime. A scheduled refresh failure may retain unexpired authority.
208    pub async fn token(&self) -> Result<String> {
209        Ok(self.capability().await?.token)
210    }
211
212    /// Invalidate only the credential rejected by a receiver. A delayed response
213    /// for an older token must not evict a newer cached credential or replace
214    /// the rejection recorded while that newer generation is being refreshed.
215    pub async fn reject(&self, rejected: &str) {
216        let mut state = self.state.lock().await;
217        if state
218            .cached
219            .as_ref()
220            .is_some_and(|current| current.credential.token == rejected)
221        {
222            state.cached = None;
223            state.rejected = Some(rejected.to_owned());
224        }
225    }
226
227    /// Retry callers share a replacement when they rejected the same token.
228    pub async fn token_after_rejection(&self, rejected: &str) -> Result<String> {
229        self.reject(rejected).await;
230        self.token().await
231    }
232
233    /// The deadline accompanies tokens written to child credential files.
234    pub async fn capability(&self) -> Result<RuntimeCapability> {
235        let mut state = self.state.lock().await;
236        if let CredentialSource::File(path) = &self.source {
237            let file = tokio::fs::File::open(path)
238                .await
239                .map_err(|_| invalid("cannot open capability file"))?;
240            let metadata = file
241                .metadata()
242                .await
243                .map_err(|_| invalid("cannot inspect capability file"))?;
244            if !metadata.is_file() || metadata.len() > 16 * 1024 {
245                return Err(invalid("invalid capability file"));
246            }
247            #[cfg(unix)]
248            {
249                let mode = metadata.permissions().mode();
250                // Projected Secrets are root-owned and readable by the pod's
251                // fsGroup. That group shares the credential trust boundary.
252                if mode & 0o037 != 0 || (mode & 0o040 != 0 && metadata.uid() != 0) {
253                    return Err(invalid(
254                        "capability file must be private or root-owned group-read-only",
255                    ));
256                }
257            }
258            let mut bytes = Vec::new();
259            file.take(16 * 1024 + 1)
260                .read_to_end(&mut bytes)
261                .await
262                .map_err(|_| invalid("cannot read capability file"))?;
263            if bytes.len() > 16 * 1024 {
264                return Err(invalid("capability file exceeds limit"));
265            }
266            let capability: RuntimeCapability =
267                serde_json::from_slice(&bytes).map_err(|_| invalid("invalid capability file"))?;
268            let remaining = capability.expires_at - (self.clock)();
269            if !capability.token.starts_with("ax_cap_v1_")
270                || remaining <= chrono::Duration::zero()
271                || remaining > max_lifetime(&self.purpose) + ALLOWED_CLOCK_SKEW
272            {
273                return Err(invalid("invalid capability file lifetime"));
274            }
275            if state.rejected.as_deref() == Some(capability.token.as_str()) {
276                return Err(invalid(
277                    "capability file still contains a rejected credential",
278                ));
279            }
280            state.cached = Some(CachedCredential {
281                credential: capability.clone(),
282                refresh_at: capability.expires_at,
283            });
284            return Ok(capability);
285        }
286        let now = (self.clock)();
287        if let Some(current) = state.cached.as_ref() {
288            if now < current.refresh_at && now < current.credential.expires_at {
289                return Ok(current.credential.clone());
290            }
291        }
292        match self.exchange().await {
293            Ok(credential) => {
294                if state.rejected.as_deref() == Some(credential.token.as_str()) {
295                    return Err(invalid("identity exchange returned a rejected credential"));
296                }
297                let now = (self.clock)();
298                if now >= credential.expires_at {
299                    return Err(invalid("identity exchange credential expired before use"));
300                }
301                state.cached = Some(CachedCredential {
302                    refresh_at: now + (credential.expires_at - now) / 2,
303                    credential: credential.clone(),
304                });
305                Ok(credential)
306            }
307            Err(error) => {
308                // Only scheduled refresh can retain authority. Explicit rejection
309                // removed its exact cached token before reaching this branch.
310                if let Some(current) = state.cached.as_ref() {
311                    if (self.clock)() < current.credential.expires_at {
312                        return Ok(current.credential.clone());
313                    }
314                }
315                state.cached = None;
316                Err(error)
317            }
318        }
319    }
320
321    async fn exchange(&self) -> Result<RuntimeCapability> {
322        let CredentialSource::Exchange(proof) = &self.source else {
323            unreachable!()
324        };
325        let proof = proof.proof(&self.audience).await?;
326        let mut response = self.client.post(self.endpoint.clone())
327            .json(&json!({"deploymentId": self.deployment_id, "purpose": self.purpose, "proof": proof}))
328            .send().await.map_err(|_| invalid("identity exchange transport failed"))?;
329        let status = response.status();
330        if !status.is_success() {
331            let message = format!("identity exchange returned HTTP {}", status.as_u16());
332            // A server error, a timeout or throttling can succeed on the next try; any other
333            // 4xx is the manager's decision about this identity.
334            return Err(
335                if status.is_server_error() || matches!(status.as_u16(), 408 | 429) {
336                    invalid(&message)
337                } else {
338                    AlienError::new(ErrorData::RuntimeAuthorityRefused { message })
339                },
340            );
341        }
342        let mut bytes = Vec::new();
343        while let Some(chunk) = response
344            .chunk()
345            .await
346            .map_err(|_| invalid("cannot read identity exchange response"))?
347        {
348            if bytes.len() + chunk.len() > 16 * 1024 {
349                return Err(invalid("identity exchange response exceeds limit"));
350            }
351            bytes.extend_from_slice(&chunk);
352        }
353        // Never preserve provider/decoder errors which may echo credential data.
354        let credential: RuntimeCapability = serde_json::from_slice(&bytes)
355            .map_err(|_| invalid("invalid identity exchange response"))?;
356        let now = (self.clock)();
357        let remaining = credential.expires_at - now;
358        if credential.token.is_empty()
359            || remaining <= chrono::Duration::zero()
360            || remaining > max_lifetime(&self.purpose) + ALLOWED_CLOCK_SKEW
361        {
362            return Err(invalid(
363                "identity exchange returned an invalid credential lifetime",
364            ));
365        }
366        Ok(credential)
367    }
368}
369
370fn secure_endpoint(endpoint: &str) -> Result<Url> {
371    let url = Url::parse(endpoint).map_err(|_| invalid("invalid credential endpoint"))?;
372    if !url.username().is_empty()
373        || url.password().is_some()
374        || url.query().is_some()
375        || url.fragment().is_some()
376        || !(url.scheme() == "https"
377            || (url.scheme() == "http"
378                && matches!(url.host_str(), Some("localhost" | "127.0.0.1" | "[::1]"))))
379    {
380        return Err(invalid(
381            "credential endpoint must use HTTPS (HTTP is allowed only on loopback)",
382        ));
383    }
384    Ok(url)
385}
386
387fn invalid(message: &str) -> crate::Error {
388    AlienError::new(ErrorData::AuthenticationError {
389        message: message.to_owned(),
390    })
391}
392
393#[cfg(test)]
394mod tests {
395    use super::*;
396    use std::{
397        io::{Read, Write},
398        net::TcpListener,
399        sync::atomic::{AtomicI64, AtomicUsize, Ordering},
400        thread,
401    };
402
403    struct Proof(AtomicUsize);
404    #[async_trait]
405    impl RuntimeProofProvider for Proof {
406        async fn proof(&self, audience: &str) -> Result<Value> {
407            assert_eq!(audience, "manager-identity");
408            self.0.fetch_add(1, Ordering::SeqCst);
409            Ok(json!({"type":"session", "token":"fixture-proof"}))
410        }
411    }
412
413    fn fixture(
414        replies: Vec<(u16, Value)>,
415    ) -> (
416        RuntimeCredentials,
417        Arc<AtomicI64>,
418        Arc<Proof>,
419        thread::JoinHandle<()>,
420    ) {
421        fixture_for(replies, None, json!({"type":"telemetry"}))
422    }
423
424    fn fixture_for_receiver(
425        replies: Vec<(u16, Value)>,
426        receiver: Option<&str>,
427    ) -> (
428        RuntimeCredentials,
429        Arc<AtomicI64>,
430        Arc<Proof>,
431        thread::JoinHandle<()>,
432    ) {
433        fixture_for(replies, receiver, json!({"type":"telemetry"}))
434    }
435
436    fn fixture_for(
437        replies: Vec<(u16, Value)>,
438        receiver: Option<&str>,
439        purpose: Value,
440    ) -> (
441        RuntimeCredentials,
442        Arc<AtomicI64>,
443        Arc<Proof>,
444        thread::JoinHandle<()>,
445    ) {
446        let expected_purpose = purpose.clone();
447        let exchange_path = if receiver.is_some() {
448            "/v1/identity/scoped"
449        } else {
450            "/v1/identity/exchange"
451        };
452        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
453        let url = format!("http://{}", listener.local_addr().unwrap());
454        let server = thread::spawn(move || {
455            for (status, body) in replies {
456                let (mut stream, _) = listener.accept().unwrap();
457                stream
458                    .set_read_timeout(Some(Duration::from_secs(5)))
459                    .unwrap();
460                let mut request = Vec::new();
461                loop {
462                    let mut chunk = [0; 4096];
463                    let count = stream.read(&mut chunk).unwrap();
464                    assert!(count > 0);
465                    request.extend_from_slice(&chunk[..count]);
466                    if let Some(header_end) =
467                        request.windows(4).position(|part| part == b"\r\n\r\n")
468                    {
469                        let headers = String::from_utf8_lossy(&request[..header_end]);
470                        let length: usize = headers
471                            .lines()
472                            .find_map(|line| {
473                                line.to_ascii_lowercase()
474                                    .strip_prefix("content-length: ")
475                                    .map(str::to_owned)
476                            })
477                            .unwrap()
478                            .parse()
479                            .unwrap();
480                        if request.len() >= header_end + 4 + length {
481                            assert!(headers.starts_with(&format!("POST {exchange_path} HTTP/1.1")));
482                            let request: Value =
483                                serde_json::from_slice(&request[header_end + 4..]).unwrap();
484                            assert_eq!(request["deploymentId"], "deployment-one");
485                            assert_eq!(request["purpose"], expected_purpose);
486                            assert_eq!(request["proof"]["token"], "fixture-proof");
487                            break;
488                        }
489                    }
490                }
491                let body = body.to_string();
492                write!(stream, "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap();
493            }
494        });
495        let clock = Arc::new(AtomicI64::new(1_800_000_000));
496        let proof = Arc::new(Proof(AtomicUsize::new(0)));
497        let observed_clock = clock.clone();
498        let config = RuntimeCredentialConfig {
499            manager_url: url,
500            audience: "manager-identity".into(),
501            deployment_id: "deployment-one".into(),
502            purpose,
503        };
504        let client = match receiver {
505            Some(receiver) => {
506                RuntimeCredentials::new_for_receiver(config, proof.clone(), exchange_path, receiver)
507            }
508            None => RuntimeCredentials::new(config, proof.clone()),
509        }
510        .unwrap()
511        .with_clock(Arc::new(move || {
512            DateTime::from_timestamp(observed_clock.load(Ordering::SeqCst), 0).unwrap()
513        }));
514        (client, clock, proof, server)
515    }
516
517    #[tokio::test]
518    async fn explicit_receiver_keeps_exchange_and_delivery_origins_separate() {
519        let receiver = TcpListener::bind("127.0.0.1:0").unwrap();
520        let receiver_url = format!("http://{}", receiver.local_addr().unwrap());
521        let unauthorized = TcpListener::bind("127.0.0.1:0").unwrap();
522        unauthorized.set_nonblocking(true).unwrap();
523        let wrong_url = format!("http://{}", unauthorized.local_addr().unwrap());
524        let (client, _, _, exchange) = fixture_for_receiver(
525            vec![(200, credential("scoped-child", 100))],
526            Some(&receiver_url),
527        );
528        assert!(!client.allows_endpoint(client.endpoint.as_str()));
529        assert!(!client.allows_endpoint(&wrong_url));
530        assert!(client.allows_endpoint(&receiver_url));
531        let delivery = thread::spawn(move || {
532            let (mut stream, _) = receiver.accept().unwrap();
533            stream
534                .set_read_timeout(Some(Duration::from_secs(5)))
535                .unwrap();
536            let mut bytes = [0; 4096];
537            let count = stream.read(&mut bytes).unwrap();
538            let request = String::from_utf8_lossy(&bytes[..count]);
539            assert!(request
540                .to_ascii_lowercase()
541                .contains("authorization: bearer scoped-child"));
542            write!(
543                stream,
544                "HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
545            )
546            .unwrap();
547        });
548        let token = client.token().await.unwrap();
549        let response = Client::builder()
550            .redirect(Policy::none())
551            .build()
552            .unwrap()
553            .get(&receiver_url)
554            .bearer_auth(token)
555            .send()
556            .await
557            .unwrap();
558        assert_eq!(response.status(), 204);
559        assert_eq!(
560            unauthorized.accept().unwrap_err().kind(),
561            std::io::ErrorKind::WouldBlock
562        );
563        exchange.join().unwrap();
564        delivery.join().unwrap();
565    }
566
567    /// A manager that is briefly down must not look like a refusal: a transport failure or a
568    /// server error can succeed on the next try, while a 4xx is the manager's answer.
569    #[tokio::test]
570    async fn an_unavailable_manager_is_retryable_and_a_refusal_is_not() {
571        let (client, _, _, server) = fixture(vec![(503, json!({}))]);
572        let error = client.capability().await.err().expect("the exchange fails");
573        assert!(error.retryable, "a 503 can succeed later: {error:?}");
574        server.join().unwrap();
575
576        let (client, _, _, server) = fixture(vec![(403, json!({"message": "no grant"}))]);
577        let error = client.capability().await.err().expect("the exchange fails");
578        assert!(!error.retryable, "a 403 is a refusal: {error:?}");
579        assert!(error.message.contains("HTTP 403"), "{}", error.message);
580        server.join().unwrap();
581
582        // Nothing listens: the connection fails.
583        let closed = TcpListener::bind("127.0.0.1:0").unwrap();
584        let manager_url = format!("http://{}", closed.local_addr().unwrap());
585        drop(closed);
586        let client = RuntimeCredentials::new(
587            RuntimeCredentialConfig {
588                manager_url,
589                audience: "manager-identity".into(),
590                deployment_id: "deployment-one".into(),
591                purpose: json!({"type":"telemetry"}),
592            },
593            Arc::new(Proof(AtomicUsize::new(0))),
594        )
595        .unwrap();
596        let error = client.capability().await.err().expect("the exchange fails");
597        assert!(
598            error.retryable,
599            "a refused connection can succeed later: {error:?}"
600        );
601    }
602
603    #[tokio::test]
604    async fn explicit_exchange_refuses_redirect_to_another_listener() {
605        let destination = TcpListener::bind("127.0.0.1:0").unwrap();
606        destination.set_nonblocking(true).unwrap();
607        let destination_url = format!("http://{}", destination.local_addr().unwrap());
608        let manager = TcpListener::bind("127.0.0.1:0").unwrap();
609        let manager_url = format!("http://{}", manager.local_addr().unwrap());
610        let redirect = destination_url.clone();
611        let server = thread::spawn(move || {
612            let (mut stream, _) = manager.accept().unwrap();
613            stream
614                .set_read_timeout(Some(Duration::from_secs(5)))
615                .unwrap();
616            let mut bytes = [0; 4096];
617            assert!(stream.read(&mut bytes).unwrap() > 0);
618            write!(stream, "HTTP/1.1 307 Temporary Redirect\r\nLocation: {redirect}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap();
619        });
620        let client = RuntimeCredentials::new_for_receiver(
621            RuntimeCredentialConfig {
622                manager_url,
623                audience: "manager-identity".into(),
624                deployment_id: "deployment-one".into(),
625                purpose: json!({"type":"telemetry"}),
626            },
627            Arc::new(Proof(AtomicUsize::new(0))),
628            "/v1/identity/scoped",
629            &destination_url,
630        )
631        .unwrap();
632        assert!(client.token().await.is_err());
633        server.join().unwrap();
634        assert_eq!(
635            destination.accept().unwrap_err().kind(),
636            std::io::ErrorKind::WouldBlock
637        );
638    }
639
640    #[test]
641    fn explicit_destinations_refuse_ambiguous_or_insecure_configuration() {
642        for (path, receiver) in [
643            ("//other.example/exchange", "https://receiver.example"),
644            ("https://other.example/exchange", "https://receiver.example"),
645            ("/exchange?next=other", "https://receiver.example"),
646            ("/exchange#other", "https://receiver.example"),
647            ("/a/../exchange", "https://receiver.example"),
648            ("/exchange", "http://receiver.example"),
649            ("/exchange", "https://user:pass@receiver.example"),
650            ("/exchange", "https://receiver.example?secret=value"),
651        ] {
652            let result = RuntimeCredentials::new_for_receiver(
653                RuntimeCredentialConfig {
654                    manager_url: "https://manager.example".into(),
655                    audience: "manager-identity".into(),
656                    deployment_id: "deployment-one".into(),
657                    purpose: json!({"type":"telemetry"}),
658                },
659                Arc::new(Proof(AtomicUsize::new(0))),
660                path,
661                receiver,
662            );
663            assert!(result.is_err(), "unexpected accepted destination");
664        }
665    }
666
667    fn credential(token: &str, seconds: i64) -> Value {
668        json!({"token": token, "expiresAt": DateTime::from_timestamp(1_800_000_000 + seconds, 0).unwrap()})
669    }
670
671    #[tokio::test]
672    async fn concurrent_requests_share_refresh_and_refresh_before_expiry() {
673        let (client, clock, proof, server) = fixture(vec![
674            (200, credential("first", 100)),
675            (200, credential("second", 200)),
676        ]);
677        let (one, two) = tokio::join!(client.token(), client.token());
678        assert_eq!(one.unwrap(), "first");
679        assert_eq!(two.unwrap(), "first");
680        assert_eq!(proof.0.load(Ordering::SeqCst), 1);
681        clock.store(1_800_000_049, Ordering::SeqCst);
682        assert_eq!(client.token().await.unwrap(), "first");
683        clock.store(1_800_000_050, Ordering::SeqCst);
684        assert_eq!(client.token().await.unwrap(), "second");
685        assert_eq!(proof.0.load(Ordering::SeqCst), 2);
686        server.join().unwrap();
687    }
688
689    #[tokio::test]
690    async fn early_refresh_outage_retains_only_unexpired_nonrejected_token() {
691        let (client, clock, _, server) = fixture(vec![
692            (200, credential("first", 100)),
693            (503, json!({})),
694            (503, json!({})),
695        ]);
696        assert_eq!(client.token().await.unwrap(), "first");
697        clock.store(1_800_000_050, Ordering::SeqCst);
698        assert_eq!(client.token().await.unwrap(), "first");
699        clock.store(1_800_000_100, Ordering::SeqCst);
700        assert!(client.token().await.is_err());
701        server.join().unwrap();
702    }
703
704    #[tokio::test]
705    async fn refresh_failure_rechecks_expiry_after_io() {
706        struct FailingProof(Arc<AtomicI64>);
707        #[async_trait]
708        impl RuntimeProofProvider for FailingProof {
709            async fn proof(&self, _: &str) -> Result<Value> {
710                self.0.store(1_800_000_100, Ordering::SeqCst);
711                Err(invalid("fixture provider unavailable"))
712            }
713        }
714        let (mut client, clock, _, server) = fixture(vec![(200, credential("first", 100))]);
715        assert_eq!(client.token().await.unwrap(), "first");
716        server.join().unwrap();
717        clock.store(1_800_000_050, Ordering::SeqCst);
718        client.source = CredentialSource::Exchange(Arc::new(FailingProof(clock)));
719        assert!(client.token().await.is_err());
720    }
721
722    #[tokio::test]
723    async fn concurrent_rejections_preserve_the_replacement() {
724        let (client, _, proof, server) = fixture(vec![
725            (200, credential("first", 100)),
726            (200, credential("second", 100)),
727        ]);
728        assert_eq!(client.token().await.unwrap(), "first");
729        let (one, two) = tokio::join!(
730            client.token_after_rejection("first"),
731            client.token_after_rejection("first")
732        );
733        assert_eq!(one.unwrap(), "second");
734        assert_eq!(two.unwrap(), "second");
735        assert_eq!(proof.0.load(Ordering::SeqCst), 2);
736        client.reject("first").await;
737        assert_eq!(client.token().await.unwrap(), "second");
738        assert_eq!(proof.0.load(Ordering::SeqCst), 2);
739        server.join().unwrap();
740    }
741
742    #[tokio::test]
743    async fn stale_rejection_cannot_replace_newer_rejection_during_refresh() {
744        let (client, _, _, server) = fixture(vec![
745            (200, credential("first", 100)),
746            (200, credential("second", 100)),
747            (200, credential("second", 100)),
748            (200, credential("third", 100)),
749        ]);
750        assert_eq!(client.token().await.unwrap(), "first");
751        assert_eq!(
752            client.token_after_rejection("first").await.unwrap(),
753            "second"
754        );
755        client.reject("second").await;
756        client.reject("first").await;
757        assert!(
758            client.token().await.is_err(),
759            "late first rejection restored rejected second credential"
760        );
761        assert_eq!(client.token().await.unwrap(), "third");
762        server.join().unwrap();
763    }
764
765    #[cfg(unix)]
766    #[tokio::test]
767    async fn stale_rejection_cannot_restore_rejected_projected_file() {
768        let directory = tempfile::tempdir().unwrap();
769        let path = directory.path().join("credential");
770        let write = |token: &str| {
771            std::fs::write(
772                &path,
773                serde_json::to_vec(&RuntimeCapability {
774                    token: token.into(),
775                    expires_at: Utc::now() + chrono::Duration::minutes(30),
776                })
777                .unwrap(),
778            )
779            .unwrap();
780            std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
781        };
782        let client = RuntimeCredentials::from_capability_file(
783            RuntimeCredentialConfig {
784                manager_url: "https://manager.example".into(),
785                audience: "manager-one".into(),
786                deployment_id: "deployment-one".into(),
787                purpose: json!({"type":"telemetry"}),
788            },
789            path.clone(),
790        )
791        .unwrap();
792        write("ax_cap_v1_first");
793        assert_eq!(client.token().await.unwrap(), "ax_cap_v1_first");
794        client.reject("ax_cap_v1_first").await;
795        write("ax_cap_v1_second");
796        assert_eq!(client.token().await.unwrap(), "ax_cap_v1_second");
797        client.reject("ax_cap_v1_second").await;
798        client.reject("ax_cap_v1_first").await;
799        assert!(
800            client.token().await.is_err(),
801            "late first rejection restored rejected file bytes"
802        );
803        write("ax_cap_v1_third");
804        assert_eq!(client.token().await.unwrap(), "ax_cap_v1_third");
805    }
806
807    #[tokio::test]
808    async fn rejected_refresh_does_not_restore_cached_or_expired_authority() {
809        let (client, _, _, server) = fixture(vec![
810            (200, credential("first", 100)),
811            (401, json!({"secret":"must-not-appear"})),
812            (200, credential("replacement", 200)),
813        ]);
814        assert_eq!(client.token().await.unwrap(), "first");
815        let error = client.token_after_rejection("first").await.unwrap_err();
816        assert!(!format!("{error:?}").contains("must-not-appear"));
817        assert_eq!(client.token().await.unwrap(), "replacement");
818        server.join().unwrap();
819    }
820
821    #[tokio::test]
822    async fn exchange_cannot_restore_the_rejected_token() {
823        let (client, _, _, server) = fixture(vec![
824            (200, credential("first", 100)),
825            (200, credential("first", 100)),
826            (200, credential("replacement", 100)),
827        ]);
828        assert_eq!(client.token().await.unwrap(), "first");
829        assert!(client.token_after_rejection("first").await.is_err());
830        assert_eq!(client.token().await.unwrap(), "replacement");
831        server.join().unwrap();
832    }
833
834    #[tokio::test]
835    async fn refuses_expired_and_overlong_responses() {
836        let (client, _, _, server) = fixture(vec![
837            (200, credential("expired", 0)),
838            (200, credential("overlong", 3600 + 301)),
839        ]);
840        assert!(client.token().await.is_err());
841        assert!(client.token().await.is_err());
842        server.join().unwrap();
843    }
844
845    #[tokio::test]
846    async fn credential_from_a_manager_clock_two_minutes_ahead_is_accepted() {
847        // The manager issued a one hour credential by its own clock, which
848        // runs two minutes ahead of ours.
849        let (client, _, _, server) = fixture(vec![(200, credential("ahead", 3600 + 120))]);
850        assert_eq!(client.token().await.unwrap(), "ahead");
851        server.join().unwrap();
852    }
853    /// The operator writes the pull credential into a Secret that pods read
854    /// while the operator may be down. It accepts one that lasts as long as
855    /// the daily key, and still refuses longer ones or long credentials for
856    /// any other purpose.
857    #[tokio::test]
858    async fn registry_pull_credential_may_last_as_long_as_the_daily_key() {
859        let day = 24 * 3600;
860        let (client, _, _, server) = fixture_for(
861            vec![
862                (200, credential("pull-day", day)),
863                (200, credential("pull-overlong", day + 301)),
864            ],
865            None,
866            json!({"type":"registryPull"}),
867        );
868        assert_eq!(client.token().await.unwrap(), "pull-day");
869        client.reject("pull-day").await;
870        assert!(client.token().await.is_err());
871        server.join().unwrap();
872
873        let (client, _, _, server) = fixture(vec![(200, credential("telemetry-day", day))]);
874        assert!(client.token().await.is_err());
875        server.join().unwrap();
876    }
877
878    #[cfg(unix)]
879    #[tokio::test]
880    async fn capability_file_reopens_replacement_and_refuses_exact_expiry() {
881        let directory = tempfile::tempdir().unwrap();
882        let path = directory.path().join("telemetry");
883        let clock = Arc::new(std::sync::atomic::AtomicI64::new(1_800_000_000));
884        let write = |name: &str, token: &str, expires: i64| {
885            let file = directory.path().join(name);
886            let capability = RuntimeCapability {
887                token: token.into(),
888                expires_at: DateTime::from_timestamp(expires, 0).unwrap(),
889            };
890            std::fs::write(&file, serde_json::to_vec(&capability).unwrap()).unwrap();
891            std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o600)).unwrap();
892            file
893        };
894        write("telemetry", "legacy-deployment-token", 1_800_000_100);
895        let observed = clock.clone();
896        let credentials = RuntimeCredentials::from_capability_file(
897            RuntimeCredentialConfig {
898                manager_url: "https://manager.example".into(),
899                audience: "manager-one".into(),
900                deployment_id: "deployment-one".into(),
901                purpose: json!({"type":"telemetry"}),
902            },
903            path.clone(),
904        )
905        .unwrap()
906        .with_clock(Arc::new(move || {
907            DateTime::from_timestamp(observed.load(std::sync::atomic::Ordering::SeqCst), 0).unwrap()
908        }));
909        assert!(
910            credentials.token().await.is_err(),
911            "legacy token must not enter capability-file mode"
912        );
913        write("telemetry", "ax_cap_v1_first", 1_800_000_100);
914        assert_eq!(credentials.token().await.unwrap(), "ax_cap_v1_first");
915        credentials.reject("ax_cap_v1_first").await;
916        assert!(credentials.token().await.is_err());
917        clock.store(1_800_000_100, std::sync::atomic::Ordering::SeqCst);
918        assert!(credentials.token().await.is_err());
919        let next = write("next", "ax_cap_v1_second", 1_800_000_200);
920        std::fs::rename(next, &path).unwrap();
921        assert_eq!(credentials.token().await.unwrap(), "ax_cap_v1_second");
922        credentials.reject("ax_cap_v1_first").await;
923        assert_eq!(credentials.token().await.unwrap(), "ax_cap_v1_second");
924        for mode in [0o644, 0o620, 0o610, 0o601] {
925            std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode)).unwrap();
926            assert!(credentials.token().await.is_err());
927        }
928        if std::fs::metadata(&path).unwrap().uid() != 0 {
929            std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o440)).unwrap();
930            assert!(credentials.token().await.is_err());
931        }
932        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
933        assert_eq!(credentials.token().await.unwrap(), "ax_cap_v1_second");
934        std::fs::remove_file(&path).unwrap();
935        assert!(credentials.token().await.is_err());
936    }
937}