Skip to main content

alien_client_core/
runtime_credentials.rs

1//! Refresh bounded runtime credentials without retaining expired authority.
2
3#[cfg(unix)]
4use std::os::unix::fs::{MetadataExt, PermissionsExt};
5use std::{fmt, path::PathBuf, sync::Arc, time::Duration};
6
7use alien_error::AlienError;
8use async_trait::async_trait;
9use chrono::{DateTime, Utc};
10use reqwest::{redirect::Policy, Client, Url};
11use serde::{Deserialize, Serialize};
12use serde_json::{json, Value};
13use tokio::{io::AsyncReadExt, sync::Mutex};
14
15use crate::{ErrorData, Result};
16
17/// How far apart this process's clock and the manager's clock may be when we
18/// check the lifetime of a credential the manager issued. Without it, a
19/// manager clock one second ahead makes a one hour credential look too long
20/// and locks the caller out. AWS uses the same 5 minute tolerance for SigV4.
21pub const ALLOWED_CLOCK_SKEW: chrono::TimeDelta = chrono::TimeDelta::minutes(5);
22
23/// Longest lifetime of a registry pull credential. Pods pull images with it
24/// from a Kubernetes Secret that the operator refreshes, so it may last as
25/// long as the operator's daily key. Pods then still start while the operator
26/// is down. The manager caps it at the key's own expiry.
27pub const REGISTRY_PULL_LIFETIME: chrono::TimeDelta = chrono::TimeDelta::hours(24);
28
29/// Every purpose other than registry pull lasts at most one hour.
30fn max_lifetime(purpose: &Value) -> chrono::TimeDelta {
31    if purpose.get("type").and_then(Value::as_str) == Some("registryPull") {
32        REGISTRY_PULL_LIFETIME
33    } else {
34        chrono::TimeDelta::hours(1)
35    }
36}
37
38/// Supplies fresh identity evidence. Implementations must not return a runtime
39/// capability as proof or fall back to an unrelated stored credential.
40#[async_trait]
41pub trait RuntimeProofProvider: Send + Sync {
42    async fn proof(&self, audience: &str) -> Result<Value>;
43}
44
45/// A bounded credential suitable for an atomically replaced private file.
46/// It deliberately has no Debug implementation.
47#[derive(Clone, Deserialize, Serialize)]
48#[serde(rename_all = "camelCase", deny_unknown_fields)]
49pub struct RuntimeCapability {
50    pub token: String,
51    pub expires_at: DateTime<Utc>,
52}
53
54enum CredentialSource {
55    Exchange(Arc<dyn RuntimeProofProvider>),
56    File(PathBuf),
57}
58
59#[derive(Default)]
60struct CredentialState {
61    cached: Option<CachedCredential>,
62    rejected: Option<String>,
63}
64
65struct CachedCredential {
66    credential: RuntimeCapability,
67    refresh_at: DateTime<Utc>,
68}
69
70/// Nonsecret configuration supplied by deployment provisioning.
71pub struct RuntimeCredentialConfig {
72    pub manager_url: String,
73    /// Exact configured manager identity, independent of its transport URL.
74    pub audience: String,
75    pub deployment_id: String,
76    pub purpose: Value,
77}
78
79/// A single-purpose credential source shared by concurrent requests.
80/// Refresh errors propagate; expired credentials are never returned.
81pub struct RuntimeCredentials {
82    client: Client,
83    endpoint: Url,
84    receiver: Url,
85    audience: String,
86    deployment_id: String,
87    purpose: Value,
88    source: CredentialSource,
89    clock: Arc<dyn Fn() -> DateTime<Utc> + Send + Sync>,
90    state: Mutex<CredentialState>,
91}
92
93impl fmt::Debug for RuntimeCredentials {
94    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
95        formatter
96            .debug_struct("RuntimeCredentials")
97            .finish_non_exhaustive()
98    }
99}
100
101impl RuntimeCredentials {
102    /// Connect to a manager using a fixed deployment and purpose. The manager
103    /// independently verifies membership and never trusts these selectors.
104    pub fn new(
105        config: RuntimeCredentialConfig,
106        proof: Arc<dyn RuntimeProofProvider>,
107    ) -> Result<Self> {
108        Self::with_source(config, CredentialSource::Exchange(proof))
109    }
110
111    /// Exchange proof at a configured manager path for a credential restricted
112    /// to an independently configured receiver. Neither destination is selected
113    /// by the proof provider or an exchange response.
114    pub fn new_for_receiver(
115        config: RuntimeCredentialConfig,
116        proof: Arc<dyn RuntimeProofProvider>,
117        exchange_path: &str,
118        receiver_url: &str,
119    ) -> Result<Self> {
120        let mut credentials = Self::new(config, proof)?;
121        if !exchange_path.starts_with('/') || exchange_path.starts_with("//") {
122            return Err(invalid("exchange path must be an absolute manager path"));
123        }
124        let endpoint = credentials
125            .endpoint
126            .join(exchange_path)
127            .map_err(|_| invalid("invalid exchange path"))?;
128        if endpoint.origin() != credentials.endpoint.origin()
129            || endpoint.path() != exchange_path
130            || endpoint.query().is_some()
131            || endpoint.fragment().is_some()
132            || !endpoint.username().is_empty()
133            || endpoint.password().is_some()
134        {
135            return Err(invalid(
136                "exchange path must remain on the configured manager",
137            ));
138        }
139        credentials.receiver = secure_endpoint(receiver_url)?;
140        credentials.endpoint = endpoint;
141        Ok(credentials)
142    }
143
144    /// Read an externally refreshed capability on each request. This source
145    /// never exchanges or renews the supplied credential.
146    pub fn from_capability_file(config: RuntimeCredentialConfig, path: PathBuf) -> Result<Self> {
147        Self::with_source(config, CredentialSource::File(path))
148    }
149
150    fn with_source(config: RuntimeCredentialConfig, source: CredentialSource) -> Result<Self> {
151        let manager_url = &config.manager_url;
152        let base = secure_endpoint(manager_url)?;
153        let endpoint = Url::parse(&format!(
154            "{}/v1/identity/exchange",
155            manager_url.trim_end_matches('/')
156        ))
157        .map_err(|_| invalid("invalid exchange URL"))?;
158        let client = Client::builder()
159            .redirect(Policy::none())
160            .http1_only()
161            .timeout(Duration::from_secs(15))
162            .build()
163            .map_err(|_| invalid("cannot initialize credential transport"))?;
164        Ok(Self {
165            client,
166            endpoint,
167            receiver: base,
168            audience: config.audience,
169            deployment_id: config.deployment_id,
170            purpose: config.purpose,
171            source,
172            clock: Arc::new(Utc::now),
173            state: Mutex::new(CredentialState::default()),
174        })
175    }
176
177    /// Credentials may only be attached to their configured receiver origin.
178    pub fn allows_endpoint(&self, endpoint: &str) -> bool {
179        Url::parse(endpoint)
180            .map(|url| {
181                url.username().is_empty()
182                    && url.password().is_none()
183                    && url.origin() == self.receiver.origin()
184            })
185            .unwrap_or(false)
186    }
187
188    /// Inject a clock for precise expiry validation.
189    pub fn with_clock(mut self, clock: Arc<dyn Fn() -> DateTime<Utc> + Send + Sync>) -> Self {
190        self.clock = clock;
191        self
192    }
193
194    /// Return a current credential, refreshing halfway through its remaining
195    /// lifetime. A scheduled refresh failure may retain unexpired authority.
196    pub async fn token(&self) -> Result<String> {
197        Ok(self.capability().await?.token)
198    }
199
200    /// Invalidate only the credential rejected by a receiver. A delayed response
201    /// for an older token must not evict a newer cached credential or replace
202    /// the rejection recorded while that newer generation is being refreshed.
203    pub async fn reject(&self, rejected: &str) {
204        let mut state = self.state.lock().await;
205        if state
206            .cached
207            .as_ref()
208            .is_some_and(|current| current.credential.token == rejected)
209        {
210            state.cached = None;
211            state.rejected = Some(rejected.to_owned());
212        }
213    }
214
215    /// Retry callers share a replacement when they rejected the same token.
216    pub async fn token_after_rejection(&self, rejected: &str) -> Result<String> {
217        self.reject(rejected).await;
218        self.token().await
219    }
220
221    /// The deadline accompanies tokens written to child credential files.
222    pub async fn capability(&self) -> Result<RuntimeCapability> {
223        let mut state = self.state.lock().await;
224        if let CredentialSource::File(path) = &self.source {
225            let file = tokio::fs::File::open(path)
226                .await
227                .map_err(|_| invalid("cannot open capability file"))?;
228            let metadata = file
229                .metadata()
230                .await
231                .map_err(|_| invalid("cannot inspect capability file"))?;
232            if !metadata.is_file() || metadata.len() > 16 * 1024 {
233                return Err(invalid("invalid capability file"));
234            }
235            #[cfg(unix)]
236            {
237                let mode = metadata.permissions().mode();
238                // Projected Secrets are root-owned and readable by the pod's
239                // fsGroup. That group shares the credential trust boundary.
240                if mode & 0o037 != 0 || (mode & 0o040 != 0 && metadata.uid() != 0) {
241                    return Err(invalid(
242                        "capability file must be private or root-owned group-read-only",
243                    ));
244                }
245            }
246            let mut bytes = Vec::new();
247            file.take(16 * 1024 + 1)
248                .read_to_end(&mut bytes)
249                .await
250                .map_err(|_| invalid("cannot read capability file"))?;
251            if bytes.len() > 16 * 1024 {
252                return Err(invalid("capability file exceeds limit"));
253            }
254            let capability: RuntimeCapability =
255                serde_json::from_slice(&bytes).map_err(|_| invalid("invalid capability file"))?;
256            let remaining = capability.expires_at - (self.clock)();
257            if !capability.token.starts_with("ax_cap_v1_")
258                || remaining <= chrono::Duration::zero()
259                || remaining > max_lifetime(&self.purpose) + ALLOWED_CLOCK_SKEW
260            {
261                return Err(invalid("invalid capability file lifetime"));
262            }
263            if state.rejected.as_deref() == Some(capability.token.as_str()) {
264                return Err(invalid(
265                    "capability file still contains a rejected credential",
266                ));
267            }
268            state.cached = Some(CachedCredential {
269                credential: capability.clone(),
270                refresh_at: capability.expires_at,
271            });
272            return Ok(capability);
273        }
274        let now = (self.clock)();
275        if let Some(current) = state.cached.as_ref() {
276            if now < current.refresh_at && now < current.credential.expires_at {
277                return Ok(current.credential.clone());
278            }
279        }
280        match self.exchange().await {
281            Ok(credential) => {
282                if state.rejected.as_deref() == Some(credential.token.as_str()) {
283                    return Err(invalid("identity exchange returned a rejected credential"));
284                }
285                let now = (self.clock)();
286                if now >= credential.expires_at {
287                    return Err(invalid("identity exchange credential expired before use"));
288                }
289                state.cached = Some(CachedCredential {
290                    refresh_at: now + (credential.expires_at - now) / 2,
291                    credential: credential.clone(),
292                });
293                Ok(credential)
294            }
295            Err(error) => {
296                // Only scheduled refresh can retain authority. Explicit rejection
297                // removed its exact cached token before reaching this branch.
298                if let Some(current) = state.cached.as_ref() {
299                    if (self.clock)() < current.credential.expires_at {
300                        return Ok(current.credential.clone());
301                    }
302                }
303                state.cached = None;
304                Err(error)
305            }
306        }
307    }
308
309    async fn exchange(&self) -> Result<RuntimeCapability> {
310        let CredentialSource::Exchange(proof) = &self.source else {
311            unreachable!()
312        };
313        let proof = proof.proof(&self.audience).await?;
314        let mut response = self.client.post(self.endpoint.clone())
315            .json(&json!({"deploymentId": self.deployment_id, "purpose": self.purpose, "proof": proof}))
316            .send().await.map_err(|_| invalid("identity exchange transport failed"))?;
317        if !response.status().is_success() {
318            return Err(invalid(&format!(
319                "identity exchange returned HTTP {}",
320                response.status().as_u16()
321            )));
322        }
323        let mut bytes = Vec::new();
324        while let Some(chunk) = response
325            .chunk()
326            .await
327            .map_err(|_| invalid("cannot read identity exchange response"))?
328        {
329            if bytes.len() + chunk.len() > 16 * 1024 {
330                return Err(invalid("identity exchange response exceeds limit"));
331            }
332            bytes.extend_from_slice(&chunk);
333        }
334        // Never preserve provider/decoder errors which may echo credential data.
335        let credential: RuntimeCapability = serde_json::from_slice(&bytes)
336            .map_err(|_| invalid("invalid identity exchange response"))?;
337        let now = (self.clock)();
338        let remaining = credential.expires_at - now;
339        if credential.token.is_empty()
340            || remaining <= chrono::Duration::zero()
341            || remaining > max_lifetime(&self.purpose) + ALLOWED_CLOCK_SKEW
342        {
343            return Err(invalid(
344                "identity exchange returned an invalid credential lifetime",
345            ));
346        }
347        Ok(credential)
348    }
349}
350
351fn secure_endpoint(endpoint: &str) -> Result<Url> {
352    let url = Url::parse(endpoint).map_err(|_| invalid("invalid credential endpoint"))?;
353    if !url.username().is_empty()
354        || url.password().is_some()
355        || url.query().is_some()
356        || url.fragment().is_some()
357        || !(url.scheme() == "https"
358            || (url.scheme() == "http"
359                && matches!(url.host_str(), Some("localhost" | "127.0.0.1" | "[::1]"))))
360    {
361        return Err(invalid(
362            "credential endpoint must use HTTPS (HTTP is allowed only on loopback)",
363        ));
364    }
365    Ok(url)
366}
367
368fn invalid(message: &str) -> crate::Error {
369    AlienError::new(ErrorData::AuthenticationError {
370        message: message.to_owned(),
371    })
372}
373
374#[cfg(test)]
375mod tests {
376    use super::*;
377    use std::{
378        io::{Read, Write},
379        net::TcpListener,
380        sync::atomic::{AtomicI64, AtomicUsize, Ordering},
381        thread,
382    };
383
384    struct Proof(AtomicUsize);
385    #[async_trait]
386    impl RuntimeProofProvider for Proof {
387        async fn proof(&self, audience: &str) -> Result<Value> {
388            assert_eq!(audience, "manager-identity");
389            self.0.fetch_add(1, Ordering::SeqCst);
390            Ok(json!({"type":"session", "token":"fixture-proof"}))
391        }
392    }
393
394    fn fixture(
395        replies: Vec<(u16, Value)>,
396    ) -> (
397        RuntimeCredentials,
398        Arc<AtomicI64>,
399        Arc<Proof>,
400        thread::JoinHandle<()>,
401    ) {
402        fixture_for(replies, None, json!({"type":"telemetry"}))
403    }
404
405    fn fixture_for_receiver(
406        replies: Vec<(u16, Value)>,
407        receiver: Option<&str>,
408    ) -> (
409        RuntimeCredentials,
410        Arc<AtomicI64>,
411        Arc<Proof>,
412        thread::JoinHandle<()>,
413    ) {
414        fixture_for(replies, receiver, json!({"type":"telemetry"}))
415    }
416
417    fn fixture_for(
418        replies: Vec<(u16, Value)>,
419        receiver: Option<&str>,
420        purpose: Value,
421    ) -> (
422        RuntimeCredentials,
423        Arc<AtomicI64>,
424        Arc<Proof>,
425        thread::JoinHandle<()>,
426    ) {
427        let expected_purpose = purpose.clone();
428        let exchange_path = if receiver.is_some() {
429            "/v1/identity/scoped"
430        } else {
431            "/v1/identity/exchange"
432        };
433        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
434        let url = format!("http://{}", listener.local_addr().unwrap());
435        let server = thread::spawn(move || {
436            for (status, body) in replies {
437                let (mut stream, _) = listener.accept().unwrap();
438                stream
439                    .set_read_timeout(Some(Duration::from_secs(5)))
440                    .unwrap();
441                let mut request = Vec::new();
442                loop {
443                    let mut chunk = [0; 4096];
444                    let count = stream.read(&mut chunk).unwrap();
445                    assert!(count > 0);
446                    request.extend_from_slice(&chunk[..count]);
447                    if let Some(header_end) =
448                        request.windows(4).position(|part| part == b"\r\n\r\n")
449                    {
450                        let headers = String::from_utf8_lossy(&request[..header_end]);
451                        let length: usize = headers
452                            .lines()
453                            .find_map(|line| {
454                                line.to_ascii_lowercase()
455                                    .strip_prefix("content-length: ")
456                                    .map(str::to_owned)
457                            })
458                            .unwrap()
459                            .parse()
460                            .unwrap();
461                        if request.len() >= header_end + 4 + length {
462                            assert!(headers.starts_with(&format!("POST {exchange_path} HTTP/1.1")));
463                            let request: Value =
464                                serde_json::from_slice(&request[header_end + 4..]).unwrap();
465                            assert_eq!(request["deploymentId"], "deployment-one");
466                            assert_eq!(request["purpose"], expected_purpose);
467                            assert_eq!(request["proof"]["token"], "fixture-proof");
468                            break;
469                        }
470                    }
471                }
472                let body = body.to_string();
473                write!(stream, "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap();
474            }
475        });
476        let clock = Arc::new(AtomicI64::new(1_800_000_000));
477        let proof = Arc::new(Proof(AtomicUsize::new(0)));
478        let observed_clock = clock.clone();
479        let config = RuntimeCredentialConfig {
480            manager_url: url,
481            audience: "manager-identity".into(),
482            deployment_id: "deployment-one".into(),
483            purpose,
484        };
485        let client = match receiver {
486            Some(receiver) => {
487                RuntimeCredentials::new_for_receiver(config, proof.clone(), exchange_path, receiver)
488            }
489            None => RuntimeCredentials::new(config, proof.clone()),
490        }
491        .unwrap()
492        .with_clock(Arc::new(move || {
493            DateTime::from_timestamp(observed_clock.load(Ordering::SeqCst), 0).unwrap()
494        }));
495        (client, clock, proof, server)
496    }
497
498    #[tokio::test]
499    async fn explicit_receiver_keeps_exchange_and_delivery_origins_separate() {
500        let receiver = TcpListener::bind("127.0.0.1:0").unwrap();
501        let receiver_url = format!("http://{}", receiver.local_addr().unwrap());
502        let unauthorized = TcpListener::bind("127.0.0.1:0").unwrap();
503        unauthorized.set_nonblocking(true).unwrap();
504        let wrong_url = format!("http://{}", unauthorized.local_addr().unwrap());
505        let (client, _, _, exchange) = fixture_for_receiver(
506            vec![(200, credential("scoped-child", 100))],
507            Some(&receiver_url),
508        );
509        assert!(!client.allows_endpoint(client.endpoint.as_str()));
510        assert!(!client.allows_endpoint(&wrong_url));
511        assert!(client.allows_endpoint(&receiver_url));
512        let delivery = thread::spawn(move || {
513            let (mut stream, _) = receiver.accept().unwrap();
514            stream
515                .set_read_timeout(Some(Duration::from_secs(5)))
516                .unwrap();
517            let mut bytes = [0; 4096];
518            let count = stream.read(&mut bytes).unwrap();
519            let request = String::from_utf8_lossy(&bytes[..count]);
520            assert!(request
521                .to_ascii_lowercase()
522                .contains("authorization: bearer scoped-child"));
523            write!(
524                stream,
525                "HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
526            )
527            .unwrap();
528        });
529        let token = client.token().await.unwrap();
530        let response = Client::builder()
531            .redirect(Policy::none())
532            .build()
533            .unwrap()
534            .get(&receiver_url)
535            .bearer_auth(token)
536            .send()
537            .await
538            .unwrap();
539        assert_eq!(response.status(), 204);
540        assert_eq!(
541            unauthorized.accept().unwrap_err().kind(),
542            std::io::ErrorKind::WouldBlock
543        );
544        exchange.join().unwrap();
545        delivery.join().unwrap();
546    }
547
548    #[tokio::test]
549    async fn explicit_exchange_refuses_redirect_to_another_listener() {
550        let destination = TcpListener::bind("127.0.0.1:0").unwrap();
551        destination.set_nonblocking(true).unwrap();
552        let destination_url = format!("http://{}", destination.local_addr().unwrap());
553        let manager = TcpListener::bind("127.0.0.1:0").unwrap();
554        let manager_url = format!("http://{}", manager.local_addr().unwrap());
555        let redirect = destination_url.clone();
556        let server = thread::spawn(move || {
557            let (mut stream, _) = manager.accept().unwrap();
558            stream
559                .set_read_timeout(Some(Duration::from_secs(5)))
560                .unwrap();
561            let mut bytes = [0; 4096];
562            assert!(stream.read(&mut bytes).unwrap() > 0);
563            write!(stream, "HTTP/1.1 307 Temporary Redirect\r\nLocation: {redirect}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap();
564        });
565        let client = RuntimeCredentials::new_for_receiver(
566            RuntimeCredentialConfig {
567                manager_url,
568                audience: "manager-identity".into(),
569                deployment_id: "deployment-one".into(),
570                purpose: json!({"type":"telemetry"}),
571            },
572            Arc::new(Proof(AtomicUsize::new(0))),
573            "/v1/identity/scoped",
574            &destination_url,
575        )
576        .unwrap();
577        assert!(client.token().await.is_err());
578        server.join().unwrap();
579        assert_eq!(
580            destination.accept().unwrap_err().kind(),
581            std::io::ErrorKind::WouldBlock
582        );
583    }
584
585    #[test]
586    fn explicit_destinations_refuse_ambiguous_or_insecure_configuration() {
587        for (path, receiver) in [
588            ("//other.example/exchange", "https://receiver.example"),
589            ("https://other.example/exchange", "https://receiver.example"),
590            ("/exchange?next=other", "https://receiver.example"),
591            ("/exchange#other", "https://receiver.example"),
592            ("/a/../exchange", "https://receiver.example"),
593            ("/exchange", "http://receiver.example"),
594            ("/exchange", "https://user:pass@receiver.example"),
595            ("/exchange", "https://receiver.example?secret=value"),
596        ] {
597            let result = RuntimeCredentials::new_for_receiver(
598                RuntimeCredentialConfig {
599                    manager_url: "https://manager.example".into(),
600                    audience: "manager-identity".into(),
601                    deployment_id: "deployment-one".into(),
602                    purpose: json!({"type":"telemetry"}),
603                },
604                Arc::new(Proof(AtomicUsize::new(0))),
605                path,
606                receiver,
607            );
608            assert!(result.is_err(), "unexpected accepted destination");
609        }
610    }
611
612    fn credential(token: &str, seconds: i64) -> Value {
613        json!({"token": token, "expiresAt": DateTime::from_timestamp(1_800_000_000 + seconds, 0).unwrap()})
614    }
615
616    #[tokio::test]
617    async fn concurrent_requests_share_refresh_and_refresh_before_expiry() {
618        let (client, clock, proof, server) = fixture(vec![
619            (200, credential("first", 100)),
620            (200, credential("second", 200)),
621        ]);
622        let (one, two) = tokio::join!(client.token(), client.token());
623        assert_eq!(one.unwrap(), "first");
624        assert_eq!(two.unwrap(), "first");
625        assert_eq!(proof.0.load(Ordering::SeqCst), 1);
626        clock.store(1_800_000_049, Ordering::SeqCst);
627        assert_eq!(client.token().await.unwrap(), "first");
628        clock.store(1_800_000_050, Ordering::SeqCst);
629        assert_eq!(client.token().await.unwrap(), "second");
630        assert_eq!(proof.0.load(Ordering::SeqCst), 2);
631        server.join().unwrap();
632    }
633
634    #[tokio::test]
635    async fn early_refresh_outage_retains_only_unexpired_nonrejected_token() {
636        let (client, clock, _, server) = fixture(vec![
637            (200, credential("first", 100)),
638            (503, json!({})),
639            (503, json!({})),
640        ]);
641        assert_eq!(client.token().await.unwrap(), "first");
642        clock.store(1_800_000_050, Ordering::SeqCst);
643        assert_eq!(client.token().await.unwrap(), "first");
644        clock.store(1_800_000_100, Ordering::SeqCst);
645        assert!(client.token().await.is_err());
646        server.join().unwrap();
647    }
648
649    #[tokio::test]
650    async fn refresh_failure_rechecks_expiry_after_io() {
651        struct FailingProof(Arc<AtomicI64>);
652        #[async_trait]
653        impl RuntimeProofProvider for FailingProof {
654            async fn proof(&self, _: &str) -> Result<Value> {
655                self.0.store(1_800_000_100, Ordering::SeqCst);
656                Err(invalid("fixture provider unavailable"))
657            }
658        }
659        let (mut client, clock, _, server) = fixture(vec![(200, credential("first", 100))]);
660        assert_eq!(client.token().await.unwrap(), "first");
661        server.join().unwrap();
662        clock.store(1_800_000_050, Ordering::SeqCst);
663        client.source = CredentialSource::Exchange(Arc::new(FailingProof(clock)));
664        assert!(client.token().await.is_err());
665    }
666
667    #[tokio::test]
668    async fn concurrent_rejections_preserve_the_replacement() {
669        let (client, _, proof, server) = fixture(vec![
670            (200, credential("first", 100)),
671            (200, credential("second", 100)),
672        ]);
673        assert_eq!(client.token().await.unwrap(), "first");
674        let (one, two) = tokio::join!(
675            client.token_after_rejection("first"),
676            client.token_after_rejection("first")
677        );
678        assert_eq!(one.unwrap(), "second");
679        assert_eq!(two.unwrap(), "second");
680        assert_eq!(proof.0.load(Ordering::SeqCst), 2);
681        client.reject("first").await;
682        assert_eq!(client.token().await.unwrap(), "second");
683        assert_eq!(proof.0.load(Ordering::SeqCst), 2);
684        server.join().unwrap();
685    }
686
687    #[tokio::test]
688    async fn stale_rejection_cannot_replace_newer_rejection_during_refresh() {
689        let (client, _, _, server) = fixture(vec![
690            (200, credential("first", 100)),
691            (200, credential("second", 100)),
692            (200, credential("second", 100)),
693            (200, credential("third", 100)),
694        ]);
695        assert_eq!(client.token().await.unwrap(), "first");
696        assert_eq!(
697            client.token_after_rejection("first").await.unwrap(),
698            "second"
699        );
700        client.reject("second").await;
701        client.reject("first").await;
702        assert!(
703            client.token().await.is_err(),
704            "late first rejection restored rejected second credential"
705        );
706        assert_eq!(client.token().await.unwrap(), "third");
707        server.join().unwrap();
708    }
709
710    #[cfg(unix)]
711    #[tokio::test]
712    async fn stale_rejection_cannot_restore_rejected_projected_file() {
713        let directory = tempfile::tempdir().unwrap();
714        let path = directory.path().join("credential");
715        let write = |token: &str| {
716            std::fs::write(
717                &path,
718                serde_json::to_vec(&RuntimeCapability {
719                    token: token.into(),
720                    expires_at: Utc::now() + chrono::Duration::minutes(30),
721                })
722                .unwrap(),
723            )
724            .unwrap();
725            std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
726        };
727        let client = RuntimeCredentials::from_capability_file(
728            RuntimeCredentialConfig {
729                manager_url: "https://manager.example".into(),
730                audience: "manager-one".into(),
731                deployment_id: "deployment-one".into(),
732                purpose: json!({"type":"telemetry"}),
733            },
734            path.clone(),
735        )
736        .unwrap();
737        write("ax_cap_v1_first");
738        assert_eq!(client.token().await.unwrap(), "ax_cap_v1_first");
739        client.reject("ax_cap_v1_first").await;
740        write("ax_cap_v1_second");
741        assert_eq!(client.token().await.unwrap(), "ax_cap_v1_second");
742        client.reject("ax_cap_v1_second").await;
743        client.reject("ax_cap_v1_first").await;
744        assert!(
745            client.token().await.is_err(),
746            "late first rejection restored rejected file bytes"
747        );
748        write("ax_cap_v1_third");
749        assert_eq!(client.token().await.unwrap(), "ax_cap_v1_third");
750    }
751
752    #[tokio::test]
753    async fn rejected_refresh_does_not_restore_cached_or_expired_authority() {
754        let (client, _, _, server) = fixture(vec![
755            (200, credential("first", 100)),
756            (401, json!({"secret":"must-not-appear"})),
757            (200, credential("replacement", 200)),
758        ]);
759        assert_eq!(client.token().await.unwrap(), "first");
760        let error = client.token_after_rejection("first").await.unwrap_err();
761        assert!(!format!("{error:?}").contains("must-not-appear"));
762        assert_eq!(client.token().await.unwrap(), "replacement");
763        server.join().unwrap();
764    }
765
766    #[tokio::test]
767    async fn exchange_cannot_restore_the_rejected_token() {
768        let (client, _, _, server) = fixture(vec![
769            (200, credential("first", 100)),
770            (200, credential("first", 100)),
771            (200, credential("replacement", 100)),
772        ]);
773        assert_eq!(client.token().await.unwrap(), "first");
774        assert!(client.token_after_rejection("first").await.is_err());
775        assert_eq!(client.token().await.unwrap(), "replacement");
776        server.join().unwrap();
777    }
778
779    #[tokio::test]
780    async fn refuses_expired_and_overlong_responses() {
781        let (client, _, _, server) = fixture(vec![
782            (200, credential("expired", 0)),
783            (200, credential("overlong", 3600 + 301)),
784        ]);
785        assert!(client.token().await.is_err());
786        assert!(client.token().await.is_err());
787        server.join().unwrap();
788    }
789
790    #[tokio::test]
791    async fn credential_from_a_manager_clock_two_minutes_ahead_is_accepted() {
792        // The manager issued a one hour credential by its own clock, which
793        // runs two minutes ahead of ours.
794        let (client, _, _, server) = fixture(vec![(200, credential("ahead", 3600 + 120))]);
795        assert_eq!(client.token().await.unwrap(), "ahead");
796        server.join().unwrap();
797    }
798    /// The operator writes the pull credential into a Secret that pods read
799    /// while the operator may be down. It accepts one that lasts as long as
800    /// the daily key, and still refuses longer ones or long credentials for
801    /// any other purpose.
802    #[tokio::test]
803    async fn registry_pull_credential_may_last_as_long_as_the_daily_key() {
804        let day = 24 * 3600;
805        let (client, _, _, server) = fixture_for(
806            vec![
807                (200, credential("pull-day", day)),
808                (200, credential("pull-overlong", day + 301)),
809            ],
810            None,
811            json!({"type":"registryPull"}),
812        );
813        assert_eq!(client.token().await.unwrap(), "pull-day");
814        client.reject("pull-day").await;
815        assert!(client.token().await.is_err());
816        server.join().unwrap();
817
818        let (client, _, _, server) = fixture(vec![(200, credential("telemetry-day", day))]);
819        assert!(client.token().await.is_err());
820        server.join().unwrap();
821    }
822
823    #[cfg(unix)]
824    #[tokio::test]
825    async fn capability_file_reopens_replacement_and_refuses_exact_expiry() {
826        let directory = tempfile::tempdir().unwrap();
827        let path = directory.path().join("telemetry");
828        let clock = Arc::new(std::sync::atomic::AtomicI64::new(1_800_000_000));
829        let write = |name: &str, token: &str, expires: i64| {
830            let file = directory.path().join(name);
831            let capability = RuntimeCapability {
832                token: token.into(),
833                expires_at: DateTime::from_timestamp(expires, 0).unwrap(),
834            };
835            std::fs::write(&file, serde_json::to_vec(&capability).unwrap()).unwrap();
836            std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o600)).unwrap();
837            file
838        };
839        write("telemetry", "legacy-deployment-token", 1_800_000_100);
840        let observed = clock.clone();
841        let credentials = RuntimeCredentials::from_capability_file(
842            RuntimeCredentialConfig {
843                manager_url: "https://manager.example".into(),
844                audience: "manager-one".into(),
845                deployment_id: "deployment-one".into(),
846                purpose: json!({"type":"telemetry"}),
847            },
848            path.clone(),
849        )
850        .unwrap()
851        .with_clock(Arc::new(move || {
852            DateTime::from_timestamp(observed.load(std::sync::atomic::Ordering::SeqCst), 0).unwrap()
853        }));
854        assert!(
855            credentials.token().await.is_err(),
856            "legacy token must not enter capability-file mode"
857        );
858        write("telemetry", "ax_cap_v1_first", 1_800_000_100);
859        assert_eq!(credentials.token().await.unwrap(), "ax_cap_v1_first");
860        credentials.reject("ax_cap_v1_first").await;
861        assert!(credentials.token().await.is_err());
862        clock.store(1_800_000_100, std::sync::atomic::Ordering::SeqCst);
863        assert!(credentials.token().await.is_err());
864        let next = write("next", "ax_cap_v1_second", 1_800_000_200);
865        std::fs::rename(next, &path).unwrap();
866        assert_eq!(credentials.token().await.unwrap(), "ax_cap_v1_second");
867        credentials.reject("ax_cap_v1_first").await;
868        assert_eq!(credentials.token().await.unwrap(), "ax_cap_v1_second");
869        for mode in [0o644, 0o620, 0o610, 0o601] {
870            std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode)).unwrap();
871            assert!(credentials.token().await.is_err());
872        }
873        if std::fs::metadata(&path).unwrap().uid() != 0 {
874            std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o440)).unwrap();
875            assert!(credentials.token().await.is_err());
876        }
877        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap();
878        assert_eq!(credentials.token().await.unwrap(), "ax_cap_v1_second");
879        std::fs::remove_file(&path).unwrap();
880        assert!(credentials.token().await.is_err());
881    }
882}