Skip to main content

cachekit/backend/
cachekitio.rs

1use std::collections::HashMap;
2use std::sync::OnceLock;
3use std::time::Duration;
4
5use async_trait::async_trait;
6use bytes::Bytes;
7use reqwest::header::{HeaderName, HeaderValue, InvalidHeaderValue};
8use zeroize::Zeroizing;
9
10use crate::backend::{
11    delete_succeeded, encode_key, ttl_header, Backend, Freshness, HealthStatus, LockableBackend,
12};
13use crate::error::{BackendError, BackendErrorKind};
14use crate::metrics::{metrics_headers_named, MetricsProvider, METRICS_HEADER_NAMES};
15use crate::session::session_headers;
16use crate::url_validator::validate_cachekitio_url;
17
18// ── CachekitIO ────────────────────────────────────────────────────────────────
19
20/// HTTP backend that talks to the cachekit.io SaaS API.
21pub struct CachekitIO {
22    client: reqwest::Client,
23    api_key: Zeroizing<String>,
24    api_url: String,
25    /// `Authorization` and the session headers, built once so no request
26    /// re-formats or re-parses them. The bearer value is marked sensitive, so
27    /// a request's `Debug` output redacts it, and its buffer is wiped when the
28    /// last clone drops.
29    static_headers: [(HeaderName, HeaderValue); 3],
30    /// The telemetry header names, parsed once, in [`METRICS_HEADER_NAMES`] order.
31    metrics_header_names: [HeaderName; 5],
32    /// Source of the `X-CacheKit-*` telemetry headers. Set once: by the
33    /// builder when the user supplies one, otherwise by the client at build
34    /// time via [`Backend::attach_metrics`] (first writer wins).
35    metrics_provider: OnceLock<MetricsProvider>,
36}
37
38/// Redact `api_key` from debug output.
39impl std::fmt::Debug for CachekitIO {
40    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
41        f.debug_struct("CachekitIO")
42            .field("api_url", &self.api_url)
43            .field("api_key", &"<redacted>")
44            .finish()
45    }
46}
47
48impl CachekitIO {
49    /// Start building a [`CachekitIO`] instance.
50    pub fn builder() -> CachekitIOBuilder {
51        CachekitIOBuilder::default()
52    }
53
54    /// Return the configured API URL (useful in tests / introspection).
55    pub fn api_url(&self) -> &str {
56        &self.api_url
57    }
58
59    /// Return a reference to the underlying HTTP client.
60    pub(crate) fn client(&self) -> &reqwest::Client {
61        &self.client
62    }
63
64    /// Return the API key as a string slice (for error sanitizing in sibling modules).
65    pub(crate) fn api_key_str(&self) -> &str {
66        self.api_key.as_str()
67    }
68
69    /// Build the full URL for a cache key path segment.
70    ///
71    /// Keys are percent-encoded via [`encode_key`](crate::backend::encode_key) so
72    /// slashes or special characters do not break the URL structure. Fallible: it
73    /// rejects exactly the keys `encode_key` rejects (CWE-22, spec rule 2).
74    fn url(&self, key: &str) -> Result<String, BackendError> {
75        Ok(format!("{}/v1/cache/{}", self.api_url, encode_key(key)?))
76    }
77
78    /// Build the TTL URL for a cache key (`…/ttl`). Composes on [`Self::url`], so
79    /// it inherits the same [`encode_key`](crate::backend::encode_key) guard and
80    /// the `/v1/cache/` prefix lives in one place (matching the wasm `workers`
81    /// backend). `pub(crate)` so the [`TtlInspectable`](super::TtlInspectable)
82    /// impl in the sibling `cachekitio_ttl` module builds its path through it.
83    pub(crate) fn ttl_url(&self, key: &str) -> Result<String, BackendError> {
84        Ok(format!("{}/ttl", self.url(key)?))
85    }
86
87    /// Build the lock URL for a cache key (`…/lock`). Composes on [`Self::url`]
88    /// (same guard, same single prefix). `pub(crate)` so the
89    /// [`LockableBackend`](super::LockableBackend) impl in the sibling
90    /// `cachekitio_lock` module builds its path through it.
91    pub(crate) fn lock_url(&self, key: &str) -> Result<String, BackendError> {
92        Ok(format!("{}/lock", self.url(key)?))
93    }
94
95    /// Build the health-check URL.
96    fn health_url(&self) -> String {
97        format!("{}/v1/cache/health", self.api_url)
98    }
99
100    /// Apply the bearer token, session and metrics headers to a request builder.
101    pub(crate) fn with_standard_headers(
102        &self,
103        mut req: reqwest::RequestBuilder,
104    ) -> reqwest::RequestBuilder {
105        for (name, value) in &self.static_headers {
106            req = req.header(name.clone(), value.clone());
107        }
108        for (name, value) in
109            metrics_headers_named(self.metrics_provider.get(), &self.metrics_header_names)
110        {
111            req = req.header(name, value);
112        }
113        req
114    }
115
116    /// Build the `PUT` for [`Backend::set`] without sending it, so tests can
117    /// pin its headers.
118    fn put_request(
119        &self,
120        key: &str,
121        value: Vec<u8>,
122        ttl: Option<Duration>,
123    ) -> Result<reqwest::RequestBuilder, BackendError> {
124        let mut req = self
125            .client
126            .put(self.url(key)?)
127            .header(reqwest::header::CONTENT_TYPE, "application/octet-stream")
128            .body(value);
129
130        if let Some(ttl) = ttl {
131            let (name, value) = ttl_header(ttl);
132            req = req.header(name, value);
133        }
134
135        Ok(self.with_standard_headers(req))
136    }
137
138    /// Read error body from response and build a sanitized BackendError.
139    pub(crate) async fn error_from_response(&self, resp: reqwest::Response) -> BackendError {
140        let status = resp.status().as_u16();
141        let quota_denied = status == 429 && resp.headers().contains_key(DENY_REASON_HEADER);
142        let body = resp.bytes().await.unwrap_or_default();
143        let err = from_http_status_sanitized(status, &body, self.api_key.as_str());
144        if quota_denied {
145            err.with_quota_denied()
146        } else {
147            err
148        }
149    }
150}
151
152// ── Error helpers ────────────────────────────────────────────────────────────
153
154/// Convert a reqwest error into a BackendError, sanitizing the API key from the message.
155pub(crate) fn reqwest_err_sanitized(e: reqwest::Error, api_key: &str) -> BackendError {
156    let kind = if e.is_timeout() {
157        BackendErrorKind::Timeout
158    } else {
159        BackendErrorKind::Transient
160    };
161    BackendError {
162        kind,
163        message: BackendError::sanitize_message(&e.to_string(), api_key),
164        source: Some(Box::new(e)),
165    }
166}
167
168/// Build a [`BackendError`] from an HTTP status + body, sanitizing the API key from output.
169pub(crate) fn from_http_status_sanitized(status: u16, body: &[u8], api_key: &str) -> BackendError {
170    let sanitized =
171        BackendError::sanitize_message(std::str::from_utf8(body).unwrap_or(""), api_key);
172    BackendError::from_http_status(status, sanitized.as_bytes())
173}
174
175// ── Freshness headers ────────────────────────────────────────────────────────
176
177/// Read a `GET 200`'s [`Freshness`] from its headers (`spec/saas-api.md`
178/// § Stale-While-Revalidate, § Remaining Freshness).
179///
180/// An absent `X-CacheKit-Freshness` means fresh (a pre-SWR server). Any value
181/// other than exactly `fresh` means stale, because revalidating is the safe
182/// reading of a token the client does not know; every copy of a repeated
183/// header must say `fresh`. An absent `X-CacheKit-Fresh-For` means no server
184/// bound. A repeated one has no single bound, so it counts as `0`.
185#[cfg(not(target_arch = "wasm32"))]
186pub(crate) fn freshness_from_headers(headers: &reqwest::header::HeaderMap) -> Freshness {
187    let is_stale = headers
188        .get_all("x-cachekit-freshness")
189        .iter()
190        .any(|value| value != "fresh");
191    let mut fresh_for = headers.get_all("x-cachekit-fresh-for").iter();
192    let fresh_for = match (fresh_for.next(), fresh_for.next()) {
193        (None, _) => None,
194        (Some(value), None) => Some(parse_fresh_for(value.as_bytes())),
195        (Some(_), Some(_)) => Some(Duration::ZERO),
196    };
197    Freshness {
198        is_stale,
199        fresh_for,
200    }
201}
202
203/// `X-CacheKit-Fresh-For` must be 1–7 ASCII digits and at most 2,592,000 (the
204/// 30-day TTL cap); anything else counts as `0`. The length check runs first,
205/// so the range check never sees a value that a fixed-width parse could wrap
206/// back into range (`4297559296` as a `u32` is `2592000`).
207#[cfg(not(target_arch = "wasm32"))]
208fn parse_fresh_for(value: &[u8]) -> Duration {
209    const MAX_FRESH_FOR_SECS: u64 = 2_592_000;
210    if !(1..=7).contains(&value.len()) || !value.iter().all(u8::is_ascii_digit) {
211        return Duration::ZERO;
212    }
213    match std::str::from_utf8(value).map(str::parse::<u64>) {
214        Ok(Ok(secs)) if secs <= MAX_FRESH_FOR_SECS => Duration::from_secs(secs),
215        _ => Duration::ZERO,
216    }
217}
218
219// ── Backend impl ──────────────────────────────────────────────────────────────
220
221#[cfg(not(target_arch = "wasm32"))]
222#[cfg_attr(not(feature = "unsync"), async_trait)]
223#[cfg_attr(feature = "unsync", async_trait(?Send))]
224impl Backend for CachekitIO {
225    fn attach_metrics(&self, provider: MetricsProvider) {
226        // A provider supplied on the builder is already set and wins.
227        self.metrics_provider.get_or_init(|| provider);
228    }
229
230    async fn get(&self, key: &str) -> Result<Option<Vec<u8>>, BackendError> {
231        Ok(self.get_with_freshness(key).await?.map(|(bytes, _)| bytes))
232    }
233
234    async fn get_with_freshness(
235        &self,
236        key: &str,
237    ) -> Result<Option<(Vec<u8>, Freshness)>, BackendError> {
238        let req = self.with_standard_headers(self.client.get(self.url(key)?));
239
240        let resp = req
241            .send()
242            .await
243            .map_err(|e| reqwest_err_sanitized(e, self.api_key.as_str()))?;
244
245        match resp.status().as_u16() {
246            200 => {
247                let freshness = freshness_from_headers(resp.headers());
248                let bytes = resp
249                    .bytes()
250                    .await
251                    .map_err(|e| reqwest_err_sanitized(e, self.api_key.as_str()))?;
252                Ok(Some((Vec::from(bytes), freshness)))
253            }
254            404 => Ok(None),
255            _ => Err(self.error_from_response(resp).await),
256        }
257    }
258
259    async fn set(
260        &self,
261        key: &str,
262        value: Vec<u8>,
263        ttl: Option<Duration>,
264    ) -> Result<(), BackendError> {
265        let resp = self
266            .put_request(key, value, ttl)?
267            .timeout(WRITE_TIMEOUT)
268            .send()
269            .await
270            .map_err(|e| reqwest_err_sanitized(e, self.api_key.as_str()))?;
271
272        let status = resp.status().as_u16();
273        if (200..300).contains(&status) {
274            Ok(())
275        } else {
276            Err(self.error_from_response(resp).await)
277        }
278    }
279
280    /// `true` on every successful delete, whether or not the key existed: the
281    /// server does not report existence on `DELETE`.
282    async fn delete(&self, key: &str) -> Result<bool, BackendError> {
283        let req = self.with_standard_headers(self.client.delete(self.url(key)?));
284
285        let resp = req
286            .timeout(WRITE_TIMEOUT)
287            .send()
288            .await
289            .map_err(|e| reqwest_err_sanitized(e, self.api_key.as_str()))?;
290
291        if delete_succeeded(resp.status().as_u16()) {
292            Ok(true)
293        } else {
294            Err(self.error_from_response(resp).await)
295        }
296    }
297
298    async fn exists(&self, key: &str) -> Result<bool, BackendError> {
299        let req = self.with_standard_headers(self.client.head(self.url(key)?));
300
301        let resp = req
302            .send()
303            .await
304            .map_err(|e| reqwest_err_sanitized(e, self.api_key.as_str()))?;
305
306        match resp.status().as_u16() {
307            200 => Ok(true),
308            404 => Ok(false),
309            // Through the header-aware path, so a quota-deny `429` is marked.
310            _ => Err(self.error_from_response(resp).await),
311        }
312    }
313
314    async fn health(&self) -> Result<HealthStatus, BackendError> {
315        let start = std::time::Instant::now();
316
317        let req = self.with_standard_headers(self.client.get(self.health_url()));
318
319        let resp = req
320            .send()
321            .await
322            .map_err(|e| reqwest_err_sanitized(e, self.api_key.as_str()))?;
323
324        let latency = start.elapsed();
325        let status = resp.status().as_u16();
326
327        if (200..300).contains(&status) {
328            let mut details = HashMap::new();
329            details.insert("http_status".to_string(), status.to_string());
330            Ok(HealthStatus {
331                is_healthy: true,
332                latency_ms: latency.as_secs_f64() * 1000.0,
333                backend_type: "cachekitio".to_string(),
334                details,
335            })
336        } else {
337            Err(self.error_from_response(resp).await)
338        }
339    }
340
341    fn as_lockable(&self) -> Option<&dyn LockableBackend> {
342        Some(self)
343    }
344}
345
346// ── Prebuilt headers ──────────────────────────────────────────────────────────
347
348/// Build the per-client constant headers: `Authorization: Bearer <key>`,
349/// marked sensitive as `bearer_auth` would, and the process's session headers.
350/// The bearer outlives every request and each request clones it, so its buffer
351/// is a zeroising owner those clones share: the last drop wipes the key.
352///
353/// # Errors
354///
355/// [`CachekitError::Config`](crate::error::CachekitError::Config) when the key
356/// holds bytes an HTTP header cannot carry. The message never echoes the key.
357fn static_headers(
358    api_key: &str,
359) -> Result<[(HeaderName, HeaderValue); 3], crate::error::CachekitError> {
360    use crate::error::CachekitError;
361
362    let mut bearer = Zeroizing::new(Vec::with_capacity("Bearer ".len() + api_key.len()));
363    bearer.extend_from_slice(b"Bearer ");
364    bearer.extend_from_slice(api_key.as_bytes());
365    let mut auth = header_over(bearer).map_err(|_| {
366        CachekitError::Config("api_key holds characters an HTTP header cannot carry".to_string())
367    })?;
368    auth.set_sensitive(true);
369
370    let [(id_name, id), (start_name, start)] = session_headers();
371    let header = |name: &str, value: &str| {
372        Ok::<_, CachekitError>((
373            HeaderName::from_bytes(name.as_bytes())
374                .map_err(|e| CachekitError::Config(format!("header name {name}: {e}")))?,
375            HeaderValue::from_str(value)
376                .map_err(|e| CachekitError::Config(format!("header {name}: {e}")))?,
377        ))
378    };
379    Ok([
380        (reqwest::header::AUTHORIZATION, auth),
381        header(id_name, id)?,
382        header(start_name, start)?,
383    ])
384}
385
386/// A header value over `owner`'s buffer, not a copy of it: clones share the
387/// buffer, and `owner` drops with the last of them.
388fn header_over<T: AsRef<[u8]> + Send + 'static>(
389    owner: T,
390) -> Result<HeaderValue, InvalidHeaderValue> {
391    HeaderValue::from_maybe_shared(Bytes::from_owner(owner))
392}
393
394/// Parse [`METRICS_HEADER_NAMES`] once per client.
395fn metrics_header_names() -> Result<[HeaderName; 5], crate::error::CachekitError> {
396    let [a, b, c, d, e] = METRICS_HEADER_NAMES.map(|name| {
397        HeaderName::from_bytes(name.as_bytes())
398            .map_err(|e| crate::error::CachekitError::Config(format!("header name {name}: {e}")))
399    });
400    Ok([a?, b?, c?, d?, e?])
401}
402
403// ── Builder ───────────────────────────────────────────────────────────────────
404
405/// `User-Agent` on every native CachekitIO request, so the service and its edge
406/// can tell this SDK and version apart from other HTTP clients.
407#[cfg(not(target_arch = "wasm32"))]
408const USER_AGENT: &str = concat!("cachekit-rs/", env!("CARGO_PKG_VERSION"));
409
410/// How long an idle pooled connection is kept for reuse. Cloudflare closes an
411/// idle client HTTP/1.1 connection after 400 s, not configurable; reqwest's
412/// 90 s default dropped connections the edge would still serve, so a request
413/// after a 90-390 s gap paid DNS, TCP and TLS again. 390 s leaves a 10 s
414/// margin, and reqwest's 15 s TCP keepalive keeps NAT mappings alive meanwhile.
415/// Per-attempt budget for every request but a write, connect and TLS
416/// included: the protocol's `CACHEKIT_TIMEOUT` default (`spec/saas-api.md`),
417/// as in cachekit-py and cachekit-ts. It was 30 s, so under the retrying
418/// presets a request the service accepted and never answered held a call for
419/// ~90 s before it failed open.
420#[cfg(not(target_arch = "wasm32"))]
421const READ_TIMEOUT: Duration = Duration::from_secs(5);
422
423/// Per-attempt budget for a write (`PUT`, `DELETE`): twice the read budget,
424/// because a write carries the payload and the service commits it before it
425/// answers, so a slow but healthy write must not time out and be re-sent.
426#[cfg(not(target_arch = "wasm32"))]
427const WRITE_TIMEOUT: Duration = Duration::from_secs(10);
428
429/// Header on a `429` that denies a spent quota or balance rather than a rate
430/// limit; such an error is never retried within the call (see
431/// [`crate::error::QuotaDenied`]).
432#[cfg_attr(target_arch = "wasm32", allow(dead_code))] // wasm32 uses the Workers backend
433const DENY_REASON_HEADER: &str = "x-cachekit-deny-reason";
434
435#[cfg(not(target_arch = "wasm32"))]
436const POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(390);
437
438/// The settings behind every native [`CachekitIO`] client, unbuilt so tests can
439/// add `.no_proxy()` and reach a loopback stub whatever the proxy env says.
440#[cfg(not(target_arch = "wasm32"))]
441fn http_client_builder() -> reqwest::ClientBuilder {
442    reqwest::Client::builder()
443        .use_rustls_tls()
444        .user_agent(USER_AGENT)
445        .redirect(reqwest::redirect::Policy::none())
446        .timeout(READ_TIMEOUT)
447        .connect_timeout(READ_TIMEOUT)
448        .pool_idle_timeout(POOL_IDLE_TIMEOUT)
449}
450
451/// The HTTP client every native [`CachekitIO`] uses.
452#[cfg(not(target_arch = "wasm32"))]
453fn http_client() -> reqwest::Result<reqwest::Client> {
454    http_client_builder().build()
455}
456
457/// wasm32: the platform's `fetch` owns pooling and timeouts; no User-Agent is set.
458#[cfg(target_arch = "wasm32")]
459fn http_client() -> reqwest::Result<reqwest::Client> {
460    reqwest::Client::builder().build()
461}
462
463/// Builder for [`CachekitIO`].
464#[derive(Default)]
465#[must_use]
466pub struct CachekitIOBuilder {
467    api_key: Option<Zeroizing<String>>,
468    api_url: Option<String>,
469    allow_custom_host: bool,
470    metrics_provider: Option<MetricsProvider>,
471}
472
473impl CachekitIOBuilder {
474    /// Set the API key (required).
475    pub fn api_key(mut self, key: impl Into<String>) -> Self {
476        self.api_key = Some(Zeroizing::new(key.into()));
477        self
478    }
479
480    /// Override the API base URL (default: `https://api.cachekit.io`).
481    pub fn api_url(mut self, url: impl Into<String>) -> Self {
482        self.api_url = Some(url.into());
483        self
484    }
485
486    /// Allow non-standard hostnames (e.g. custom proxies). Private IPs are still blocked.
487    pub fn allow_custom_host(mut self, allow: bool) -> Self {
488        self.allow_custom_host = allow;
489        self
490    }
491
492    /// Override the source of the `X-CacheKit-*` telemetry headers.
493    ///
494    /// Not needed for normal use: `CacheKitBuilder::build` attaches the
495    /// client's own live counters to any backend without one. Set this only
496    /// to report numbers from somewhere else; it takes precedence.
497    pub fn metrics_provider(mut self, provider: MetricsProvider) -> Self {
498        self.metrics_provider = Some(provider);
499        self
500    }
501
502    /// Consume the builder and construct a [`CachekitIO`].
503    ///
504    /// # Errors
505    ///
506    /// Returns an error if:
507    /// - `api_key` was not set, or holds bytes an HTTP header cannot carry
508    ///   (a control character such as a newline).
509    /// - the resolved URL scheme is not `https`.
510    /// - the URL hostname is not permitted (see [`validate_cachekitio_url`]).
511    pub fn build(self) -> Result<CachekitIO, crate::error::CachekitError> {
512        use crate::error::CachekitError;
513
514        let api_key = self
515            .api_key
516            .filter(|k| !k.is_empty())
517            .ok_or_else(|| CachekitError::Config("api_key is required".to_string()))?;
518
519        let api_url = self
520            .api_url
521            .unwrap_or_else(|| "https://api.cachekit.io".to_string());
522
523        // Validate URL: HTTPS, allowed host, no private IPs.
524        validate_cachekitio_url(&api_url, self.allow_custom_host)?;
525
526        // Trim trailing slash once so url()/health_url() don't repeat it per-request.
527        let api_url = api_url.trim_end_matches('/').to_string();
528
529        let client = http_client()
530            .map_err(|e| CachekitError::Config(format!("failed to build HTTP client: {e}")))?;
531
532        let static_headers = static_headers(&api_key)?;
533        let metrics_header_names = metrics_header_names()?;
534
535        Ok(CachekitIO {
536            client,
537            api_key,
538            api_url,
539            static_headers,
540            metrics_header_names,
541            metrics_provider: self
542                .metrics_provider
543                .map_or_else(OnceLock::new, OnceLock::from),
544        })
545    }
546}
547
548// ── Freshness-header parsing tests ───────────────────────────────────────────
549
550#[cfg(all(test, not(target_arch = "wasm32")))]
551#[allow(clippy::expect_used)] // test-only: a malformed fixture header should panic loudly
552mod freshness_header_tests {
553    use std::time::Duration;
554
555    use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
556
557    use super::freshness_from_headers;
558    use crate::backend::Freshness;
559
560    /// Build a header map with the wire's mixed-case names; `append` keeps
561    /// repeats, as a proxy that duplicates a header would.
562    fn headers(pairs: &[(&str, &[u8])]) -> HeaderMap {
563        let mut map = HeaderMap::new();
564        for (name, value) in pairs {
565            map.append(
566                HeaderName::from_bytes(name.as_bytes()).expect("valid header name"),
567                HeaderValue::from_bytes(value).expect("valid header value"),
568            );
569        }
570        map
571    }
572
573    fn label(values: &[&[u8]]) -> bool {
574        let pairs: Vec<_> = values
575            .iter()
576            .map(|v| ("X-CacheKit-Freshness", *v))
577            .collect();
578        freshness_from_headers(&headers(&pairs)).is_stale
579    }
580
581    fn fresh_for(values: &[&[u8]]) -> Option<Duration> {
582        let pairs: Vec<_> = values
583            .iter()
584            .map(|v| ("X-CacheKit-Fresh-For", *v))
585            .collect();
586        freshness_from_headers(&headers(&pairs)).fresh_for
587    }
588
589    #[test]
590    fn absent_headers_are_a_fresh_unbounded_read() {
591        assert_eq!(
592            freshness_from_headers(&HeaderMap::new()),
593            Freshness::default()
594        );
595    }
596
597    #[test]
598    fn only_an_exact_fresh_label_is_fresh() {
599        assert!(!label(&[b"fresh"]));
600        assert!(!label(&[b"fresh", b"fresh"]));
601        // (d) `stale`, an unknown token, a case variant, an empty value, or any
602        // non-`fresh` copy of a repeated header all read as stale.
603        for stale in [&b"stale"[..], b"revalidating", b"Fresh", b"", b"fresh "] {
604            assert!(
605                label(&[stale]),
606                "{:?} must read as stale",
607                String::from_utf8_lossy(stale)
608            );
609        }
610        assert!(label(&[b"fresh", b"stale"]));
611    }
612
613    #[test]
614    fn fresh_for_in_grammar_parses() {
615        for (value, secs) in [
616            (&b"0"[..], 0),
617            (b"1", 1),
618            (b"0000005", 5),
619            (b"2592000", 2_592_000),
620        ] {
621            assert_eq!(fresh_for(&[value]), Some(Duration::from_secs(secs)));
622        }
623    }
624
625    /// (d) Outside 1–7 ASCII digits, or over 2,592,000, is `0`. `4297559296` is
626    /// the value a wrapping `u32` parse would turn into `2592000`.
627    #[test]
628    fn fresh_for_outside_grammar_is_zero() {
629        let invalid: [&[u8]; 11] = [
630            b"",
631            b"-1",
632            b"+5",
633            b" 5",
634            b"5s",
635            b"1.5",
636            "\u{0663}".as_bytes(), // ARABIC-INDIC DIGIT THREE: a digit, not ASCII
637            b"12345678",
638            b"2592001",
639            b"9999999",
640            b"4297559296",
641        ];
642        for value in invalid {
643            assert_eq!(
644                fresh_for(&[value]),
645                Some(Duration::ZERO),
646                "{:?} must count as 0",
647                String::from_utf8_lossy(value)
648            );
649        }
650        assert_eq!(
651            fresh_for(&[b"10", b"10"]),
652            Some(Duration::ZERO),
653            "a repeated bound counts as 0"
654        );
655    }
656}
657
658// ── Telemetry-header wiring tests ─────────────────────────────────────────────
659
660#[cfg(test)]
661#[allow(clippy::expect_used)] // test-only: a builder failure on a fixture should panic loudly
662mod metrics_wiring_tests {
663    use std::sync::Arc;
664
665    use super::CachekitIO;
666    use crate::backend::Backend;
667    use crate::metrics::{metrics_headers, L1Stats, MetricsProvider};
668
669    fn provider(l1_hits: u64) -> MetricsProvider {
670        Arc::new(move || {
671            Some(L1Stats {
672                l1_hits,
673                l2_hits: 0,
674                misses: 0,
675                l1_enabled: true,
676            })
677        })
678    }
679
680    fn l1_hits_header(backend: &CachekitIO) -> Option<String> {
681        metrics_headers(backend.metrics_provider.get())
682            .into_iter()
683            .find(|h| h.0 == "X-CacheKit-L1-Hits")
684            .map(|h| h.1)
685    }
686
687    #[test]
688    fn attach_fills_an_unset_provider() {
689        let backend = CachekitIO::builder()
690            .api_key("ck_test_key")
691            .build()
692            .expect("builder succeeds");
693        assert_eq!(l1_hits_header(&backend), None, "nothing attached yet");
694
695        backend.attach_metrics(provider(7));
696        assert_eq!(l1_hits_header(&backend).as_deref(), Some("7"));
697    }
698
699    #[test]
700    fn builder_provider_is_retained_over_attach() {
701        let backend = CachekitIO::builder()
702            .api_key("ck_test_key")
703            .metrics_provider(provider(1))
704            .build()
705            .expect("builder succeeds");
706
707        backend.attach_metrics(provider(99));
708        assert_eq!(
709            l1_hits_header(&backend).as_deref(),
710            Some("1"),
711            "user-supplied provider wins"
712        );
713    }
714}
715
716// ── Cache-key path encoding tests (CWE-22) ────────────────────────────────────
717
718#[cfg(test)]
719#[allow(clippy::expect_used)] // test-only: a builder/parse failure on a fixture should panic loudly
720mod path_encoding_tests {
721    use super::CachekitIO;
722    use crate::backend::path_encoding_vectors::{reject_keys, transmittable, RS_NEAR_MISSES};
723    use url::Url;
724
725    const API: &str = "https://api.cachekit.io";
726
727    fn backend() -> CachekitIO {
728        CachekitIO::builder()
729            .api_url(API)
730            .api_key("ck_test_key")
731            .build()
732            .expect("builder should succeed for the canonical host")
733    }
734
735    /// Repro. Before any guard, a raw-encoded `.`/`..` key collapses in
736    /// rust-url (the parser `reqwest` uses) *before* the request leaves the
737    /// process: the segment is stripped and the path escapes `/v1/cache/`.
738    /// `%2E%2E` collapses identically, which is why the fix rejects rather than
739    /// re-encodes (WHATWG treats `%2e%2e` as a dot-segment too).
740    #[test]
741    fn repro_raw_dot_key_escapes_the_cache_prefix() {
742        let cases = [
743            ("..", "", "/v1/"),
744            ("..", "/ttl", "/v1/ttl"),
745            ("..", "/lock", "/v1/lock"),
746            (".", "", "/v1/cache/"),
747        ];
748        for (key, suffix, escaped) in cases {
749            let raw = format!("{API}/v1/cache/{}{suffix}", urlencoding::encode(key));
750            let parsed = Url::parse(&raw).expect("parses");
751            assert_eq!(
752                parsed.path(),
753                escaped,
754                "raw {key:?}{suffix} should collapse to {escaped}"
755            );
756            // The %2E form the Python SDK emits collapses just the same in rust-url.
757            let pct = key.replace('.', "%2E");
758            let enc = format!("{API}/v1/cache/{pct}{suffix}");
759            assert_eq!(
760                Url::parse(&enc).expect("parses").path(),
761                escaped,
762                "%2E-encoded {key:?}{suffix} also collapses — encoding cannot fix this in rust-url"
763            );
764        }
765    }
766
767    /// Spec rule 2 — every fixture reject row (the empty key, `.`, `..`,
768    /// `health`, `ttl`, `lock`) is rejected by every builder (base, ttl, lock):
769    /// no URL is produced, so no rewritten or mis-routed request can be sent.
770    #[test]
771    fn reserved_segments_rejected_by_every_builder() {
772        let b = backend();
773        let keys = reject_keys();
774        assert!(!keys.is_empty(), "fixture has no reject rows");
775        for key in &keys {
776            assert!(b.url(key).is_err(), "url({key:?}) must be rejected");
777            assert!(b.ttl_url(key).is_err(), "ttl_url({key:?}) must be rejected");
778            assert!(
779                b.lock_url(key).is_err(),
780                "lock_url({key:?}) must be rejected"
781            );
782        }
783    }
784
785    /// Every transmittable fixture row, plus the rs near-misses, builds a URL
786    /// whose *parsed* path (the real wire path, post-normalisation) stays inside
787    /// `/v1/cache/`. Asserting on the unparsed `format!` output would pass while
788    /// still shipping a traversal, so we parse with the same `url` crate
789    /// `reqwest` uses.
790    #[test]
791    fn safe_keys_never_escape_the_cache_prefix() {
792        let b = backend();
793        let fixture_keys: Vec<String> = transmittable().into_iter().map(|v| v.key).collect();
794        let vectors = fixture_keys
795            .iter()
796            .map(String::as_str)
797            .chain(RS_NEAR_MISSES.iter().copied());
798        for key in vectors {
799            let base = Url::parse(&b.url(key).expect("url")).expect("parse base");
800            let ttl = Url::parse(&b.ttl_url(key).expect("ttl_url")).expect("parse ttl");
801            let lock = Url::parse(&b.lock_url(key).expect("lock_url")).expect("parse lock");
802
803            assert!(
804                base.path().starts_with("/v1/cache/") && base.path().len() > "/v1/cache/".len(),
805                "base path {} escaped prefix for {key:?}",
806                base.path()
807            );
808            assert_eq!(
809                base.path(),
810                format!("/v1/cache/{}", urlencoding::encode(key)),
811                "base wire path mismatch for {key:?}"
812            );
813            assert!(
814                ttl.path().starts_with("/v1/cache/") && ttl.path().ends_with("/ttl"),
815                "ttl path {} escaped prefix for {key:?}",
816                ttl.path()
817            );
818            assert!(
819                lock.path().starts_with("/v1/cache/") && lock.path().ends_with("/lock"),
820                "lock path {} escaped prefix for {key:?}",
821                lock.path()
822            );
823        }
824    }
825}
826
827// ── PUT TTL header tests ──────────────────────────────────────────────────────
828
829#[cfg(test)]
830#[allow(clippy::expect_used)] // test-only: a builder failure on a fixture should panic loudly
831mod put_ttl_header_tests {
832    use std::time::Duration;
833
834    use super::CachekitIO;
835
836    fn put(ttl: Option<Duration>) -> reqwest::Request {
837        CachekitIO::builder()
838            .api_key("ck_test_key")
839            .build()
840            .expect("builder succeeds")
841            .put_request("k", b"v".to_vec(), ttl)
842            .expect("key encodes")
843            .build()
844            .expect("request builds")
845    }
846
847    #[test]
848    fn put_sends_x_cachekit_ttl_and_no_legacy_x_ttl() {
849        let req = put(Some(Duration::from_secs(300)));
850        assert_eq!(
851            req.headers()
852                .get("X-CacheKit-TTL")
853                .and_then(|v| v.to_str().ok()),
854            Some("300")
855        );
856        assert!(req.headers().get("X-TTL").is_none(), "legacy header sent");
857    }
858
859    #[test]
860    fn put_without_ttl_sends_no_ttl_header() {
861        let req = put(None);
862        assert!(req.headers().get("X-CacheKit-TTL").is_none());
863        assert!(req.headers().get("X-TTL").is_none());
864    }
865}
866
867// ── Prebuilt header tests ─────────────────────────────────────────────────────
868
869#[cfg(test)]
870#[allow(clippy::expect_used)] // test-only: a builder failure on a fixture should panic loudly
871mod prebuilt_header_tests {
872    use std::sync::atomic::{AtomicBool, Ordering};
873    use std::sync::Arc;
874
875    use super::{header_over, CachekitIO};
876    use crate::metrics::{metrics_headers, L1Stats, MetricsProvider};
877    use crate::session::session_headers;
878
879    const KEY: &str = "ck_live_prebuilt_secret";
880
881    fn put(provider: Option<MetricsProvider>) -> reqwest::Request {
882        let mut builder = CachekitIO::builder().api_key(KEY);
883        if let Some(provider) = provider {
884            builder = builder.metrics_provider(provider);
885        }
886        builder
887            .build()
888            .expect("builder succeeds")
889            .put_request("k", b"v".to_vec(), None)
890            .expect("key encodes")
891            .build()
892            .expect("request builds")
893    }
894
895    #[test]
896    fn bearer_is_sent_and_redacted_from_debug() {
897        let req = put(None);
898        let auth = req
899            .headers()
900            .get(reqwest::header::AUTHORIZATION)
901            .expect("authorization header");
902        assert_eq!(auth.as_bytes(), format!("Bearer {KEY}").as_bytes());
903        assert!(auth.is_sensitive(), "prebuilt bearer must be sensitive");
904        let debug = format!("{req:?}");
905        assert!(!debug.contains(KEY), "api key leaked into Debug: {debug}");
906    }
907
908    /// The bearer's zeroising owner is the header's buffer, not a copy of it:
909    /// clones share it, and it drops (wiping the key) only with the last one.
910    #[test]
911    fn header_clones_share_the_owner_until_the_last_drops() {
912        struct Owner(Vec<u8>, Arc<AtomicBool>);
913        impl AsRef<[u8]> for Owner {
914            fn as_ref(&self) -> &[u8] {
915                &self.0
916            }
917        }
918        impl Drop for Owner {
919            fn drop(&mut self) {
920                self.1.store(true, Ordering::SeqCst);
921            }
922        }
923
924        let dropped = Arc::new(AtomicBool::new(false));
925        let header = header_over(Owner(b"Bearer k".to_vec(), Arc::clone(&dropped))).expect("valid");
926        let clone = header.clone();
927        drop(header);
928        assert!(
929            !dropped.load(Ordering::SeqCst),
930            "owner dropped while a clone lives"
931        );
932        assert_eq!(clone.as_bytes(), b"Bearer k");
933        drop(clone);
934        assert!(
935            dropped.load(Ordering::SeqCst),
936            "owner outlived its last clone"
937        );
938    }
939
940    #[test]
941    fn a_key_no_header_can_carry_fails_build_without_echoing_it() {
942        let err = CachekitIO::builder()
943            .api_key("ck_bad\nkey")
944            .build()
945            .expect_err("newline in key");
946        assert!(!err.to_string().contains("ck_bad"), "{err}");
947    }
948
949    /// The prebuilt headers carry the same names and values the string
950    /// builders produce (header names compare case-insensitively).
951    #[test]
952    fn session_and_metrics_headers_match_their_string_builders() {
953        let provider: MetricsProvider = Arc::new(|| {
954            Some(L1Stats {
955                l1_hits: 3,
956                l2_hits: 2,
957                misses: 5,
958                l1_enabled: true,
959            })
960        });
961        let req = put(Some(provider.clone()));
962        let expected = session_headers()
963            .into_iter()
964            .map(|(n, v)| (n, v.to_string()))
965            .chain(metrics_headers(Some(&provider)));
966        let mut count = 0;
967        for (name, value) in expected {
968            let got = req.headers().get_all(name).iter().collect::<Vec<_>>();
969            assert_eq!(got.len(), 1, "{name} sent {} times", got.len());
970            assert_eq!(got[0].as_bytes(), value.as_bytes(), "{name}");
971            count += 1;
972        }
973        assert_eq!(count, 7);
974    }
975
976    #[test]
977    fn disabled_metrics_send_only_the_status_header() {
978        let req = put(None);
979        assert_eq!(
980            req.headers()
981                .get("X-CacheKit-L1-Status")
982                .map(|v| v.as_bytes()),
983            Some(&b"disabled"[..])
984        );
985        assert!(req.headers().get("X-CacheKit-L1-Hits").is_none());
986    }
987}
988
989// ── HTTP client settings tests ────────────────────────────────────────────────
990
991#[cfg(all(test, not(target_arch = "wasm32")))]
992#[allow(clippy::expect_used)] // test-only: a stub-server failure should panic loudly
993mod http_client_tests {
994    use std::io::{BufRead, BufReader, Write};
995    use std::net::TcpListener;
996    use std::sync::{Arc, Mutex};
997    use std::time::Duration;
998
999    use super::{http_client_builder, CachekitIO, READ_TIMEOUT, WRITE_TIMEOUT};
1000    use crate::backend::Backend;
1001    use crate::error::BackendErrorKind;
1002
1003    /// The production client minus the system proxy: an `HTTP_PROXY` or
1004    /// `ALL_PROXY` that does not exempt loopback would otherwise take the
1005    /// stub's requests.
1006    fn client() -> reqwest::Client {
1007        http_client_builder()
1008            .no_proxy()
1009            .build()
1010            .expect("client builds")
1011    }
1012
1013    /// A keep-alive HTTP/1.1 stub on loopback: answers every request `200`,
1014    /// records each request's header lines, and counts accepted connections.
1015    struct Stub {
1016        url: String,
1017        connections: Arc<Mutex<usize>>,
1018        requests: Arc<Mutex<Vec<Vec<String>>>>,
1019    }
1020
1021    const OK: &[u8] = b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n";
1022
1023    fn stub() -> Stub {
1024        stub_with(|_| Some(OK))
1025    }
1026
1027    /// [`stub`], but `answer(n)` gives the reply to the stub's `n`th request
1028    /// (from 0); `None` reads the request and never answers it.
1029    fn stub_with(answer: fn(usize) -> Option<&'static [u8]>) -> Stub {
1030        let listener = TcpListener::bind("127.0.0.1:0").expect("bind stub");
1031        let url = format!("http://{}/", listener.local_addr().expect("addr"));
1032        let connections = Arc::new(Mutex::new(0));
1033        let requests = Arc::new(Mutex::new(Vec::new()));
1034        let (conns, reqs) = (connections.clone(), requests.clone());
1035        std::thread::spawn(move || {
1036            for stream in listener.incoming() {
1037                let Ok(mut stream) = stream else { return };
1038                *conns.lock().expect("lock") += 1;
1039                let reqs = reqs.clone();
1040                std::thread::spawn(move || {
1041                    let mut reader = BufReader::new(stream.try_clone().expect("clone"));
1042                    loop {
1043                        let mut head = Vec::new();
1044                        loop {
1045                            let mut line = String::new();
1046                            if reader.read_line(&mut line).unwrap_or(0) == 0 {
1047                                return;
1048                            }
1049                            if line == "\r\n" {
1050                                break;
1051                            }
1052                            head.push(line.trim_end().to_owned());
1053                        }
1054                        let n = {
1055                            let mut reqs = reqs.lock().expect("lock");
1056                            reqs.push(head);
1057                            reqs.len() - 1
1058                        };
1059                        let Some(reply) = answer(n) else {
1060                            // Hold the connection open, unanswered, for good.
1061                            loop {
1062                                std::thread::park();
1063                            }
1064                        };
1065                        if stream.write_all(reply).is_err() {
1066                            return;
1067                        }
1068                    }
1069                });
1070            }
1071        });
1072        Stub {
1073            url,
1074            connections,
1075            requests,
1076        }
1077    }
1078
1079    /// Connections the stub accepted for two GETs `idle` apart, on the
1080    /// tokio clock reqwest's pool reads.
1081    async fn connections_after_idle(idle: Duration) -> usize {
1082        let stub = stub();
1083        let client = client();
1084        client.get(&stub.url).send().await.expect("first GET");
1085        tokio::time::pause();
1086        tokio::time::advance(idle).await;
1087        tokio::time::resume();
1088        client.get(&stub.url).send().await.expect("second GET");
1089        let n = *stub.connections.lock().expect("lock");
1090        n
1091    }
1092
1093    #[tokio::test]
1094    async fn sends_cachekit_rs_user_agent() {
1095        let stub = stub();
1096        client().get(&stub.url).send().await.expect("GET");
1097        let want = format!("user-agent: cachekit-rs/{}", env!("CARGO_PKG_VERSION"));
1098        let requests = stub.requests.lock().expect("lock");
1099        assert!(
1100            requests[0].iter().any(|h| h.eq_ignore_ascii_case(&want)),
1101            "no `{want}` in {:?}",
1102            requests[0]
1103        );
1104    }
1105
1106    #[tokio::test]
1107    async fn reuses_a_connection_idle_389_s() {
1108        assert_eq!(connections_after_idle(Duration::from_secs(389)).await, 1);
1109    }
1110
1111    #[tokio::test]
1112    async fn redials_a_connection_idle_past_390_s() {
1113        assert_eq!(connections_after_idle(Duration::from_secs(391)).await, 2);
1114    }
1115
1116    /// A production [`CachekitIO`] aimed at `stub` over plain HTTP: the
1117    /// builder only accepts `https`, so the test swaps the client and URL.
1118    fn backend_at(stub: &Stub) -> CachekitIO {
1119        let mut backend = CachekitIO::builder()
1120            .api_key("ck_test_key")
1121            .build()
1122            .expect("builder succeeds for the default host");
1123        backend.client = client();
1124        backend.api_url = stub.url.trim_end_matches('/').to_owned();
1125        backend
1126    }
1127
1128    /// How long `op` takes on the tokio clock, paused once the stub's first
1129    /// request has opened the pooled connection: an unanswered request then
1130    /// advances virtual time straight to whichever timer fires first.
1131    async fn paused_elapsed<T>(op: impl std::future::Future<Output = T>) -> (T, Duration) {
1132        tokio::time::pause();
1133        let start = tokio::time::Instant::now();
1134        let out = op.await;
1135        let elapsed = start.elapsed();
1136        tokio::time::resume();
1137        (out, elapsed)
1138    }
1139
1140    /// `elapsed` is `budget`, give or take the tokio timer's 1 ms resolution.
1141    fn assert_budget(elapsed: Duration, budget: Duration) {
1142        assert!(
1143            elapsed >= budget && elapsed <= budget + Duration::from_millis(2),
1144            "took {elapsed:?}, budget {budget:?}"
1145        );
1146    }
1147
1148    /// The stub's request count once it reaches `at_least`, or after 2 s.
1149    ///
1150    /// The stub records a request on its own thread; a stalled request may
1151    /// time out on the paused clock before that thread has read it.
1152    #[cfg(feature = "reliability")]
1153    fn recorded(stub: &Stub, at_least: usize) -> usize {
1154        let deadline = std::time::Instant::now() + Duration::from_secs(2);
1155        loop {
1156            let n = stub.requests.lock().expect("lock").len();
1157            if n >= at_least || std::time::Instant::now() >= deadline {
1158                return n;
1159            }
1160            std::thread::sleep(Duration::from_millis(5));
1161        }
1162    }
1163
1164    /// Answers the first request (to pool a connection), stalls every later one.
1165    fn first_only(n: usize) -> Option<&'static [u8]> {
1166        (n == 0).then_some(OK)
1167    }
1168
1169    #[tokio::test]
1170    async fn an_unanswered_read_times_out_after_5_s() {
1171        let stub = stub_with(first_only);
1172        let backend = backend_at(&stub);
1173        backend
1174            .exists("warm")
1175            .await
1176            .expect("first request pools a connection");
1177
1178        let (result, elapsed) = paused_elapsed(backend.get("k")).await;
1179
1180        let err = result.expect_err("an unanswered GET times out");
1181        assert_eq!(err.kind, BackendErrorKind::Timeout);
1182        assert_budget(elapsed, READ_TIMEOUT);
1183    }
1184
1185    #[tokio::test]
1186    async fn an_unanswered_write_times_out_after_10_s() {
1187        let stub = stub_with(first_only);
1188        let backend = backend_at(&stub);
1189        backend
1190            .exists("warm")
1191            .await
1192            .expect("first request pools a connection");
1193
1194        let (result, elapsed) = paused_elapsed(backend.set("k", b"v".to_vec(), None)).await;
1195
1196        let err = result.expect_err("an unanswered PUT times out");
1197        assert_eq!(err.kind, BackendErrorKind::Timeout);
1198        assert_budget(elapsed, WRITE_TIMEOUT);
1199    }
1200
1201    #[cfg(feature = "reliability")]
1202    mod through_the_reliability_stack {
1203        use super::*;
1204        use crate::client::SharedBackend;
1205        use crate::reliability::{
1206            wrap_reliable, ReliabilityConfig, RetryConfig, CACHEKITIO_DEADLINES,
1207        };
1208
1209        /// The default stack with jitter off, so virtual time is exact.
1210        fn reliable(backend: CachekitIO) -> SharedBackend {
1211            reliable_with_breaker(backend).0
1212        }
1213
1214        fn reliable_with_breaker(
1215            backend: CachekitIO,
1216        ) -> (
1217            SharedBackend,
1218            Option<Arc<crate::reliability::CircuitBreaker>>,
1219        ) {
1220            #[cfg(not(feature = "unsync"))]
1221            let shared: SharedBackend = Arc::new(backend);
1222            #[cfg(feature = "unsync")]
1223            let shared: SharedBackend = std::rc::Rc::new(backend);
1224            let config = ReliabilityConfig {
1225                retry: Some(RetryConfig {
1226                    jitter: false,
1227                    ..RetryConfig::default()
1228                }),
1229                ..ReliabilityConfig::default()
1230            };
1231            wrap_reliable(shared, config, Some(CACHEKITIO_DEADLINES))
1232        }
1233
1234        const QUOTA_DENY: &[u8] = b"HTTP/1.1 429 Too Many Requests\r\n\
1235            x-cachekit-deny-reason: quota\r\ncontent-length: 0\r\n\r\n";
1236        const RATE_LIMITED: &[u8] = b"HTTP/1.1 429 Too Many Requests\r\n\
1237            retry-after: 1\r\ncontent-length: 0\r\n\r\n";
1238
1239        /// Was 3 x 30 s + backoff = 90.3 s and 3 requests.
1240        #[tokio::test]
1241        async fn an_unanswered_get_costs_one_attempt_and_5_s() {
1242            let stub = stub_with(first_only);
1243            let backend = reliable(backend_at(&stub));
1244            backend
1245                .exists("warm")
1246                .await
1247                .expect("first request pools a connection");
1248
1249            let (result, elapsed) = paused_elapsed(backend.get("k")).await;
1250
1251            assert_eq!(result.expect_err("stalled").kind, BackendErrorKind::Timeout);
1252            assert_budget(elapsed, Duration::from_secs(5));
1253            assert_eq!(recorded(&stub, 2), 2, "warm-up + one GET");
1254        }
1255
1256        #[tokio::test]
1257        async fn a_quota_denied_exists_is_sent_once() {
1258            let stub = stub_with(|_| Some(QUOTA_DENY));
1259            let backend = reliable(backend_at(&stub));
1260
1261            let err = backend.exists("k").await.expect_err("denied");
1262
1263            assert_eq!(err.kind, BackendErrorKind::Transient);
1264            assert!(err.is_quota_denied());
1265            assert_eq!(recorded(&stub, 1), 1, "no in-call retry");
1266        }
1267
1268        #[tokio::test]
1269        async fn a_quota_deny_is_sent_once_and_stays_transient() {
1270            let stub = stub_with(|_| Some(QUOTA_DENY));
1271            let backend = reliable(backend_at(&stub));
1272
1273            let err = backend.get("k").await.expect_err("denied");
1274
1275            assert_eq!(
1276                err.kind,
1277                BackendErrorKind::Transient,
1278                "fails open, counts toward the breaker"
1279            );
1280            assert!(err.is_quota_denied());
1281            assert_eq!(
1282                stub.requests.lock().expect("lock").len(),
1283                1,
1284                "no in-call retry"
1285            );
1286        }
1287
1288        #[tokio::test]
1289        async fn quota_denies_still_open_the_breaker() {
1290            let stub = stub_with(|_| Some(QUOTA_DENY));
1291            let (backend, breaker) = reliable_with_breaker(backend_at(&stub));
1292
1293            for _ in 0..5 {
1294                backend.get("k").await.expect_err("denied");
1295            }
1296
1297            let breaker = breaker.expect("default stack has a breaker");
1298            assert_eq!(breaker.state(), crate::reliability::CircuitState::Open);
1299            assert_eq!(
1300                stub.requests.lock().expect("lock").len(),
1301                5,
1302                "one request per op"
1303            );
1304        }
1305
1306        #[tokio::test]
1307        async fn a_rate_limit_429_still_retries() {
1308            let stub = stub_with(|_| Some(RATE_LIMITED));
1309            let backend = reliable(backend_at(&stub));
1310
1311            let err = backend.get("k").await.expect_err("rate limited");
1312
1313            assert_eq!(err.kind, BackendErrorKind::Transient);
1314            assert!(!err.is_quota_denied());
1315            assert_eq!(
1316                stub.requests.lock().expect("lock").len(),
1317                3,
1318                "every attempt is sent"
1319            );
1320        }
1321    }
1322}