Skip to main content

cogrs/
header_cache.rs

1//! Process-wide cache of parsed COG headers for remote sources.
2//!
3//! A tile server opens the same image for every request. Opening costs a ranged `GET` for the
4//! header plus, for COGs with many tiles, extra `GET`s for the tile offset arrays, and the default
5//! overview-quality hint samples tiles on top of that. [`CogCache`] remembers what an open
6//! learned, so the next open of the same source sends no request at all.
7//!
8//! # What is cached
9//!
10//! One entry per source: the parsed [`CogMetadata`] and [`OverviewMetadata`] (shared `Arc`s),
11//! what the server reported about the object (size, ETag, last-modified), and the computed
12//! overview-quality result. Not cached: the object store client (shared per bucket or host by
13//! the store registry), the S3 region (cached per bucket), and decoded tiles (the byte-bounded
14//! tile cache, keyed by the same identity, see [`tile_cache`]).
15//!
16//! An entry weighs about 1.5 KiB plus 16 bytes per tile (offset and byte count, all levels): a
17//! 10980 x 10980 COG with 512 px tiles and three overviews is about 12 KB, a 100k x 100k one
18//! 0.8 MB (512 px tiles) to 3.3 MB (256 px tiles). The cache is bounded by the sum of the
19//! weights ([`CacheConfig::capacity_bytes`], 64 MiB by default, least recently used first).
20//!
21//! # Keeping it correct
22//!
23//! The lookup key is known before any I/O: the origin (S3 bucket, endpoint and credentials, or
24//! HTTP scheme, host and port), the object path and the [`IoOptions`]. The query string of an
25//! HTTP URL is not part of it, so presigned URLs of one object share an entry. What the cache
26//! stored is only valid for the version of the object it was learned from, and
27//! [`Validation`] decides how that is enforced:
28//!
29//! - [`Validation::IfMatch`] (default): every read of the object is conditional on its ETag
30//!   (`If-Match`, or `If-Unmodified-Since` without a strong ETag). A replaced object answers
31//!   `412`; the entry and the old version's decoded tiles are dropped and the operation runs
32//!   again once against a fresh open. Entries live for [`CacheConfig::ttl`]; entries whose
33//!   server gives no validator at all expire after [`CacheConfig::ttl_unvalidated`].
34//! - [`Validation::Ttl`]: no conditions; entries expire after [`CacheConfig::ttl`].
35//! - [`Validation::Immutable`]: no conditions and no expiry, for objects that are never
36//!   overwritten (date-versioned keys).
37//!
38//! [`CogCache::invalidate`] drops a source explicitly, [`CogCache::clear`] everything.
39//!
40//! Concurrent opens of the same source share one request ("single flight"), and a missing object
41//! (`404`) is remembered for [`CacheConfig::negative_ttl`].
42
43use std::collections::HashMap;
44use std::sync::atomic::{AtomicU64, Ordering};
45use std::sync::{Arc, LazyLock};
46use std::time::Duration;
47
48use futures::future::{BoxFuture, FutureExt, Shared, WeakShared};
49use lru::LruCache;
50use parking_lot::Mutex;
51use tokio::time::Instant;
52
53use crate::async_io::{AsyncRangeReader, IoOptions, is_source_changed};
54use crate::cog_reader::{CogMetadata, CogReader, OverviewMetadata, OverviewQualityHint, is_remote_source, parse_cog_structure};
55use crate::remote::{ObjectIdentity, ObjectStoreRangeReader, SourceKeyParts, SourceNotFound, Validation, source_key_parts};
56use crate::tile_cache::{self, TileCacheStats};
57use crate::tiff_utils::AnyResult;
58
59/// Fixed part of an entry's weight (structs, strings, allocator overhead), in bytes.
60const ENTRY_BASE_BYTES: usize = 1536;
61/// Most missing sources remembered at once.
62const NEGATIVE_CAPACITY: usize = 1024;
63
64/// Tuning of a [`CogCache`].
65#[derive(Debug, Clone, PartialEq, Eq)]
66pub struct CacheConfig {
67    /// Upper bound on the summed weight of the entries, in bytes (64 MiB).
68    pub capacity_bytes: usize,
69    /// How long an entry that reads can be validated against (see [`Validation`]) is trusted
70    /// (1 hour). Entries are also dropped as soon as a read finds the object changed.
71    pub ttl: Duration,
72    /// How long an entry is trusted when the server gave no usable validator, so a replaced
73    /// object cannot be detected on read (60 seconds). Only used with [`Validation::IfMatch`].
74    pub ttl_unvalidated: Duration,
75    /// How long a `404` is remembered (5 seconds; zero turns negative caching off). Only "not
76    /// found" is remembered, never `401`/`403`, server errors or timeouts.
77    pub negative_ttl: Duration,
78    /// How reads are tied to the cached version.
79    pub validation: Validation,
80}
81
82impl Default for CacheConfig {
83    fn default() -> Self {
84        Self {
85            capacity_bytes: 64 * 1024 * 1024,
86            ttl: Duration::from_secs(3600),
87            ttl_unvalidated: Duration::from_secs(60),
88            negative_ttl: Duration::from_secs(5),
89            validation: Validation::IfMatch,
90        }
91    }
92}
93
94/// Whether an open may use and fill the header cache.
95#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
96pub enum CacheMode {
97    /// Use a cached header, or open and cache it (default).
98    #[default]
99    Use,
100    /// Open the source from scratch and do not cache anything.
101    Bypass,
102    /// Drop what is cached for the source, open it from scratch and cache the result.
103    Refresh,
104}
105
106/// Point-in-time statistics of the header cache. Counters only grow; they are atomics and cost
107/// nothing on the hot path.
108#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
109pub struct HeaderCacheStats {
110    pub entries: usize,
111    /// Summed weight of the entries
112    pub bytes: usize,
113    pub capacity_bytes: usize,
114    /// Opens served from a cached entry (no request)
115    pub hits: u64,
116    /// Opens that found no usable entry and no open in flight (a request was made)
117    pub misses: u64,
118    /// Entries evicted to stay within the capacity
119    pub evictions: u64,
120    /// Entries dropped because their time to live ran out
121    pub expirations: u64,
122    /// Opens that waited for an identical open already in flight instead of requesting
123    pub coalesced_opens: u64,
124    /// Entries dropped because a read found the object changed (`412`)
125    pub stale_evictions: u64,
126    /// Opens answered from a remembered `404`
127    pub negative_hits: u64,
128}
129
130/// Statistics of both caches, for server metrics.
131#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
132pub struct CacheStats {
133    pub headers: HeaderCacheStats,
134    /// The process-wide decoded-tile cache, shared by every [`CogCache`].
135    pub tiles: TileCacheStats,
136}
137
138/// Lookup key: known before any I/O.
139#[derive(Clone, PartialEq, Eq, Hash)]
140struct HeaderKey {
141    origin: String,
142    path: String,
143    options: IoOptions,
144}
145
146/// What an open learned about one version of a source.
147struct Entry {
148    identifier: String,
149    metadata: Arc<CogMetadata>,
150    overviews: Arc<[OverviewMetadata]>,
151    identity: ObjectIdentity,
152    /// `identity`'s version token
153    version: String,
154    expires_at: Option<Instant>,
155    weight: usize,
156    quality: QualityCell,
157}
158
159impl Entry {
160    fn expired(&self, now: Instant) -> bool {
161        self.expires_at.is_some_and(|at| now >= at)
162    }
163}
164
165type QualityFuture = Shared<BoxFuture<'static, (Option<usize>, bool)>>;
166
167/// The overview-quality result of an entry, computed once: concurrent first requests share one
168/// computation, and an incomplete one (some sample tile failed to read) is used by the callers
169/// that waited for it but never stored.
170enum QualityState {
171    Empty,
172    Computing(WeakShared<BoxFuture<'static, (Option<usize>, bool)>>),
173    Done(Option<usize>),
174}
175
176struct QualityCell(Mutex<QualityState>);
177
178impl QualityCell {
179    async fn get(&self, reader: &CogReader) -> Option<usize> {
180        let shared: QualityFuture = {
181            let mut state = self.0.lock();
182            if let QualityState::Done(value) = &*state {
183                return *value;
184            }
185            let running = match &*state {
186                QualityState::Computing(weak) => weak.upgrade(),
187                _ => None,
188            };
189            if let Some(shared) = running {
190                shared
191            } else {
192                let sampler = reader.clone();
193                let shared = async move { sampler.analyze_overview_quality_async().await }.boxed().shared();
194                *state = QualityState::Computing(shared.downgrade().expect("not polled yet"));
195                shared
196            }
197        };
198        let (value, complete) = shared.await;
199        if complete {
200            *self.0.lock() = QualityState::Done(value);
201        }
202        value
203    }
204}
205
206/// An open that failed, shared by every caller that waited for it.
207struct OpenFailure {
208    message: String,
209    not_found: bool,
210}
211
212impl OpenFailure {
213    fn of(error: &(dyn std::error::Error + 'static)) -> Self {
214        Self { message: error.to_string(), not_found: error.is::<SourceNotFound>() }
215    }
216
217    fn error(&self) -> Box<dyn std::error::Error + Send + Sync> {
218        if self.not_found { Box::new(SourceNotFound::new(self.message.clone())) } else { self.message.clone().into() }
219    }
220}
221
222type OpenOutcome = Result<Arc<Entry>, Arc<OpenFailure>>;
223type OpenFuture = Shared<BoxFuture<'static, OpenOutcome>>;
224type InflightOpens = HashMap<HeaderKey, WeakShared<BoxFuture<'static, OpenOutcome>>>;
225
226struct NegativeEntry {
227    identifier: String,
228    message: String,
229    until: Instant,
230}
231
232struct Headers {
233    entries: LruCache<HeaderKey, Arc<Entry>>,
234    bytes: usize,
235    negatives: LruCache<HeaderKey, NegativeEntry>,
236}
237
238#[derive(Default)]
239struct Counters {
240    hits: AtomicU64,
241    misses: AtomicU64,
242    evictions: AtomicU64,
243    expirations: AtomicU64,
244    coalesced_opens: AtomicU64,
245    stale_evictions: AtomicU64,
246    negative_hits: AtomicU64,
247}
248
249struct Inner {
250    config: CacheConfig,
251    enabled: bool,
252    headers: Mutex<Headers>,
253    inflight: Mutex<InflightOpens>,
254    counters: Counters,
255}
256
257/// Where a [`CogReader`] came from, so a stale one can be reopened (see
258/// [`CogReader::reopen_after_change`](crate::CogReader)).
259pub(crate) struct ReaderOrigin {
260    pub cache: CogCache,
261    /// The source as the caller gave it (including any presigned query string)
262    pub source: String,
263    pub options: IoOptions,
264}
265
266/// Cache of parsed COG headers; cheap to clone (clones share one cache).
267///
268/// [`CogCache::global`] is the process-wide default every `CogReader::open*` call uses for remote
269/// sources. Create your own with [`CogCache::new`] and pass it to
270/// [`CogReader::builder`](crate::CogReader::builder), or use [`CogCache::disabled`] to turn
271/// caching off. Decoded tiles live in one process-wide cache regardless of the `CogCache` used
272/// (see [`tile_cache`]); [`invalidate`](Self::invalidate),
273/// [`clear`](Self::clear) and [`stats`](Self::stats) cover both.
274///
275/// The global cache is disabled by `COGRS_HEADER_CACHE=off` and sized by
276/// `COGRS_HEADER_CACHE_MB`.
277#[derive(Clone)]
278pub struct CogCache {
279    inner: Arc<Inner>,
280}
281
282static GLOBAL: LazyLock<CogCache> = LazyLock::new(|| {
283    let disabled = std::env::var("COGRS_HEADER_CACHE")
284        .is_ok_and(|v| matches!(v.trim().to_ascii_lowercase().as_str(), "off" | "0" | "false" | "disabled"));
285    if disabled {
286        return CogCache::disabled();
287    }
288    let mut config = CacheConfig::default();
289    if let Some(mb) = std::env::var("COGRS_HEADER_CACHE_MB").ok().and_then(|v| v.trim().parse::<usize>().ok()) {
290        config.capacity_bytes = mb.saturating_mul(1024 * 1024);
291    }
292    CogCache::new(config)
293});
294
295static DISABLED: LazyLock<CogCache> = LazyLock::new(|| CogCache::build(CacheConfig::default(), false));
296
297impl CogCache {
298    /// A cache with its own entries and `config`.
299    #[must_use]
300    pub fn new(config: CacheConfig) -> Self {
301        Self::build(config, true)
302    }
303
304    fn build(config: CacheConfig, enabled: bool) -> Self {
305        Self {
306            inner: Arc::new(Inner {
307                config,
308                enabled,
309                headers: Mutex::new(Headers {
310                    entries: LruCache::unbounded(),
311                    bytes: 0,
312                    negatives: LruCache::new(std::num::NonZeroUsize::new(NEGATIVE_CAPACITY).expect("non-zero")),
313                }),
314                inflight: Mutex::new(HashMap::new()),
315                counters: Counters::default(),
316            }),
317        }
318    }
319
320    /// The process-wide default cache.
321    #[must_use]
322    pub fn global() -> Self {
323        GLOBAL.clone()
324    }
325
326    /// A cache that caches nothing: opens through it always read the source (the decoded-tile
327    /// cache is unaffected). Reads are still conditional on the opened version.
328    #[must_use]
329    pub fn disabled() -> Self {
330        DISABLED.clone()
331    }
332
333    /// This cache's configuration.
334    #[must_use]
335    pub fn config(&self) -> &CacheConfig {
336        &self.inner.config
337    }
338
339    /// Statistics of the header cache and of the decoded-tile cache.
340    #[must_use]
341    pub fn stats(&self) -> CacheStats {
342        let (entries, bytes) = {
343            let headers = self.inner.headers.lock();
344            (headers.entries.len(), headers.bytes)
345        };
346        let c = &self.inner.counters;
347        CacheStats {
348            headers: HeaderCacheStats {
349                entries,
350                bytes,
351                capacity_bytes: self.inner.config.capacity_bytes,
352                hits: c.hits.load(Ordering::Relaxed),
353                misses: c.misses.load(Ordering::Relaxed),
354                evictions: c.evictions.load(Ordering::Relaxed),
355                expirations: c.expirations.load(Ordering::Relaxed),
356                coalesced_opens: c.coalesced_opens.load(Ordering::Relaxed),
357                stale_evictions: c.stale_evictions.load(Ordering::Relaxed),
358                negative_hits: c.negative_hits.load(Ordering::Relaxed),
359            },
360            tiles: tile_cache::snapshot(),
361        }
362    }
363
364    /// Forget everything cached for `source` (an `s3://` or `http(s)://` URL, or a path): its
365    /// header entries (every origin and version), a remembered `404`, and its decoded tiles.
366    /// The next open reads the source. Returns the number of header entries dropped.
367    pub fn invalidate(&self, source: &str) -> usize {
368        let identifier = source_key_parts(source).map_or_else(|_| source.to_string(), |parts| parts.identifier);
369        let removed = {
370            let mut headers = self.inner.headers.lock();
371            let keys: Vec<HeaderKey> =
372                headers.entries.iter().filter(|(_, e)| e.identifier == identifier).map(|(k, _)| k.clone()).collect();
373            for key in &keys {
374                headers.remove(key);
375            }
376            let negatives: Vec<HeaderKey> =
377                headers.negatives.iter().filter(|(_, n)| n.identifier == identifier).map(|(k, _)| k.clone()).collect();
378            for key in &negatives {
379                headers.negatives.pop(key);
380            }
381            keys.len()
382        };
383        tile_cache::invalidate_source(&identifier);
384        removed
385    }
386
387    /// Drop every header entry and remembered `404` of this cache, and the decoded tiles of the
388    /// sources they describe. Tiles of other sources stay; empty the whole tile cache with
389    /// [`tile_cache::clear`].
390    pub fn clear(&self) {
391        let identifiers: Vec<String> = {
392            let mut headers = self.inner.headers.lock();
393            let ids = headers
394                .entries
395                .iter()
396                .map(|(_, e)| e.identifier.clone())
397                .chain(headers.negatives.iter().map(|(_, n)| n.identifier.clone()))
398                .collect();
399            headers.entries.clear();
400            headers.negatives.clear();
401            headers.bytes = 0;
402            ids
403        };
404        for identifier in identifiers {
405            tile_cache::invalidate_source(&identifier);
406        }
407    }
408
409    /// Open `source` through this cache.
410    pub(crate) async fn open(
411        &self,
412        source: &str,
413        hint: OverviewQualityHint,
414        options: &IoOptions,
415        mode: CacheMode,
416    ) -> AnyResult<CogReader> {
417        let validation = self.inner.config.validation;
418        let origin = is_remote_source(source).then(|| {
419            Arc::new(ReaderOrigin { cache: self.clone(), source: source.to_string(), options: options.clone() })
420        });
421        if !self.inner.enabled || mode == CacheMode::Bypass || origin.is_none() {
422            return CogReader::open_uncached(source, hint, options, validation, origin).await;
423        }
424
425        let parts = source_key_parts(source)?;
426        let key = HeaderKey { origin: parts.origin.clone(), path: parts.path.clone(), options: options.clone() };
427        if mode == CacheMode::Refresh {
428            self.drop_source(&key, &parts.identifier);
429        }
430        let entry = self.entry(source, &key, &parts, options).await?;
431        self.reader_from(source, options, &entry, hint, origin).await
432    }
433
434    /// Reopen `source` after a read found it changed: drop the entry for `stale_version` (unless
435    /// someone already replaced it with a newer one), then open through the cache. Concurrent
436    /// callers share one fresh open.
437    pub(crate) async fn reopen_stale(
438        &self,
439        source: &str,
440        hint: OverviewQualityHint,
441        options: &IoOptions,
442        stale_version: Option<&str>,
443    ) -> AnyResult<CogReader> {
444        if let Ok(parts) = source_key_parts(source) {
445            let key = HeaderKey { origin: parts.origin, path: parts.path, options: options.clone() };
446            let mut headers = self.inner.headers.lock();
447            if headers.entries.peek(&key).is_some_and(|e| stale_version.is_none_or(|v| e.version == v)) {
448                headers.remove(&key);
449                self.inner.counters.stale_evictions.fetch_add(1, Ordering::Relaxed);
450            }
451        }
452        self.open(source, hint, options, CacheMode::Use).await
453    }
454
455    /// A cached entry, or the result of one open shared by every concurrent caller.
456    async fn entry(
457        &self,
458        source: &str,
459        key: &HeaderKey,
460        parts: &SourceKeyParts,
461        options: &IoOptions,
462    ) -> AnyResult<Arc<Entry>> {
463        let counters = &self.inner.counters;
464        if let Some(entry) = self.lookup(key) {
465            counters.hits.fetch_add(1, Ordering::Relaxed);
466            return Ok(entry);
467        }
468        if let Some(message) = self.remembered_not_found(key) {
469            counters.negative_hits.fetch_add(1, Ordering::Relaxed);
470            return Err(Box::new(SourceNotFound::new(message)));
471        }
472
473        let open: OpenFuture = {
474            let mut inflight = self.inner.inflight.lock();
475            if let Some(shared) = inflight.get(key).and_then(WeakShared::upgrade) {
476                counters.coalesced_opens.fetch_add(1, Ordering::Relaxed);
477                shared
478            } else {
479                counters.misses.fetch_add(1, Ordering::Relaxed);
480                let (cache, owned_key, identifier) = (self.clone(), key.clone(), parts.identifier.clone());
481                let (source, options) = (source.to_string(), options.clone());
482                let shared = async move {
483                    let result = cache.open_entry(&source, &owned_key, &identifier, &options).await;
484                    cache.inner.inflight.lock().remove(&owned_key);
485                    result
486                }
487                .boxed()
488                .shared();
489                inflight.insert(key.clone(), shared.downgrade().expect("not polled yet"));
490                shared
491            }
492        };
493        open.await.map_err(|failure| failure.error())
494    }
495
496    /// Read the source and cache what was learned. A conditional read that finds the object
497    /// replaced while the header is being parsed (a torn open) is tried once more.
498    async fn open_entry(
499        &self,
500        source: &str,
501        key: &HeaderKey,
502        identifier: &str,
503        options: &IoOptions,
504    ) -> Result<Arc<Entry>, Arc<OpenFailure>> {
505        let mut attempt = 0;
506        let result = loop {
507            attempt += 1;
508            match self.read_entry(source, key, identifier, options).await {
509                Err(e) if attempt == 1 && is_source_changed(&*e) => {}
510                other => break other,
511            }
512        };
513        match result {
514            Ok(entry) => {
515                let entry = Arc::new(entry);
516                self.insert(key, Arc::clone(&entry));
517                Ok(entry)
518            }
519            Err(e) => {
520                let failure = OpenFailure::of(&*e);
521                if failure.not_found {
522                    self.remember_not_found(key, identifier, &failure.message);
523                }
524                Err(Arc::new(failure))
525            }
526        }
527    }
528
529    async fn read_entry(
530        &self,
531        source: &str,
532        key: &HeaderKey,
533        identifier: &str,
534        options: &IoOptions,
535    ) -> AnyResult<Entry> {
536        let validation = self.inner.config.validation;
537        let reader = ObjectStoreRangeReader::open_with_validation(source, options, validation).await?;
538        let identity = reader.object_identity().clone();
539        let structure = parse_cog_structure(&reader).await?;
540        let (metadata, overviews) = (Arc::new(structure.metadata), Arc::<[OverviewMetadata]>::from(structure.overviews));
541
542        let config = &self.inner.config;
543        let ttl = match validation {
544            Validation::Immutable => None,
545            Validation::Ttl => Some(config.ttl),
546            Validation::IfMatch if identity.is_validatable(validation) => Some(config.ttl),
547            Validation::IfMatch => Some(config.ttl_unvalidated),
548        };
549        let tiles: usize = metadata.tile_offsets.len()
550            + metadata.tile_byte_counts.len()
551            + overviews.iter().map(|o| o.tile_offsets.len() + o.tile_byte_counts.len()).sum::<usize>();
552        let weight = ENTRY_BASE_BYTES
553            + tiles * std::mem::size_of::<u64>()
554            + overviews.len() * std::mem::size_of::<OverviewMetadata>()
555            + identifier.len()
556            + key.origin.len()
557            + key.path.len();
558        Ok(Entry {
559            identifier: identifier.to_string(),
560            metadata,
561            overviews,
562            version: identity.version(),
563            identity,
564            expires_at: ttl.map(|ttl| Instant::now() + ttl),
565            weight,
566            quality: QualityCell(Mutex::new(QualityState::Empty)),
567        })
568    }
569
570    /// A reader for `source` backed by `entry`: no request is sent.
571    async fn reader_from(
572        &self,
573        source: &str,
574        options: &IoOptions,
575        entry: &Arc<Entry>,
576        hint: OverviewQualityHint,
577        origin: Option<Arc<ReaderOrigin>>,
578    ) -> AnyResult<CogReader> {
579        let reader =
580            ObjectStoreRangeReader::open_known(source, options, self.inner.config.validation, &entry.identity).await?;
581        let io: Arc<dyn AsyncRangeReader> = Arc::new(reader);
582        let mut cog = CogReader::from_cached(io, Arc::clone(&entry.metadata), Arc::clone(&entry.overviews), hint, origin);
583        if matches!(hint, OverviewQualityHint::ComputeAtRuntime) {
584            cog.min_usable_overview = entry.quality.get(&cog).await;
585        }
586        Ok(cog)
587    }
588
589    /// A fresh entry for `key`, dropping it if its time to live ran out.
590    fn lookup(&self, key: &HeaderKey) -> Option<Arc<Entry>> {
591        let mut headers = self.inner.headers.lock();
592        let entry = Arc::clone(headers.entries.get(key)?);
593        if entry.expired(Instant::now()) {
594            headers.remove(key);
595            self.inner.counters.expirations.fetch_add(1, Ordering::Relaxed);
596            return None;
597        }
598        Some(entry)
599    }
600
601    fn insert(&self, key: &HeaderKey, entry: Arc<Entry>) {
602        let capacity = self.inner.config.capacity_bytes;
603        let mut headers = self.inner.headers.lock();
604        headers.negatives.pop(key);
605        headers.remove(key);
606        if entry.weight > capacity {
607            return;
608        }
609        while headers.bytes + entry.weight > capacity {
610            let Some((_, evicted)) = headers.entries.pop_lru() else { break };
611            headers.bytes = headers.bytes.saturating_sub(evicted.weight);
612            self.inner.counters.evictions.fetch_add(1, Ordering::Relaxed);
613        }
614        headers.bytes += entry.weight;
615        headers.entries.put(key.clone(), entry);
616    }
617
618    /// Remove the entry and any remembered `404` for `key`, and every decoded tile of the source.
619    fn drop_source(&self, key: &HeaderKey, identifier: &str) {
620        {
621            let mut headers = self.inner.headers.lock();
622            headers.remove(key);
623            headers.negatives.pop(key);
624        }
625        tile_cache::invalidate_source(identifier);
626    }
627
628    fn remember_not_found(&self, key: &HeaderKey, identifier: &str, message: &str) {
629        let ttl = self.inner.config.negative_ttl;
630        if ttl.is_zero() {
631            return;
632        }
633        let entry = NegativeEntry { identifier: identifier.to_string(), message: message.to_string(), until: Instant::now() + ttl };
634        self.inner.headers.lock().negatives.put(key.clone(), entry);
635    }
636
637    /// The message of a remembered `404` for `key` that has not expired.
638    fn remembered_not_found(&self, key: &HeaderKey) -> Option<String> {
639        let mut headers = self.inner.headers.lock();
640        match headers.negatives.get(key) {
641            Some(n) if Instant::now() < n.until => Some(n.message.clone()),
642            Some(_) => {
643                headers.negatives.pop(key);
644                None
645            }
646            None => None,
647        }
648    }
649}
650
651impl Headers {
652    fn remove(&mut self, key: &HeaderKey) {
653        if let Some(entry) = self.entries.pop(key) {
654            self.bytes = self.bytes.saturating_sub(entry.weight);
655        }
656    }
657}
658
659#[cfg(test)]
660mod tests {
661    use super::*;
662    use crate::cog_reader::OverviewQualityHint as Hint;
663    use crate::remote::SourceNotFound;
664    use crate::test_support::{CogSpec, ObjectServer, Sample, ServedObject, build_cog};
665
666    fn pattern_a(_: usize, x: usize, y: usize) -> f64 {
667        ((x * 31 + y * 17) % 4000) as f64 + 1.0
668    }
669
670    fn pattern_b(_: usize, x: usize, y: usize) -> f64 {
671        ((x * 7 + y * 13) % 3000) as f64 + 1001.0
672    }
673
674    /// Uncompressed 64 px tiles with two overviews: the tile data lies far beyond the 16 KiB
675    /// prefix, so tile reads are real requests.
676    fn spec(size: usize, pixel: fn(usize, usize, usize) -> f64) -> CogSpec {
677        CogSpec {
678            width: size,
679            height: size,
680            tile: 64,
681            bands: 1,
682            sample: Sample::U16,
683            deflate: false,
684            predictor: false,
685            epsg: 3857,
686            origin: (0.0, 6400.0),
687            pixel_size: (10.0, 10.0),
688            nodata: None,
689            overviews: 2,
690            sparse: vec![],
691            corrupt: vec![],
692            pixel,
693        }
694    }
695
696    fn object(size: usize, pixel: fn(usize, usize, usize) -> f64, etag: &str) -> ServedObject {
697        ServedObject::new(build_cog(&spec(size, pixel))).etag(Some(etag))
698    }
699
700    fn server(object: ServedObject) -> ObjectServer {
701        ObjectServer::start(Some(object), Duration::ZERO)
702    }
703
704    fn options() -> IoOptions {
705        IoOptions { max_retries: 0, ..IoOptions::default() }
706    }
707
708    fn cache_with(config: impl FnOnce(&mut CacheConfig)) -> CogCache {
709        let mut c = CacheConfig::default();
710        config(&mut c);
711        CogCache::new(c)
712    }
713
714    async fn open(cache: &CogCache, url: &str, hint: Hint) -> AnyResult<CogReader> {
715        CogReader::builder(url).cache(cache).hint(hint).io_options(options()).open_async().await
716    }
717
718    fn requests_for(server: &ObjectServer, path: &str) -> usize {
719        server.requests().iter().filter(|r| r.path == path).count()
720    }
721
722    #[tokio::test]
723    async fn the_second_open_sends_no_request() {
724        let server = server(object(512, pattern_a, "\"a1\""));
725        let url = format!("{}/a.tif", server.base());
726        let cache = cache_with(|_| {});
727
728        let first = open(&cache, &url, Hint::ComputeAtRuntime).await.unwrap();
729        let after_first = server.requests().len();
730        assert!(after_first > 1, "the open reads the header and samples tiles: {after_first}");
731        assert_eq!(first.min_usable_overview, Some(1));
732
733        let second = open(&cache, &url, Hint::ComputeAtRuntime).await.unwrap();
734        assert_eq!(server.requests().len(), after_first, "a cache hit must not touch the network");
735        assert!(Arc::ptr_eq(&first.metadata, &second.metadata));
736        assert!(Arc::ptr_eq(&first.overviews, &second.overviews));
737        assert_eq!(second.min_usable_overview, Some(1), "the overview quality is computed once per source");
738
739        let stats = cache.stats().headers;
740        assert_eq!((stats.entries, stats.hits, stats.misses), (1, 1, 1));
741        assert!(stats.bytes > 1536 && stats.bytes < 64 * 1024, "weight {}", stats.bytes);
742
743        // Explicit hints apply on top of the cached structure, still without a request.
744        assert_eq!(open(&cache, &url, Hint::AllUsable).await.unwrap().min_usable_overview, Some(1));
745        assert_eq!(open(&cache, &url, Hint::NoneUsable).await.unwrap().min_usable_overview, None);
746        assert_eq!(open(&cache, &url, Hint::MinUsable(0)).await.unwrap().min_usable_overview, Some(0));
747        assert_eq!(server.requests().len(), after_first);
748
749        // The reader works: a tile read is a conditional request for the cached version.
750        let tile = second.read_tile_async(0).await.unwrap();
751        assert!((tile[0] - pattern_a(0, 0, 0) as f32).abs() < 1e-3);
752        let last = server.requests().pop().unwrap();
753        assert_eq!(last.if_match.as_deref(), Some("\"a1\""));
754    }
755
756    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
757    async fn concurrent_cold_opens_share_one_request() {
758        let server = ObjectServer::start(Some(object(512, pattern_a, "\"a1\"")), Duration::from_millis(60));
759        let cache = cache_with(|_| {});
760        // What one sequential open costs
761        open(&cache, &format!("{}/reference.tif", server.base()), Hint::AllUsable).await.unwrap();
762        let single = requests_for(&server, "/reference.tif");
763
764        let url = format!("{}/shared.tif", server.base());
765        let opens = futures::future::join_all((0..32).map(|_| open(&cache, &url, Hint::AllUsable))).await;
766        assert!(opens.iter().all(Result::is_ok));
767        assert_eq!(requests_for(&server, "/shared.tif"), single, "32 concurrent opens, one open's requests");
768        let stats = cache.stats().headers;
769        assert_eq!(stats.misses, 2, "reference + shared");
770        assert_eq!(stats.coalesced_opens + stats.hits, 31);
771    }
772
773    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
774    async fn concurrent_cold_opens_compute_the_quality_once() {
775        let server = ObjectServer::start(Some(object(512, pattern_a, "\"a1\"")), Duration::from_millis(40));
776        let url = format!("{}/q.tif", server.base());
777        let cache = cache_with(|_| {});
778        // Header plus the sample tiles of one open, measured on its own
779        open(&cache, &format!("{}/ref.tif", server.base()), Hint::ComputeAtRuntime).await.unwrap();
780        let single = requests_for(&server, "/ref.tif");
781
782        let opens = futures::future::join_all((0..16).map(|_| open(&cache, &url, Hint::ComputeAtRuntime))).await;
783        assert!(opens.iter().all(|r| r.as_ref().unwrap().min_usable_overview == Some(1)));
784        assert_eq!(requests_for(&server, "/q.tif"), single);
785    }
786
787    #[tokio::test]
788    async fn evicts_least_recently_used_entries_by_bytes() {
789        let server = server(object(512, pattern_a, "\"a1\""));
790        let url = |name: &str| format!("{}/{name}.tif", server.base());
791        let probe = cache_with(|_| {});
792        open(&probe, &url("probe"), Hint::AllUsable).await.unwrap();
793        let weight = probe.stats().headers.bytes;
794
795        // Room for two entries, not three
796        let cache = cache_with(|c| c.capacity_bytes = 2 * weight + weight / 2);
797        for name in ["one", "two", "three"] {
798            open(&cache, &url(name), Hint::AllUsable).await.unwrap();
799        }
800        let stats = cache.stats().headers;
801        assert_eq!((stats.entries, stats.evictions), (2, 1));
802        assert!(stats.bytes <= stats.capacity_bytes);
803
804        let before = server.requests().len();
805        open(&cache, &url("three"), Hint::AllUsable).await.unwrap();
806        open(&cache, &url("two"), Hint::AllUsable).await.unwrap();
807        assert_eq!(server.requests().len(), before, "the two most recent are still cached");
808        open(&cache, &url("one"), Hint::AllUsable).await.unwrap();
809        assert!(server.requests().len() > before, "the least recently used entry was evicted");
810
811        // An entry larger than the whole cache is not kept
812        let tiny = cache_with(|c| c.capacity_bytes = weight / 2);
813        open(&tiny, &url("tiny"), Hint::AllUsable).await.unwrap();
814        assert_eq!(tiny.stats().headers.entries, 0);
815    }
816
817    // Virtual time: the cache's clock is tokio's, so `sleep` jumps the clock instead of waiting,
818    // and nothing here depends on how fast the machine is. (The I/O itself runs on the crate's
819    // own runtime and is not affected.)
820    #[tokio::test(start_paused = true)]
821    async fn entries_expire_after_their_time_to_live() {
822        let ttl = Duration::from_millis(60);
823        let wait = Duration::from_millis(150);
824
825        // A validated source uses `ttl`
826        let validated = server(object(512, pattern_a, "\"a1\""));
827        let url = format!("{}/v.tif", validated.base());
828        let cache = cache_with(|c| {
829            c.ttl = ttl;
830            c.ttl_unvalidated = Duration::from_secs(3600);
831        });
832        open(&cache, &url, Hint::AllUsable).await.unwrap();
833        tokio::time::sleep(wait).await;
834        let before = validated.requests().len();
835        open(&cache, &url, Hint::AllUsable).await.unwrap();
836        assert!(validated.requests().len() > before);
837        assert_eq!(cache.stats().headers.expirations, 1);
838
839        // A source without any validator uses `ttl_unvalidated`
840        let blind = server(ServedObject::new(build_cog(&spec(512, pattern_a))).etag(None).modified(None));
841        let url = format!("{}/b.tif", blind.base());
842        let cache = cache_with(|c| {
843            c.ttl = Duration::from_secs(3600);
844            c.ttl_unvalidated = ttl;
845        });
846        open(&cache, &url, Hint::AllUsable).await.unwrap();
847        tokio::time::sleep(wait).await;
848        let before = blind.requests().len();
849        open(&cache, &url, Hint::AllUsable).await.unwrap();
850        assert!(blind.requests().len() > before);
851
852        // Immutable sources never expire (and send no conditions)
853        let fixed = server(object(512, pattern_a, "\"a1\""));
854        let url = format!("{}/i.tif", fixed.base());
855        let cache = cache_with(|c| {
856            c.ttl = ttl;
857            c.ttl_unvalidated = ttl;
858            c.validation = Validation::Immutable;
859        });
860        open(&cache, &url, Hint::AllUsable).await.unwrap().read_tile_async(0).await.unwrap();
861        tokio::time::sleep(wait).await;
862        let before = fixed.requests().len();
863        let reader = open(&cache, &url, Hint::AllUsable).await.unwrap();
864        assert_eq!(fixed.requests().len(), before);
865        reader.read_tile_async(1).await.unwrap();
866        assert!(fixed.requests().iter().all(|r| r.if_match.is_none()));
867    }
868
869    #[tokio::test]
870    async fn reopening_a_replaced_object_evicts_the_stale_entry_once() {
871        let server = server(object(512, pattern_a, "\"v1\""));
872        let url = format!("{}/r.tif", server.base());
873        let cache = cache_with(|_| {});
874        let old = open(&cache, &url, Hint::AllUsable).await.unwrap();
875        assert_eq!(old.metadata.width, 512);
876
877        server.replace_default(Some(object(576, pattern_b, "\"v2\"")));
878        let err = old.io().read_range(30_000, 16).await.unwrap_err();
879        assert!(crate::async_io::is_source_changed(&*err), "{err}");
880        // The cache still believes in v1 until a read has shown otherwise
881        assert_eq!(open(&cache, &url, Hint::AllUsable).await.unwrap().metadata.width, 512);
882
883        let before = server.requests().len();
884        let fresh = futures::future::join_all((0..8).map(|_| old.reopen_after_change())).await;
885        let fresh: Vec<CogReader> = fresh.into_iter().map(Result::unwrap).collect();
886        assert!(fresh.iter().all(|r| r.metadata.width == 576));
887        assert!(fresh.iter().all(|r| Arc::ptr_eq(&r.metadata, &fresh[0].metadata)));
888        assert_eq!(server.requests().len() - before, 1, "8 concurrent reopens, one request");
889        let stats = cache.stats().headers;
890        assert_eq!((stats.stale_evictions, stats.entries), (1, 1));
891
892        let tile = fresh[0].read_tile_async(0).await.unwrap();
893        assert!((tile[0] - pattern_b(0, 0, 0) as f32).abs() < 1e-3);
894    }
895
896    #[tokio::test(start_paused = true)]
897    async fn a_missing_object_is_remembered_briefly() {
898        let server = ObjectServer::start(None, Duration::ZERO);
899        let url = format!("{}/missing.tif", server.base());
900        let cache = cache_with(|c| c.negative_ttl = Duration::from_millis(150));
901
902        let err = open(&cache, &url, Hint::AllUsable).await.err().unwrap();
903        assert!(err.downcast_ref::<SourceNotFound>().is_some(), "{err}");
904        assert!(err.to_string().contains("Failed to open"), "{err}");
905        assert_eq!(server.requests().len(), 1);
906
907        let again = open(&cache, &url, Hint::AllUsable).await.err().unwrap();
908        assert!(again.downcast_ref::<SourceNotFound>().is_some());
909        assert_eq!(again.to_string(), err.to_string());
910        assert_eq!(server.requests().len(), 1, "remembered: no request");
911        assert_eq!(cache.stats().headers.negative_hits, 1);
912
913        // After the TTL the object is asked for again, and once it exists it is cached
914        tokio::time::sleep(Duration::from_millis(250)).await;
915        server.replace_default(Some(object(512, pattern_a, "\"a1\"")));
916        open(&cache, &url, Hint::AllUsable).await.unwrap();
917        assert_eq!(server.requests().len(), 2);
918        open(&cache, &url, Hint::AllUsable).await.unwrap();
919        assert_eq!(server.requests().len(), 2);
920
921        // A zero TTL turns it off
922        let off = cache_with(|c| c.negative_ttl = Duration::ZERO);
923        let gone = ObjectServer::start(None, Duration::ZERO);
924        let url = format!("{}/gone.tif", gone.base());
925        for _ in 0..2 {
926            assert!(open(&off, &url, Hint::AllUsable).await.is_err());
927        }
928        assert_eq!(gone.requests().len(), 2);
929    }
930
931    #[tokio::test]
932    async fn an_incomplete_quality_analysis_is_not_remembered() {
933        let server = server(object(512, pattern_a, "\"a1\""));
934        let url = format!("{}/flaky.tif", server.base());
935        let cache = cache_with(|_| {});
936
937        // Tile data (beyond the header prefix) fails: the open still succeeds, as it always did,
938        // with the sparse-looking fallback for this reader only.
939        server.fail_from(Some(crate::remote::PREFIX_BYTES as usize));
940        let degraded = open(&cache, &url, Hint::ComputeAtRuntime).await.unwrap();
941        assert_eq!(degraded.min_usable_overview, None);
942
943        server.fail_from(None);
944        let before = server.requests().len();
945        let healthy = open(&cache, &url, Hint::ComputeAtRuntime).await.unwrap();
946        assert_eq!(healthy.min_usable_overview, Some(1));
947        assert!(server.requests().len() > before, "the analysis ran again");
948
949        let before = server.requests().len();
950        assert_eq!(open(&cache, &url, Hint::ComputeAtRuntime).await.unwrap().min_usable_overview, Some(1));
951        assert_eq!(server.requests().len(), before, "and its complete result is cached");
952    }
953
954    #[tokio::test]
955    async fn invalidate_clear_bypass_refresh_and_disabled() {
956        let server = server(object(512, pattern_a, "\"a1\""));
957        let url = format!("{}/m.tif", server.base());
958        let cache = cache_with(|_| {});
959
960        let reader = open(&cache, &url, Hint::AllUsable).await.unwrap();
961        reader.read_tile_async(0).await.unwrap();
962        assert_eq!(cache.stats().headers.entries, 1);
963        assert_eq!(cache.invalidate(&format!("{url}?ignored=1")), 1, "the query string is not part of the source");
964        assert_eq!(cache.stats().headers.entries, 0);
965        let n = server.requests().len();
966        open(&cache, &url, Hint::AllUsable).await.unwrap();
967        assert!(server.requests().len() > n, "invalidated: the next open reads the source");
968
969        // Bypass neither reads nor fills the cache
970        let n = server.requests().len();
971        let bypass = CogReader::builder(&url).cache(&cache).cache_mode(CacheMode::Bypass).io_options(options());
972        bypass.clone().hint(Hint::AllUsable).open_async().await.unwrap();
973        assert!(server.requests().len() > n);
974        assert_eq!(cache.stats().headers.hits, 0);
975
976        // Refresh reads the source and replaces the entry
977        let n = server.requests().len();
978        let refresh = CogReader::builder(&url).cache(&cache).cache_mode(CacheMode::Refresh).io_options(options());
979        refresh.hint(Hint::AllUsable).open_async().await.unwrap();
980        assert!(server.requests().len() > n);
981        assert_eq!(cache.stats().headers.entries, 1);
982
983        cache.clear();
984        assert_eq!(cache.stats().headers.entries, 0);
985        assert_eq!(cache.stats().headers.bytes, 0);
986
987        // A disabled cache caches nothing, and its readers still validate their reads
988        let off = CogCache::disabled();
989        for tile in [2, 3] {
990            let n = server.requests().len();
991            let reader = open(&off, &url, Hint::AllUsable).await.unwrap();
992            assert!(server.requests().len() > n);
993            reader.read_tile_async(tile).await.unwrap();
994        }
995        assert_eq!(off.stats().headers.entries, 0);
996        assert_eq!(server.requests().last().unwrap().if_match.as_deref(), Some("\"a1\""));
997    }
998
999    #[tokio::test]
1000    async fn presigned_urls_share_an_entry_and_each_read_uses_its_own_query() {
1001        let server = server(object(512, pattern_a, "\"a1\""));
1002        let url = |sig: &str| format!("{}/p.tif?sig={sig}", server.base());
1003        let cache = cache_with(|_| {});
1004
1005        open(&cache, &url("one"), Hint::AllUsable).await.unwrap();
1006        let before = server.requests().len();
1007        let second = open(&cache, &url("two"), Hint::AllUsable).await.unwrap();
1008        assert_eq!(server.requests().len(), before, "same object, same entry");
1009        second.read_tile_async(0).await.unwrap();
1010        let read = server.requests().pop().unwrap();
1011        assert_eq!(read.query.as_deref(), Some("sig=two"), "{read:?}");
1012    }
1013
1014    #[tokio::test]
1015    async fn local_files_are_not_cached() {
1016        let dir = tempfile::tempdir().unwrap();
1017        let path = dir.path().join("local.tif");
1018        std::fs::write(&path, build_cog(&spec(256, pattern_a))).unwrap();
1019        let cache = cache_with(|_| {});
1020        for _ in 0..2 {
1021            let reader = CogReader::builder(path.to_str().unwrap()).cache(&cache).hint(Hint::NoneUsable).open_async().await.unwrap();
1022            assert_eq!(reader.metadata.width, 256);
1023        }
1024        let stats = cache.stats().headers;
1025        assert_eq!((stats.entries, stats.hits, stats.misses), (0, 0, 0));
1026    }
1027
1028    // ---- Operations that find the object replaced run again once on a fresh open ----
1029
1030    use crate::cog_reader::TileRef;
1031    use crate::range_reader::MemoryRangeReader;
1032    use crate::xyz_tile::{BoundingBox, Reprojector, TileExtractor};
1033
1034    /// Two source windows over disjoint tiles of the 512 px raster (and of its 576 px successor).
1035    fn window_a() -> BoundingBox {
1036        BoundingBox::new(200.0, 4600.0, 1400.0, 5800.0)
1037    }
1038
1039    fn window_b() -> BoundingBox {
1040        BoundingBox::new(3000.0, 1500.0, 4500.0, 3000.0)
1041    }
1042
1043    async fn extract(reader: &CogReader, bounds: BoundingBox) -> AnyResult<Vec<f32>> {
1044        Ok(TileExtractor::new(reader).bounds(bounds).size(128).extract().await?.pixels)
1045    }
1046
1047    fn same(a: &[f32], b: &[f32]) -> bool {
1048        a.len() == b.len() && a.iter().zip(b).all(|(x, y)| x.to_bits() == y.to_bits() || (x.is_nan() && y.is_nan()))
1049    }
1050
1051    /// The object as an uncached, in-memory reader would read it: the reference for what an
1052    /// extraction must return.
1053    fn direct(bytes: &[u8], id: &str) -> CogReader {
1054        CogReader::from_reader_with_hint(Arc::new(MemoryRangeReader::new(bytes.to_vec(), id.to_string())), Hint::NoneUsable)
1055            .unwrap()
1056    }
1057
1058    fn unconditional_requests(server: &ObjectServer) -> usize {
1059        server.requests().iter().filter(|r| r.if_match.is_none() && r.if_unmodified_since.is_none()).count()
1060    }
1061
1062    fn cached_tiles(reader: &CogReader) -> usize {
1063        (0..reader.overviews.first().map_or(0, |_| 0) + reader.metadata.tile_offsets.len())
1064            .filter(|&i| reader.cached_tile(TileRef { overview: None, index: i }).is_some())
1065            .count()
1066    }
1067
1068    #[tokio::test]
1069    async fn extraction_after_an_overwrite_returns_exactly_the_new_objects_pixels() {
1070        let v1 = build_cog(&spec(512, pattern_a));
1071        let v2 = build_cog(&spec(576, pattern_b));
1072        let server = server(ServedObject::new(v1.clone()).etag(Some("\"v1\"")));
1073        let url = format!("{}/o.tif", server.base());
1074        let cache = cache_with(|_| {});
1075        let reader = open(&cache, &url, Hint::NoneUsable).await.unwrap();
1076
1077        let a_old = extract(&reader, window_a()).await.unwrap();
1078        assert!(same(&a_old, &extract(&direct(&v1, "mem://reval/v1"), window_a()).await.unwrap()));
1079        assert!(cached_tiles(&reader) > 0, "the first extraction filled the tile cache");
1080
1081        // The object is overwritten: a different size, different pixels, a new ETag.
1082        server.replace_default(Some(ServedObject::new(v2.clone()).etag(Some("\"v2\""))));
1083        let opens_before = unconditional_requests(&server);
1084
1085        // Window B needs tiles that are not cached: the conditional read fails (412) and the
1086        // extraction runs again on a fresh open of the new object.
1087        let b_new = extract(&reader, window_b()).await.unwrap();
1088        let truth = direct(&v2, "mem://reval/v2");
1089        let expected = extract(&truth, window_b()).await.unwrap();
1090        assert!(same(&b_new, &expected), "pixels must be exactly the new object's");
1091        let b_old = extract(&direct(&v1, "mem://reval/v1b"), window_b()).await.unwrap();
1092        assert!(!same(&b_new, &b_old), "the test must tell the two objects apart");
1093
1094        assert_eq!(unconditional_requests(&server) - opens_before, 1, "one fresh open");
1095        assert_eq!(cache.stats().headers.stale_evictions, 1);
1096        assert_eq!(cached_tiles(&reader), 0, "the old version's decoded tiles were dropped");
1097
1098        // The stale reader stays pinned to the old version and heals itself on each use: window A
1099        // was cached for the old version only, so it is now read (and checked) again.
1100        let a_new = extract(&reader, window_a()).await.unwrap();
1101        assert!(same(&a_new, &extract(&truth, window_a()).await.unwrap()));
1102        assert!(!same(&a_new, &a_old));
1103        let fresh = open(&cache, &url, Hint::NoneUsable).await.unwrap();
1104        assert_eq!(fresh.metadata.width, 576);
1105        assert!(same(&extract(&fresh, window_b()).await.unwrap(), &expected));
1106    }
1107
1108    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1109    async fn concurrent_extractions_after_an_overwrite_share_one_reopen() {
1110        let v2 = build_cog(&spec(576, pattern_b));
1111        let server = ObjectServer::start(Some(object(512, pattern_a, "\"v1\"")), Duration::from_millis(20));
1112        let url = format!("{}/c.tif", server.base());
1113        let cache = cache_with(|_| {});
1114        let reader = open(&cache, &url, Hint::NoneUsable).await.unwrap();
1115
1116        server.replace_default(Some(ServedObject::new(v2.clone()).etag(Some("\"v2\""))));
1117        let before = unconditional_requests(&server);
1118        let results = futures::future::join_all((0..32).map(|_| extract(&reader, window_b()))).await;
1119        let expected = extract(&direct(&v2, "mem://reval/concurrent"), window_b()).await.unwrap();
1120        for result in results {
1121            assert!(same(&result.unwrap(), &expected));
1122        }
1123        assert_eq!(unconditional_requests(&server) - before, 1, "32 operations, one reopen");
1124        assert_eq!(cache.stats().headers.stale_evictions, 1);
1125    }
1126
1127    #[tokio::test]
1128    async fn a_source_that_keeps_changing_fails_after_one_retry() {
1129        let server = server(object(512, pattern_a, "\"v1\""));
1130        let url = format!("{}/f.tif", server.base());
1131        let cache = cache_with(|_| {});
1132        let reader = open(&cache, &url, Hint::NoneUsable).await.unwrap();
1133
1134        // Every response carries a new ETag and every conditional read fails: a retry cannot help.
1135        server.set_flapping(true);
1136        let before = server.requests().len();
1137        let err = extract(&reader, window_b()).await.unwrap_err();
1138        assert!(crate::async_io::is_source_changed(&*err), "{err}");
1139        let requests = server.requests();
1140        let new = &requests[before..];
1141        assert_eq!(new.iter().filter(|r| r.if_match.is_none()).count(), 1, "exactly one reopen: {new:?}");
1142        assert!(new.len() < 40, "bounded work: {} requests", new.len());
1143        assert!(new.iter().filter(|r| r.if_match.is_some()).all(|r| r.status == 412));
1144    }
1145
1146    #[tokio::test]
1147    async fn point_queries_after_an_overwrite_read_the_new_object() {
1148        let v2 = build_cog(&spec(576, pattern_b));
1149        let server = server(object(512, pattern_a, "\"v1\""));
1150        let url = format!("{}/p.tif", server.base());
1151        let cache = cache_with(|_| {});
1152        let reader = open(&cache, &url, Hint::NoneUsable).await.unwrap();
1153        assert_eq!(reader.sample_async(0, 20, 20).await.unwrap(), Some(pattern_a(0, 20, 20) as f32));
1154
1155        server.replace_default(Some(ServedObject::new(v2).etag(Some("\"v2\""))));
1156        // Async: pixel (500, 500) lies in a tile that is not cached; the grids differ (8 v. 9 tiles across)
1157        assert_eq!(reader.sample_async(0, 500, 500).await.unwrap(), Some(pattern_b(0, 500, 500) as f32));
1158        // Sync, from a blocking thread
1159        let sync_reader = reader.clone();
1160        let value = tokio::task::spawn_blocking(move || sync_reader.sample(0, 100, 100)).await.unwrap().unwrap();
1161        assert_eq!(value, Some(pattern_b(0, 100, 100) as f32));
1162    }
1163
1164    #[tokio::test]
1165    async fn reprojection_after_an_overwrite_uses_the_new_geometry() {
1166        let v2 = build_cog(&spec(576, pattern_b));
1167        let server = server(object(512, pattern_a, "\"v1\""));
1168        let url = format!("{}/rp.tif", server.base());
1169        let cache = cache_with(|_| {});
1170        let reader = open(&cache, &url, Hint::NoneUsable).await.unwrap();
1171
1172        server.replace_default(Some(ServedObject::new(v2.clone()).etag(Some("\"v2\""))));
1173        // Bounds from the new object's extent (it grew to the south)
1174        let bounds = BoundingBox::new(3000.0, 700.0, 4500.0, 2200.0);
1175        let run = |r: CogReader| async move {
1176            Reprojector::new(&r).to_crs(3857).bounds(bounds).size(96, 96).extract().await.unwrap().pixels
1177        };
1178        let got = run(reader.clone()).await;
1179        let expected = run(direct(&v2, "mem://reval/reproject")).await;
1180        assert!(same(&got, &expected));
1181        assert!(got.iter().any(|v| !v.is_nan()), "the window must be inside the new, larger extent");
1182    }
1183}