Skip to main content

cogrs/
range_reader.rs

1//! Range-based reader interface for COG files
2//!
3//! # Sync readers and async runtimes
4//!
5//! [`RangeReader`] is the *synchronous* interface: each read blocks the calling thread. It is
6//! what plain-thread users and custom readers implement, and what [`LocalRangeReader`] and
7//! [`MemoryRangeReader`] provide. Remote sources are natively asynchronous
8//! ([`AsyncRangeReader`](crate::AsyncRangeReader), see [`crate::remote`]); the sync remote
9//! readers here ([`HttpRangeReader`], [`S3RangeReaderSync`](crate::S3RangeReaderSync)) wrap
10//! them with [`AsyncToSync`], which runs the request on a private runtime and blocks the
11//! calling thread. That is safe from plain threads and `spawn_blocking` threads and never
12//! panics or deadlocks inside a runtime, but on an async worker thread it blocks that worker:
13//! from async code use [`CogReader::open_async`](crate::CogReader::open_async) and
14//! [`TileExtractor::extract`](crate::TileExtractor::extract), which never block a thread on
15//! the network.
16//!
17//! This module provides a unified interface for reading byte ranges from various sources
18//! (local files, S3, HTTP). This is essential for efficient COG reading since COGs are
19//! designed to be read via HTTP Range requests.
20
21use std::fs::File;
22use std::path::{Path, PathBuf};
23use std::sync::Arc;
24
25#[cfg(unix)]
26use std::os::unix::fs::FileExt;
27#[cfg(not(unix))]
28use std::io::{Read, Seek, SeekFrom};
29
30use crate::async_io::{block_on_io, AsyncToSync};
31use crate::remote::ObjectStoreRangeReader;
32use crate::tiff_utils::AnyResult;
33
34/// Trait for reading byte ranges from any source
35///
36/// This abstraction allows the same COG reading code to work with:
37/// - Local files (using seek + read)
38/// - S3 objects (using `GetObject` with Range header)
39/// - HTTP URLs (using Range header)
40pub trait RangeReader: Send + Sync {
41    /// Read a range of bytes from the source
42    ///
43    /// # Errors
44    /// Returns an error if the read operation fails due to I/O errors, network issues, or invalid ranges.
45    fn read_range(&self, offset: u64, length: usize) -> AnyResult<Vec<u8>>;
46
47    /// Get the total size of the source in bytes
48    fn size(&self) -> u64;
49
50    /// Get a human-readable identifier for this source (for logging/errors)
51    fn identifier(&self) -> &str;
52
53    /// Token that changes when the content changes (see
54    /// [`AsyncRangeReader::version`](crate::AsyncRangeReader::version)); `None` if unknown.
55    fn version(&self) -> Option<&str> {
56        None
57    }
58
59    /// Check if this is a local file (fast random access) or remote (expensive reads)
60    fn is_local(&self) -> bool {
61        let id = self.identifier();
62        !id.starts_with("http://") && !id.starts_with("https://") && !id.starts_with("s3://")
63    }
64
65    /// True when `read_range` never blocks on I/O (e.g. in-memory data), so async adapters
66    /// may call it directly instead of moving it to a blocking thread.
67    fn reads_inline(&self) -> bool {
68        false
69    }
70}
71
72/// Local file range reader
73pub struct LocalRangeReader {
74    path: PathBuf,
75    size: u64,
76    /// `"{size}:{modification time in ns}"` as of opening
77    version: String,
78    /// Kept open so each range read is a positioned read, not a new `open`.
79    #[cfg(unix)]
80    file: File,
81    #[cfg(not(unix))]
82    file: std::sync::Mutex<File>,
83}
84
85impl LocalRangeReader {
86    /// # Errors
87    /// Returns an error if the file does not exist or metadata cannot be read.
88    pub fn new(path: impl AsRef<Path>) -> AnyResult<Self> {
89        let path = path.as_ref().to_path_buf();
90        let metadata = std::fs::metadata(&path)?;
91        let modified_ns = metadata
92            .modified()
93            .ok()
94            .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
95            .map_or(0, |d| d.as_nanos());
96        let file = File::open(&path)?;
97        Ok(Self {
98            path,
99            size: metadata.len(),
100            version: format!("{}:{modified_ns}", metadata.len()),
101            #[cfg(unix)]
102            file,
103            #[cfg(not(unix))]
104            file: std::sync::Mutex::new(file),
105        })
106    }
107}
108
109/// In-memory range reader for bytes already loaded into memory
110///
111/// This is useful when you have already fetched COG data (e.g., from a cache
112/// or network) and want to parse it without writing to disk.
113///
114/// # Example
115///
116/// ```rust,no_run
117/// use cogrs::{CogReader, MemoryRangeReader};
118/// use std::sync::Arc;
119///
120/// fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
121///     let bytes = std::fs::read("path/to/file.tif")?;
122///     let reader = MemoryRangeReader::new(bytes, "cached://my-layer.tif".to_string());
123///     let cog = CogReader::from_reader(Arc::new(reader))?;
124///     Ok(())
125/// }
126/// ```
127pub struct MemoryRangeReader {
128    data: Arc<Vec<u8>>,
129    identifier: String,
130}
131
132impl MemoryRangeReader {
133    /// Create a new `MemoryRangeReader` from a byte vector
134    ///
135    /// # Arguments
136    /// * `data` - The COG file bytes
137    /// * `identifier` - A human-readable identifier for logging (e.g., "<memory://layer.tif>")
138    #[must_use] 
139    pub fn new(data: Vec<u8>, identifier: String) -> Self {
140        Self {
141            data: Arc::new(data),
142            identifier,
143        }
144    }
145
146    /// Create from an `Arc<Vec<u8>>` to avoid cloning large buffers
147    #[must_use] 
148    pub fn from_arc(data: Arc<Vec<u8>>, identifier: String) -> Self {
149        Self { data, identifier }
150    }
151}
152
153impl RangeReader for MemoryRangeReader {
154    fn read_range(&self, offset: u64, length: usize) -> AnyResult<Vec<u8>> {
155        // Cast is safe for in-memory data: usize is always sufficient for memory addresses
156        #[allow(clippy::cast_possible_truncation)]
157        let start = offset as usize;
158        let end = (start + length).min(self.data.len());
159        if start >= self.data.len() {
160            return Ok(vec![]);
161        }
162        Ok(self.data[start..end].to_vec())
163    }
164
165    fn size(&self) -> u64 {
166        self.data.len() as u64
167    }
168
169    fn identifier(&self) -> &str {
170        &self.identifier
171    }
172
173    fn is_local(&self) -> bool {
174        true // Memory is fast, treat as local
175    }
176
177    fn reads_inline(&self) -> bool {
178        true
179    }
180}
181
182impl RangeReader for LocalRangeReader {
183    fn read_range(&self, offset: u64, length: usize) -> AnyResult<Vec<u8>> {
184        let mut buffer = vec![0u8; length];
185        #[cfg(unix)]
186        {
187            self.file.read_exact_at(&mut buffer, offset)?;
188        }
189        #[cfg(not(unix))]
190        {
191            let mut file = self.file.lock().map_err(|e| format!("local file lock: {e}"))?;
192            file.seek(SeekFrom::Start(offset))?;
193            file.read_exact(&mut buffer)?;
194        }
195        Ok(buffer)
196    }
197
198    fn size(&self) -> u64 {
199        self.size
200    }
201
202    fn identifier(&self) -> &str {
203        self.path.to_str().unwrap_or("<invalid path>")
204    }
205
206    fn version(&self) -> Option<&str> {
207        Some(&self.version)
208    }
209}
210
211/// Blocking HTTP(S) range reader for remote COG files.
212///
213/// A synchronous front for [`ObjectStoreRangeReader`] (see the module docs for what that means
214/// on async threads). Opening costs one ranged `GET`; the first 16 KiB are kept in memory.
215pub struct HttpRangeReader {
216    inner: AsyncToSync,
217}
218
219impl HttpRangeReader {
220    /// Open `url` (`http://` or `https://`).
221    ///
222    /// # Errors
223    /// Returns an error if the URL is invalid, the server does not support range requests, or
224    /// the object cannot be read.
225    pub fn new(url: &str) -> AnyResult<Self> {
226        if !(url.starts_with("http://") || url.starts_with("https://")) {
227            return Err(format!("Expected an http:// or https:// URL, got: {url}").into());
228        }
229        let url = url.to_string();
230        let reader = block_on_io(async move { ObjectStoreRangeReader::open(&url).await })??;
231        Ok(Self { inner: AsyncToSync::new(Arc::new(reader)) })
232    }
233}
234
235impl RangeReader for HttpRangeReader {
236    fn read_range(&self, offset: u64, length: usize) -> AnyResult<Vec<u8>> {
237        self.inner.read_range(offset, length)
238    }
239
240    fn size(&self) -> u64 {
241        self.inner.size()
242    }
243
244    fn identifier(&self) -> &str {
245        self.inner.identifier()
246    }
247
248    fn version(&self) -> Option<&str> {
249        self.inner.version()
250    }
251
252    fn is_local(&self) -> bool {
253        false
254    }
255}
256
257/// Create a range reader from a path or URL
258///
259/// # Errors
260/// Returns an error if the source cannot be opened or is invalid.
261pub fn create_range_reader(source: &str) -> AnyResult<Arc<dyn RangeReader>> {
262    if source.starts_with("s3://") {
263        // Use the proper S3 reader that supports credentials and custom endpoints
264        Ok(Arc::new(crate::s3::S3RangeReaderSync::new(source)?))
265    } else if source.starts_with("http://") || source.starts_with("https://") {
266        Ok(Arc::new(HttpRangeReader::new(source)?))
267    } else {
268        Ok(Arc::new(LocalRangeReader::new(source)?))
269    }
270}
271
272#[cfg(test)]
273mod tests {
274    use super::*;
275    use std::io::Write;
276    use tempfile::NamedTempFile;
277
278    #[test]
279    fn test_local_range_reader() {
280        let mut file = NamedTempFile::new().unwrap();
281        file.write_all(b"Hello, World!").unwrap();
282
283        let reader = LocalRangeReader::new(file.path()).unwrap();
284        assert_eq!(reader.size(), 13);
285
286        let data = reader.read_range(0, 5).unwrap();
287        assert_eq!(&data, b"Hello");
288
289        let data = reader.read_range(7, 5).unwrap();
290        assert_eq!(&data, b"World");
291    }
292
293    #[test]
294    fn test_memory_range_reader() {
295        let data = b"Hello, World!".to_vec();
296        let reader = MemoryRangeReader::new(data, "test://memory".to_string());
297
298        assert_eq!(reader.size(), 13);
299        assert_eq!(reader.identifier(), "test://memory");
300        assert!(reader.is_local());
301
302        // Test reading ranges
303        let range1 = reader.read_range(0, 5).unwrap();
304        assert_eq!(&range1, b"Hello");
305
306        let range2 = reader.read_range(7, 5).unwrap();
307        assert_eq!(&range2, b"World");
308
309        // Test reading past end (should return partial data)
310        let range3 = reader.read_range(10, 10).unwrap();
311        assert_eq!(&range3, b"ld!");
312
313        // Test reading from beyond end (should return empty)
314        let range4 = reader.read_range(100, 10).unwrap();
315        assert!(range4.is_empty());
316    }
317
318    #[test]
319    fn test_memory_range_reader_from_arc() {
320        let data = Arc::new(b"Test data".to_vec());
321        let reader = MemoryRangeReader::from_arc(data.clone(), "arc://test".to_string());
322
323        assert_eq!(reader.size(), 9);
324        let result = reader.read_range(0, 4).unwrap();
325        assert_eq!(&result, b"Test");
326    }
327}
328
329#[cfg(test)]
330mod http_tests {
331    use super::*;
332    use crate::test_support::serve_bytes;
333    use std::time::Duration;
334
335    fn data() -> Vec<u8> {
336        (0..50_000usize).map(|i| (i % 253) as u8).collect()
337    }
338
339    #[test]
340    fn http_range_reader_from_a_plain_thread() {
341        let d = data();
342        let (base, log) = serve_bytes(d.clone(), Duration::ZERO);
343        let reader = HttpRangeReader::new(&format!("{base}/sync.bin")).unwrap();
344        assert_eq!((reader.size(), reader.is_local()), (50_000, false));
345        assert_eq!(reader.identifier(), format!("{base}/sync.bin"));
346        assert_eq!(reader.read_range(10, 100).unwrap(), d[10..110]);
347        assert_eq!(reader.read_range(30_000, 500).unwrap(), d[30_000..30_500]);
348        // open (prefix) + one request outside the prefix
349        assert_eq!(log.lock().len(), 2);
350    }
351
352    #[test]
353    fn http_range_reader_rejects_other_schemes() {
354        assert!(HttpRangeReader::new("s3://bucket/key").is_err());
355        assert!(HttpRangeReader::new("/tmp/file").is_err());
356    }
357
358    #[test]
359    fn create_range_reader_routes_http_sources() {
360        let (base, _log) = serve_bytes(data(), Duration::ZERO);
361        let reader = create_range_reader(&format!("{base}/x.bin")).unwrap();
362        assert!(!reader.is_local());
363        assert_eq!(reader.size(), 50_000);
364    }
365
366    /// The sync reader inside a single-threaded runtime blocks it but neither panics nor
367    /// deadlocks (the request runs on the private I/O runtime).
368    #[tokio::test(flavor = "current_thread")]
369    async fn http_range_reader_inside_a_current_thread_runtime() {
370        let d = data();
371        let (base, _log) = serve_bytes(d.clone(), Duration::from_millis(10));
372        let reader = HttpRangeReader::new(&format!("{base}/rt.bin")).unwrap();
373        assert_eq!(reader.read_range(40_000, 64).unwrap(), d[40_000..40_064]);
374    }
375}