Skip to main content

rete_core/
reader.rs

1//! Range-reading abstraction (SPEC.md §9).
2//!
3//! A client doesn't need the whole file: it reads the 1 KiB header, learns
4//! where each section lives, then fetches only those byte ranges. [`RangeReader`]
5//! is the seam — back it with a local file, an in-memory slice, or an HTTP
6//! `Range` client. [`CountingReader`] wraps any reader to measure how few bytes
7//! a given access pattern actually touches.
8
9use std::sync::atomic::{AtomicU64, Ordering};
10
11/// Something that can serve arbitrary byte ranges of a `.rete` resource.
12pub trait RangeReader {
13    /// Total resource length in bytes.
14    fn len(&self) -> u64;
15
16    /// True when the resource is empty.
17    fn is_empty(&self) -> bool {
18        self.len() == 0
19    }
20
21    /// Read `len` bytes starting at `offset`. Implementations should error on
22    /// an out-of-bounds range rather than truncating.
23    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>>;
24
25    /// Read several `(offset, len)` ranges, returning each range's bytes in
26    /// request order. These ranges are independent, so a reader whose backing
27    /// store is high-latency but parallelizable (an HTTP client) overrides this
28    /// to issue the reads concurrently — turning N round trips into ~N/P. The
29    /// default fetches them sequentially. Any range failing fails the batch.
30    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
31        ranges
32            .iter()
33            .map(|&(offset, len)| self.read_at(offset, len))
34            .collect()
35    }
36
37    /// [`read_many`](Self::read_many) for ranges the caller knows lie far apart
38    /// (the tiles of a window of probes): a reader that rounds reads out to
39    /// blocks, or merges ranges across wide gaps, to save requests should fetch
40    /// these as they are, since what lies between them is not wanted. The
41    /// default is [`read_many`](Self::read_many); results are the same bytes
42    /// either way, only the requests differ.
43    fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
44        self.read_many(ranges)
45    }
46
47    /// How many ranges this reader can usefully have in flight at once — the
48    /// planner's hint for probe-vs-scan and batch-size decisions (a phone's
49    /// serial sync-XHR reader reports 1; the CLI's threaded HTTP client and the
50    /// browser's concurrent-fetch variants report their fan-out). Purely
51    /// advisory: correctness never depends on it. Defaults to 1 (sequential).
52    fn concurrency(&self) -> usize {
53        1
54    }
55
56    /// How many leading bytes this reader already holds in memory: a read that
57    /// lies wholly inside `[0, resident_prefix())` costs no round trip. An HTTP
58    /// reader that learnt the length from a ranged GET of the first 128 KiB
59    /// reports those 128 KiB. A caching layer above it serves such a read
60    /// straight through instead of fetching a block-aligned span that would
61    /// re-read what is already here (the open's first read used to cost a
62    /// second, larger request for block 0). Purely advisory; defaults to 0.
63    fn resident_prefix(&self) -> u64 {
64        0
65    }
66
67    /// Is a read cheap and local: bytes already in memory, or a local file?
68    /// False (the default) for anything where a read is a round trip. Purely
69    /// advisory: an engine may then read from several threads at once, each in
70    /// small pieces, where over a network it keeps one scan and batches its
71    /// reads. A wrapper reports what it wraps.
72    fn is_local(&self) -> bool {
73        false
74    }
75}
76
77/// Sharing a reader (e.g. keeping a counting handle while a lazily-faulting
78/// [`Rete`](crate::Rete) owns another) just delegates.
79impl<R: RangeReader + ?Sized> RangeReader for std::sync::Arc<R> {
80    fn is_local(&self) -> bool {
81        (**self).is_local()
82    }
83
84    fn len(&self) -> u64 {
85        (**self).len()
86    }
87
88    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
89        (**self).read_at(offset, len)
90    }
91
92    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
93        (**self).read_many(ranges)
94    }
95
96    fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
97        (**self).read_many_scattered(ranges)
98    }
99
100    fn concurrency(&self) -> usize {
101        (**self).concurrency()
102    }
103
104    fn resident_prefix(&self) -> u64 {
105        (**self).resident_prefix()
106    }
107}
108
109/// A [`RangeReader`] over an in-memory byte slice (tests, embedded files).
110pub struct SliceReader<'a> {
111    data: &'a [u8],
112}
113
114impl<'a> SliceReader<'a> {
115    pub fn new(data: &'a [u8]) -> Self {
116        Self { data }
117    }
118}
119
120impl RangeReader for SliceReader<'_> {
121    fn is_local(&self) -> bool {
122        true
123    }
124
125    fn len(&self) -> u64 {
126        self.data.len() as u64
127    }
128
129    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
130        // `try_from`, not `as`: on wasm32 an offset of 2^32 + k would wrap to
131        // k and return the wrong bytes instead of an out-of-bounds error.
132        let range = usize::try_from(offset).ok().and_then(|start| {
133            let end = start.checked_add(usize::try_from(len).ok()?)?;
134            (end <= self.data.len()).then_some(start..end)
135        });
136        let range = range.ok_or_else(|| {
137            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "range out of bounds")
138        })?;
139        Ok(self.data[range].to_vec())
140    }
141}
142
143/// Wraps a reader and tallies how many ranges were requested and how many bytes
144/// were returned — the metric that matters for a range-streamed format.
145/// Atomically counted, so it stays `Sync` (a lazily-faulting remote index holds
146/// its reader behind a shared loader).
147pub struct CountingReader<R> {
148    inner: R,
149    requests: AtomicU64,
150    bytes: AtomicU64,
151}
152
153impl<R: RangeReader> CountingReader<R> {
154    pub fn new(inner: R) -> Self {
155        Self {
156            inner,
157            requests: AtomicU64::new(0),
158            bytes: AtomicU64::new(0),
159        }
160    }
161
162    /// Number of `read_at` calls made so far.
163    pub fn requests(&self) -> u64 {
164        self.requests.load(Ordering::Relaxed)
165    }
166
167    /// Total bytes returned so far.
168    pub fn bytes_read(&self) -> u64 {
169        self.bytes.load(Ordering::Relaxed)
170    }
171}
172
173impl<R: RangeReader> RangeReader for CountingReader<R> {
174    fn is_local(&self) -> bool {
175        self.inner.is_local()
176    }
177
178    fn len(&self) -> u64 {
179        self.inner.len()
180    }
181
182    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
183        let out = self.inner.read_at(offset, len)?;
184        self.requests.fetch_add(1, Ordering::Relaxed);
185        self.bytes.fetch_add(out.len() as u64, Ordering::Relaxed);
186        Ok(out)
187    }
188
189    /// Delegate to the inner reader (preserving its parallelism) and tally each
190    /// returned range as one request — the count reflects the coalesced spans
191    /// actually fetched, however the inner reader issues them.
192    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
193        let out = self.inner.read_many(ranges)?;
194        self.requests.fetch_add(out.len() as u64, Ordering::Relaxed);
195        self.bytes
196            .fetch_add(out.iter().map(|b| b.len() as u64).sum(), Ordering::Relaxed);
197        Ok(out)
198    }
199
200    fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
201        let out = self.inner.read_many_scattered(ranges)?;
202        self.requests.fetch_add(out.len() as u64, Ordering::Relaxed);
203        self.bytes
204            .fetch_add(out.iter().map(|b| b.len() as u64).sum(), Ordering::Relaxed);
205        Ok(out)
206    }
207
208    fn concurrency(&self) -> usize {
209        self.inner.concurrency()
210    }
211
212    fn resident_prefix(&self) -> u64 {
213        self.inner.resident_prefix()
214    }
215}
216
217/// Wraps a [`RangeReader`] so every access is shifted by `base` bytes: a `.rete`
218/// that starts at byte `base` of the backing resource — e.g. behind an HTML shell
219/// in a **polyglot** file that is both a web page and a graph — reads exactly as
220/// if it began at offset 0, lazily, touching only the `.rete` bytes a query needs
221/// and never the prefix. Pair with [`detect_polyglot_base`] to find `base`.
222pub struct OffsetReader<R> {
223    inner: R,
224    base: u64,
225}
226
227impl<R> OffsetReader<R> {
228    pub fn new(inner: R, base: u64) -> Self {
229        Self { inner, base }
230    }
231    pub fn base(&self) -> u64 {
232        self.base
233    }
234    pub fn into_inner(self) -> R {
235        self.inner
236    }
237}
238
239impl<R: RangeReader> RangeReader for OffsetReader<R> {
240    fn is_local(&self) -> bool {
241        self.inner.is_local()
242    }
243
244    fn len(&self) -> u64 {
245        self.inner.len().saturating_sub(self.base)
246    }
247
248    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
249        self.inner.read_at(self.base + offset, len)
250    }
251
252    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
253        let shifted: Vec<(u64, u64)> = ranges.iter().map(|&(o, l)| (self.base + o, l)).collect();
254        self.inner.read_many(&shifted)
255    }
256
257    fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
258        let shifted: Vec<(u64, u64)> = ranges.iter().map(|&(o, l)| (self.base + o, l)).collect();
259        self.inner.read_many_scattered(&shifted)
260    }
261
262    fn concurrency(&self) -> usize {
263        self.inner.concurrency()
264    }
265
266    fn resident_prefix(&self) -> u64 {
267        self.inner.resident_prefix().saturating_sub(self.base)
268    }
269}
270
271/// The marker a polyglot (HTML + `.rete`) file carries in its first bytes so a
272/// reader can locate the embedded `.rete` without knowing the file size: ASCII
273/// `RETE-BASE:` followed by [`POLYGLOT_DIGITS`] zero-padded decimal digits — the
274/// byte offset where the `.rete` begins. It is emitted inside an HTML comment so
275/// browsers ignore it, and it sits within the first header window a reader fetches.
276pub const POLYGLOT_MARKER: &[u8] = b"RETE-BASE:";
277/// Fixed decimal width of the offset that follows [`POLYGLOT_MARKER`].
278pub const POLYGLOT_DIGITS: usize = 16;
279
280/// If `head` (the first bytes of a resource whose byte 0 is NOT the `RETE` magic)
281/// carries a [`POLYGLOT_MARKER`], return the byte offset of the embedded `.rete`.
282pub fn detect_polyglot_base(head: &[u8]) -> Option<u64> {
283    let pos = head
284        .windows(POLYGLOT_MARKER.len())
285        .position(|w| w == POLYGLOT_MARKER)?;
286    let start = pos + POLYGLOT_MARKER.len();
287    let digits = head.get(start..start + POLYGLOT_DIGITS)?;
288    std::str::from_utf8(digits).ok()?.parse::<u64>().ok()
289}
290
291/// A [`RangeReader`] over owned, shared bytes (a whole file image held in
292/// memory — the resident opens of a `0x06` file go through the lazy machinery
293/// over one of these).
294pub struct BytesReader(pub std::sync::Arc<[u8]>);
295
296impl RangeReader for BytesReader {
297    fn is_local(&self) -> bool {
298        true
299    }
300
301    fn len(&self) -> u64 {
302        self.0.len() as u64
303    }
304
305    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
306        let end = offset
307            .checked_add(len)
308            .filter(|&e| e <= self.0.len() as u64)
309            .ok_or_else(|| {
310                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "range out of bounds")
311            })?;
312        Ok(self.0[offset as usize..end as usize].to_vec())
313    }
314}
315
316/// A reader **seeded** with the file's leading bytes: any range inside them is
317/// served from memory, the rest goes to `inner`. The open reads the first
318/// [`crate::v06::FIRST_READ`] bytes once; a `0x05` open's header, card and
319/// first container prefixes then cost nothing more.
320pub struct PrefixReader<R> {
321    inner: R,
322    prefix: std::sync::Arc<[u8]>,
323}
324
325impl<R: RangeReader> PrefixReader<R> {
326    /// Seed `inner` with `prefix` (its bytes from offset 0).
327    pub fn new(inner: R, prefix: std::sync::Arc<[u8]>) -> Self {
328        PrefixReader { inner, prefix }
329    }
330
331    fn cut(&self, offset: u64, len: u64) -> Option<Vec<u8>> {
332        let end = offset.checked_add(len)?;
333        (end <= self.prefix.len() as u64)
334            .then(|| self.prefix[offset as usize..end as usize].to_vec())
335    }
336}
337
338impl<R: RangeReader> PrefixReader<R> {
339    /// Serve what the prefix holds, fetch the rest through `fetch`.
340    fn serve(
341        &self,
342        ranges: &[(u64, u64)],
343        fetch: impl FnOnce(&[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>>,
344    ) -> std::io::Result<Vec<Vec<u8>>> {
345        let mut out: Vec<Option<Vec<u8>>> = ranges.iter().map(|&(o, l)| self.cut(o, l)).collect();
346        let rest: Vec<(u64, u64)> = ranges
347            .iter()
348            .zip(&out)
349            .filter(|(_, b)| b.is_none())
350            .map(|(r, _)| *r)
351            .collect();
352        if !rest.is_empty() {
353            let mut fetched = fetch(&rest)?.into_iter();
354            for slot in out.iter_mut().filter(|b| b.is_none()) {
355                *slot = fetched.next();
356            }
357        }
358        out.into_iter()
359            .map(|b| b.ok_or_else(|| std::io::Error::other("read_many left a range unfetched")))
360            .collect()
361    }
362}
363
364impl<R: RangeReader> RangeReader for PrefixReader<R> {
365    fn is_local(&self) -> bool {
366        self.inner.is_local()
367    }
368
369    fn len(&self) -> u64 {
370        self.inner.len()
371    }
372
373    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
374        match self.cut(offset, len) {
375            Some(b) => Ok(b),
376            None => self.inner.read_at(offset, len),
377        }
378    }
379
380    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
381        self.serve(ranges, |rest| self.inner.read_many(rest))
382    }
383
384    fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
385        self.serve(ranges, |rest| self.inner.read_many_scattered(rest))
386    }
387
388    fn concurrency(&self) -> usize {
389        self.inner.concurrency()
390    }
391
392    fn resident_prefix(&self) -> u64 {
393        (self.prefix.len() as u64).max(self.inner.resident_prefix())
394    }
395}
396
397#[cfg(test)]
398mod tests {
399    use super::*;
400
401    #[test]
402    fn slice_reader_serves_ranges_and_bounds_check() {
403        let data = (0u8..=255).collect::<Vec<_>>();
404        let r = SliceReader::new(&data);
405        assert_eq!(r.len(), 256);
406        assert_eq!(r.read_at(10, 4).unwrap(), vec![10, 11, 12, 13]);
407        assert!(r.read_at(254, 10).is_err()); // overruns
408    }
409
410    #[test]
411    fn counting_reader_tallies() {
412        let data = vec![0u8; 100];
413        let r = CountingReader::new(SliceReader::new(&data));
414        r.read_at(0, 10).unwrap();
415        r.read_at(50, 20).unwrap();
416        assert_eq!(r.requests(), 2);
417        assert_eq!(r.bytes_read(), 30);
418    }
419}