Skip to main content

rete_core/
reader.rs

1//! Range-reading abstraction (SPEC.md §9).
2//!
3//! A client doesn't need the whole file: it reads the 1 KiB header, learns
4//! where each section lives, then fetches only those byte ranges. [`RangeReader`]
5//! is the seam — back it with a local file, an in-memory slice, or an HTTP
6//! `Range` client. [`CountingReader`] wraps any reader to measure how few bytes
7//! a given access pattern actually touches.
8
9use std::sync::atomic::{AtomicU64, Ordering};
10
11/// Something that can serve arbitrary byte ranges of a `.rete` resource.
12pub trait RangeReader {
13    /// Total resource length in bytes.
14    fn len(&self) -> u64;
15
16    /// True when the resource is empty.
17    fn is_empty(&self) -> bool {
18        self.len() == 0
19    }
20
21    /// Read `len` bytes starting at `offset`. Implementations should error on
22    /// an out-of-bounds range rather than truncating.
23    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>>;
24
25    /// Read several `(offset, len)` ranges, returning each range's bytes in
26    /// request order. These ranges are independent, so a reader whose backing
27    /// store is high-latency but parallelizable (an HTTP client) overrides this
28    /// to issue the reads concurrently — turning N round trips into ~N/P. The
29    /// default fetches them sequentially. Any range failing fails the batch.
30    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
31        ranges
32            .iter()
33            .map(|&(offset, len)| self.read_at(offset, len))
34            .collect()
35    }
36
37    /// [`read_many`](Self::read_many) for ranges the caller knows lie far apart
38    /// (the tiles of a window of probes): a reader that rounds reads out to
39    /// blocks, or merges ranges across wide gaps, to save requests should fetch
40    /// these as they are, since what lies between them is not wanted. The
41    /// default is [`read_many`](Self::read_many); results are the same bytes
42    /// either way, only the requests differ.
43    fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
44        self.read_many(ranges)
45    }
46
47    /// How many ranges this reader can usefully have in flight at once — the
48    /// planner's hint for probe-vs-scan and batch-size decisions (a phone's
49    /// serial sync-XHR reader reports 1; the CLI's threaded HTTP client and the
50    /// browser's concurrent-fetch variants report their fan-out). Purely
51    /// advisory: correctness never depends on it. Defaults to 1 (sequential).
52    fn concurrency(&self) -> usize {
53        1
54    }
55
56    /// How many leading bytes this reader already holds in memory: a read that
57    /// lies wholly inside `[0, resident_prefix())` costs no round trip. An HTTP
58    /// reader that learnt the length from a ranged GET of the first 128 KiB
59    /// reports those 128 KiB. A caching layer above it serves such a read
60    /// straight through instead of fetching a block-aligned span that would
61    /// re-read what is already here (the open's first read used to cost a
62    /// second, larger request for block 0). Purely advisory; defaults to 0.
63    fn resident_prefix(&self) -> u64 {
64        0
65    }
66}
67
68/// Sharing a reader (e.g. keeping a counting handle while a lazily-faulting
69/// [`Rete`](crate::Rete) owns another) just delegates.
70impl<R: RangeReader + ?Sized> RangeReader for std::sync::Arc<R> {
71    fn len(&self) -> u64 {
72        (**self).len()
73    }
74
75    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
76        (**self).read_at(offset, len)
77    }
78
79    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
80        (**self).read_many(ranges)
81    }
82
83    fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
84        (**self).read_many_scattered(ranges)
85    }
86
87    fn concurrency(&self) -> usize {
88        (**self).concurrency()
89    }
90
91    fn resident_prefix(&self) -> u64 {
92        (**self).resident_prefix()
93    }
94}
95
96/// A [`RangeReader`] over an in-memory byte slice (tests, embedded files).
97pub struct SliceReader<'a> {
98    data: &'a [u8],
99}
100
101impl<'a> SliceReader<'a> {
102    pub fn new(data: &'a [u8]) -> Self {
103        Self { data }
104    }
105}
106
107impl RangeReader for SliceReader<'_> {
108    fn len(&self) -> u64 {
109        self.data.len() as u64
110    }
111
112    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
113        // `try_from`, not `as`: on wasm32 an offset of 2^32 + k would wrap to
114        // k and return the wrong bytes instead of an out-of-bounds error.
115        let range = usize::try_from(offset).ok().and_then(|start| {
116            let end = start.checked_add(usize::try_from(len).ok()?)?;
117            (end <= self.data.len()).then_some(start..end)
118        });
119        let range = range.ok_or_else(|| {
120            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "range out of bounds")
121        })?;
122        Ok(self.data[range].to_vec())
123    }
124}
125
126/// Wraps a reader and tallies how many ranges were requested and how many bytes
127/// were returned — the metric that matters for a range-streamed format.
128/// Atomically counted, so it stays `Sync` (a lazily-faulting remote index holds
129/// its reader behind a shared loader).
130pub struct CountingReader<R> {
131    inner: R,
132    requests: AtomicU64,
133    bytes: AtomicU64,
134}
135
136impl<R: RangeReader> CountingReader<R> {
137    pub fn new(inner: R) -> Self {
138        Self {
139            inner,
140            requests: AtomicU64::new(0),
141            bytes: AtomicU64::new(0),
142        }
143    }
144
145    /// Number of `read_at` calls made so far.
146    pub fn requests(&self) -> u64 {
147        self.requests.load(Ordering::Relaxed)
148    }
149
150    /// Total bytes returned so far.
151    pub fn bytes_read(&self) -> u64 {
152        self.bytes.load(Ordering::Relaxed)
153    }
154}
155
156impl<R: RangeReader> RangeReader for CountingReader<R> {
157    fn len(&self) -> u64 {
158        self.inner.len()
159    }
160
161    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
162        let out = self.inner.read_at(offset, len)?;
163        self.requests.fetch_add(1, Ordering::Relaxed);
164        self.bytes.fetch_add(out.len() as u64, Ordering::Relaxed);
165        Ok(out)
166    }
167
168    /// Delegate to the inner reader (preserving its parallelism) and tally each
169    /// returned range as one request — the count reflects the coalesced spans
170    /// actually fetched, however the inner reader issues them.
171    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
172        let out = self.inner.read_many(ranges)?;
173        self.requests.fetch_add(out.len() as u64, Ordering::Relaxed);
174        self.bytes
175            .fetch_add(out.iter().map(|b| b.len() as u64).sum(), Ordering::Relaxed);
176        Ok(out)
177    }
178
179    fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
180        let out = self.inner.read_many_scattered(ranges)?;
181        self.requests.fetch_add(out.len() as u64, Ordering::Relaxed);
182        self.bytes
183            .fetch_add(out.iter().map(|b| b.len() as u64).sum(), Ordering::Relaxed);
184        Ok(out)
185    }
186
187    fn concurrency(&self) -> usize {
188        self.inner.concurrency()
189    }
190
191    fn resident_prefix(&self) -> u64 {
192        self.inner.resident_prefix()
193    }
194}
195
196/// Wraps a [`RangeReader`] so every access is shifted by `base` bytes: a `.rete`
197/// that starts at byte `base` of the backing resource — e.g. behind an HTML shell
198/// in a **polyglot** file that is both a web page and a graph — reads exactly as
199/// if it began at offset 0, lazily, touching only the `.rete` bytes a query needs
200/// and never the prefix. Pair with [`detect_polyglot_base`] to find `base`.
201pub struct OffsetReader<R> {
202    inner: R,
203    base: u64,
204}
205
206impl<R> OffsetReader<R> {
207    pub fn new(inner: R, base: u64) -> Self {
208        Self { inner, base }
209    }
210    pub fn base(&self) -> u64 {
211        self.base
212    }
213    pub fn into_inner(self) -> R {
214        self.inner
215    }
216}
217
218impl<R: RangeReader> RangeReader for OffsetReader<R> {
219    fn len(&self) -> u64 {
220        self.inner.len().saturating_sub(self.base)
221    }
222
223    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
224        self.inner.read_at(self.base + offset, len)
225    }
226
227    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
228        let shifted: Vec<(u64, u64)> = ranges.iter().map(|&(o, l)| (self.base + o, l)).collect();
229        self.inner.read_many(&shifted)
230    }
231
232    fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
233        let shifted: Vec<(u64, u64)> = ranges.iter().map(|&(o, l)| (self.base + o, l)).collect();
234        self.inner.read_many_scattered(&shifted)
235    }
236
237    fn concurrency(&self) -> usize {
238        self.inner.concurrency()
239    }
240
241    fn resident_prefix(&self) -> u64 {
242        self.inner.resident_prefix().saturating_sub(self.base)
243    }
244}
245
246/// The marker a polyglot (HTML + `.rete`) file carries in its first bytes so a
247/// reader can locate the embedded `.rete` without knowing the file size: ASCII
248/// `RETE-BASE:` followed by [`POLYGLOT_DIGITS`] zero-padded decimal digits — the
249/// byte offset where the `.rete` begins. It is emitted inside an HTML comment so
250/// browsers ignore it, and it sits within the first header window a reader fetches.
251pub const POLYGLOT_MARKER: &[u8] = b"RETE-BASE:";
252/// Fixed decimal width of the offset that follows [`POLYGLOT_MARKER`].
253pub const POLYGLOT_DIGITS: usize = 16;
254
255/// If `head` (the first bytes of a resource whose byte 0 is NOT the `RETE` magic)
256/// carries a [`POLYGLOT_MARKER`], return the byte offset of the embedded `.rete`.
257pub fn detect_polyglot_base(head: &[u8]) -> Option<u64> {
258    let pos = head
259        .windows(POLYGLOT_MARKER.len())
260        .position(|w| w == POLYGLOT_MARKER)?;
261    let start = pos + POLYGLOT_MARKER.len();
262    let digits = head.get(start..start + POLYGLOT_DIGITS)?;
263    std::str::from_utf8(digits).ok()?.parse::<u64>().ok()
264}
265
266/// A [`RangeReader`] over owned, shared bytes (a whole file image held in
267/// memory — the resident opens of a `0x06` file go through the lazy machinery
268/// over one of these).
269pub struct BytesReader(pub std::sync::Arc<[u8]>);
270
271impl RangeReader for BytesReader {
272    fn len(&self) -> u64 {
273        self.0.len() as u64
274    }
275
276    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
277        let end = offset
278            .checked_add(len)
279            .filter(|&e| e <= self.0.len() as u64)
280            .ok_or_else(|| {
281                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "range out of bounds")
282            })?;
283        Ok(self.0[offset as usize..end as usize].to_vec())
284    }
285}
286
287/// A reader **seeded** with the file's leading bytes: any range inside them is
288/// served from memory, the rest goes to `inner`. The open reads the first
289/// [`crate::v06::FIRST_READ`] bytes once; a `0x05` open's header, card and
290/// first container prefixes then cost nothing more.
291pub struct PrefixReader<R> {
292    inner: R,
293    prefix: std::sync::Arc<[u8]>,
294}
295
296impl<R: RangeReader> PrefixReader<R> {
297    /// Seed `inner` with `prefix` (its bytes from offset 0).
298    pub fn new(inner: R, prefix: std::sync::Arc<[u8]>) -> Self {
299        PrefixReader { inner, prefix }
300    }
301
302    fn cut(&self, offset: u64, len: u64) -> Option<Vec<u8>> {
303        let end = offset.checked_add(len)?;
304        (end <= self.prefix.len() as u64)
305            .then(|| self.prefix[offset as usize..end as usize].to_vec())
306    }
307}
308
309impl<R: RangeReader> PrefixReader<R> {
310    /// Serve what the prefix holds, fetch the rest through `fetch`.
311    fn serve(
312        &self,
313        ranges: &[(u64, u64)],
314        fetch: impl FnOnce(&[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>>,
315    ) -> std::io::Result<Vec<Vec<u8>>> {
316        let mut out: Vec<Option<Vec<u8>>> = ranges.iter().map(|&(o, l)| self.cut(o, l)).collect();
317        let rest: Vec<(u64, u64)> = ranges
318            .iter()
319            .zip(&out)
320            .filter(|(_, b)| b.is_none())
321            .map(|(r, _)| *r)
322            .collect();
323        if !rest.is_empty() {
324            let mut fetched = fetch(&rest)?.into_iter();
325            for slot in out.iter_mut().filter(|b| b.is_none()) {
326                *slot = fetched.next();
327            }
328        }
329        out.into_iter()
330            .map(|b| b.ok_or_else(|| std::io::Error::other("read_many left a range unfetched")))
331            .collect()
332    }
333}
334
335impl<R: RangeReader> RangeReader for PrefixReader<R> {
336    fn len(&self) -> u64 {
337        self.inner.len()
338    }
339
340    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
341        match self.cut(offset, len) {
342            Some(b) => Ok(b),
343            None => self.inner.read_at(offset, len),
344        }
345    }
346
347    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
348        self.serve(ranges, |rest| self.inner.read_many(rest))
349    }
350
351    fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
352        self.serve(ranges, |rest| self.inner.read_many_scattered(rest))
353    }
354
355    fn concurrency(&self) -> usize {
356        self.inner.concurrency()
357    }
358
359    fn resident_prefix(&self) -> u64 {
360        (self.prefix.len() as u64).max(self.inner.resident_prefix())
361    }
362}
363
364#[cfg(test)]
365mod tests {
366    use super::*;
367
368    #[test]
369    fn slice_reader_serves_ranges_and_bounds_check() {
370        let data = (0u8..=255).collect::<Vec<_>>();
371        let r = SliceReader::new(&data);
372        assert_eq!(r.len(), 256);
373        assert_eq!(r.read_at(10, 4).unwrap(), vec![10, 11, 12, 13]);
374        assert!(r.read_at(254, 10).is_err()); // overruns
375    }
376
377    #[test]
378    fn counting_reader_tallies() {
379        let data = vec![0u8; 100];
380        let r = CountingReader::new(SliceReader::new(&data));
381        r.read_at(0, 10).unwrap();
382        r.read_at(50, 20).unwrap();
383        assert_eq!(r.requests(), 2);
384        assert_eq!(r.bytes_read(), 30);
385    }
386}