Skip to main content

rete_core/
reader.rs

1//! Range-reading abstraction (SPEC.md §9).
2//!
3//! A client doesn't need the whole file: it reads the 1 KiB header, learns
4//! where each section lives, then fetches only those byte ranges. [`RangeReader`]
5//! is the seam — back it with a local file, an in-memory slice, or an HTTP
6//! `Range` client. [`CountingReader`] wraps any reader to measure how few bytes
7//! a given access pattern actually touches.
8
9use std::sync::atomic::{AtomicU64, Ordering};
10
11/// Something that can serve arbitrary byte ranges of a `.rete` resource.
12pub trait RangeReader {
13    /// Total resource length in bytes.
14    fn len(&self) -> u64;
15
16    /// True when the resource is empty.
17    fn is_empty(&self) -> bool {
18        self.len() == 0
19    }
20
21    /// Read `len` bytes starting at `offset`. Implementations should error on
22    /// an out-of-bounds range rather than truncating.
23    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>>;
24
25    /// Read several `(offset, len)` ranges, returning each range's bytes in
26    /// request order. These ranges are independent, so a reader whose backing
27    /// store is high-latency but parallelizable (an HTTP client) overrides this
28    /// to issue the reads concurrently — turning N round trips into ~N/P. The
29    /// default fetches them sequentially. Any range failing fails the batch.
30    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
31        ranges
32            .iter()
33            .map(|&(offset, len)| self.read_at(offset, len))
34            .collect()
35    }
36
37    /// How many ranges this reader can usefully have in flight at once — the
38    /// planner's hint for probe-vs-scan and batch-size decisions (a phone's
39    /// serial sync-XHR reader reports 1; the CLI's threaded HTTP client and the
40    /// browser's concurrent-fetch variants report their fan-out). Purely
41    /// advisory: correctness never depends on it. Defaults to 1 (sequential).
42    fn concurrency(&self) -> usize {
43        1
44    }
45
46    /// How many leading bytes this reader already holds in memory: a read that
47    /// lies wholly inside `[0, resident_prefix())` costs no round trip. An HTTP
48    /// reader that learnt the length from a ranged GET of the first 64 KiB
49    /// reports those 64 KiB. A caching layer above it serves such a read
50    /// straight through instead of fetching a block-aligned span that would
51    /// re-read what is already here (the open's first read used to cost a
52    /// second, larger request for block 0). Purely advisory; defaults to 0.
53    fn resident_prefix(&self) -> u64 {
54        0
55    }
56}
57
58/// Sharing a reader (e.g. keeping a counting handle while a lazily-faulting
59/// [`Rete`](crate::Rete) owns another) just delegates.
60impl<R: RangeReader + ?Sized> RangeReader for std::sync::Arc<R> {
61    fn len(&self) -> u64 {
62        (**self).len()
63    }
64
65    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
66        (**self).read_at(offset, len)
67    }
68
69    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
70        (**self).read_many(ranges)
71    }
72
73    fn concurrency(&self) -> usize {
74        (**self).concurrency()
75    }
76
77    fn resident_prefix(&self) -> u64 {
78        (**self).resident_prefix()
79    }
80}
81
82/// A [`RangeReader`] over an in-memory byte slice (tests, embedded files).
83pub struct SliceReader<'a> {
84    data: &'a [u8],
85}
86
87impl<'a> SliceReader<'a> {
88    pub fn new(data: &'a [u8]) -> Self {
89        Self { data }
90    }
91}
92
93impl RangeReader for SliceReader<'_> {
94    fn len(&self) -> u64 {
95        self.data.len() as u64
96    }
97
98    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
99        // `try_from`, not `as`: on wasm32 an offset of 2^32 + k would wrap to
100        // k and return the wrong bytes instead of an out-of-bounds error.
101        let range = usize::try_from(offset).ok().and_then(|start| {
102            let end = start.checked_add(usize::try_from(len).ok()?)?;
103            (end <= self.data.len()).then_some(start..end)
104        });
105        let range = range.ok_or_else(|| {
106            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "range out of bounds")
107        })?;
108        Ok(self.data[range].to_vec())
109    }
110}
111
112/// Wraps a reader and tallies how many ranges were requested and how many bytes
113/// were returned — the metric that matters for a range-streamed format.
114/// Atomically counted, so it stays `Sync` (a lazily-faulting remote index holds
115/// its reader behind a shared loader).
116pub struct CountingReader<R> {
117    inner: R,
118    requests: AtomicU64,
119    bytes: AtomicU64,
120}
121
122impl<R: RangeReader> CountingReader<R> {
123    pub fn new(inner: R) -> Self {
124        Self {
125            inner,
126            requests: AtomicU64::new(0),
127            bytes: AtomicU64::new(0),
128        }
129    }
130
131    /// Number of `read_at` calls made so far.
132    pub fn requests(&self) -> u64 {
133        self.requests.load(Ordering::Relaxed)
134    }
135
136    /// Total bytes returned so far.
137    pub fn bytes_read(&self) -> u64 {
138        self.bytes.load(Ordering::Relaxed)
139    }
140}
141
142impl<R: RangeReader> RangeReader for CountingReader<R> {
143    fn len(&self) -> u64 {
144        self.inner.len()
145    }
146
147    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
148        let out = self.inner.read_at(offset, len)?;
149        self.requests.fetch_add(1, Ordering::Relaxed);
150        self.bytes.fetch_add(out.len() as u64, Ordering::Relaxed);
151        Ok(out)
152    }
153
154    /// Delegate to the inner reader (preserving its parallelism) and tally each
155    /// returned range as one request — the count reflects the coalesced spans
156    /// actually fetched, however the inner reader issues them.
157    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
158        let out = self.inner.read_many(ranges)?;
159        self.requests.fetch_add(out.len() as u64, Ordering::Relaxed);
160        self.bytes
161            .fetch_add(out.iter().map(|b| b.len() as u64).sum(), Ordering::Relaxed);
162        Ok(out)
163    }
164
165    fn concurrency(&self) -> usize {
166        self.inner.concurrency()
167    }
168
169    fn resident_prefix(&self) -> u64 {
170        self.inner.resident_prefix()
171    }
172}
173
174/// Wraps a [`RangeReader`] so every access is shifted by `base` bytes: a `.rete`
175/// that starts at byte `base` of the backing resource — e.g. behind an HTML shell
176/// in a **polyglot** file that is both a web page and a graph — reads exactly as
177/// if it began at offset 0, lazily, touching only the `.rete` bytes a query needs
178/// and never the prefix. Pair with [`detect_polyglot_base`] to find `base`.
179pub struct OffsetReader<R> {
180    inner: R,
181    base: u64,
182}
183
184impl<R> OffsetReader<R> {
185    pub fn new(inner: R, base: u64) -> Self {
186        Self { inner, base }
187    }
188    pub fn base(&self) -> u64 {
189        self.base
190    }
191    pub fn into_inner(self) -> R {
192        self.inner
193    }
194}
195
196impl<R: RangeReader> RangeReader for OffsetReader<R> {
197    fn len(&self) -> u64 {
198        self.inner.len().saturating_sub(self.base)
199    }
200
201    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
202        self.inner.read_at(self.base + offset, len)
203    }
204
205    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
206        let shifted: Vec<(u64, u64)> = ranges.iter().map(|&(o, l)| (self.base + o, l)).collect();
207        self.inner.read_many(&shifted)
208    }
209
210    fn concurrency(&self) -> usize {
211        self.inner.concurrency()
212    }
213
214    fn resident_prefix(&self) -> u64 {
215        self.inner.resident_prefix().saturating_sub(self.base)
216    }
217}
218
219/// The marker a polyglot (HTML + `.rete`) file carries in its first bytes so a
220/// reader can locate the embedded `.rete` without knowing the file size: ASCII
221/// `RETE-BASE:` followed by [`POLYGLOT_DIGITS`] zero-padded decimal digits — the
222/// byte offset where the `.rete` begins. It is emitted inside an HTML comment so
223/// browsers ignore it, and it sits within the first header window a reader fetches.
224pub const POLYGLOT_MARKER: &[u8] = b"RETE-BASE:";
225/// Fixed decimal width of the offset that follows [`POLYGLOT_MARKER`].
226pub const POLYGLOT_DIGITS: usize = 16;
227
228/// If `head` (the first bytes of a resource whose byte 0 is NOT the `RETE` magic)
229/// carries a [`POLYGLOT_MARKER`], return the byte offset of the embedded `.rete`.
230pub fn detect_polyglot_base(head: &[u8]) -> Option<u64> {
231    let pos = head
232        .windows(POLYGLOT_MARKER.len())
233        .position(|w| w == POLYGLOT_MARKER)?;
234    let start = pos + POLYGLOT_MARKER.len();
235    let digits = head.get(start..start + POLYGLOT_DIGITS)?;
236    std::str::from_utf8(digits).ok()?.parse::<u64>().ok()
237}
238
239/// A [`RangeReader`] over owned, shared bytes (a whole file image held in
240/// memory — the resident opens of a `0x06` file go through the lazy machinery
241/// over one of these).
242pub struct BytesReader(pub std::sync::Arc<[u8]>);
243
244impl RangeReader for BytesReader {
245    fn len(&self) -> u64 {
246        self.0.len() as u64
247    }
248
249    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
250        let end = offset
251            .checked_add(len)
252            .filter(|&e| e <= self.0.len() as u64)
253            .ok_or_else(|| {
254                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "range out of bounds")
255            })?;
256        Ok(self.0[offset as usize..end as usize].to_vec())
257    }
258}
259
260/// A reader **seeded** with the file's leading bytes: any range inside them is
261/// served from memory, the rest goes to `inner`. The open reads the first
262/// [`crate::v06::FIRST_READ`] bytes once; a `0x05` open's header, card and
263/// first container prefixes then cost nothing more.
264pub struct PrefixReader<R> {
265    inner: R,
266    prefix: std::sync::Arc<[u8]>,
267}
268
269impl<R: RangeReader> PrefixReader<R> {
270    /// Seed `inner` with `prefix` (its bytes from offset 0).
271    pub fn new(inner: R, prefix: std::sync::Arc<[u8]>) -> Self {
272        PrefixReader { inner, prefix }
273    }
274
275    fn cut(&self, offset: u64, len: u64) -> Option<Vec<u8>> {
276        let end = offset.checked_add(len)?;
277        (end <= self.prefix.len() as u64)
278            .then(|| self.prefix[offset as usize..end as usize].to_vec())
279    }
280}
281
282impl<R: RangeReader> RangeReader for PrefixReader<R> {
283    fn len(&self) -> u64 {
284        self.inner.len()
285    }
286
287    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
288        match self.cut(offset, len) {
289            Some(b) => Ok(b),
290            None => self.inner.read_at(offset, len),
291        }
292    }
293
294    fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
295        let mut out: Vec<Option<Vec<u8>>> = ranges.iter().map(|&(o, l)| self.cut(o, l)).collect();
296        let rest: Vec<(u64, u64)> = ranges
297            .iter()
298            .zip(&out)
299            .filter(|(_, b)| b.is_none())
300            .map(|(r, _)| *r)
301            .collect();
302        if !rest.is_empty() {
303            let mut fetched = self.inner.read_many(&rest)?.into_iter();
304            for slot in out.iter_mut().filter(|b| b.is_none()) {
305                *slot = fetched.next();
306            }
307        }
308        out.into_iter()
309            .map(|b| b.ok_or_else(|| std::io::Error::other("read_many left a range unfetched")))
310            .collect()
311    }
312
313    fn concurrency(&self) -> usize {
314        self.inner.concurrency()
315    }
316
317    fn resident_prefix(&self) -> u64 {
318        (self.prefix.len() as u64).max(self.inner.resident_prefix())
319    }
320}
321
322#[cfg(test)]
323mod tests {
324    use super::*;
325
326    #[test]
327    fn slice_reader_serves_ranges_and_bounds_check() {
328        let data = (0u8..=255).collect::<Vec<_>>();
329        let r = SliceReader::new(&data);
330        assert_eq!(r.len(), 256);
331        assert_eq!(r.read_at(10, 4).unwrap(), vec![10, 11, 12, 13]);
332        assert!(r.read_at(254, 10).is_err()); // overruns
333    }
334
335    #[test]
336    fn counting_reader_tallies() {
337        let data = vec![0u8; 100];
338        let r = CountingReader::new(SliceReader::new(&data));
339        r.read_at(0, 10).unwrap();
340        r.read_at(50, 20).unwrap();
341        assert_eq!(r.requests(), 2);
342        assert_eq!(r.bytes_read(), 30);
343    }
344}