1use std::sync::atomic::{AtomicU64, Ordering};
10
11pub trait RangeReader {
13 fn len(&self) -> u64;
15
16 fn is_empty(&self) -> bool {
18 self.len() == 0
19 }
20
21 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>>;
24
25 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
31 ranges
32 .iter()
33 .map(|&(offset, len)| self.read_at(offset, len))
34 .collect()
35 }
36
37 fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
44 self.read_many(ranges)
45 }
46
47 fn concurrency(&self) -> usize {
53 1
54 }
55
56 fn resident_prefix(&self) -> u64 {
64 0
65 }
66}
67
68impl<R: RangeReader + ?Sized> RangeReader for std::sync::Arc<R> {
71 fn len(&self) -> u64 {
72 (**self).len()
73 }
74
75 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
76 (**self).read_at(offset, len)
77 }
78
79 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
80 (**self).read_many(ranges)
81 }
82
83 fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
84 (**self).read_many_scattered(ranges)
85 }
86
87 fn concurrency(&self) -> usize {
88 (**self).concurrency()
89 }
90
91 fn resident_prefix(&self) -> u64 {
92 (**self).resident_prefix()
93 }
94}
95
96pub struct SliceReader<'a> {
98 data: &'a [u8],
99}
100
101impl<'a> SliceReader<'a> {
102 pub fn new(data: &'a [u8]) -> Self {
103 Self { data }
104 }
105}
106
107impl RangeReader for SliceReader<'_> {
108 fn len(&self) -> u64 {
109 self.data.len() as u64
110 }
111
112 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
113 let range = usize::try_from(offset).ok().and_then(|start| {
116 let end = start.checked_add(usize::try_from(len).ok()?)?;
117 (end <= self.data.len()).then_some(start..end)
118 });
119 let range = range.ok_or_else(|| {
120 std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "range out of bounds")
121 })?;
122 Ok(self.data[range].to_vec())
123 }
124}
125
126pub struct CountingReader<R> {
131 inner: R,
132 requests: AtomicU64,
133 bytes: AtomicU64,
134}
135
136impl<R: RangeReader> CountingReader<R> {
137 pub fn new(inner: R) -> Self {
138 Self {
139 inner,
140 requests: AtomicU64::new(0),
141 bytes: AtomicU64::new(0),
142 }
143 }
144
145 pub fn requests(&self) -> u64 {
147 self.requests.load(Ordering::Relaxed)
148 }
149
150 pub fn bytes_read(&self) -> u64 {
152 self.bytes.load(Ordering::Relaxed)
153 }
154}
155
156impl<R: RangeReader> RangeReader for CountingReader<R> {
157 fn len(&self) -> u64 {
158 self.inner.len()
159 }
160
161 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
162 let out = self.inner.read_at(offset, len)?;
163 self.requests.fetch_add(1, Ordering::Relaxed);
164 self.bytes.fetch_add(out.len() as u64, Ordering::Relaxed);
165 Ok(out)
166 }
167
168 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
172 let out = self.inner.read_many(ranges)?;
173 self.requests.fetch_add(out.len() as u64, Ordering::Relaxed);
174 self.bytes
175 .fetch_add(out.iter().map(|b| b.len() as u64).sum(), Ordering::Relaxed);
176 Ok(out)
177 }
178
179 fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
180 let out = self.inner.read_many_scattered(ranges)?;
181 self.requests.fetch_add(out.len() as u64, Ordering::Relaxed);
182 self.bytes
183 .fetch_add(out.iter().map(|b| b.len() as u64).sum(), Ordering::Relaxed);
184 Ok(out)
185 }
186
187 fn concurrency(&self) -> usize {
188 self.inner.concurrency()
189 }
190
191 fn resident_prefix(&self) -> u64 {
192 self.inner.resident_prefix()
193 }
194}
195
196pub struct OffsetReader<R> {
202 inner: R,
203 base: u64,
204}
205
206impl<R> OffsetReader<R> {
207 pub fn new(inner: R, base: u64) -> Self {
208 Self { inner, base }
209 }
210 pub fn base(&self) -> u64 {
211 self.base
212 }
213 pub fn into_inner(self) -> R {
214 self.inner
215 }
216}
217
218impl<R: RangeReader> RangeReader for OffsetReader<R> {
219 fn len(&self) -> u64 {
220 self.inner.len().saturating_sub(self.base)
221 }
222
223 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
224 self.inner.read_at(self.base + offset, len)
225 }
226
227 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
228 let shifted: Vec<(u64, u64)> = ranges.iter().map(|&(o, l)| (self.base + o, l)).collect();
229 self.inner.read_many(&shifted)
230 }
231
232 fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
233 let shifted: Vec<(u64, u64)> = ranges.iter().map(|&(o, l)| (self.base + o, l)).collect();
234 self.inner.read_many_scattered(&shifted)
235 }
236
237 fn concurrency(&self) -> usize {
238 self.inner.concurrency()
239 }
240
241 fn resident_prefix(&self) -> u64 {
242 self.inner.resident_prefix().saturating_sub(self.base)
243 }
244}
245
246pub const POLYGLOT_MARKER: &[u8] = b"RETE-BASE:";
252pub const POLYGLOT_DIGITS: usize = 16;
254
255pub fn detect_polyglot_base(head: &[u8]) -> Option<u64> {
258 let pos = head
259 .windows(POLYGLOT_MARKER.len())
260 .position(|w| w == POLYGLOT_MARKER)?;
261 let start = pos + POLYGLOT_MARKER.len();
262 let digits = head.get(start..start + POLYGLOT_DIGITS)?;
263 std::str::from_utf8(digits).ok()?.parse::<u64>().ok()
264}
265
266pub struct BytesReader(pub std::sync::Arc<[u8]>);
270
271impl RangeReader for BytesReader {
272 fn len(&self) -> u64 {
273 self.0.len() as u64
274 }
275
276 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
277 let end = offset
278 .checked_add(len)
279 .filter(|&e| e <= self.0.len() as u64)
280 .ok_or_else(|| {
281 std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "range out of bounds")
282 })?;
283 Ok(self.0[offset as usize..end as usize].to_vec())
284 }
285}
286
287pub struct PrefixReader<R> {
292 inner: R,
293 prefix: std::sync::Arc<[u8]>,
294}
295
296impl<R: RangeReader> PrefixReader<R> {
297 pub fn new(inner: R, prefix: std::sync::Arc<[u8]>) -> Self {
299 PrefixReader { inner, prefix }
300 }
301
302 fn cut(&self, offset: u64, len: u64) -> Option<Vec<u8>> {
303 let end = offset.checked_add(len)?;
304 (end <= self.prefix.len() as u64)
305 .then(|| self.prefix[offset as usize..end as usize].to_vec())
306 }
307}
308
309impl<R: RangeReader> PrefixReader<R> {
310 fn serve(
312 &self,
313 ranges: &[(u64, u64)],
314 fetch: impl FnOnce(&[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>>,
315 ) -> std::io::Result<Vec<Vec<u8>>> {
316 let mut out: Vec<Option<Vec<u8>>> = ranges.iter().map(|&(o, l)| self.cut(o, l)).collect();
317 let rest: Vec<(u64, u64)> = ranges
318 .iter()
319 .zip(&out)
320 .filter(|(_, b)| b.is_none())
321 .map(|(r, _)| *r)
322 .collect();
323 if !rest.is_empty() {
324 let mut fetched = fetch(&rest)?.into_iter();
325 for slot in out.iter_mut().filter(|b| b.is_none()) {
326 *slot = fetched.next();
327 }
328 }
329 out.into_iter()
330 .map(|b| b.ok_or_else(|| std::io::Error::other("read_many left a range unfetched")))
331 .collect()
332 }
333}
334
335impl<R: RangeReader> RangeReader for PrefixReader<R> {
336 fn len(&self) -> u64 {
337 self.inner.len()
338 }
339
340 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
341 match self.cut(offset, len) {
342 Some(b) => Ok(b),
343 None => self.inner.read_at(offset, len),
344 }
345 }
346
347 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
348 self.serve(ranges, |rest| self.inner.read_many(rest))
349 }
350
351 fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
352 self.serve(ranges, |rest| self.inner.read_many_scattered(rest))
353 }
354
355 fn concurrency(&self) -> usize {
356 self.inner.concurrency()
357 }
358
359 fn resident_prefix(&self) -> u64 {
360 (self.prefix.len() as u64).max(self.inner.resident_prefix())
361 }
362}
363
364#[cfg(test)]
365mod tests {
366 use super::*;
367
368 #[test]
369 fn slice_reader_serves_ranges_and_bounds_check() {
370 let data = (0u8..=255).collect::<Vec<_>>();
371 let r = SliceReader::new(&data);
372 assert_eq!(r.len(), 256);
373 assert_eq!(r.read_at(10, 4).unwrap(), vec![10, 11, 12, 13]);
374 assert!(r.read_at(254, 10).is_err()); }
376
377 #[test]
378 fn counting_reader_tallies() {
379 let data = vec![0u8; 100];
380 let r = CountingReader::new(SliceReader::new(&data));
381 r.read_at(0, 10).unwrap();
382 r.read_at(50, 20).unwrap();
383 assert_eq!(r.requests(), 2);
384 assert_eq!(r.bytes_read(), 30);
385 }
386}