1use std::sync::atomic::{AtomicU64, Ordering};
10
11pub trait RangeReader {
13 fn len(&self) -> u64;
15
16 fn is_empty(&self) -> bool {
18 self.len() == 0
19 }
20
21 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>>;
24
25 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
31 ranges
32 .iter()
33 .map(|&(offset, len)| self.read_at(offset, len))
34 .collect()
35 }
36
37 fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
44 self.read_many(ranges)
45 }
46
47 fn concurrency(&self) -> usize {
53 1
54 }
55
56 fn resident_prefix(&self) -> u64 {
64 0
65 }
66
67 fn is_local(&self) -> bool {
73 false
74 }
75}
76
77impl<R: RangeReader + ?Sized> RangeReader for std::sync::Arc<R> {
80 fn is_local(&self) -> bool {
81 (**self).is_local()
82 }
83
84 fn len(&self) -> u64 {
85 (**self).len()
86 }
87
88 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
89 (**self).read_at(offset, len)
90 }
91
92 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
93 (**self).read_many(ranges)
94 }
95
96 fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
97 (**self).read_many_scattered(ranges)
98 }
99
100 fn concurrency(&self) -> usize {
101 (**self).concurrency()
102 }
103
104 fn resident_prefix(&self) -> u64 {
105 (**self).resident_prefix()
106 }
107}
108
109pub struct SliceReader<'a> {
111 data: &'a [u8],
112}
113
114impl<'a> SliceReader<'a> {
115 pub fn new(data: &'a [u8]) -> Self {
116 Self { data }
117 }
118}
119
120impl RangeReader for SliceReader<'_> {
121 fn is_local(&self) -> bool {
122 true
123 }
124
125 fn len(&self) -> u64 {
126 self.data.len() as u64
127 }
128
129 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
130 let range = usize::try_from(offset).ok().and_then(|start| {
133 let end = start.checked_add(usize::try_from(len).ok()?)?;
134 (end <= self.data.len()).then_some(start..end)
135 });
136 let range = range.ok_or_else(|| {
137 std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "range out of bounds")
138 })?;
139 Ok(self.data[range].to_vec())
140 }
141}
142
143pub struct CountingReader<R> {
148 inner: R,
149 requests: AtomicU64,
150 bytes: AtomicU64,
151}
152
153impl<R: RangeReader> CountingReader<R> {
154 pub fn new(inner: R) -> Self {
155 Self {
156 inner,
157 requests: AtomicU64::new(0),
158 bytes: AtomicU64::new(0),
159 }
160 }
161
162 pub fn requests(&self) -> u64 {
164 self.requests.load(Ordering::Relaxed)
165 }
166
167 pub fn bytes_read(&self) -> u64 {
169 self.bytes.load(Ordering::Relaxed)
170 }
171}
172
173impl<R: RangeReader> RangeReader for CountingReader<R> {
174 fn is_local(&self) -> bool {
175 self.inner.is_local()
176 }
177
178 fn len(&self) -> u64 {
179 self.inner.len()
180 }
181
182 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
183 let out = self.inner.read_at(offset, len)?;
184 self.requests.fetch_add(1, Ordering::Relaxed);
185 self.bytes.fetch_add(out.len() as u64, Ordering::Relaxed);
186 Ok(out)
187 }
188
189 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
193 let out = self.inner.read_many(ranges)?;
194 self.requests.fetch_add(out.len() as u64, Ordering::Relaxed);
195 self.bytes
196 .fetch_add(out.iter().map(|b| b.len() as u64).sum(), Ordering::Relaxed);
197 Ok(out)
198 }
199
200 fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
201 let out = self.inner.read_many_scattered(ranges)?;
202 self.requests.fetch_add(out.len() as u64, Ordering::Relaxed);
203 self.bytes
204 .fetch_add(out.iter().map(|b| b.len() as u64).sum(), Ordering::Relaxed);
205 Ok(out)
206 }
207
208 fn concurrency(&self) -> usize {
209 self.inner.concurrency()
210 }
211
212 fn resident_prefix(&self) -> u64 {
213 self.inner.resident_prefix()
214 }
215}
216
217pub struct OffsetReader<R> {
223 inner: R,
224 base: u64,
225}
226
227impl<R> OffsetReader<R> {
228 pub fn new(inner: R, base: u64) -> Self {
229 Self { inner, base }
230 }
231 pub fn base(&self) -> u64 {
232 self.base
233 }
234 pub fn into_inner(self) -> R {
235 self.inner
236 }
237}
238
239impl<R: RangeReader> RangeReader for OffsetReader<R> {
240 fn is_local(&self) -> bool {
241 self.inner.is_local()
242 }
243
244 fn len(&self) -> u64 {
245 self.inner.len().saturating_sub(self.base)
246 }
247
248 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
249 self.inner.read_at(self.base + offset, len)
250 }
251
252 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
253 let shifted: Vec<(u64, u64)> = ranges.iter().map(|&(o, l)| (self.base + o, l)).collect();
254 self.inner.read_many(&shifted)
255 }
256
257 fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
258 let shifted: Vec<(u64, u64)> = ranges.iter().map(|&(o, l)| (self.base + o, l)).collect();
259 self.inner.read_many_scattered(&shifted)
260 }
261
262 fn concurrency(&self) -> usize {
263 self.inner.concurrency()
264 }
265
266 fn resident_prefix(&self) -> u64 {
267 self.inner.resident_prefix().saturating_sub(self.base)
268 }
269}
270
271pub const POLYGLOT_MARKER: &[u8] = b"RETE-BASE:";
277pub const POLYGLOT_DIGITS: usize = 16;
279
280pub fn detect_polyglot_base(head: &[u8]) -> Option<u64> {
283 let pos = head
284 .windows(POLYGLOT_MARKER.len())
285 .position(|w| w == POLYGLOT_MARKER)?;
286 let start = pos + POLYGLOT_MARKER.len();
287 let digits = head.get(start..start + POLYGLOT_DIGITS)?;
288 std::str::from_utf8(digits).ok()?.parse::<u64>().ok()
289}
290
291pub struct BytesReader(pub std::sync::Arc<[u8]>);
295
296impl RangeReader for BytesReader {
297 fn is_local(&self) -> bool {
298 true
299 }
300
301 fn len(&self) -> u64 {
302 self.0.len() as u64
303 }
304
305 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
306 let end = offset
307 .checked_add(len)
308 .filter(|&e| e <= self.0.len() as u64)
309 .ok_or_else(|| {
310 std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "range out of bounds")
311 })?;
312 Ok(self.0[offset as usize..end as usize].to_vec())
313 }
314}
315
316pub struct PrefixReader<R> {
321 inner: R,
322 prefix: std::sync::Arc<[u8]>,
323}
324
325impl<R: RangeReader> PrefixReader<R> {
326 pub fn new(inner: R, prefix: std::sync::Arc<[u8]>) -> Self {
328 PrefixReader { inner, prefix }
329 }
330
331 fn cut(&self, offset: u64, len: u64) -> Option<Vec<u8>> {
332 let end = offset.checked_add(len)?;
333 (end <= self.prefix.len() as u64)
334 .then(|| self.prefix[offset as usize..end as usize].to_vec())
335 }
336}
337
338impl<R: RangeReader> PrefixReader<R> {
339 fn serve(
341 &self,
342 ranges: &[(u64, u64)],
343 fetch: impl FnOnce(&[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>>,
344 ) -> std::io::Result<Vec<Vec<u8>>> {
345 let mut out: Vec<Option<Vec<u8>>> = ranges.iter().map(|&(o, l)| self.cut(o, l)).collect();
346 let rest: Vec<(u64, u64)> = ranges
347 .iter()
348 .zip(&out)
349 .filter(|(_, b)| b.is_none())
350 .map(|(r, _)| *r)
351 .collect();
352 if !rest.is_empty() {
353 let mut fetched = fetch(&rest)?.into_iter();
354 for slot in out.iter_mut().filter(|b| b.is_none()) {
355 *slot = fetched.next();
356 }
357 }
358 out.into_iter()
359 .map(|b| b.ok_or_else(|| std::io::Error::other("read_many left a range unfetched")))
360 .collect()
361 }
362}
363
364impl<R: RangeReader> RangeReader for PrefixReader<R> {
365 fn is_local(&self) -> bool {
366 self.inner.is_local()
367 }
368
369 fn len(&self) -> u64 {
370 self.inner.len()
371 }
372
373 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
374 match self.cut(offset, len) {
375 Some(b) => Ok(b),
376 None => self.inner.read_at(offset, len),
377 }
378 }
379
380 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
381 self.serve(ranges, |rest| self.inner.read_many(rest))
382 }
383
384 fn read_many_scattered(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
385 self.serve(ranges, |rest| self.inner.read_many_scattered(rest))
386 }
387
388 fn concurrency(&self) -> usize {
389 self.inner.concurrency()
390 }
391
392 fn resident_prefix(&self) -> u64 {
393 (self.prefix.len() as u64).max(self.inner.resident_prefix())
394 }
395}
396
397#[cfg(test)]
398mod tests {
399 use super::*;
400
401 #[test]
402 fn slice_reader_serves_ranges_and_bounds_check() {
403 let data = (0u8..=255).collect::<Vec<_>>();
404 let r = SliceReader::new(&data);
405 assert_eq!(r.len(), 256);
406 assert_eq!(r.read_at(10, 4).unwrap(), vec![10, 11, 12, 13]);
407 assert!(r.read_at(254, 10).is_err()); }
409
410 #[test]
411 fn counting_reader_tallies() {
412 let data = vec![0u8; 100];
413 let r = CountingReader::new(SliceReader::new(&data));
414 r.read_at(0, 10).unwrap();
415 r.read_at(50, 20).unwrap();
416 assert_eq!(r.requests(), 2);
417 assert_eq!(r.bytes_read(), 30);
418 }
419}