lance_encoding/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4use std::{ops::Range, sync::Arc};
5
6use bytes::Bytes;
7use futures::{FutureExt, TryFutureExt, future::BoxFuture};
8
9use lance_core::Result;
10
11mod array_encoding;
12pub mod buffer;
13pub mod compression;
14pub mod compression_config;
15pub mod constants;
16pub mod data;
17pub mod decoder;
18pub mod encoder;
19pub mod encodings;
20pub mod format;
21pub mod repdef;
22pub mod statistics;
23#[cfg(test)]
24pub mod testing;
25pub mod utils;
26
27// We can definitely add support for big-endian machines someday.  However, it's not a priority and
28// would involve extensive testing (probably through emulation) to ensure that the encodings are
29// correct.
30#[cfg(not(target_endian = "little"))]
31compile_error!("Lance encodings only support little-endian systems.");
32
33/// A trait for an I/O service
34///
35/// This represents the I/O API that the encoders and decoders need in order to operate.
36/// We specify this as a trait so that lance-encodings does not need to depend on lance-io
37///
38/// In general, it is assumed that this trait will be implemented by some kind of "file reader"
39/// or "file scheduler".  The encodings here are all limited to accessing a single file.
40pub trait EncodingsIo: std::fmt::Debug + Send + Sync {
41    /// Submit an I/O request
42    ///
43    /// The response must contain a `Bytes` object for each range requested even if the underlying
44    /// I/O was coalesced into fewer actual requests.
45    ///
46    /// # Arguments
47    ///
48    /// * `ranges` - the byte ranges to request
49    /// * `priority` - the priority of the request
50    ///
51    /// Priority should be set to the lowest row number that this request is delivering data for.
52    /// This is important in cases where indirect I/O causes high priority requests to be submitted
53    /// after low priority requests.  We want to fulfill the indirect I/O more quickly so that we
54    /// can decode as quickly as possible.
55    ///
56    /// The implementation should be able to handle empty ranges, and should return an empty
57    /// byte buffer for each empty range.
58    fn submit_request(
59        &self,
60        range: Vec<Range<u64>>,
61        priority: u64,
62    ) -> BoxFuture<'static, Result<Vec<Bytes>>>;
63
64    /// Submit an I/O request with a single range
65    ///
66    /// This is just a utitliy function that wraps [`EncodingsIo::submit_request`] for the common
67    /// case of a single range request.
68    fn submit_single(
69        &self,
70        range: std::ops::Range<u64>,
71        priority: u64,
72    ) -> BoxFuture<'static, lance_core::Result<bytes::Bytes>> {
73        self.submit_request(vec![range], priority)
74            .map_ok(|mut v| v.pop().unwrap())
75            .boxed()
76    }
77
78    /// Returns a version of this I/O service that bypasses backpressure for all requests.
79    ///
80    /// This is intended for indirect I/O (e.g. fetching items after decoding offsets) where
81    /// blocking on backpressure could cause deadlocks or excessive latency.
82    ///
83    /// Returns `None` if this implementation does not support bypass (e.g. in-memory or test
84    /// schedulers), in which case the caller should fall back to using self.
85    fn with_bypass_backpressure(&self) -> Option<Arc<dyn EncodingsIo>> {
86        None
87    }
88
89    /// Returns a version of this I/O service that additionally records the I/O it
90    /// performs into `stats`, on top of any global accounting.  This is the seam
91    /// used to measure exact per-scope (e.g. per-query) I/O without re-opening
92    /// files: wrap a reader's I/O service, perform the reads, then inspect the
93    /// recorder.
94    ///
95    /// Returns `None` if this implementation does not support per-scope I/O
96    /// statistics (e.g. in-memory or test schedulers), in which case the caller
97    /// should fall back to using self (and no statistics are recorded).
98    fn with_io_stats(
99        &self,
100        _stats: Arc<dyn lance_core::utils::io_stats::IoStatsRecorder>,
101    ) -> Option<Arc<dyn EncodingsIo>> {
102        None
103    }
104}
105
106/// An implementation of EncodingsIo that serves data from an in-memory buffer
107#[derive(Debug)]
108pub struct BufferScheduler {
109    data: Bytes,
110}
111
112impl BufferScheduler {
113    pub fn new(data: Bytes) -> Self {
114        Self { data }
115    }
116
117    fn satisfy_request(&self, req: Range<u64>) -> Bytes {
118        self.data.slice(req.start as usize..req.end as usize)
119    }
120}
121
122impl EncodingsIo for BufferScheduler {
123    fn submit_request(
124        &self,
125        ranges: Vec<Range<u64>>,
126        _priority: u64,
127    ) -> BoxFuture<'static, Result<Vec<Bytes>>> {
128        std::future::ready(Ok(ranges
129            .into_iter()
130            .map(|range| self.satisfy_request(range))
131            .collect::<Vec<_>>()))
132        .boxed()
133    }
134}
lance_encoding/lib.rs

lance_encoding/
lib.rs
