genegraph_storage/lancefmt/pb/
lance.encodings.rs

1// Vendored from the Lance v2.1 spec (lance crate v11.0.0), generated with
2// prost 0.14 from the .proto files in src/lancefmt/protos/ (Apache-2.0,
3// Copyright The Lance Authors). Regenerate rather than hand-editing; see
4// src/lancefmt/protos/README.md.
5// This file is @generated by prost-build.
6/// A pointer to a buffer in a Lance file
7///
8/// A writer can place a buffer in three different locations.  The buffer
9/// can go in the data page, in the column metadata, or in the file metadata.
10/// The writer is free to choose whatever is most appropriate (for example, a dictionary
11/// that is shared across all pages in a column will probably go in the column
12/// metadata).  This specification does not dictate where the buffer should go.
13#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
14pub struct Buffer {
15    /// The index of the buffer in the collection of buffers
16    #[prost(uint32, tag = "1")]
17    pub buffer_index: u32,
18    #[prost(enumeration = "buffer::BufferType", tag = "2")]
19    pub buffer_type: i32,
20}
21/// Nested message and enum types in `Buffer`.
22pub mod buffer {
23    /// The collection holding the buffer
24    #[derive(
25        Clone,
26        Copy,
27        Debug,
28        PartialEq,
29        Eq,
30        Hash,
31        PartialOrd,
32        Ord,
33        ::prost::Enumeration
34    )]
35    #[repr(i32)]
36    pub enum BufferType {
37        /// The buffer is stored in the data page itself
38        Page = 0,
39        /// The buffer is stored in the column metadata
40        Column = 1,
41        /// The buffer is stored in the file metadata
42        File = 2,
43    }
44    impl BufferType {
45        /// String value of the enum field names used in the ProtoBuf definition.
46        ///
47        /// The values are not transformed in any way and thus are considered stable
48        /// (if the ProtoBuf definition does not change) and safe for programmatic use.
49        pub fn as_str_name(&self) -> &'static str {
50            match self {
51                Self::Page => "page",
52                Self::Column => "column",
53                Self::File => "file",
54            }
55        }
56        /// Creates an enum from field names used in the ProtoBuf definition.
57        pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
58            match value {
59                "page" => Some(Self::Page),
60                "column" => Some(Self::Column),
61                "file" => Some(Self::File),
62                _ => None,
63            }
64        }
65    }
66}
67/// An encoding that adds nullability to another array encoding
68///
69/// This can wrap any array encoding and add nullability information
70#[derive(Clone, PartialEq, ::prost::Message)]
71pub struct Nullable {
72    #[prost(oneof = "nullable::Nullability", tags = "1, 2, 3")]
73    pub nullability: ::core::option::Option<nullable::Nullability>,
74}
75/// Nested message and enum types in `Nullable`.
76pub mod nullable {
77    #[derive(Clone, PartialEq, ::prost::Message)]
78    pub struct NoNull {
79        #[prost(message, optional, boxed, tag = "1")]
80        pub values: ::core::option::Option<
81            ::prost::alloc::boxed::Box<super::ArrayEncoding>,
82        >,
83    }
84    #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
85    pub struct AllNull {}
86    #[derive(Clone, PartialEq, ::prost::Message)]
87    pub struct SomeNull {
88        #[prost(message, optional, boxed, tag = "1")]
89        pub validity: ::core::option::Option<
90            ::prost::alloc::boxed::Box<super::ArrayEncoding>,
91        >,
92        #[prost(message, optional, boxed, tag = "2")]
93        pub values: ::core::option::Option<
94            ::prost::alloc::boxed::Box<super::ArrayEncoding>,
95        >,
96    }
97    #[derive(Clone, PartialEq, ::prost::Oneof)]
98    pub enum Nullability {
99        /// The array has no nulls and there is a single buffer needed
100        #[prost(message, tag = "1")]
101        NoNulls(::prost::alloc::boxed::Box<NoNull>),
102        /// The array may have nulls and we need two buffers
103        #[prost(message, tag = "2")]
104        SomeNulls(::prost::alloc::boxed::Box<SomeNull>),
105        /// All values are null (no buffers needed)
106        #[prost(message, tag = "3")]
107        AllNulls(AllNull),
108    }
109}
110/// An array encoding for variable-length list fields
111#[derive(Clone, PartialEq, ::prost::Message)]
112pub struct List {
113    /// An array containing the offsets into an items array.
114    ///
115    /// This array will have num_rows items and will never
116    /// have nulls.
117    ///
118    /// If the list at index i is not null then offsets\[i\] will
119    /// contain `base + len(list)` where `base` is defined as:
120    ///    i == 0: 0
121    ///    i >  0: (offsets\[i-1\] % null_offset_adjustment)
122    ///
123    /// To help understand we can consider the following example list:
124    /// \[ [A, B\], null, \[\], \[C, D, E\] ]
125    ///
126    /// The offsets will be \[2, ?, 2, 5\]
127    ///
128    /// If the incoming list at index i IS null then offsets\[i\] will
129    /// contain `base + len(list) + null_offset_adjustment` where `base`
130    /// is defined the same as above.
131    ///
132    /// To complete the above example let's assume that `null_offset_adjustment`
133    /// is 7.  Then the offsets will be \[2, 9, 2, 5\]
134    ///
135    /// If there are no nulls then the offsets we write here are exactly the
136    /// same as the offsets in an Arrow list array (except we omit the leading
137    /// 0 which is redundant)
138    ///
139    /// The reason we do this is so that reading a single list at index i only
140    /// requires us to load the indices at i and i-1.
141    ///
142    /// If the offset at index i is greater than `null_offset_adjustment``
143    /// then the list at index i is null.
144    ///
145    /// Otherwise the length of the list is `offsets\[i\] - base` where
146    /// base is defined the same as above.
147    ///
148    /// Let's consider our example offsets: \[2, 9, 2, 5\]
149    ///
150    /// We can take any range of lists and determine how many list items are
151    /// referenced by the sublist.
152    ///
153    /// 0..3: \[_, 5\] -> items 0..5 (base = 0* and end is 5)
154    /// 0..2: \[_, 2\] -> items 0..2 (base = 0* and end is 2)
155    /// 0..1: \[_, 9\] -> items 0..2 (base = 0* and end is 9 % 7)
156    /// 1..3: \[2, 5\] -> items 2..5 (base = 2 and end is 5)
157    /// 1..2: \[2, 2\] -> items 2..2 (base = 2 and end is 2)
158    /// 2..3: \[9, 5\] -> items 2..5 (base = 9 % 7 and end is 5)
159    ///
160    /// * When the start of our range is the 0th item the base is always 0 and we only
161    ///    need to load a single index from disk to determine the range.
162    ///
163    /// The data type of the offsets array is flexible and does not need
164    /// to match the data type of the destination array.  Please note that the offsets
165    /// array is very likely to be efficiently encoded by bit packing deltas.
166    #[prost(message, optional, boxed, tag = "1")]
167    pub offsets: ::core::option::Option<::prost::alloc::boxed::Box<ArrayEncoding>>,
168    /// If a list is null then we add this value to the offset
169    ///
170    /// This value must be greater than the length of the items so that
171    /// (offset + null_offset_adjustment) is never used by a non-null list.
172    ///
173    /// Note that this value cannot be equal to the length of the items
174    /// because then a page with a single list would store \[ X \] and we
175    /// couldn't know if that is a null list or a list with X items.
176    ///
177    /// Therefore, the best choice for this value is 1 + # of items.
178    /// Choosing this will maximize the bit packing that we can apply to the offsets.
179    #[prost(uint64, tag = "2")]
180    pub null_offset_adjustment: u64,
181    /// How many items are referenced by these offsets.  This is needed in
182    /// order to determine which items pages map to this offsets page.
183    #[prost(uint64, tag = "3")]
184    pub num_items: u64,
185}
186/// An array encoding for fixed-size list fields
187#[derive(Clone, PartialEq, ::prost::Message)]
188pub struct FixedSizeList {
189    /// / The number of items in each list
190    #[prost(uint32, tag = "1")]
191    pub dimension: u32,
192    /// / True if the list is nullable
193    #[prost(bool, tag = "3")]
194    pub has_validity: bool,
195    /// / The items in the list
196    #[prost(message, optional, boxed, tag = "2")]
197    pub items: ::core::option::Option<::prost::alloc::boxed::Box<ArrayEncoding>>,
198}
199#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
200pub struct Compression {
201    #[prost(string, tag = "1")]
202    pub scheme: ::prost::alloc::string::String,
203    #[prost(int32, optional, tag = "2")]
204    pub level: ::core::option::Option<i32>,
205}
206/// Fixed width items placed contiguously in a buffer
207#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
208pub struct Flat {
209    /// the number of bits per value, must be greater than 0, does
210    /// not need to be a multiple of 8
211    #[prost(uint64, tag = "1")]
212    pub bits_per_value: u64,
213    /// the buffer of values
214    #[prost(message, optional, tag = "2")]
215    pub buffer: ::core::option::Option<Buffer>,
216    /// The Compression message can specify the compression scheme (e.g. zstd) and any
217    /// other information that is needed for decompression.
218    ///
219    /// If this array is compressed then the bits_per_value refers to the uncompressed
220    /// data.
221    #[prost(message, optional, tag = "3")]
222    pub compression: ::core::option::Option<Compression>,
223}
224/// Compression algorithm where all values have a constant value
225#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
226pub struct Constant {
227    /// The value (TODO: define encoding for literals?)
228    #[prost(bytes = "vec", tag = "1")]
229    pub value: ::prost::alloc::vec::Vec<u8>,
230}
231/// Items are bitpacked in a buffer
232#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
233pub struct Bitpacked {
234    /// the number of bits used for a value in the buffer
235    #[prost(uint64, tag = "1")]
236    pub compressed_bits_per_value: u64,
237    /// the number of bits of the uncompressed value. e.g. for a u32, this will be 32
238    #[prost(uint64, tag = "2")]
239    pub uncompressed_bits_per_value: u64,
240    /// The items in the list
241    #[prost(message, optional, tag = "3")]
242    pub buffer: ::core::option::Option<Buffer>,
243    /// Whether or not a sign bit is included in the bitpacked value
244    #[prost(bool, tag = "4")]
245    pub signed: bool,
246}
247/// Items are bitpacked in a buffer
248#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
249pub struct BitpackedForNonNeg {
250    /// the number of bits used for a value in the buffer
251    #[prost(uint64, tag = "1")]
252    pub compressed_bits_per_value: u64,
253    /// the number of bits of the uncompressed value. e.g. for a u32, this will be 32
254    #[prost(uint64, tag = "2")]
255    pub uncompressed_bits_per_value: u64,
256    /// The items in the list
257    #[prost(message, optional, tag = "3")]
258    pub buffer: ::core::option::Option<Buffer>,
259}
260/// Opaque bitpacking variant where the bits per value are stored inline in the chunks themselves
261#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
262pub struct InlineBitpacking {
263    /// the number of bits of the uncompressed value. e.g. for a u32, this will be 32
264    #[prost(uint64, tag = "2")]
265    pub uncompressed_bits_per_value: u64,
266}
267/// Transparent bitpacking variant where the number of bits per value is fixed through the whole buffer
268#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
269pub struct OutOfLineBitpacking {
270    /// the number of bits of the uncompressed value. e.g. for a u32, this will be 32
271    #[prost(uint64, tag = "2")]
272    pub uncompressed_bits_per_value: u64,
273    /// The number of compressed bits per value, fixed across the entire buffer
274    #[prost(uint64, tag = "3")]
275    pub compressed_bits_per_value: u64,
276}
277/// An array encoding for shredded structs that will never be null
278///
279/// There is no actual data in this column.
280///
281/// TODO: Struct validity bitmaps will be placed here.
282#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
283pub struct SimpleStruct {}
284/// An array encoding for binary fields
285#[derive(Clone, PartialEq, ::prost::Message)]
286pub struct Binary {
287    #[prost(message, optional, boxed, tag = "1")]
288    pub indices: ::core::option::Option<::prost::alloc::boxed::Box<ArrayEncoding>>,
289    #[prost(message, optional, boxed, tag = "2")]
290    pub bytes: ::core::option::Option<::prost::alloc::boxed::Box<ArrayEncoding>>,
291    #[prost(uint64, tag = "3")]
292    pub null_adjustment: u64,
293}
294#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
295pub struct Variable {
296    #[prost(uint32, tag = "1")]
297    pub bits_per_offset: u32,
298}
299#[derive(Clone, PartialEq, ::prost::Message)]
300pub struct Fsst {
301    #[prost(message, optional, boxed, tag = "1")]
302    pub binary: ::core::option::Option<::prost::alloc::boxed::Box<ArrayEncoding>>,
303    #[prost(bytes = "vec", tag = "2")]
304    pub symbol_table: ::prost::alloc::vec::Vec<u8>,
305}
306/// An array encoding for dictionary-encoded fields
307#[derive(Clone, PartialEq, ::prost::Message)]
308pub struct Dictionary {
309    #[prost(message, optional, boxed, tag = "1")]
310    pub indices: ::core::option::Option<::prost::alloc::boxed::Box<ArrayEncoding>>,
311    #[prost(message, optional, boxed, tag = "2")]
312    pub items: ::core::option::Option<::prost::alloc::boxed::Box<ArrayEncoding>>,
313    #[prost(uint32, tag = "3")]
314    pub num_dictionary_items: u32,
315}
316#[derive(Clone, PartialEq, ::prost::Message)]
317pub struct PackedStruct {
318    #[prost(message, repeated, tag = "1")]
319    pub inner: ::prost::alloc::vec::Vec<ArrayEncoding>,
320    #[prost(message, optional, tag = "2")]
321    pub buffer: ::core::option::Option<Buffer>,
322}
323#[derive(Clone, PartialEq, ::prost::Message)]
324pub struct PackedStructFixedWidthMiniBlock {
325    #[prost(message, optional, boxed, tag = "1")]
326    pub flat: ::core::option::Option<::prost::alloc::boxed::Box<ArrayEncoding>>,
327    #[prost(uint32, repeated, tag = "2")]
328    pub bits_per_values: ::prost::alloc::vec::Vec<u32>,
329}
330#[derive(Clone, PartialEq, ::prost::Message)]
331pub struct FixedSizeBinary {
332    #[prost(message, optional, boxed, tag = "1")]
333    pub bytes: ::core::option::Option<::prost::alloc::boxed::Box<ArrayEncoding>>,
334    #[prost(uint32, tag = "2")]
335    pub byte_width: u32,
336}
337#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
338pub struct Block {
339    #[prost(string, tag = "1")]
340    pub scheme: ::prost::alloc::string::String,
341}
342/// Run-Length Encoding for miniblock format
343#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
344pub struct Rle {
345    /// Number of bits per value (8, 16, 32, 64, or 128)
346    #[prost(uint64, tag = "1")]
347    pub bits_per_value: u64,
348}
349/// Byte Stream Split encoding for floating point values
350#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
351pub struct ByteStreamSplit {
352    /// Number of bits per value (32 for float, 64 for double)
353    #[prost(uint64, tag = "1")]
354    pub bits_per_value: u64,
355}
356/// General miniblock encoding - wraps another miniblock encoding with compression
357#[derive(Clone, PartialEq, ::prost::Message)]
358pub struct GeneralMiniBlock {
359    /// The inner miniblock encoding (e.g., Rle, Bitpacked, etc.)
360    #[prost(message, optional, boxed, tag = "1")]
361    pub inner: ::core::option::Option<::prost::alloc::boxed::Box<ArrayEncoding>>,
362    /// The compression scheme to apply to the miniblock buffers
363    #[prost(message, optional, tag = "2")]
364    pub compression: ::core::option::Option<Compression>,
365}
366/// Encodings that decode into an Arrow array
367#[derive(Clone, PartialEq, ::prost::Message)]
368pub struct ArrayEncoding {
369    #[prost(
370        oneof = "array_encoding::ArrayEncoding",
371        tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21"
372    )]
373    pub array_encoding: ::core::option::Option<array_encoding::ArrayEncoding>,
374}
375/// Nested message and enum types in `ArrayEncoding`.
376pub mod array_encoding {
377    #[derive(Clone, PartialEq, ::prost::Oneof)]
378    pub enum ArrayEncoding {
379        #[prost(message, tag = "1")]
380        Flat(super::Flat),
381        #[prost(message, tag = "2")]
382        Nullable(::prost::alloc::boxed::Box<super::Nullable>),
383        #[prost(message, tag = "3")]
384        FixedSizeList(::prost::alloc::boxed::Box<super::FixedSizeList>),
385        #[prost(message, tag = "4")]
386        List(::prost::alloc::boxed::Box<super::List>),
387        #[prost(message, tag = "5")]
388        Struct(super::SimpleStruct),
389        #[prost(message, tag = "6")]
390        Binary(::prost::alloc::boxed::Box<super::Binary>),
391        #[prost(message, tag = "7")]
392        Dictionary(::prost::alloc::boxed::Box<super::Dictionary>),
393        #[prost(message, tag = "8")]
394        Fsst(::prost::alloc::boxed::Box<super::Fsst>),
395        #[prost(message, tag = "9")]
396        PackedStruct(super::PackedStruct),
397        #[prost(message, tag = "10")]
398        Bitpacked(super::Bitpacked),
399        #[prost(message, tag = "11")]
400        FixedSizeBinary(::prost::alloc::boxed::Box<super::FixedSizeBinary>),
401        #[prost(message, tag = "12")]
402        BitpackedForNonNeg(super::BitpackedForNonNeg),
403        #[prost(message, tag = "13")]
404        Constant(super::Constant),
405        #[prost(message, tag = "14")]
406        InlineBitpacking(super::InlineBitpacking),
407        #[prost(message, tag = "15")]
408        OutOfLineBitpacking(super::OutOfLineBitpacking),
409        #[prost(message, tag = "16")]
410        Variable(super::Variable),
411        #[prost(message, tag = "17")]
412        PackedStructFixedWidthMiniBlock(
413            ::prost::alloc::boxed::Box<super::PackedStructFixedWidthMiniBlock>,
414        ),
415        #[prost(message, tag = "18")]
416        Block(super::Block),
417        #[prost(message, tag = "19")]
418        Rle(super::Rle),
419        #[prost(message, tag = "20")]
420        GeneralMiniBlock(::prost::alloc::boxed::Box<super::GeneralMiniBlock>),
421        #[prost(message, tag = "21")]
422        ByteStreamSplit(super::ByteStreamSplit),
423    }
424}
425/// Wraps a column with a zone map index that can be used
426/// to apply pushdown filters
427#[derive(Clone, PartialEq, ::prost::Message)]
428pub struct ZoneIndex {
429    #[prost(uint32, tag = "1")]
430    pub rows_per_zone: u32,
431    #[prost(message, optional, tag = "2")]
432    pub zone_map_buffer: ::core::option::Option<Buffer>,
433    #[prost(message, optional, boxed, tag = "3")]
434    pub inner: ::core::option::Option<::prost::alloc::boxed::Box<ColumnEncoding>>,
435}
436/// Marks a column as blob data.  It will contain a packed struct
437/// with fields position and size (u64)
438#[derive(Clone, PartialEq, ::prost::Message)]
439pub struct Blob {
440    #[prost(message, optional, boxed, tag = "1")]
441    pub inner: ::core::option::Option<::prost::alloc::boxed::Box<ColumnEncoding>>,
442}
443/// Encodings that describe a column of values
444#[derive(Clone, PartialEq, ::prost::Message)]
445pub struct ColumnEncoding {
446    #[prost(oneof = "column_encoding::ColumnEncoding", tags = "1, 2, 3")]
447    pub column_encoding: ::core::option::Option<column_encoding::ColumnEncoding>,
448}
449/// Nested message and enum types in `ColumnEncoding`.
450pub mod column_encoding {
451    #[derive(Clone, PartialEq, ::prost::Oneof)]
452    pub enum ColumnEncoding {
453        /// No special encoding, just column values
454        #[prost(message, tag = "1")]
455        Values(()),
456        #[prost(message, tag = "2")]
457        ZoneIndex(::prost::alloc::boxed::Box<super::ZoneIndex>),
458        #[prost(message, tag = "3")]
459        Blob(::prost::alloc::boxed::Box<super::Blob>),
460    }
461}
genegraph_storage/lancefmt/pb/lance.encodings.rs

genegraph_storage/lancefmt/pb/
lance.encodings.rs
