lance_encoding/format/
metadata.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4//! Execution metadata for shared structural engines. Page descriptors are not
5//! wire messages. Stable protobufs are converted at the file boundary; current
6//! compression descriptors are shared with the independent 2.3 grammar.
7
8use prost::Message;
9
10pub use super::pb23::{
11    BufferCompression, ByteStreamSplit, CompressionScheme, CompressiveEncoding, Constant, Delta,
12    Dictionary, FixedSizeList, Flat, Fsst, General, InlineBitpacking, OutOfLineBitpacking,
13    PackedStruct, Range, RepDefLayer, Rle, Variable, VariablePackedStruct, compressive_encoding,
14    variable_packed_struct,
15};
16
17/// A structural engine's execution description, independent of its wire envelope.
18#[derive(Clone, Debug, Default, PartialEq)]
19pub struct PageLayout {
20    /// The selected engine. Missing layouts are rejected before execution.
21    pub layout: Option<page_layout::Layout>,
22}
23
24/// Structural engine choices.
25pub mod page_layout {
26    use super::*;
27    /// Physical engines shared by stable and planar pages.
28    #[derive(Clone, Debug, PartialEq)]
29    pub enum Layout {
30        /// Independently compressed chunks, with optional levels and dictionary.
31        MiniBlockLayout(MiniBlockLayout),
32        /// Individually addressable values interleaved with optional control words.
33        FullZipLayout(FullZipLayout),
34        /// An inline scalar or no materialized values, with optional level buffers.
35        ConstantLayout(ConstantLayout),
36        /// Stable Blob v1 descriptors and their null semantics.
37        BlobLayout(Box<BlobLayout>),
38    }
39}
40
41/// Buffer and row accounting needed by the mini-block engine.
42#[derive(Clone, Debug, Default, PartialEq)]
43pub struct MiniBlockLayout {
44    /// Repetition stream encoding; absent when no repetition buffer exists.
45    pub rep_compression: Option<CompressiveEncoding>,
46    /// Definition stream encoding; absent when no definition buffer exists.
47    pub def_compression: Option<CompressiveEncoding>,
48    /// Value encoding, required for engines that materialize values.
49    pub value_compression: Option<CompressiveEncoding>,
50    /// Page dictionary encoding; absent when chunks do not share a dictionary.
51    pub dictionary: Option<CompressiveEncoding>,
52    /// Number of entries in the page dictionary.
53    pub num_dictionary_items: u64,
54    /// Inner-most to outer-most repetition/definition interpretations.
55    pub layers: Vec<i32>,
56    /// Value buffers per chunk, excluding repetition and definition streams.
57    pub num_buffers: u64,
58    /// Number of list layers represented by the repetition index.
59    pub repetition_index_depth: u32,
60    /// Number of structural events, including invisible entries.
61    pub num_items: u64,
62    /// Selects u32 chunk metadata; false selects the stable u16 representation.
63    pub has_large_chunk: bool,
64}
65
66/// Addressing and level information needed by the zipped engine.
67#[derive(Clone, Debug, Default, PartialEq)]
68pub struct FullZipLayout {
69    /// Repetition bits in each control word.
70    pub bits_rep: u32,
71    /// Definition bits in each control word.
72    pub bits_def: u32,
73    /// Fixed value width or variable length-prefix width; required for zipped values.
74    pub details: Option<full_zip_layout::Details>,
75    /// Number of structural events, including invisible entries.
76    pub num_items: u32,
77    /// Number of leaf slots that are visible to the parent.
78    pub num_visible_items: u32,
79    /// Value encoding, required for engines that materialize values.
80    pub value_compression: Option<CompressiveEncoding>,
81    /// Inner-most to outer-most repetition/definition interpretations.
82    pub layers: Vec<i32>,
83}
84
85/// Zipped value addressing.
86pub mod full_zip_layout {
87    /// How to locate values within the zipped buffer.
88    #[derive(Clone, Debug, PartialEq)]
89    pub enum Details {
90        /// Fixed width of each encoded value, in bits.
91        BitsPerValue(u32),
92        /// Width of each variable-value length prefix, in bits.
93        BitsPerOffset(u32),
94    }
95}
96
97/// Scalar and level information needed by the constant engine.
98#[derive(Clone, Debug, Default, PartialEq)]
99pub struct ConstantLayout {
100    /// Inner-most to outer-most repetition/definition interpretations.
101    pub layers: Vec<i32>,
102    /// Encoded scalar; absent for pages without materialized values.
103    pub inline_value: Option<bytes::Bytes>,
104    /// Repetition stream encoding; absent when no repetition buffer exists.
105    pub rep_compression: Option<CompressiveEncoding>,
106    /// Definition stream encoding; absent when no definition buffer exists.
107    pub def_compression: Option<CompressiveEncoding>,
108    /// Number of entries in the repetition stream.
109    pub num_rep_values: u64,
110    /// Number of entries in the definition stream.
111    pub num_def_values: u64,
112}
113
114/// Frozen Blob v1 execution description used by stable readers and writers.
115#[derive(Clone, Debug, Default, PartialEq)]
116pub struct BlobLayout {
117    /// Descriptor layout, required by the stable Blob v1 engine.
118    pub inner_layout: Option<Box<PageLayout>>,
119    /// Inner-most to outer-most repetition/definition interpretations.
120    pub layers: Vec<i32>,
121}
122
123impl PageLayout {
124    /// Estimate retained metadata size using the same conservative compression
125    /// tree multiplier as the wire-backed page metadata.
126    pub(crate) fn estimated_size(&self) -> usize {
127        let compression =
128            |value: &Option<CompressiveEncoding>| value.as_ref().map_or(0, |v| v.encoded_len() * 4);
129        std::mem::size_of::<Self>()
130            + match &self.layout {
131                Some(page_layout::Layout::MiniBlockLayout(v)) => {
132                    v.layers.capacity() * 4
133                        + compression(&v.rep_compression)
134                        + compression(&v.def_compression)
135                        + compression(&v.value_compression)
136                        + compression(&v.dictionary)
137                }
138                Some(page_layout::Layout::FullZipLayout(v)) => {
139                    v.layers.capacity() * 4 + compression(&v.value_compression)
140                }
141                Some(page_layout::Layout::ConstantLayout(v)) => {
142                    v.layers.capacity() * 4
143                        + v.inline_value.as_ref().map_or(0, |v| v.len())
144                        + compression(&v.rep_compression)
145                        + compression(&v.def_compression)
146                }
147                Some(page_layout::Layout::BlobLayout(v)) => {
148                    std::mem::size_of::<BlobLayout>()
149                        + v.layers.capacity() * 4
150                        + v.inner_layout.as_ref().map_or(0, |v| v.estimated_size())
151                }
152                None => 0,
153            }
154    }
155}
lance_encoding/format/metadata.rs

lance_encoding/format/
metadata.rs
