lance_encoding/format/
stable.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4//! Explicit stable wire boundaries. No serialization round trip is used to
5//! translate metadata, and a current-only codec cannot be emitted as stable.
6
7use super::{metadata, pb21};
8use lance_core::{Error, Result};
9
10macro_rules! message {
11    ($old:path, $new:path, direct: [$($direct:ident),*], optional: [$($optional:ident),*], boxed: [$($boxed:ident),*], repeated: [$($repeated:ident),*]) => {
12        impl From<$old> for $new {
13            fn from(value: $old) -> Self {
14                Self {
15                    $($direct: value.$direct,)*
16                    $($optional: value.$optional.map(Into::into),)*
17                    $($boxed: value.$boxed.map(|v| Box::new((*v).into())),)*
18                    $($repeated: value.$repeated.into_iter().map(Into::into).collect(),)*
19                }
20            }
21        }
22        impl TryFrom<$new> for $old {
23            type Error = Error;
24            fn try_from(value: $new) -> Result<Self> {
25                Ok(Self {
26                    $($direct: value.$direct,)*
27                    $($optional: value.$optional.map(TryInto::try_into).transpose()?,)*
28                    $($boxed: value.$boxed.map(|v| (*v).try_into().map(Box::new)).transpose()?,)*
29                    $($repeated: value.$repeated.into_iter().map(TryInto::try_into).collect::<Result<_>>()?,)*
30                })
31            }
32        }
33    };
34}
35
36macro_rules! scalar_variants {
37    ($old:path, $new:path, [$($variant:ident),*]) => {
38        impl From<$old> for $new {
39            fn from(value: $old) -> Self {
40                use $old as Old;
41                match value { $(Old::$variant(v) => Self::$variant(v),)* }
42            }
43        }
44        impl TryFrom<$new> for $old {
45            type Error = Error;
46            fn try_from(value: $new) -> Result<Self> {
47                use $new as New;
48                Ok(match value { $(New::$variant(v) => Self::$variant(v),)* })
49            }
50        }
51    };
52}
53
54macro_rules! variants {
55    ($old:path, $new:path, [$($variant:ident),*] $(, unsupported: [$($unsupported:ident),*])?) => {
56        impl From<$old> for $new {
57            fn from(value: $old) -> Self {
58                use $old as Old;
59                match value { $(Old::$variant(v) => Self::$variant(v.into()),)* }
60            }
61        }
62        impl TryFrom<$new> for $old {
63            type Error = Error;
64            fn try_from(value: $new) -> Result<Self> {
65                use $new as New;
66                match value {
67                    $(New::$variant(v) => Ok(Self::$variant(v.try_into()?)),)*
68                    $($(New::$unsupported(_) => Err(Error::invalid_input(concat!(stringify!($unsupported), " is not part of the stable encoding grammar"))),)*)?
69                }
70            }
71        }
72    };
73}
74
75// Recursive protobuf variants carry boxes; convert their contents explicitly.
76macro_rules! boxed {
77    ($old:path, $new:path) => {
78        impl From<Box<$old>> for Box<$new> {
79            fn from(value: Box<$old>) -> Self {
80                Box::new((*value).into())
81            }
82        }
83        impl TryFrom<Box<$new>> for Box<$old> {
84            type Error = Error;
85            fn try_from(value: Box<$new>) -> Result<Self> {
86                (*value).try_into().map(Box::new)
87            }
88        }
89    };
90}
91message!(
92    pb21::BufferCompression,
93    metadata::BufferCompression,
94    direct: [scheme, level],
95    optional: [],
96    boxed: [],
97    repeated: []
98);
99message!(
100    pb21::Flat,
101    metadata::Flat,
102    direct: [bits_per_value],
103    optional: [data],
104    boxed: [],
105    repeated: []
106);
107message!(
108    pb21::Variable,
109    metadata::Variable,
110    direct: [],
111    optional: [values],
112    boxed: [offsets],
113    repeated: []
114);
115message!(
116    pb21::Constant,
117    metadata::Constant,
118    direct: [value],
119    optional: [],
120    boxed: [],
121    repeated: []
122);
123message!(
124    pb21::OutOfLineBitpacking,
125    metadata::OutOfLineBitpacking,
126    direct: [uncompressed_bits_per_value],
127    optional: [],
128    boxed: [values],
129    repeated: []
130);
131message!(
132    pb21::InlineBitpacking,
133    metadata::InlineBitpacking,
134    direct: [uncompressed_bits_per_value],
135    optional: [values],
136    boxed: [],
137    repeated: []
138);
139message!(
140    pb21::Fsst,
141    metadata::Fsst,
142    direct: [symbol_table],
143    optional: [],
144    boxed: [values],
145    repeated: []
146);
147message!(
148    pb21::Dictionary,
149    metadata::Dictionary,
150    direct: [num_dictionary_items],
151    optional: [],
152    boxed: [indices, items],
153    repeated: []
154);
155message!(
156    pb21::Rle,
157    metadata::Rle,
158    direct: [],
159    optional: [],
160    boxed: [values, run_lengths],
161    repeated: []
162);
163message!(
164    pb21::FixedSizeList,
165    metadata::FixedSizeList,
166    direct: [items_per_value, has_validity],
167    optional: [],
168    boxed: [values],
169    repeated: []
170);
171message!(
172    pb21::PackedStruct,
173    metadata::PackedStruct,
174    direct: [bits_per_value],
175    optional: [],
176    boxed: [values],
177    repeated: []
178);
179message!(
180    pb21::VariablePackedStruct,
181    metadata::VariablePackedStruct,
182    direct: [],
183    optional: [],
184    boxed: [],
185    repeated: [fields]
186);
187message!(
188    pb21::General,
189    metadata::General,
190    direct: [],
191    optional: [compression],
192    boxed: [values],
193    repeated: []
194);
195message!(
196    pb21::ByteStreamSplit,
197    metadata::ByteStreamSplit,
198    direct: [],
199    optional: [],
200    boxed: [values],
201    repeated: []
202);
203message!(
204    pb21::CompressiveEncoding,
205    metadata::CompressiveEncoding,
206    direct: [],
207    optional: [compression],
208    boxed: [],
209    repeated: []
210);
211message!(
212    pb21::MiniBlockLayout,
213    metadata::MiniBlockLayout,
214    direct: [num_dictionary_items, layers, num_buffers, repetition_index_depth, num_items, has_large_chunk],
215    optional: [rep_compression, def_compression, value_compression, dictionary],
216    boxed: [],
217    repeated: []
218);
219message!(
220    pb21::FullZipLayout,
221    metadata::FullZipLayout,
222    direct: [bits_rep, bits_def, num_items, num_visible_items, layers],
223    optional: [details, value_compression],
224    boxed: [],
225    repeated: []
226);
227message!(
228    pb21::ConstantLayout,
229    metadata::ConstantLayout,
230    direct: [layers, inline_value, num_rep_values, num_def_values],
231    optional: [rep_compression, def_compression],
232    boxed: [],
233    repeated: []
234);
235message!(
236    pb21::BlobLayout,
237    metadata::BlobLayout,
238    direct: [layers],
239    optional: [],
240    boxed: [inner_layout],
241    repeated: []
242);
243message!(
244    pb21::PageLayout,
245    metadata::PageLayout,
246    direct: [],
247    optional: [layout],
248    boxed: [],
249    repeated: []
250);
251message!(
252    pb21::variable_packed_struct::FieldEncoding,
253    metadata::variable_packed_struct::FieldEncoding,
254    direct: [],
255    optional: [value, layout],
256    boxed: [],
257    repeated: []
258);
259boxed!(pb21::Variable, metadata::Variable);
260boxed!(pb21::OutOfLineBitpacking, metadata::OutOfLineBitpacking);
261boxed!(pb21::Fsst, metadata::Fsst);
262boxed!(pb21::Dictionary, metadata::Dictionary);
263boxed!(pb21::Rle, metadata::Rle);
264boxed!(pb21::FixedSizeList, metadata::FixedSizeList);
265boxed!(pb21::PackedStruct, metadata::PackedStruct);
266boxed!(pb21::General, metadata::General);
267boxed!(pb21::ByteStreamSplit, metadata::ByteStreamSplit);
268boxed!(pb21::BlobLayout, metadata::BlobLayout);
269variants!(
270    pb21::compressive_encoding::Compression,
271    metadata::compressive_encoding::Compression,
272    [Flat, Variable, Constant, OutOfLineBitpacking, InlineBitpacking, Fsst, Dictionary,
273     Rle, ByteStreamSplit, General, FixedSizeList, PackedStruct, VariablePackedStruct],
274    unsupported: [Range, Delta]
275);
276variants!(
277    pb21::page_layout::Layout,
278    metadata::page_layout::Layout,
279    [MiniBlockLayout, FullZipLayout, ConstantLayout, BlobLayout]
280);
281scalar_variants!(
282    pb21::full_zip_layout::Details,
283    metadata::full_zip_layout::Details,
284    [BitsPerValue, BitsPerOffset]
285);
286scalar_variants!(
287    pb21::variable_packed_struct::field_encoding::Layout,
288    metadata::variable_packed_struct::field_encoding::Layout,
289    [BitsPerValue, BitsPerLength]
290);
lance_encoding/format/stable.rs

lance_encoding/format/
stable.rs
