Skip to main content

lance_encoding/
format.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright The Lance Authors
3
4/// Protobuf definitions for encodings
5///
6/// These are the messages used for describing encoding in the 2.0 format
7pub mod pb {
8    #![allow(clippy::all)]
9    #![allow(non_upper_case_globals)]
10    #![allow(non_camel_case_types)]
11    #![allow(non_snake_case)]
12    #![allow(unused)]
13    #![allow(improper_ctypes)]
14    #![allow(clippy::upper_case_acronyms)]
15    #![allow(clippy::use_self)]
16    include!(concat!(env!("OUT_DIR"), "/lance.encodings.rs"));
17}
18
19/// Protobuf definitions for encodings21
20///
21/// These are the messages used for describing encoding in the 2.1 format
22/// and the stable 2.2 format.
23pub mod pb21 {
24    #![allow(clippy::all)]
25    #![allow(non_upper_case_globals)]
26    #![allow(non_camel_case_types)]
27    #![allow(non_snake_case)]
28    #![allow(unused)]
29    #![allow(improper_ctypes)]
30    #![allow(clippy::upper_case_acronyms)]
31    #![allow(clippy::use_self)]
32    include!(concat!(env!("OUT_DIR"), "/lance.encodings21.rs"));
33}
34
35/// Metadata for the unstable 2.3 planar page format.
36pub mod pb23 {
37    #![allow(clippy::all)]
38    include!(concat!(env!("OUT_DIR"), "/lance.encodings23.rs"));
39}
40
41pub mod metadata;
42pub mod planar;
43mod stable;
44
45use pb::{
46    ArrayEncoding, Binary, Bitpacked, BitpackedForNonNeg, Block, Dictionary, FixedSizeBinary,
47    FixedSizeList, Flat, Fsst, InlineBitpacking, Nullable, OutOfLineBitpacking, PackedStruct,
48    PackedStructFixedWidthMiniBlock, Rle, Variable,
49    array_encoding::ArrayEncoding as ArrayEncodingEnum,
50    buffer::BufferType,
51    nullable::{AllNull, NoNull, Nullability, SomeNull},
52};
53
54use crate::{encodings::physical::block::CompressionConfig, repdef::DefinitionInterpretation};
55
56use self::pb::Constant;
57use lance_core::Result;
58
59// Utility functions for creating complex protobuf objects
60pub struct ProtobufUtils {}
61
62impl ProtobufUtils {
63    pub fn constant(value: Vec<u8>) -> ArrayEncoding {
64        ArrayEncoding {
65            array_encoding: Some(ArrayEncodingEnum::Constant(Constant {
66                value: value.into(),
67            })),
68        }
69    }
70
71    pub fn basic_all_null_encoding() -> ArrayEncoding {
72        ArrayEncoding {
73            array_encoding: Some(ArrayEncodingEnum::Nullable(Box::new(Nullable {
74                nullability: Some(Nullability::AllNulls(AllNull {})),
75            }))),
76        }
77    }
78
79    pub fn basic_some_null_encoding(
80        validity: ArrayEncoding,
81        values: ArrayEncoding,
82    ) -> ArrayEncoding {
83        ArrayEncoding {
84            array_encoding: Some(ArrayEncodingEnum::Nullable(Box::new(Nullable {
85                nullability: Some(Nullability::SomeNulls(Box::new(SomeNull {
86                    validity: Some(Box::new(validity)),
87                    values: Some(Box::new(values)),
88                }))),
89            }))),
90        }
91    }
92
93    pub fn basic_no_null_encoding(values: ArrayEncoding) -> ArrayEncoding {
94        ArrayEncoding {
95            array_encoding: Some(ArrayEncodingEnum::Nullable(Box::new(Nullable {
96                nullability: Some(Nullability::NoNulls(Box::new(NoNull {
97                    values: Some(Box::new(values)),
98                }))),
99            }))),
100        }
101    }
102
103    pub fn block(scheme: &str) -> ArrayEncoding {
104        ArrayEncoding {
105            array_encoding: Some(ArrayEncodingEnum::Block(Block {
106                scheme: scheme.to_string(),
107            })),
108        }
109    }
110
111    pub fn flat_encoding(
112        bits_per_value: u64,
113        buffer_index: u32,
114        compression: Option<CompressionConfig>,
115    ) -> ArrayEncoding {
116        ArrayEncoding {
117            array_encoding: Some(ArrayEncodingEnum::Flat(Flat {
118                bits_per_value,
119                buffer: Some(pb::Buffer {
120                    buffer_index,
121                    buffer_type: BufferType::Page as i32,
122                }),
123                compression: compression.map(|compression_config| pb::Compression {
124                    scheme: compression_config.scheme.to_string(),
125                    level: compression_config.level,
126                }),
127            })),
128        }
129    }
130
131    pub fn fsl_encoding(dimension: u64, items: ArrayEncoding, has_validity: bool) -> ArrayEncoding {
132        ArrayEncoding {
133            array_encoding: Some(ArrayEncodingEnum::FixedSizeList(Box::new(FixedSizeList {
134                dimension: dimension.try_into().unwrap(),
135                items: Some(Box::new(items)),
136                has_validity,
137            }))),
138        }
139    }
140
141    pub fn bitpacked_encoding(
142        compressed_bits_per_value: u64,
143        uncompressed_bits_per_value: u64,
144        buffer_index: u32,
145        signed: bool,
146    ) -> ArrayEncoding {
147        ArrayEncoding {
148            array_encoding: Some(ArrayEncodingEnum::Bitpacked(Bitpacked {
149                compressed_bits_per_value,
150                buffer: Some(pb::Buffer {
151                    buffer_index,
152                    buffer_type: BufferType::Page as i32,
153                }),
154                uncompressed_bits_per_value,
155                signed,
156            })),
157        }
158    }
159
160    pub fn bitpacked_for_non_neg_encoding(
161        compressed_bits_per_value: u64,
162        uncompressed_bits_per_value: u64,
163        buffer_index: u32,
164    ) -> ArrayEncoding {
165        ArrayEncoding {
166            array_encoding: Some(ArrayEncodingEnum::BitpackedForNonNeg(BitpackedForNonNeg {
167                compressed_bits_per_value,
168                buffer: Some(pb::Buffer {
169                    buffer_index,
170                    buffer_type: BufferType::Page as i32,
171                }),
172                uncompressed_bits_per_value,
173            })),
174        }
175    }
176    pub fn inline_bitpacking(uncompressed_bits_per_value: u64) -> ArrayEncoding {
177        ArrayEncoding {
178            array_encoding: Some(ArrayEncodingEnum::InlineBitpacking(InlineBitpacking {
179                uncompressed_bits_per_value,
180            })),
181        }
182    }
183    pub fn out_of_line_bitpacking(
184        uncompressed_bits_per_value: u64,
185        compressed_bits_per_value: u64,
186    ) -> ArrayEncoding {
187        ArrayEncoding {
188            array_encoding: Some(ArrayEncodingEnum::OutOfLineBitpacking(
189                OutOfLineBitpacking {
190                    uncompressed_bits_per_value,
191                    compressed_bits_per_value,
192                },
193            )),
194        }
195    }
196
197    pub fn variable(bits_per_offset: u8) -> ArrayEncoding {
198        ArrayEncoding {
199            array_encoding: Some(ArrayEncodingEnum::Variable(Variable {
200                bits_per_offset: bits_per_offset as u32,
201            })),
202        }
203    }
204
205    // Construct a `FsstMiniBlock` ArrayEncoding, the inner `binary_mini_block` encoding is actually
206    // not used and `FsstMiniBlockDecompressor` constructs a `binary_mini_block` in a `hard-coded` fashion.
207    // This can be an optimization later.
208    pub fn fsst(data: ArrayEncoding, symbol_table: Vec<u8>) -> ArrayEncoding {
209        ArrayEncoding {
210            array_encoding: Some(ArrayEncodingEnum::Fsst(Box::new(Fsst {
211                binary: Some(Box::new(data)),
212                symbol_table: symbol_table.into(),
213            }))),
214        }
215    }
216
217    pub fn rle(bits_per_value: u64) -> ArrayEncoding {
218        ArrayEncoding {
219            array_encoding: Some(ArrayEncodingEnum::Rle(Rle { bits_per_value })),
220        }
221    }
222
223    pub fn byte_stream_split(bits_per_value: u64) -> ArrayEncoding {
224        ArrayEncoding {
225            array_encoding: Some(ArrayEncodingEnum::ByteStreamSplit(pb::ByteStreamSplit {
226                bits_per_value,
227            })),
228        }
229    }
230
231    pub fn general_mini_block(
232        inner: ArrayEncoding,
233        compression: CompressionConfig,
234    ) -> ArrayEncoding {
235        ArrayEncoding {
236            array_encoding: Some(ArrayEncodingEnum::GeneralMiniBlock(Box::new(
237                pb::GeneralMiniBlock {
238                    inner: Some(Box::new(inner)),
239                    compression: Some(pb::Compression {
240                        scheme: compression.scheme.to_string(),
241                        level: compression.level,
242                    }),
243                },
244            ))),
245        }
246    }
247
248    pub fn packed_struct(
249        child_encodings: Vec<ArrayEncoding>,
250        packed_buffer_index: u32,
251    ) -> ArrayEncoding {
252        ArrayEncoding {
253            array_encoding: Some(ArrayEncodingEnum::PackedStruct(PackedStruct {
254                inner: child_encodings,
255                buffer: Some(pb::Buffer {
256                    buffer_index: packed_buffer_index,
257                    buffer_type: BufferType::Page as i32,
258                }),
259            })),
260        }
261    }
262
263    pub fn packed_struct_fixed_width_mini_block(
264        data: ArrayEncoding,
265        bits_per_values: Vec<u32>,
266    ) -> ArrayEncoding {
267        ArrayEncoding {
268            array_encoding: Some(ArrayEncodingEnum::PackedStructFixedWidthMiniBlock(
269                Box::new(PackedStructFixedWidthMiniBlock {
270                    flat: Some(Box::new(data)),
271                    bits_per_values,
272                }),
273            )),
274        }
275    }
276
277    pub fn binary(
278        indices_encoding: ArrayEncoding,
279        bytes_encoding: ArrayEncoding,
280        null_adjustment: u64,
281    ) -> ArrayEncoding {
282        ArrayEncoding {
283            array_encoding: Some(ArrayEncodingEnum::Binary(Box::new(Binary {
284                bytes: Some(Box::new(bytes_encoding)),
285                indices: Some(Box::new(indices_encoding)),
286                null_adjustment,
287            }))),
288        }
289    }
290
291    pub fn dict_encoding(
292        indices: ArrayEncoding,
293        items: ArrayEncoding,
294        num_items: u32,
295    ) -> ArrayEncoding {
296        ArrayEncoding {
297            array_encoding: Some(ArrayEncodingEnum::Dictionary(Box::new(Dictionary {
298                indices: Some(Box::new(indices)),
299                items: Some(Box::new(items)),
300                num_dictionary_items: num_items,
301            }))),
302        }
303    }
304
305    pub fn fixed_size_binary(data: ArrayEncoding, byte_width: u32) -> ArrayEncoding {
306        ArrayEncoding {
307            array_encoding: Some(ArrayEncodingEnum::FixedSizeBinary(Box::new(
308                FixedSizeBinary {
309                    bytes: Some(Box::new(data)),
310                    byte_width,
311                },
312            ))),
313        }
314    }
315}
316
317/// Constructors for compression trees and structural execution descriptions.
318pub struct EncodingUtils {}
319
320impl EncodingUtils {
321    pub fn flat(
322        bits_per_value: u64,
323        values_compression: Option<metadata::BufferCompression>,
324    ) -> metadata::CompressiveEncoding {
325        metadata::CompressiveEncoding {
326            compression: Some(metadata::compressive_encoding::Compression::Flat(
327                metadata::Flat {
328                    bits_per_value,
329                    data: values_compression,
330                },
331            )),
332        }
333    }
334
335    pub fn constant(value: Option<bytes::Bytes>) -> metadata::CompressiveEncoding {
336        metadata::CompressiveEncoding {
337            compression: Some(metadata::compressive_encoding::Compression::Constant(
338                metadata::Constant { value },
339            )),
340        }
341    }
342
343    pub fn fsl(
344        items_per_value: u64,
345        has_validity: bool,
346        values: metadata::CompressiveEncoding,
347    ) -> metadata::CompressiveEncoding {
348        metadata::CompressiveEncoding {
349            compression: Some(metadata::compressive_encoding::Compression::FixedSizeList(
350                Box::new(metadata::FixedSizeList {
351                    items_per_value,
352                    has_validity,
353                    values: Some(Box::new(values)),
354                }),
355            )),
356        }
357    }
358
359    pub fn variable(
360        offsets_desc: metadata::CompressiveEncoding,
361        values_compression: Option<metadata::BufferCompression>,
362    ) -> metadata::CompressiveEncoding {
363        metadata::CompressiveEncoding {
364            compression: Some(metadata::compressive_encoding::Compression::Variable(
365                Box::new(metadata::Variable {
366                    offsets: Some(Box::new(offsets_desc)),
367                    values: values_compression,
368                }),
369            )),
370        }
371    }
372
373    pub fn inline_bitpacking(
374        uncompressed_bits_per_value: u64,
375        values_compression: Option<metadata::BufferCompression>,
376    ) -> metadata::CompressiveEncoding {
377        metadata::CompressiveEncoding {
378            compression: Some(
379                metadata::compressive_encoding::Compression::InlineBitpacking(
380                    metadata::InlineBitpacking {
381                        uncompressed_bits_per_value,
382                        values: values_compression,
383                    },
384                ),
385            ),
386        }
387    }
388
389    pub fn out_of_line_bitpacking(
390        uncompressed_bits_per_value: u64,
391        values: metadata::CompressiveEncoding,
392    ) -> metadata::CompressiveEncoding {
393        metadata::CompressiveEncoding {
394            compression: Some(
395                metadata::compressive_encoding::Compression::OutOfLineBitpacking(Box::new(
396                    metadata::OutOfLineBitpacking {
397                        uncompressed_bits_per_value,
398                        values: Some(Box::new(values)),
399                    },
400                )),
401            ),
402        }
403    }
404
405    pub fn buffer_compression(
406        compression: CompressionConfig,
407    ) -> Result<metadata::BufferCompression> {
408        Ok(metadata::BufferCompression {
409            scheme: metadata::CompressionScheme::try_from(compression.scheme)? as i32,
410            level: compression.level,
411        })
412    }
413
414    pub fn wrapped(
415        compression: CompressionConfig,
416        values: metadata::CompressiveEncoding,
417    ) -> Result<metadata::CompressiveEncoding> {
418        Ok(metadata::CompressiveEncoding {
419            compression: Some(metadata::compressive_encoding::Compression::General(
420                Box::new(metadata::General {
421                    compression: Some(Self::buffer_compression(compression)?),
422                    values: Some(Box::new(values)),
423                }),
424            )),
425        })
426    }
427
428    pub fn rle(
429        values: metadata::CompressiveEncoding,
430        run_lengths: metadata::CompressiveEncoding,
431    ) -> metadata::CompressiveEncoding {
432        metadata::CompressiveEncoding {
433            compression: Some(metadata::compressive_encoding::Compression::Rle(Box::new(
434                metadata::Rle {
435                    values: Some(Box::new(values)),
436                    run_lengths: Some(Box::new(run_lengths)),
437                },
438            ))),
439        }
440    }
441
442    pub fn byte_stream_split(
443        values: metadata::CompressiveEncoding,
444    ) -> metadata::CompressiveEncoding {
445        metadata::CompressiveEncoding {
446            compression: Some(
447                metadata::compressive_encoding::Compression::ByteStreamSplit(Box::new(
448                    metadata::ByteStreamSplit {
449                        values: Some(Box::new(values)),
450                    },
451                )),
452            ),
453        }
454    }
455
456    pub fn fsst(
457        data: metadata::CompressiveEncoding,
458        symbol_table: Vec<u8>,
459    ) -> metadata::CompressiveEncoding {
460        metadata::CompressiveEncoding {
461            compression: Some(metadata::compressive_encoding::Compression::Fsst(Box::new(
462                metadata::Fsst {
463                    symbol_table: symbol_table.into(),
464                    values: Some(Box::new(data)),
465                },
466            ))),
467        }
468    }
469
470    fn def_inter_to_repdef_layer(def: DefinitionInterpretation) -> i32 {
471        match def {
472            DefinitionInterpretation::AllValidItem => {
473                metadata::RepDefLayer::RepdefAllValidItem as i32
474            }
475            DefinitionInterpretation::AllValidList => {
476                metadata::RepDefLayer::RepdefAllValidList as i32
477            }
478            DefinitionInterpretation::NullableItem => {
479                metadata::RepDefLayer::RepdefNullableItem as i32
480            }
481            DefinitionInterpretation::NullableList => {
482                metadata::RepDefLayer::RepdefNullableList as i32
483            }
484            DefinitionInterpretation::EmptyableList => {
485                metadata::RepDefLayer::RepdefEmptyableList as i32
486            }
487            DefinitionInterpretation::NullableAndEmptyableList => {
488                metadata::RepDefLayer::RepdefNullAndEmptyList as i32
489            }
490        }
491    }
492
493    pub fn repdef_layer_to_def_interp(layer: i32) -> DefinitionInterpretation {
494        let layer = metadata::RepDefLayer::try_from(layer).unwrap();
495        match layer {
496            metadata::RepDefLayer::RepdefAllValidItem => DefinitionInterpretation::AllValidItem,
497            metadata::RepDefLayer::RepdefAllValidList => DefinitionInterpretation::AllValidList,
498            metadata::RepDefLayer::RepdefNullableItem => DefinitionInterpretation::NullableItem,
499            metadata::RepDefLayer::RepdefNullableList => DefinitionInterpretation::NullableList,
500            metadata::RepDefLayer::RepdefEmptyableList => DefinitionInterpretation::EmptyableList,
501            metadata::RepDefLayer::RepdefNullAndEmptyList => {
502                DefinitionInterpretation::NullableAndEmptyableList
503            }
504            metadata::RepDefLayer::RepdefUnspecified => {
505                panic!("Unspecified repdef layer")
506            }
507        }
508    }
509
510    #[allow(clippy::too_many_arguments)]
511    pub fn miniblock_layout(
512        rep_encoding: Option<metadata::CompressiveEncoding>,
513        def_encoding: Option<metadata::CompressiveEncoding>,
514        value_encoding: metadata::CompressiveEncoding,
515        repetition_index_depth: u32,
516        num_buffers: u64,
517        dictionary_encoding: Option<(metadata::CompressiveEncoding, u64)>,
518        def_meaning: &[DefinitionInterpretation],
519        num_items: u64,
520        has_large_chunk: bool,
521    ) -> metadata::PageLayout {
522        assert!(!def_meaning.is_empty());
523        let (dictionary, num_dictionary_items) = dictionary_encoding
524            .map(|(d, i)| (Some(d), i))
525            .unwrap_or((None, 0));
526        metadata::PageLayout {
527            layout: Some(metadata::page_layout::Layout::MiniBlockLayout(
528                metadata::MiniBlockLayout {
529                    def_compression: def_encoding,
530                    rep_compression: rep_encoding,
531                    value_compression: Some(value_encoding),
532                    repetition_index_depth,
533                    num_buffers,
534                    dictionary,
535                    num_dictionary_items,
536                    layers: def_meaning
537                        .iter()
538                        .map(|&def| Self::def_inter_to_repdef_layer(def))
539                        .collect(),
540                    num_items,
541                    has_large_chunk,
542                },
543            )),
544        }
545    }
546
547    fn full_zip_layout(
548        bits_rep: u8,
549        bits_def: u8,
550        details: metadata::full_zip_layout::Details,
551        value_encoding: metadata::CompressiveEncoding,
552        def_meaning: &[DefinitionInterpretation],
553        num_items: u32,
554        num_visible_items: u32,
555    ) -> metadata::PageLayout {
556        metadata::PageLayout {
557            layout: Some(metadata::page_layout::Layout::FullZipLayout(
558                metadata::FullZipLayout {
559                    bits_rep: bits_rep as u32,
560                    bits_def: bits_def as u32,
561                    details: Some(details),
562                    value_compression: Some(value_encoding),
563                    num_items,
564                    num_visible_items,
565                    layers: def_meaning
566                        .iter()
567                        .map(|&def| Self::def_inter_to_repdef_layer(def))
568                        .collect(),
569                },
570            )),
571        }
572    }
573
574    pub fn fixed_full_zip_layout(
575        bits_rep: u8,
576        bits_def: u8,
577        bits_per_value: u32,
578        value_encoding: metadata::CompressiveEncoding,
579        def_meaning: &[DefinitionInterpretation],
580        num_items: u32,
581        num_visible_items: u32,
582    ) -> metadata::PageLayout {
583        Self::full_zip_layout(
584            bits_rep,
585            bits_def,
586            metadata::full_zip_layout::Details::BitsPerValue(bits_per_value),
587            value_encoding,
588            def_meaning,
589            num_items,
590            num_visible_items,
591        )
592    }
593
594    pub fn variable_full_zip_layout(
595        bits_rep: u8,
596        bits_def: u8,
597        bits_per_offset: u32,
598        value_encoding: metadata::CompressiveEncoding,
599        def_meaning: &[DefinitionInterpretation],
600        num_items: u32,
601        num_visible_items: u32,
602    ) -> metadata::PageLayout {
603        Self::full_zip_layout(
604            bits_rep,
605            bits_def,
606            metadata::full_zip_layout::Details::BitsPerOffset(bits_per_offset),
607            value_encoding,
608            def_meaning,
609            num_items,
610            num_visible_items,
611        )
612    }
613
614    pub fn blob_layout(
615        inner_layout: metadata::PageLayout,
616        def_meaning: &[DefinitionInterpretation],
617    ) -> metadata::PageLayout {
618        metadata::PageLayout {
619            layout: Some(metadata::page_layout::Layout::BlobLayout(Box::new(
620                metadata::BlobLayout {
621                    inner_layout: Some(Box::new(inner_layout)),
622                    layers: def_meaning
623                        .iter()
624                        .map(|&def| Self::def_inter_to_repdef_layer(def))
625                        .collect(),
626                },
627            ))),
628        }
629    }
630
631    pub fn range(
632        uncompressed_bits_per_value: u64,
633        start: u64,
634        step: u64,
635    ) -> metadata::CompressiveEncoding {
636        metadata::CompressiveEncoding {
637            compression: Some(metadata::compressive_encoding::Compression::Range(
638                metadata::Range {
639                    uncompressed_bits_per_value,
640                    start,
641                    step,
642                },
643            )),
644        }
645    }
646
647    pub fn delta(
648        uncompressed_bits_per_value: u64,
649        base: u64,
650        deltas: metadata::CompressiveEncoding,
651    ) -> metadata::CompressiveEncoding {
652        metadata::CompressiveEncoding {
653            compression: Some(metadata::compressive_encoding::Compression::Delta(
654                Box::new(metadata::Delta {
655                    uncompressed_bits_per_value,
656                    base,
657                    deltas: Some(Box::new(deltas)),
658                }),
659            )),
660        }
661    }
662
663    pub fn dictionary(
664        indices: metadata::CompressiveEncoding,
665        items: metadata::CompressiveEncoding,
666        num_dictionary_items: u32,
667    ) -> metadata::CompressiveEncoding {
668        metadata::CompressiveEncoding {
669            compression: Some(metadata::compressive_encoding::Compression::Dictionary(
670                Box::new(metadata::Dictionary {
671                    indices: Some(Box::new(indices)),
672                    items: Some(Box::new(items)),
673                    num_dictionary_items,
674                }),
675            )),
676        }
677    }
678
679    pub fn constant_layout(
680        def_meaning: &[DefinitionInterpretation],
681        inline_value: Option<Vec<u8>>,
682    ) -> metadata::PageLayout {
683        metadata::PageLayout {
684            layout: Some(metadata::page_layout::Layout::ConstantLayout(
685                metadata::ConstantLayout {
686                    inline_value: inline_value.map(bytes::Bytes::from),
687                    rep_compression: None,
688                    def_compression: None,
689                    num_rep_values: 0,
690                    num_def_values: 0,
691                    layers: def_meaning
692                        .iter()
693                        .map(|&def| Self::def_inter_to_repdef_layer(def))
694                        .collect(),
695                },
696            )),
697        }
698    }
699
700    pub fn compressed_all_null_constant_layout(
701        def_meaning: &[DefinitionInterpretation],
702        rep_compression: Option<metadata::CompressiveEncoding>,
703        def_compression: Option<metadata::CompressiveEncoding>,
704        num_rep_values: u64,
705        num_def_values: u64,
706    ) -> metadata::PageLayout {
707        metadata::PageLayout {
708            layout: Some(metadata::page_layout::Layout::ConstantLayout(
709                metadata::ConstantLayout {
710                    inline_value: None,
711                    rep_compression,
712                    def_compression,
713                    num_rep_values,
714                    num_def_values,
715                    layers: def_meaning
716                        .iter()
717                        .map(|&def| Self::def_inter_to_repdef_layer(def))
718                        .collect(),
719                },
720            )),
721        }
722    }
723
724    pub fn packed_struct(
725        values: metadata::CompressiveEncoding,
726        bits_per_values: Vec<u64>,
727    ) -> metadata::CompressiveEncoding {
728        metadata::CompressiveEncoding {
729            compression: Some(metadata::compressive_encoding::Compression::PackedStruct(
730                Box::new(metadata::PackedStruct {
731                    bits_per_value: bits_per_values,
732                    values: Some(Box::new(values)),
733                }),
734            )),
735        }
736    }
737
738    pub fn packed_struct_variable(
739        fields: Vec<metadata::variable_packed_struct::FieldEncoding>,
740    ) -> metadata::CompressiveEncoding {
741        metadata::CompressiveEncoding {
742            compression: Some(
743                metadata::compressive_encoding::Compression::VariablePackedStruct(
744                    metadata::VariablePackedStruct { fields },
745                ),
746            ),
747        }
748    }
749
750    pub fn packed_struct_field_fixed(
751        value_encoding: metadata::CompressiveEncoding,
752        bits_per_value: u64,
753    ) -> metadata::variable_packed_struct::FieldEncoding {
754        metadata::variable_packed_struct::FieldEncoding {
755            value: Some(value_encoding),
756            layout: Some(
757                metadata::variable_packed_struct::field_encoding::Layout::BitsPerValue(
758                    bits_per_value,
759                ),
760            ),
761        }
762    }
763
764    pub fn packed_struct_field_variable(
765        value_encoding: metadata::CompressiveEncoding,
766        bits_per_length: u64,
767    ) -> metadata::variable_packed_struct::FieldEncoding {
768        metadata::variable_packed_struct::FieldEncoding {
769            value: Some(value_encoding),
770            layout: Some(
771                metadata::variable_packed_struct::field_encoding::Layout::BitsPerLength(
772                    bits_per_length,
773                ),
774            ),
775        }
776    }
777}