vortex_zstd/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Zstd-backed compression encodings for variable-width Vortex arrays.
5//!
6//! [`ZstdArray`] stores UTF-8 or binary values as one or more zstd frames, optionally sharing a
7//! trained dictionary across frames. Frame metadata lets slices decompress only the frames that can
8//! contribute values to the requested row range.
9//!
10//! With the `unstable_encodings` feature, `ZstdBuffers` stores the buffers of another encoding as
11//! independently compressed zstd buffers while preserving the inner encoding metadata.
12//!
13//! This crate exposes array encodings only. Compression scheme selection is wired through
14//! `vortex-btrblocks` and file writing. To deserialize arrays manually, register the encoding in the
15//! array session:
16//!
17//! ```rust
18//! use vortex_array::session::ArraySessionExt;
19//!
20//! let session = vortex_array::array_session();
21//! session.arrays().register(vortex_zstd::Zstd);
22//! ```
23
24pub use array::*;
25use vortex_array::dtype::proto::dtype as pb;
26#[cfg(feature = "unstable_encodings")]
27pub use zstd_buffers::*;
28
29mod array;
30mod compute;
31mod rules;
32mod slice;
33#[cfg(feature = "unstable_encodings")]
34mod zstd_buffers;
35
36#[cfg(test)]
37mod test;
38
39#[derive(Clone, prost::Message)]
40/// Metadata for one zstd frame.
41pub struct ZstdFrameMetadata {
42    /// Uncompressed byte size of this frame.
43    #[prost(uint64, tag = "1")]
44    pub uncompressed_size: u64,
45    /// Number of valid values stored in this frame.
46    #[prost(uint64, tag = "2")]
47    pub n_values: u64,
48}
49
50#[derive(Clone, prost::Message)]
51/// Serialized metadata for a [`ZstdArray`].
52pub struct ZstdMetadata {
53    // optional, will be 0 if there's no dictionary
54    /// Dictionary size in bytes, or `0` when no dictionary is present.
55    #[prost(uint32, tag = "1")]
56    pub dictionary_size: u32,
57    /// Metadata for each compressed frame.
58    #[prost(message, repeated, tag = "2")]
59    pub frames: Vec<ZstdFrameMetadata>,
60}
61
62#[derive(Clone, prost::Message)]
63/// Serialized metadata for the unstable `ZstdBuffers` encoding.
64pub struct ZstdBuffersMetadata {
65    /// Encoding id of the inner array whose buffers were compressed.
66    #[prost(string, tag = "1")]
67    pub inner_encoding_id: String,
68    /// Serialized metadata of the inner array.
69    #[prost(bytes = "vec", tag = "2")]
70    pub inner_metadata: Vec<u8>,
71    /// Uncompressed byte size of each compressed buffer.
72    #[prost(uint64, repeated, tag = "3")]
73    pub uncompressed_sizes: Vec<u64>,
74    /// Alignment of each buffer in bytes (must be a power of two).
75    #[prost(uint32, repeated, tag = "4")]
76    pub buffer_alignments: Vec<u32>,
77    /// DType of child arrays. Children belong to inner encodings, and their
78    /// dtypes don't persist after serialization, so we need to retrieve them
79    /// from metadata.
80    #[prost(message, repeated, tag = "5")]
81    pub child_dtypes: Vec<pb::DType>,
82    /// Length of each child array, ordered as "child_dtypes"
83    #[prost(uint64, repeated, tag = "6")]
84    pub child_lens: Vec<u64>,
85}
vortex_zstd/lib.rs

vortex_zstd/
lib.rs
