vortex_btrblocks/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4#![deny(missing_docs)]
5
6//! Vortex's [BtrBlocks]-inspired adaptive compression framework.
7//!
8//! This crate provides a sophisticated multi-level compression system that adaptively selects
9//! optimal compression schemes based on data characteristics. The compressor analyzes arrays
10//! to determine the best encoding strategy, supporting cascaded compression with multiple
11//! encoding layers for maximum efficiency.
12//!
13//! # Key Features
14//!
15//! - **Adaptive Compression**: Automatically selects the best compression scheme based on data
16//!   patterns.
17//! - **Unified Scheme Trait**: A single [`Scheme`] trait covers all data types (integers, floats,
18//!   strings, etc.) with a [`SchemeId`] for identity.
19//! - **Cascaded Encoding**: Multiple compression layers can be applied for optimal results.
20//! - **Statistical Analysis**: Uses data sampling and statistics to predict compression ratios.
21//! - **Recursive Structure Handling**: Compresses nested structures like structs and lists.
22//!
23//! # How It Works
24//!
25//! [`BtrBlocksCompressor::compress()`] takes an `&ArrayRef` plus a mutable execution context and
26//! returns an `ArrayRef` that may use a different encoding. It first canonicalizes the input, then dispatches by type.
27//! Primitives and strings go through `choose_and_compress`, which evaluates every enabled
28//! [`Scheme`] and picks the one with the best compression ratio. Compound types like structs
29//! and lists recurse into their fields and elements.
30//!
31//! Each `Scheme` implementation declares whether it [`matches`](Scheme::matches) a given
32//! canonical form and, if so, estimates the compression ratio (often by compressing a ~1%
33//! sample). The available schemes are registered on the session's [`CompressionSession`], and
34//! [`BtrBlocksCompressorBuilder::from_session`] builds a compressor from them.
35//!
36//! Schemes can produce arrays that are themselves further compressed (e.g. FoR then BitPacking),
37//! up to [`MAX_CASCADE`] (3) layers deep. Descendant exclusion rules for of [`SchemeId`] prevents
38//! the same scheme from being applied twice in a chain.
39//!
40//! # Example
41//!
42//! ```rust
43//! use vortex_array::{IntoArray, VortexSessionExecute, array_session};
44//! use vortex_array::arrays::PrimitiveArray;
45//! use vortex_array::validity::Validity;
46//! use vortex_btrblocks::{BtrBlocksCompressor, BtrBlocksCompressorBuilder, Scheme, SchemeExt};
47//! use vortex_btrblocks::schemes::integer::IntDictScheme;
48//! use vortex_buffer::buffer;
49//!
50//! # fn example() -> vortex_error::VortexResult<()> {
51//! let session = array_session();
52//! let array = PrimitiveArray::new(buffer![42u64; 1024], Validity::NonNullable).into_array();
53//!
54//! // Only schemes producing encodings the session registers and its enabled editions permit
55//! // are kept, so a session with no enabled editions leaves the array uncompressed.
56//! let compressor = BtrBlocksCompressor::from_session(&session);
57//! let compressed = compressor.compress(&array, &mut session.create_execution_ctx())?;
58//! assert_eq!(compressed.dtype(), array.dtype());
59//!
60//! // Remove specific schemes using the builder.
61//! let compressor = BtrBlocksCompressorBuilder::from_session(&session)
62//!     .exclude_schemes([IntDictScheme.id()])
63//!     .build();
64//! # let _ = compressor;
65//! # Ok(())
66//! # }
67//! ```
68//!
69//! [BtrBlocks]: https://www.cs.cit.tum.de/fileadmin/w00cfj/dis/papers/btrblocks.pdf
70
71mod allowed_ids;
72mod builder;
73mod canonical_compressor;
74/// Compression scheme implementations.
75pub mod schemes;
76mod session;
77
78// Re-export framework types from vortex-compressor for backwards compatibility.
79// Btrblocks-specific exports.
80pub use builder::BtrBlocksCompressorBuilder;
81pub use canonical_compressor::BtrBlocksCompressor;
82pub use schemes::patches::compress_patches;
83pub use session::CompressionSession;
84pub use session::CompressionSessionExt;
85pub use vortex_compressor::CascadingCompressor;
86pub use vortex_compressor::scheme::CompressorContext;
87pub use vortex_compressor::scheme::MAX_CASCADE;
88pub use vortex_compressor::scheme::Scheme;
89pub use vortex_compressor::scheme::SchemeExt;
90pub use vortex_compressor::scheme::SchemeId;
91pub use vortex_compressor::stats::ArrayAndStats;
92pub use vortex_compressor::stats::BoolStats;
93pub use vortex_compressor::stats::FloatStats;
94pub use vortex_compressor::stats::GenerateStatsOptions;
95pub use vortex_compressor::stats::IntegerStats;
96pub use vortex_compressor::stats::StringStats;
97
98#[cfg(test)]
99static SESSION: std::sync::LazyLock<vortex_session::VortexSession> =
100    std::sync::LazyLock::new(vortex_array::array_session);
101
102#[cfg(test)]
103mod tests;
vortex_btrblocks/lib.rs

vortex_btrblocks/
lib.rs
