vortex_compressor/lib.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4#![deny(missing_docs)]
5#![warn(clippy::missing_docs_in_private_items)]
6#![warn(clippy::missing_errors_doc)]
7#![warn(clippy::missing_panics_doc)]
8#![warn(clippy::missing_safety_doc)]
9
10//! Encoding-agnostic compression framework for Vortex arrays.
11//!
12//! This crate provides the core compression engine: the [`Scheme`](scheme::Scheme) trait,
13//! sampling-based ratio estimation, cascaded compression, and statistics infrastructure for
14//! deciding the best encoding scheme for an array.
15//!
16//! This crate contains no encoding dependencies. Batteries-included compressors are provided by
17//! downstream crates like `vortex-btrblocks`, which register different encodings to the compressor.
18//!
19//! # Example
20//!
21//! A [`CascadingCompressor`] can be created directly with a fixed scheme list. With no schemes it
22//! still canonicalizes supported inputs, recursively handles nested structure, and encodes
23//! constant leaves (constant detection is built into the compressor), but no other leaf
24//! compression is selected.
25//!
26//! ```rust
27//! use vortex_array::{IntoArray, VortexSessionExecute, array_session};
28//! use vortex_array::arrays::PrimitiveArray;
29//! use vortex_array::validity::Validity;
30//! use vortex_buffer::buffer;
31//! use vortex_compressor::CascadingCompressor;
32//!
33//! # fn example() -> vortex_error::VortexResult<()> {
34//! let session = array_session();
35//! let array = PrimitiveArray::new(buffer![1i32, 2, 3], Validity::NonNullable).into_array();
36//! let compressor = CascadingCompressor::new(Vec::new());
37//!
38//! let result = compressor.compress(&array, &mut session.create_execution_ctx())?;
39//! assert_eq!(result.dtype(), array.dtype());
40//! assert_eq!(result.len(), array.len());
41//! # Ok(())
42//! # }
43//! ```
44//!
45//! # Observability
46//!
47//! The compressor emits a small set of `tracing` spans and events on a single target so you can
48//! see what it's doing without attaching a profiler.
49//!
50//! For example, set `RUST_LOG=vortex_compressor::encode=debug` to see compression decision spans
51//! and exceptional events. The `vortex_compressor::encode` target carries the top-level `compress`
52//! span, per-scheme evaluation and winning-compression spans, the `cascade_exhausted` event,
53//! `sample.result` events for zero-byte sample outputs, and both `*.compress_failed` events.
54//!
55//! The winning-compression span carries `scheme_chosen`, `input_nbytes`, `compressed_nbytes`,
56//! `estimated_ratio` (absent when the scheme returned `AlwaysUse` or sampled to 0 bytes),
57//! `achieved_ratio` (absent when the compressed output is 0 bytes), and `accepted`.
58//!
59//! Failure events additionally carry `cascade_path` and `cascade_depth`, so nested compression
60//! errors can be tied back to the ancestor branch that triggered them.
61//!
62//! From those fields you can derive per-scheme savings, rejection counts, and estimator accuracy
63//! with a short `jq` query.
64
65pub mod builtins;
66pub mod ctx;
67pub mod estimate;
68pub mod scheme;
69pub mod stats;
70
71mod constant;
72mod sample;
73
74mod compressor;
75mod trace;
76pub use compressor::CascadingCompressor;