vortex_compressor/lib.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4#![deny(missing_docs)]
5#![warn(clippy::missing_docs_in_private_items)]
6#![warn(clippy::missing_errors_doc)]
7#![warn(clippy::missing_panics_doc)]
8#![warn(clippy::missing_safety_doc)]
9
10//! Encoding-agnostic compression framework for Vortex arrays.
11//!
12//! This crate provides the core compression engine: the [`Scheme`](scheme::Scheme) trait,
13//! sampling-based ratio estimation, cascaded compression, and statistics infrastructure for
14//! deciding the best encoding scheme for an array.
15//!
16//! This crate contains no encoding dependencies. Batteries-included compressors are provided by
17//! downstream crates like `vortex-btrblocks`, which register different encodings to the compressor.
18//!
19//! # Example
20//!
21//! A [`CascadingCompressor`] can be created directly with a fixed scheme list. With no schemes it
22//! still canonicalizes supported inputs and recursively handles nested structure, but no leaf
23//! compression is selected.
24//!
25//! ```rust
26//! use vortex_array::{IntoArray, VortexSessionExecute, array_session};
27//! use vortex_array::arrays::PrimitiveArray;
28//! use vortex_array::validity::Validity;
29//! use vortex_buffer::buffer;
30//! use vortex_compressor::CascadingCompressor;
31//!
32//! # fn example() -> vortex_error::VortexResult<()> {
33//! let session = array_session();
34//! let array = PrimitiveArray::new(buffer![1i32, 2, 3], Validity::NonNullable).into_array();
35//! let compressor = CascadingCompressor::new(Vec::new());
36//!
37//! let result = compressor.compress(&array, &mut session.create_execution_ctx())?;
38//! assert_eq!(result.dtype(), array.dtype());
39//! assert_eq!(result.len(), array.len());
40//! # Ok(())
41//! # }
42//! ```
43//!
44//! # Observability
45//!
46//! The compressor emits a small set of `tracing` spans and events on a single target so you can
47//! see what it's doing without attaching a profiler.
48//!
49//! For example, set `RUST_LOG=vortex_compressor::encode=debug` to see compression decision spans
50//! and exceptional events. The `vortex_compressor::encode` target carries the top-level `compress`
51//! span, per-scheme evaluation and winning-compression spans, the `cascade_exhausted` event,
52//! `sample.result` events for zero-byte sample outputs, and both `*.compress_failed` events.
53//!
54//! The winning-compression span carries `scheme_chosen`, `input_nbytes`, `compressed_nbytes`,
55//! `estimated_ratio` (absent when the scheme returned `AlwaysUse` or sampled to 0 bytes),
56//! `achieved_ratio` (absent when the compressed output is 0 bytes), and `accepted`.
57//!
58//! Failure events additionally carry `cascade_path` and `cascade_depth`, so nested compression
59//! errors can be tied back to the ancestor branch that triggered them.
60//!
61//! From those fields you can derive per-scheme savings, rejection counts, and estimator accuracy
62//! with a short `jq` query.
63
64pub mod builtins;
65pub mod ctx;
66pub mod estimate;
67pub mod scheme;
68pub mod stats;
69
70mod sample;
71
72mod compressor;
73mod trace;
74pub use compressor::CascadingCompressor;