vortex_compressor/lib.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4#![deny(missing_docs)]
5#![warn(clippy::missing_docs_in_private_items)]
6#![warn(clippy::missing_errors_doc)]
7#![warn(clippy::missing_panics_doc)]
8#![warn(clippy::missing_safety_doc)]
9
10//! Encoding-agnostic compression framework for Vortex arrays.
11//!
12//! This crate provides the core compression engine: the [`Scheme`](scheme::Scheme) trait,
13//! sampling-based ratio estimation, cascaded compression, and statistics infrastructure for
14//! deciding the best encoding scheme for an array.
15//!
16//! This crate contains no encoding dependencies. Batteries-included compressors are provided by
17//! downstream crates like `vortex-btrblocks`, which register different encodings to the compressor.
18//!
19//! # Observability
20//!
21//! The compressor emits a small set of `tracing` spans and events on a single target so you can
22//! see what it's doing without attaching a profiler.
23//!
24//! For example, set `RUST_LOG=vortex_compressor::encode=debug` to see compression decision spans
25//! and exceptional events. The `vortex_compressor::encode` target carries the top-level `compress`
26//! span, per-scheme evaluation and winning-compression spans, the `cascade_exhausted` event,
27//! `sample.result` events for zero-byte sample outputs, and both `*.compress_failed` events.
28//!
29//! The winning-compression span carries `scheme_chosen`, `input_nbytes`, `compressed_nbytes`,
30//! `estimated_ratio` (absent when the scheme returned `AlwaysUse` or sampled to 0 bytes),
31//! `achieved_ratio` (absent when the compressed output is 0 bytes), and `accepted`.
32//!
33//! Failure events additionally carry `cascade_path` and `cascade_depth`, so nested compression
34//! errors can be tied back to the ancestor branch that triggered them.
35//!
36//! From those fields you can derive per-scheme savings, rejection counts, and estimator accuracy
37//! with a short `jq` query.
38
39pub mod builtins;
40pub mod ctx;
41pub mod estimate;
42pub mod scheme;
43pub mod stats;
44
45mod sample;
46
47mod compressor;
48mod trace;
49pub use compressor::CascadingCompressor;