vortex_pco/lib.rs
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: Copyright the Vortex contributors
3
4//! Pco-backed numeric compression encoding for Vortex arrays.
5//!
6//! [`PcoArray`] stores valid primitive numeric values in Pco chunks and pages, while Vortex
7//! validity tracks null rows separately. Page metadata lets slices decompress only the components
8//! required for the requested row range.
9//!
10//! Pco supports integer and floating-point primitive dtypes handled by the upstream `pco` crate.
11//! It is normally selected through the BtrBlocks compressor when the `pco` feature is enabled.
12//! To deserialize arrays manually, register the encoding in the array session:
13//!
14//! ```rust
15//! use vortex_array::session::ArraySessionExt;
16//!
17//! let session = vortex_array::array_session();
18//! session.arrays().register(vortex_pco::Pco);
19//! ```
20
21mod array;
22mod compute;
23mod rules;
24mod slice;
25
26pub use array::*;
27
28#[derive(Clone, prost::Message)]
29/// Metadata for one Pco page.
30pub struct PcoPageInfo {
31 // Since pco limits to 2^24 values per chunk, u32 is sufficient for the
32 // count of values.
33 /// Number of valid primitive values stored in this page.
34 #[prost(uint32, tag = "1")]
35 pub n_values: u32,
36}
37
38// We're calling this Info instead of Metadata because ChunkMeta refers to a specific
39// component of a Pco file.
40#[derive(Clone, prost::Message)]
41/// Metadata for one Pco chunk.
42pub struct PcoChunkInfo {
43 /// Pages contained in this chunk.
44 #[prost(message, repeated, tag = "1")]
45 pub pages: Vec<PcoPageInfo>,
46}
47
48#[derive(Clone, prost::Message)]
49/// Serialized metadata for a [`PcoArray`].
50pub struct PcoMetadata {
51 // would be nice to reuse one header per vortex file, but it's really only 1 byte, so
52 // no issue duplicating it here per PcoArray
53 /// Pco file header bytes.
54 #[prost(bytes, tag = "1")]
55 pub header: Vec<u8>,
56 /// Metadata for each compressed chunk.
57 #[prost(message, repeated, tag = "2")]
58 pub chunks: Vec<PcoChunkInfo>,
59}
60
61#[cfg(test)]
62mod tests;