nodedb_codec/
lib.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! Compression codecs for NodeDB timeseries columnar storage.
4//!
5//! Provides per-column codec selection with **cascading compression**:
6//! type-aware encoding (ALP, FastLanes, FSST, Pcodec) followed by a terminal
7//! byte compressor (lz4_flex for hot/warm, rANS for cold/S3).
8//!
9//! Cascading chains (hot/warm — lz4 terminal):
10//! - `AlpFastLanesLz4`:   f64 metrics → ALP → FastLanes → lz4
11//! - `DeltaFastLanesLz4`: i64 timestamps/counters → Delta → FastLanes → lz4
12//! - `FastLanesLz4`:      i64 raw integers → FastLanes → lz4
13//! - `FsstLz4`:           strings/logs → FSST → lz4
14//! - `PcodecLz4`:         complex numerics → Pcodec → lz4
15//! - `AlpRdLz4`:          true doubles → ALP-RD → lz4
16//!
17//! Cold/S3 tier chains (rANS terminal):
18//! - `AlpFastLanesRans`, `DeltaFastLanesRans`, `FsstRans`
19//!
20//! Shared by Origin and Lite. Compiles to WASM.
21
22pub mod alp;
23pub mod alp_rd;
24mod bounds;
25pub mod codec_types;
26pub mod crdt_compress;
27pub mod delta;
28pub mod detect;
29pub mod double_delta;
30pub mod error;
31pub mod fastlanes;
32pub mod fsst;
33pub mod gorilla;
34pub mod lz4;
35pub mod pcodec;
36pub mod pipeline;
37pub mod rans;
38pub mod raw;
39pub mod spherical;
40pub mod vector_quant;
41pub mod zstd_codec;
42
43/// Number of values to sample for codec auto-detection and exponent selection.
44/// Used by ALP, ALP-RD, and the codec detector.
45pub const CODEC_SAMPLE_SIZE: usize = 1024;
46
47pub use codec_types::{
48    ColumnCodec, ColumnStatistics, ColumnTypeHint, ResolvedColumnCodec, parse_codec_name,
49};
50pub use crdt_compress::CrdtOp;
51pub use delta::{DeltaDecoder, DeltaEncoder};
52pub use detect::detect_codec;
53pub use double_delta::{DoubleDeltaDecoder, DoubleDeltaEncoder};
54pub use error::CodecError;
55pub use gorilla::{GorillaDecoder, GorillaEncoder};
56pub use lz4::{Lz4Decoder, Lz4Encoder};
57pub use pipeline::{
58    decode_bytes_pipeline, decode_f64_pipeline, decode_i64_pipeline, encode_bytes_pipeline,
59    encode_f64_pipeline, encode_i64_pipeline,
60};
61pub use raw::{RawDecoder, RawEncoder};
62pub use zstd_codec::{ZstdDecoder, ZstdEncoder};
63
64#[cfg(test)]
65mod tests {
66    use super::*;
67
68    /// Frozen canonical codec name surface. Locks the lowercase, snake_case
69    /// forms before any user DDL exposes them. Adding a codec means appending
70    /// here and to `as_str()`; renaming any existing entry is a wire break.
71    #[test]
72    fn canonical_codec_names_frozen() {
73        let canonical: &[(ColumnCodec, &str)] = &[
74            (ColumnCodec::Auto, "auto"),
75            (ColumnCodec::AlpFastLanesLz4, "alp_fastlanes_lz4"),
76            (ColumnCodec::AlpRdLz4, "alp_rd_lz4"),
77            (ColumnCodec::PcodecLz4, "pcodec_lz4"),
78            (ColumnCodec::DeltaFastLanesLz4, "delta_fastlanes_lz4"),
79            (ColumnCodec::FastLanesLz4, "fastlanes_lz4"),
80            (ColumnCodec::FsstLz4, "fsst_lz4"),
81            (ColumnCodec::AlpFastLanesRans, "alp_fastlanes_rans"),
82            (ColumnCodec::DeltaFastLanesRans, "delta_fastlanes_rans"),
83            (ColumnCodec::FsstRans, "fsst_rans"),
84            (ColumnCodec::Gorilla, "gorilla"),
85            (ColumnCodec::DoubleDelta, "double_delta"),
86            (ColumnCodec::Delta, "delta"),
87            (ColumnCodec::Lz4, "lz4"),
88            (ColumnCodec::Zstd, "zstd"),
89            (ColumnCodec::Raw, "raw"),
90        ];
91        for (codec, expected) in canonical {
92            assert_eq!(codec.as_str(), *expected, "codec name drift: {codec:?}");
93            assert!(
94                expected
95                    .chars()
96                    .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_'),
97                "codec name '{expected}' is not lowercase snake_case"
98            );
99        }
100    }
101
102    // ── ResolvedColumnCodec tests ──────────────────────────────────────────────
103
104    /// Discriminants of ResolvedColumnCodec must exactly match those of the
105    /// corresponding ColumnCodec variants so on-disk byte values are unchanged.
106    #[test]
107    fn resolved_codec_discriminants_match_column_codec() {
108        let pairs: &[(ResolvedColumnCodec, ColumnCodec)] = &[
109            (
110                ResolvedColumnCodec::AlpFastLanesLz4,
111                ColumnCodec::AlpFastLanesLz4,
112            ),
113            (ResolvedColumnCodec::AlpRdLz4, ColumnCodec::AlpRdLz4),
114            (ResolvedColumnCodec::PcodecLz4, ColumnCodec::PcodecLz4),
115            (
116                ResolvedColumnCodec::DeltaFastLanesLz4,
117                ColumnCodec::DeltaFastLanesLz4,
118            ),
119            (ResolvedColumnCodec::FastLanesLz4, ColumnCodec::FastLanesLz4),
120            (ResolvedColumnCodec::FsstLz4, ColumnCodec::FsstLz4),
121            (
122                ResolvedColumnCodec::AlpFastLanesRans,
123                ColumnCodec::AlpFastLanesRans,
124            ),
125            (
126                ResolvedColumnCodec::DeltaFastLanesRans,
127                ColumnCodec::DeltaFastLanesRans,
128            ),
129            (ResolvedColumnCodec::FsstRans, ColumnCodec::FsstRans),
130            (ResolvedColumnCodec::Gorilla, ColumnCodec::Gorilla),
131            (ResolvedColumnCodec::DoubleDelta, ColumnCodec::DoubleDelta),
132            (ResolvedColumnCodec::Delta, ColumnCodec::Delta),
133            (ResolvedColumnCodec::Lz4, ColumnCodec::Lz4),
134            (ResolvedColumnCodec::Zstd, ColumnCodec::Zstd),
135            (ResolvedColumnCodec::Raw, ColumnCodec::Raw),
136        ];
137
138        for &(resolved, column) in pairs {
139            let resolved_bytes = zerompk::to_msgpack_vec(&resolved).unwrap();
140            let column_bytes = zerompk::to_msgpack_vec(&column).unwrap();
141            assert_eq!(
142                resolved_bytes, column_bytes,
143                "discriminant mismatch for {resolved} vs {column}"
144            );
145
146            assert_eq!(
147                resolved.into_column_codec(),
148                column,
149                "into_column_codec mismatch for {resolved}"
150            );
151        }
152    }
153
154    /// Auto resolves to an error; all concrete variants resolve successfully.
155    #[test]
156    fn try_resolve_auto_returns_error() {
157        assert!(
158            matches!(
159                ColumnCodec::Auto.try_resolve(),
160                Err(crate::error::CodecError::UnresolvedAuto)
161            ),
162            "Auto.try_resolve() must return UnresolvedAuto error"
163        );
164    }
165
166    #[test]
167    fn try_resolve_concrete_succeeds() {
168        let concretes = [
169            ColumnCodec::AlpFastLanesLz4,
170            ColumnCodec::Gorilla,
171            ColumnCodec::Delta,
172            ColumnCodec::Raw,
173            ColumnCodec::Lz4,
174        ];
175        for codec in concretes {
176            assert!(
177                codec.try_resolve().is_ok(),
178                "{codec} should resolve successfully"
179            );
180        }
181    }
182
183    #[test]
184    fn resolved_codec_serde_roundtrip() {
185        for codec in [
186            ResolvedColumnCodec::AlpFastLanesLz4,
187            ResolvedColumnCodec::AlpRdLz4,
188            ResolvedColumnCodec::PcodecLz4,
189            ResolvedColumnCodec::DeltaFastLanesLz4,
190            ResolvedColumnCodec::FastLanesLz4,
191            ResolvedColumnCodec::FsstLz4,
192            ResolvedColumnCodec::AlpFastLanesRans,
193            ResolvedColumnCodec::DeltaFastLanesRans,
194            ResolvedColumnCodec::FsstRans,
195            ResolvedColumnCodec::Gorilla,
196            ResolvedColumnCodec::DoubleDelta,
197            ResolvedColumnCodec::Delta,
198            ResolvedColumnCodec::Lz4,
199            ResolvedColumnCodec::Zstd,
200            ResolvedColumnCodec::Raw,
201        ] {
202            let json = sonic_rs::to_string(&codec).unwrap();
203            let back: ResolvedColumnCodec = sonic_rs::from_str(&json).unwrap();
204            assert_eq!(codec, back, "serde roundtrip failed for {codec}");
205        }
206    }
207
208    #[test]
209    fn column_codec_serde_roundtrip() {
210        for codec in [
211            ColumnCodec::Auto,
212            ColumnCodec::AlpFastLanesLz4,
213            ColumnCodec::AlpRdLz4,
214            ColumnCodec::PcodecLz4,
215            ColumnCodec::DeltaFastLanesLz4,
216            ColumnCodec::FastLanesLz4,
217            ColumnCodec::FsstLz4,
218            ColumnCodec::AlpFastLanesRans,
219            ColumnCodec::DeltaFastLanesRans,
220            ColumnCodec::FsstRans,
221            ColumnCodec::Gorilla,
222            ColumnCodec::DoubleDelta,
223            ColumnCodec::Delta,
224            ColumnCodec::Lz4,
225            ColumnCodec::Zstd,
226            ColumnCodec::Raw,
227        ] {
228            let json = sonic_rs::to_string(&codec).unwrap();
229            let back: ColumnCodec = sonic_rs::from_str(&json).unwrap();
230            assert_eq!(codec, back, "serde roundtrip failed for {codec}");
231        }
232    }
233
234    #[test]
235    fn column_statistics_i64() {
236        let values = vec![10i64, 20, 30, 40, 50];
237        let stats = ColumnStatistics::from_i64(&values, ResolvedColumnCodec::Delta, 12);
238        assert_eq!(stats.count, 5);
239        assert_eq!(stats.min, Some(10.0));
240        assert_eq!(stats.max, Some(50.0));
241        assert_eq!(stats.sum, Some(150.0));
242        assert_eq!(stats.uncompressed_bytes, 40);
243        assert_eq!(stats.compressed_bytes, 12);
244    }
245
246    #[test]
247    fn column_statistics_f64() {
248        let values = vec![1.5f64, 2.5, 3.5];
249        let stats = ColumnStatistics::from_f64(&values, ResolvedColumnCodec::Gorilla, 8);
250        assert_eq!(stats.count, 3);
251        assert_eq!(stats.min, Some(1.5));
252        assert_eq!(stats.max, Some(3.5));
253        assert_eq!(stats.sum, Some(7.5));
254    }
255
256    #[test]
257    fn column_statistics_symbols() {
258        let values = vec![0u32, 1, 2, 0, 1];
259        let stats = ColumnStatistics::from_symbols(&values, 3, ResolvedColumnCodec::Raw, 20);
260        assert_eq!(stats.count, 5);
261        assert_eq!(stats.cardinality, Some(3));
262        assert!(stats.min.is_none());
263    }
264
265    #[test]
266    fn compression_ratio_calculation() {
267        let stats = ColumnStatistics {
268            codec: ResolvedColumnCodec::Delta,
269            count: 100,
270            min: None,
271            max: None,
272            sum: None,
273            cardinality: None,
274            compressed_bytes: 200,
275            uncompressed_bytes: 800,
276        };
277        assert!((stats.compression_ratio() - 4.0).abs() < f64::EPSILON);
278    }
279
280    // ── parse_codec_name snapshot tests ───────────────────────────────────────
281
282    #[test]
283    fn parse_codec_name_all_canonical_round_trip() {
284        let cases: &[(&str, ColumnCodec)] = &[
285            ("auto", ColumnCodec::Auto),
286            ("alp_fastlanes_lz4", ColumnCodec::AlpFastLanesLz4),
287            ("alp_rd_lz4", ColumnCodec::AlpRdLz4),
288            ("pcodec_lz4", ColumnCodec::PcodecLz4),
289            ("delta_fastlanes_lz4", ColumnCodec::DeltaFastLanesLz4),
290            ("fastlanes_lz4", ColumnCodec::FastLanesLz4),
291            ("fsst_lz4", ColumnCodec::FsstLz4),
292            ("alp_fastlanes_rans", ColumnCodec::AlpFastLanesRans),
293            ("delta_fastlanes_rans", ColumnCodec::DeltaFastLanesRans),
294            ("fsst_rans", ColumnCodec::FsstRans),
295            ("gorilla", ColumnCodec::Gorilla),
296            ("double_delta", ColumnCodec::DoubleDelta),
297            ("delta", ColumnCodec::Delta),
298            ("lz4", ColumnCodec::Lz4),
299            ("zstd", ColumnCodec::Zstd),
300            ("raw", ColumnCodec::Raw),
301        ];
302        for &(name, expected) in cases {
303            let parsed = parse_codec_name(name)
304                .unwrap_or_else(|e| panic!("parse_codec_name({name:?}) failed: {e}"));
305            assert_eq!(parsed, expected, "parse mismatch for {name:?}");
306            assert_eq!(
307                parsed.as_str(),
308                name,
309                "as_str() round-trip mismatch for {name:?}"
310            );
311        }
312        assert_eq!(
313            cases.len(),
314            16,
315            "variant count changed — update parse_codec_name"
316        );
317    }
318
319    #[test]
320    fn parse_codec_name_rejects_non_canonical() {
321        let bad: &[&str] = &[
322            "LZ4",
323            "Lz4",
324            "GORILLA",
325            "Gorilla",
326            "FastLanes",
327            "fast_lanes",
328            "fast-lanes",
329            "FSST",
330            "alp-fastlanes-lz4",
331            "ALP_FASTLANES_LZ4",
332            "Delta_FastLanes_LZ4",
333            "ZSTD",
334            "RAW",
335            "",
336            " lz4",
337            "lz4 ",
338            "unknown",
339            "pcodec",
340        ];
341        for &name in bad {
342            let result = parse_codec_name(name);
343            assert!(
344                result.is_err(),
345                "parse_codec_name({name:?}) should have been rejected but returned Ok"
346            );
347            let err = result.unwrap_err();
348            assert!(
349                matches!(err, crate::error::CodecError::UnknownCodec { .. }),
350                "wrong error variant for {name:?}: {err}"
351            );
352        }
353    }
354
355    #[test]
356    fn parse_codec_name_error_message_content() {
357        let err = parse_codec_name("BadCodec").unwrap_err();
358        let msg = err.to_string();
359        assert!(
360            msg.contains("BadCodec"),
361            "error message should contain the bad name: {msg}"
362        );
363        assert!(
364            msg.contains("lz4"),
365            "error message should list at least one valid name: {msg}"
366        );
367    }
368}
nodedb_codec/lib.rs

nodedb_codec/
lib.rs
