rusty_chunkenc/
lib.rs

1//!
2//! A Rust implementation of Prometheus' [`chunkenc`](https://pkg.go.dev/github.com/prometheus/prometheus/tsdb/chunkenc) library.
3//!
4//! ## Features
5//!
6//! - Parse Prometheus XOR-encoded chunks (that are heavily inspired by [Gorilla](https://www.vldb.org/pvldb/vol8/p1816-teller.pdf)).
7//! - Serialise time series to Prometheus XOR-encoded chunks.
8//! - Read Prometheus' cold data directly from the disk.
9//! - Also comes with utilities to read and write `varint`, `uvarint`, `varbit`, `varbit_ts`, and `varbit_xor` numbers.
10//!
11//! ## Why?
12//!
13//! Prometheus uses XOR Chunks in its remote read API, and I wanted to understand how they work in detail. This crate enables [SensApp](https://github.com/sintef/sensapp) to stream data to Prometheus. SensApp is written in Rust, and I wanted a chunkenc Rust implementation.
14//!
15//! Also, writing a parser and a serialiser did sound fun.
16//!
17//! ## Acknowledgements
18//!
19//! This project is ported from Prometheus' [`chunkenc`](https://pkg.go.dev/github.com/prometheus/prometheus/tsdb/chunkenc), that used [`go-tzs`](https://github.com/dgryski/go-tsz), that is based on the [Gorilla](https://www.vldb.org/pvldb/vol8/p1816-teller.pdf) paper. The parsing heavily relies on [`nom`](https://crates.io/crates/nom).
20//!
21//! The project supports the [Smart Building Hub](https://smartbuildinghub.no/) research infrastructure project, which is funded by the [Norwegian Research Council](https://www.forskningsradet.no/).
22//!
23//! ## Example
24//!
25//! ```rust
26//! let chunk_disk_format = rusty_chunkenc::ChunksDiskFormat::new(
27//! vec![
28//!     rusty_chunkenc::Chunk::new_xor(vec![
29//!         rusty_chunkenc::XORSample {
30//!             timestamp: 7200000,
31//!             value: 12000.0,
32//!         },
33//!         rusty_chunkenc::XORSample {
34//!             timestamp: 7201000,
35//!             value: 12001.0,
36//!         },
37//!     ]),
38//!     rusty_chunkenc::Chunk::new_xor(vec![
39//!         rusty_chunkenc::XORSample {
40//!             timestamp: 7200000,
41//!             value: 123.45,
42//!         },
43//!         rusty_chunkenc::XORSample {
44//!             timestamp: 7201000,
45//!             value: 123.46,
46//!         },
47//!     ]),
48//! ],
49//! None,
50//! );
51//!
52//! // Serialise the chunks
53//! let mut buffer: Vec<u8> = Vec::new();
54//! chunk_disk_format.write(&mut buffer).unwrap();
55//!
56//! // Parse a chunk from a buffer
57//! let (_, parsed_chunk_disk_format) = rusty_chunkenc::read_chunks(&buffer, None).unwrap();
58//! println!("parsed_chunks: {:?}", parsed_chunk_disk_format);
59//! assert_eq!(parsed_chunk_disk_format, chunk_disk_format);
60//! ```
61//!
62//! Or for a single chunk:
63//!
64//! ```rust
65//! let chunk = rusty_chunkenc::Chunk::new_xor(vec![
66//!     rusty_chunkenc::XORSample {
67//!         timestamp: 7200000,
68//!         value: 12000.0,
69//!     },
70//!     rusty_chunkenc::XORSample {
71//!         timestamp: 7201000,
72//!         value: 12001.0,
73//!     },
74//! ]);
75//!
76//! // Serialise the chunk
77//! let mut buffer: Vec<u8> = Vec::new();
78//! chunk.write(&mut buffer).unwrap();
79//!
80//! assert_eq!(
81//!     buffer,
82//!     [
83//!         0x12, 0x01, 0x00, 0x02, 0x80, 0xF4, 0xEE, 0x06, 0x40, 0xC7, 0x70, 0x00, 0x00, 0x00,
84//!         0x00, 0x00, 0xE8, 0x07, 0xF0, 0x0C, 0x1F, 0xCE, 0x4F, 0xA7
85//!     ]
86//! );
87//!
88//! // Parse a chunk from a buffer
89//! let (_, parsed_chunk) = rusty_chunkenc::read_chunk(&buffer).unwrap();
90//! println!("parsed_chunk: {:?}", parsed_chunk);
91//! ```
92
93/// Single Prometheus chunk.
94pub mod chunk;
95/// Prometheus chunks disk format.
96pub mod chunks;
97/// CRC32 Castagnoli Checksum.
98pub mod crc32c;
99mod encoder;
100mod errors;
101/// WIP: Parse all prometheus data from the prometheus folder.
102pub mod folder;
103/// Histogram and Float Histogram chunks, not implemented yet.
104pub mod histogram;
105/// WIP: Prometheus index files
106pub mod index;
107mod series;
108mod symbol_table;
109mod toc;
110/// Golang's uvarint.
111pub mod uvarint;
112/// Prometheus's varbit encoding.
113pub mod varbit;
114/// Prometheus's varbit timestamp encoding.
115pub mod varbit_ts;
116/// Prometheus's varbit xor encoding.
117pub mod varbit_xor;
118/// Golang's varint.
119pub mod varint;
120/// XOR chunk.
121pub mod xor;
122
123type NomBitInput<'a> = (&'a [u8], usize);
124
125// Re-exports
126pub use chunk::read_chunk;
127pub use chunk::Chunk;
128
129pub use chunks::read_chunks;
130pub use chunks::ChunksDiskFormat;
131
132pub use xor::XORSample;
rusty_chunkenc/lib.rs

rusty_chunkenc/
lib.rs
