alopex_skulk/
lib.rs

1//! Skulk - Alopex Time Series Storage Engine
2//!
3//! This crate provides the core storage primitives for the Alopex Skulk time series database,
4//! implementing a high-performance columnar storage engine optimized for time series workloads.
5//!
6//! # Architecture Overview
7//!
8//! Skulk uses a Log-Structured Merge-tree (LSM) inspired architecture with time-based partitioning:
9//!
10//! ```text
11//! ┌─────────────────────────────────────────────────────────────────────┐
12//! │                          Write Path                                  │
13//! ├─────────────────────────────────────────────────────────────────────┤
14//! │  Client → WAL (durability) → MemTable (in-memory) → TSM (on-disk)   │
15//! └─────────────────────────────────────────────────────────────────────┘
16//! ```
17//!
18//! # Components
19//!
20//! - **[`tsm::TimeSeriesMemTable`]**: Time-partition aware in-memory buffer with
21//!   efficient series-based storage using BTreeMap for sorted timestamp access.
22//!
23//! - **[`tsm::PartitionManager`]**: Routes data points to appropriate time partitions
24//!   and manages flush lifecycle with oldest-first policy.
25//!
26//! - **[`tsm::CompressedBlock`]**: Gorilla compression achieving 10-15x compression
27//!   ratios for typical time series data through delta-of-delta timestamps and
28//!   XOR-based value encoding.
29//!
30//! - **[`tsm::TsmWriter`] / [`tsm::TsmReader`]**: File I/O for the TSM (Time Series Merge)
31//!   format with CRC32 integrity verification and Bloom filter for efficient lookups.
32//!
33//! - **[`wal::Wal`]**: Write-Ahead Log for crash recovery with configurable sync modes
34//!   (None, Fsync, Fdatasync).
35//!
36//! # File Format (TSM)
37//!
38//! ```text
39//! ┌─────────────────────────────────────────────────────────────────────┐
40//! │  File Header (32 bytes)                                              │
41//! │  - Magic: "ATSM", Version: 2, Timestamps, Series Count, Flags       │
42//! ├─────────────────────────────────────────────────────────────────────┤
43//! │  Data Blocks (per series)                                            │
44//! │  - Series ID, Gorilla-compressed timestamps & values, Block CRC     │
45//! ├─────────────────────────────────────────────────────────────────────┤
46//! │  Series Index with Bloom Filter                                      │
47//! │  - Sorted index entries with block offsets and metadata             │
48//! ├─────────────────────────────────────────────────────────────────────┤
49//! │  Footer (48 bytes)                                                   │
50//! │  - Offsets, sizes, file CRC32, reverse magic "MSTA"                 │
51//! └─────────────────────────────────────────────────────────────────────┘
52//! ```
53//!
54//! # Durability Guarantees
55//!
56//! The write path follows a strict durability contract:
57//!
58//! 1. **WAL First**: Data is written to WAL and fsync'd before MemTable mutation
59//! 2. **Atomic Flush**: MemTable flush uses tmp→fsync→rename→fsync pattern
60//! 3. **Crash Recovery**: All committed data can be recovered from WAL
61//!
62//! # Example: Basic Usage
63//!
64//! ```rust,ignore
65//! use skulk::tsm::{TimeSeriesMemTable, TimePartition, DataPoint};
66//! use skulk::wal::{Wal, WalConfig, SyncMode};
67//! use std::time::Duration;
68//!
69//! // Create WAL for durability
70//! let wal_config = WalConfig {
71//!     batch_size: 100,
72//!     batch_timeout: Duration::from_millis(10),
73//!     segment_size: 64 * 1024 * 1024,
74//!     sync_mode: SyncMode::Fsync,
75//! };
76//! let mut wal = Wal::new("./wal", wal_config)?;
77//!
78//! // Create MemTable for a 1-hour partition
79//! let partition = TimePartition::new(0, Duration::from_secs(3600));
80//! let mut memtable = TimeSeriesMemTable::new(partition);
81//!
82//! // Insert with WAL durability
83//! let point = DataPoint::new(
84//!     "cpu.usage",
85//!     vec![("host".to_string(), "server1".to_string())],
86//!     1234567890_000_000_000, // nanoseconds
87//!     0.75,
88//! );
89//! memtable.insert_with_wal(&point, &mut wal)?;
90//!
91//! // Flush to TSM file
92//! let handle = memtable.flush("./data/partition.skulk")?;
93//! println!("Wrote {} points", handle.footer.total_point_count);
94//! ```
95//!
96//! # Example: Multi-Partition Management
97//!
98//! ```rust,ignore
99//! use skulk::tsm::{PartitionManager, PartitionManagerConfig, DataPoint, TimeRange};
100//! use std::time::Duration;
101//!
102//! // Configure partition manager
103//! let config = PartitionManagerConfig::default()
104//!     .with_partition_duration(Duration::from_secs(3600))
105//!     .with_size_threshold(64 * 1024 * 1024)
106//!     .with_max_active_partitions(4);
107//!
108//! let mut manager = PartitionManager::new(config);
109//!
110//! // Insert data - automatically routed to correct partition
111//! for i in 0..10000 {
112//!     let point = DataPoint::new("metric", vec![], i * 1_000_000, i as f64);
113//!     manager.insert(&point)?;
114//! }
115//!
116//! // Scan across partitions
117//! let range = TimeRange::new(0, 5000 * 1_000_000);
118//! for point in manager.scan(range) {
119//!     println!("{}: {}", point.timestamp, point.value);
120//! }
121//!
122//! // Flush oldest partition when threshold reached
123//! if let Some(handle) = manager.flush_oldest("./data")? {
124//!     println!("Flushed partition to {:?}", handle.path);
125//! }
126//! ```
127//!
128//! # Example: Crash Recovery
129//!
130//! ```rust,ignore
131//! use skulk::wal::Wal;
132//! use skulk::tsm::{TimeSeriesMemTable, TimePartition, DataPoint};
133//! use std::time::Duration;
134//!
135//! // Recover entries from WAL after crash
136//! let recovered_entries = Wal::recover("./wal")?;
137//! println!("Recovered {} entries", recovered_entries.len());
138//!
139//! // Reconstruct MemTable from recovered entries
140//! let partition = TimePartition::new(0, Duration::from_secs(3600));
141//! let mut memtable = TimeSeriesMemTable::new(partition);
142//!
143//! for entry in recovered_entries {
144//!     if let WalEntry::DataPoint { timestamp, value, .. } = entry {
145//!         // Rebuild DataPoint from entry (requires metadata lookup)
146//!         let point = DataPoint::new(
147//!             "recovered_metric", // from metadata store
148//!             vec![],
149//!             timestamp,
150//!             value,
151//!         );
152//!         memtable.insert(&point)?;
153//!     }
154//! }
155//! ```
156//!
157//! # Performance Characteristics
158//!
159//! - **Compression**: 10-15x ratio with Gorilla encoding
160//! - **Write Latency**: <1ms for buffered writes, ~10ms with fsync
161//! - **Scan Throughput**: >1M points/sec for sequential reads
162//! - **Memory Usage**: ~16 bytes per point in MemTable
163//!
164//! # Feature Flags
165//!
166//! Currently no optional features. All functionality is included by default.
167
168#![deny(missing_docs)]
169#![warn(rustdoc::missing_crate_level_docs)]
170
171pub mod error;
172pub mod lifecycle;
173pub mod tsm;
174pub mod wal;
175
176pub use error::{Result, TsmError};
177pub use tsm::{
178    CompressedBlock, DataPoint, SeriesId, SeriesMeta, TimePartition, TimeRange, Timestamp,
179};
180pub use wal::{SyncMode, Wal, WalConfig, WalEntry};
alopex_skulk/lib.rs

alopex_skulk/
lib.rs
