laminar-storage 0.19.1

Storage layer for LaminarDB - WAL, checkpointing, and lakehouse integration
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

//! # `LaminarDB` Storage
//!
//! Durability layer for `LaminarDB` - WAL, checkpointing, and recovery.
//!
//! ## Module Overview
//!
//! - [`wal`]: Write-ahead log for durability and exactly-once semantics
//! - [`checkpoint`]: Basic checkpointing for fast recovery
//! - [`incremental`]: Incremental checkpointing
//!
//! **Note:** Lakehouse sinks (Delta Lake, Iceberg) are in `laminar-connectors` crate,
//! not here. This crate handles `LaminarDB`'s internal durability, not external storage formats.

#![deny(missing_docs)]
#![warn(clippy::all, clippy::pedantic)]
#![allow(clippy::disallowed_types)] // cold path: all storage modules (WAL, checkpoint, recovery) are off hot path

/// Write-ahead log implementation - WAL for durability and exactly-once semantics
pub mod wal;

/// Checkpointing for fast recovery
pub mod checkpoint;

/// Unified checkpoint manifest types
pub mod checkpoint_manifest;

/// Checkpoint persistence trait and filesystem store
pub mod checkpoint_store;

/// S3 storage class tiering for cost optimization
pub mod tiering;

/// Ring 1 changelog drainer
pub mod changelog_drainer;

/// Incremental checkpointing - Directory-based checkpoint architecture
pub mod incremental;

/// Object store factory — builds S3, GCS, Azure, or local backends from URL schemes.
pub mod object_store_builder;

// Re-export key types
pub use changelog_drainer::ChangelogDrainer;
pub use checkpoint::checkpointer::{verify_integrity, CheckpointerError, ObjectStoreCheckpointer};
pub use checkpoint::layout::{
    CheckpointId, CheckpointManifestV2, CheckpointPaths, OperatorSnapshotEntry,
    PartitionSnapshotEntry, SourceOffsetEntry,
};
pub use checkpoint::source_offsets::{
    DeterminismWarning, FilePosition, GenericPosition, KafkaPartitionOffset, KafkaPosition,
    MysqlCdcPosition, PostgresCdcPosition, RecoveryPlan, SourceId, SourceOffset, SourcePosition,
};
pub use checkpoint::{Checkpoint, CheckpointMetadata};
pub use checkpoint_manifest::{CheckpointManifest, ConnectorCheckpoint, OperatorCheckpoint};
pub use checkpoint_store::{
    CheckpointStore, CheckpointStoreError, FileSystemCheckpointStore, ObjectStoreCheckpointStore,
    RecoveryReport, ValidationResult,
};
pub use laminar_core::error_codes::WarningSeverity;
pub use tiering::{StorageClass, StorageTier, TieringPolicy};
pub use wal::{WalEntry, WalError, WalPosition, WriteAheadLog};

// Re-export incremental checkpoint types
pub use incremental::{
    validate_checkpoint, wal_size, CheckpointConfig, IncrementalCheckpointError,
    IncrementalCheckpointManager, IncrementalCheckpointMetadata, RecoveredState, RecoveryConfig,
    RecoveryManager, StateChangelogBuffer, StateChangelogEntry, StateOp,
};