crabka_log/
lib.rs

1//! Byte-compatible reader/writer for Apache Kafka's on-disk log format.
2//!
3//! This crate provides the append-only storage layer used by the Crabka broker.
4//! It reads and writes Kafka 4.x's on-disk log format byte-for-byte:
5//! 20-digit zero-padded segment filenames, sparse `.index` and
6//! `.timeindex` files, append-only `.log` files containing
7//! [`crabka_protocol::records::RecordBatch`] v2 streams.
8//!
9//! ## What this crate does
10//!
11//! - Open + recover existing log directories.
12//! - Append `RecordBatch`es to the active segment.
13//! - Read sequentially from an absolute offset.
14//! - Truncate the log to an offset (for replication / leader election).
15//! - Time-based and size-based retention.
16//!
17//! ## Scope and boundaries
18//!
19//! This crate is the byte-compatible segment/index layer. It exposes append,
20//! read, truncation, leader-epoch checkpoint, transaction-index, retention, and
21//! compaction primitives over a single log directory. Broker-level policy —
22//! topic configuration, leader/follower ownership, tiered-storage scheduling,
23//! transaction visibility rules, and write serialization — is applied by
24//! `crabka-broker` above this storage layer.
25//!
26//! ## Quick start
27//!
28//! ```no_run
29//! use crabka_log::{Log, LogConfig};
30//! use crabka_protocol::records::RecordBatch;
31//!
32//! let mut log = Log::open("/var/kafka/my-topic-0", LogConfig::default()).unwrap();
33//! let mut batch = RecordBatch::default();
34//! // ... fill the batch ...
35//! let assigned_offset = log.append(&mut batch).unwrap();
36//!
37//! let out = log.read(0, 1024 * 1024).unwrap();
38//! # let _ = (assigned_offset, out);
39//! ```
40//! ## Exporting a segment
41//!
42//! ```no_run
43//! use crabka_log::{Log, LogConfig};
44//!
45//! # fn run() -> Result<(), Box<dyn std::error::Error>> {
46//! let mut log = Log::open("/var/kafka/my-topic-0", LogConfig::default())?;
47//! for segment in log.tierable_segments() {
48//!     println!(
49//!         "segment {} starts at offset {}",
50//!         segment.log_path.display(),
51//!         segment.base_offset
52//!     );
53//! }
54//! # Ok(())
55//! # }
56//! ```
57
58#![doc(html_root_url = "https://docs.rs/crabka-log/0.3.5")]
59
60mod compact;
61mod config;
62mod error;
63mod index;
64mod leader_epoch_checkpoint;
65mod log;
66mod name;
67mod recovery;
68mod retention;
69mod segment;
70mod txn_index;
71
72pub use config::{CleanupPolicy, LogConfig};
73pub use error::LogError;
74pub use leader_epoch_checkpoint::{EpochEntry, LeaderEpochCheckpoint};
75pub use log::{Log, RawRead, ReadOutput, SegmentExport};
76pub use segment::{RawSegmentRead, Segment};
77pub use txn_index::{AbortedTxn, TxnIndex};
crabka_log/lib.rs

crabka_log/
lib.rs
