Skip to main content

krabka_log/
lib.rs

1//! Byte-compatible reader/writer for Apache Kafka's on-disk log format.
2//!
3//! This crate is the append-only storage layer that the Krabka broker uses.
4//! It reads and writes Kafka 4.x's on-disk log format byte-for-byte:
5//! 20-digit zero-padded segment filenames, sparse `.index` and
6//! `.timeindex` files, and append-only `.log` files that hold
7//! [`krabka_protocol::records::RecordBatch`] v2 streams.
8//!
9//! ## What this crate does
10//!
11//! - Open and recover existing log directories.
12//! - Append `RecordBatch`es to the active segment.
13//! - Read sequentially from an absolute offset.
14//! - Truncate the log to an offset, for replication and leader election.
15//! - Time-based and size-based retention.
16//! - Deliver-at-time visibility, where a batch becomes readable only once its
17//!   activation time has passed.
18//!
19//! ## Scope and boundaries
20//!
21//! This crate is the byte-compatible segment and index layer. It exposes
22//! append, read, truncation, leader-epoch checkpoint, transaction-index,
23//! retention, and compaction primitives over a single log directory.
24//! `krabka-broker` applies broker-level policy above this storage layer. That
25//! policy covers topic configuration, leader and follower ownership,
26//! tiered-storage scheduling, transaction visibility rules, and write
27//! serialization.
28//!
29//! ## Quick start
30//!
31//! ```no_run
32//! use {
33//!     krabka_ids::Offset,
34//!     krabka_log::{Log, LogConfig},
35//!     krabka_protocol::records::RecordBatch,
36//!     krabka_units::prelude::mebibytes,
37//! };
38//!
39//! let mut log = Log::open("/var/kafka/my-topic-0", LogConfig::default()).unwrap();
40//! let mut batch = RecordBatch::default();
41//! // ... fill the batch ...
42//! let assigned_offset = log.append(&mut batch).unwrap();
43//!
44//! let out = log.read(Offset(0), mebibytes(1)).unwrap();
45//! # let _ = (assigned_offset, out);
46//! ```
47//! ## Exporting a segment
48//!
49//! ```no_run
50//! use krabka_log::{Log, LogConfig};
51//!
52//! # fn run() -> Result<(), Box<dyn std::error::Error>> {
53//! let mut log = Log::open("/var/kafka/my-topic-0", LogConfig::default())?;
54//! for segment in log.tierable_segments() {
55//!     println!(
56//!         "segment {} starts at offset {}",
57//!         segment.log_path.display(),
58//!         segment.base_offset
59//!     );
60//! }
61//! # Ok(())
62//! # }
63//! ```
64
65#![doc(html_root_url = "https://docs.rs/krabka-log/1.0.8")]
66
67mod compact;
68mod config;
69mod delivery;
70mod error;
71mod filter;
72mod index;
73mod io;
74mod leader_epoch_checkpoint;
75mod log;
76mod log_start_offset_checkpoint;
77/// Shared breadth-first runner for the exhaustive Stateright models.
78#[cfg(test)]
79mod model_check;
80pub mod name;
81mod producer_snapshot;
82mod record_limit;
83mod recovery;
84mod retention;
85mod segment;
86mod stamp_index;
87mod stamp_source;
88#[cfg(test)]
89mod test_support;
90pub mod topic_name;
91mod txn_index;
92
93#[cfg(any(test, feature = "test-helpers"))]
94pub use io::{DirectFile, IoTarget, LogIo};
95#[krabka_macros::sendfile_cfg]
96pub use log::RawReadDesc;
97#[krabka_macros::sendfile_cfg]
98pub use segment::RawSegmentDesc;
99// Re-export the zero-copy fetch descriptor so broker code can name
100// `krabka_log::FileRegion` without depending on the protocol crate's path.
101#[cfg(any(test, feature = "test-helpers"))]
102pub use stamp_source::MonotonicStampSource;
103pub use {
104    config::{
105        CleanupPolicy, DEFAULT_MAX_MESSAGE_SIZE, DEFAULT_READ_AHEAD_MAX, DEFAULT_TAIL_CACHE_SIZE,
106        DeliveryPolicy, LogConfig, RemoteTierFlags, ScheduleOrder, SegmentAllocation,
107    },
108    delivery::{DeliveryAdvance, batch_is_deliverable},
109    error::LogError,
110    filter::{FilteredBatch, filter_batch},
111    io::write_file_atomic,
112    krabka_ids::{LeaderEpoch, Offset, ProducerId},
113    krabka_protocol::records::FileRegion,
114    leader_epoch_checkpoint::{EpochEntry, LeaderEpochCheckpoint, epoch_and_offset_for_entries},
115    log::{
116        ActiveProducer, ActiveSegmentExport, BARRIER_CONTROL_TYPE, CompactionContext, DueDeletions,
117        FailedDeletion, Log, RawRead, ReadOutput, SegmentExport, TransactionAppendRefusal,
118        TransactionalBatch, VerbatimBatch, VerificationGuard,
119    },
120    producer_snapshot::{
121        NUM_BATCHES_TO_RETAIN, ProducerBatchMetadata, ProducerSnapshotEntry, RecoveredProducer,
122        SnapshotDecodeError, decode as decode_producer_snapshot,
123    },
124    retention::now_ms as epoch_ms,
125    segment::{RawSegmentRead, Segment},
126    stamp_index::{StampEntry, StampIndex},
127    stamp_source::StampSource,
128    txn_index::{AbortedTxn, TxnIndex},
129};