object_log/lib.rs
1//! A buffered, multiplexing append log over pluggable object storage.
2//!
3//! object-log stores an ordered, offset-addressed log as immutable objects in any
4//! [`BlobStore`] (memory, local filesystem, or — behind a feature — S3). It deals
5//! only in **opaque payload bytes**: it knows nothing about record formats, Kafka,
6//! or brokers. Three pieces:
7//!
8//! - [`BlobStore`] — the storage *port*: durable-on-return `put`, `get`,
9//! `get_range`, `list`, `delete`. Adapters [`MemoryBlobStore`] and
10//! [`LocalBlobStore`] ship in-crate.
11//! - [`Sequencer`] — the *sequencing seam*: assigns offsets to durably-stored
12//! batches and owns the offset→location index. [`InMemorySequencer`] ships
13//! in-crate; a consumer plugs its own (e.g. a Kafka coordinator) and the engine
14//! forwards each sequencer's [`Sequencer::Meta`] uninterpreted.
15//! - [`LogEngine`] — buffers and group-commits many batches into one object, PUTs
16//! it durably, then calls the sequencer, so PUT count is decoupled from produce
17//! count. [`LogEngine::produce`] resolves at a requested [`Durability`].
18//!
19//! # Example
20//!
21//! ```
22//! use object_log::{Durability, FlushConfig, InMemorySequencer, LogEngine, MemoryBlobStore, PartitionKey};
23//! use bytes::Bytes;
24//! use std::sync::Arc;
25//!
26//! # tokio::runtime::Runtime::new().unwrap().block_on(async {
27//! let engine = LogEngine::new(
28//! Arc::new(MemoryBlobStore::new()),
29//! Arc::new(InMemorySequencer::new()),
30//! FlushConfig::default(),
31//! "log/",
32//! );
33//! let p = PartitionKey("events-0".into());
34//!
35//! let out = engine
36//! .produce(p.clone(), Bytes::from_static(b"hello"), 1, (), Durability::Sequenced)
37//! .await
38//! .unwrap();
39//! assert_eq!(out.base_offset, Some(0));
40//!
41//! let batches = engine.fetch(&p, 0, 1024).await.unwrap();
42//! assert_eq!(batches[0].payload, "hello");
43//! # });
44//! ```
45#![deny(missing_docs)]
46
47mod blob;
48mod budget;
49mod engine;
50mod error;
51mod manifest_sequencer;
52#[cfg(any(test, feature = "native"))]
53mod native_v2;
54#[cfg(feature = "s3")]
55mod s3;
56mod sequencer;
57
58pub use blob::{
59 BlobStore, BoundedMetadataStore, CasOutcome, LocalBlobStore, MediaOpStats, MemoryBlobStore,
60 MetadataLimits, MetadataOwners, MetadataSnapshot, MetadataSwap,
61};
62pub use budget::{BudgetConfig, BudgetMode, EffectiveKnob, EffectiveReason, PipelineSnapshot};
63pub use engine::{
64 AppendOutcome, AppendOwner, BufferStats, Durability, FetchedBatch, FlushConfig, LogEngine,
65 reap_orphans,
66};
67mod bounded_fetch;
68mod bounded_lookup;
69pub use bounded_fetch::FetchPlan;
70pub use bounded_lookup::BoundedSequencer;
71pub use error::ObjectLogError;
72pub use manifest_sequencer::{IndexSnapshot, ManifestSequencer, PartitionSnapshot};
73#[cfg(feature = "s3")]
74pub use s3::{S3BlobStore, S3ClientOptions};
75pub use sequencer::{
76 AuthoritativePartitionState, BatchLocation, CommitBatch, CommitOutcome,
77 ConditionalCommitOutcome, InMemorySequencer, IndexEntry, PartitionKey, Sequencer,
78};
79
80#[cfg(feature = "native")]
81pub use native_v2::facade::{
82 NativeAppend, NativeEditLimits, NativeLimits, NativeLog, NativeNamespace, NativePublication,
83 NativePublishOutcome, NativeReadCut, NativeReadPlan, NativeReadRequirements, NativeRecord,
84 NativeResolution, NativeScanLimits, NativeScanStatus, NativeSemanticContinuation,
85 NativeSemanticEdit, NativeSemanticRange, NativeSemanticRow, NativeSemanticScan,
86 NativeSemanticValue, NativeSemanticWork, NativeWriter,
87};
88#[cfg(feature = "native")]
89pub use native_v2::head::NativeHead;