object-log 0.4.0

A buffered, multiplexing append log over pluggable object storage, with a sequencing seam (opaque payloads, durable-on-return writes, group-commit).
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
//! The sequencing seam: offset assignment, the offset→location index, and
//! retention. The engine owns storage + buffering; a `Sequencer` owns offsets.

use crate::ObjectLogError;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Mutex;

/// Opaque, engine-visible identifier for an independent offset stream (one dense,
/// monotonic offset space). A Kafka broker maps `(topic, partition)` onto one of
/// these; a WAL maps a shard. object-log treats it as an opaque key.
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
pub struct PartitionKey(pub String);

impl PartitionKey {
    /// Borrow the key as a string slice.
    pub fn as_str(&self) -> &str {
        &self.0
    }
}

/// Point-in-time durable bounds; grants no exclusivity or pinned history view.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AuthoritativePartitionState {
    /// Durable writer fence, or zero for genuinely unseen state.
    pub epoch: u64,
    /// First readable logical offset.
    pub log_start: i64,
    /// Next logical offset to assign.
    pub next_offset: i64,
}

/// Where a batch's bytes live inside its object. **Authored by the engine** (it
/// owns object layout); the sequencer stores it in the index and returns it from
/// [`Sequencer::lookup`].
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BatchLocation {
    /// Key of the object holding the batch.
    pub object_id: String,
    /// Byte offset of the batch within the object.
    pub byte_start: u32,
    /// Byte length of the batch.
    pub byte_len: u32,
}

/// One batch presented to [`Sequencer::commit`]. The engine fills `partition`,
/// `record_count`, and `location` (all engine-visible); `meta` is forwarded
/// **uninterpreted** — only the sequencer reads it.
pub struct CommitBatch<'a, M> {
    /// The offset stream this batch belongs to.
    pub partition: PartitionKey,
    /// Number of records in the batch (offsets advance by this).
    pub record_count: i32,
    /// Where the batch lives in its (already-durable) object.
    pub location: BatchLocation,
    /// Sequencer-private metadata (e.g. idempotent-producer identity). Opaque to
    /// the engine.
    pub meta: &'a M,
    /// Fence epoch the caller believes owns this partition. [`ManifestSequencer`](crate::ManifestSequencer)
    /// rejects the batch when the stored index epoch differs. Other sequencers
    /// ignore it.
    pub epoch: u64,
}

/// Per-batch result of [`Sequencer::commit`].
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum CommitOutcome {
    /// A fresh, contiguous offset range was assigned starting at `base_offset`.
    Assigned {
        /// First offset assigned to the batch.
        base_offset: i64,
        /// Number of records committed.
        record_count: i32,
    },
    /// A retried batch was recognized as already committed (idempotent no-op);
    /// the original `base_offset` is returned.
    Duplicate {
        /// The originally assigned first offset.
        base_offset: i64,
    },
    /// This batch was not indexed. Other batches in the same call may already
    /// be durable. The caller must rewrite this batch in a new data object
    /// rather than reuse the offsets it expected.
    Rejected {
        /// Why the conditional update did not land.
        reason: String,
    },
}

/// Opt-in conditional results; existing CommitOutcome encoding stays unchanged.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ConditionalCommitOutcome {
    /// An ordinary assigned/duplicate/rejected sequencing outcome.
    Committed(CommitOutcome),
    /// No entry for this batch was indexed; the original condition did not hold.
    HeadRejected {
        /// Original required next offset.
        expected: i64,
        /// Authoritative next offset that did not match.
        current: i64,
    },
    /// Publication may be ambiguous. Never interpret this as a head rejection.
    Unknown(ObjectLogError),
}

/// An index entry resolving an offset range to its bytes.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct IndexEntry {
    /// Where the batch's bytes live.
    pub location: BatchLocation,
    /// First offset in the batch.
    pub base_offset: i64,
    /// Number of records in the batch.
    pub record_count: i32,
}

/// The linearization point: assigns offsets to durably-stored batches and owns
/// the offset→location index.
///
/// **Synchronous on purpose** — a lin-point is a critical section, not async I/O.
/// The engine calls it from its flush worker (a dedicated thread / blocking
/// task), so a blocking implementation (a `Mutex`, a SQL transaction) is fine.
///
/// Publication scope belongs to the implementation. [`InMemorySequencer`]
/// validates and commits the whole slice under one lock. Durable manifest
/// publication linearizes each partition independently, so an error can follow
/// a durable earlier partition; callers must not infer global rollback.
/// The engine presents each [`PartitionKey`] in arrival order and never splits
/// one partition across concurrent `commit` calls.
pub trait Sequencer: Send + Sync {
    /// Sequencer-private per-batch metadata, forwarded uninterpreted by the
    /// engine. object-log's default sequencers use `()`; a Kafka coordinator uses
    /// its producer-identity fields.
    type Meta: Send + Sync;

    /// Opt in to atomic commits containing already-durable batches from several
    /// data objects, in creation order. Default preserves one-object calls for
    /// custom sequencers that rely on that boundary.
    fn supports_multi_object_commit(&self) -> bool {
        false
    }

    /// Assign offsets to durably uploaded batches and persist the index.
    /// By default these belong to one object; an opted-in implementation may
    /// receive several objects in one ordered call.
    /// Returns one [`CommitOutcome`] per input batch, in order.
    ///
    /// An `Err` may follow indexed earlier partitions; unacknowledged publication
    /// remains ambiguous. A [`CommitOutcome::Rejected`] batch was not indexed,
    /// while other batches in the same call may already be durable.
    fn commit(
        &self,
        batches: &[CommitBatch<'_, Self::Meta>],
    ) -> Result<Vec<CommitOutcome>, ObjectLogError>;

    /// Whether this sequencer implements native durable head conditions.
    fn supports_conditional_commit(&self) -> bool {
        false
    }

    /// Opt-in head compare-and-append. Conditions are next log offsets, not
    /// record insertion counters or sparse hints. Partition publication remains
    /// independent; this is not a cross-partition transaction. The default
    /// rejects any condition before calling the legacy commit implementation.
    fn commit_conditionally(
        &self,
        batches: &[CommitBatch<'_, Self::Meta>],
        expected_heads: &[Option<i64>],
    ) -> Result<Vec<ConditionalCommitOutcome>, ObjectLogError> {
        if batches.len() != expected_heads.len()
            || expected_heads.iter().flatten().any(|head| *head < 0)
        {
            return Err(ObjectLogError::InvalidBatch(
                "invalid head conditions".into(),
            ));
        }
        if expected_heads.iter().any(Option::is_some) {
            return Err(ObjectLogError::InvalidConfig(
                "conditional commit unsupported".into(),
            ));
        }
        self.commit(batches).map(|outcomes| {
            outcomes
                .into_iter()
                .map(ConditionalCommitOutcome::Committed)
                .collect()
        })
    }

    /// Resolve `fetch_offset` to the ordered index entries covering it onward.
    fn lookup(
        &self,
        partition: &PartitionKey,
        fetch_offset: i64,
    ) -> Result<Vec<IndexEntry>, ObjectLogError>;

    /// The next offset to be assigned (index-only; no object reads).
    fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError>;

    /// The first readable offset (advances on [`truncate_before`](Sequencer::truncate_before)).
    fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError>;

    /// Retention MECHANISM (not policy): drop index entries below `offset` and
    /// return object ids that now have **no** live references from **any**
    /// partition (objects are multiplexed and shared), for the engine to delete.
    fn truncate_before(
        &self,
        partition: &PartitionKey,
        offset: i64,
    ) -> Result<Vec<String>, ObjectLogError>;

    /// Move the stored fence epoch for `partition` from `expected` to `new_epoch`.
    ///
    /// The default succeeds without storing anything. [`ManifestSequencer`](crate::ManifestSequencer)
    /// reads that partition's durable index, fails when its epoch is neither
    /// `expected` nor `new_epoch`, and otherwise stores `new_epoch`, creating the
    /// index when it is missing. Once this returns, a writer that has not seen
    /// the new epoch cannot commit to the partition.
    fn fence_epoch(
        &self,
        partition: &PartitionKey,
        expected: u64,
        new_epoch: u64,
    ) -> Result<(), ObjectLogError> {
        let _ = (partition, expected, new_epoch);
        Ok(())
    }

    /// The fence epoch this handle last read for `partition`, if it tracks one.
    ///
    /// The default returns `None`.
    #[deprecated(
        since = "0.3.6",
        note = "the engine no longer reads fence epochs; implement `rejection_error` instead"
    )]
    fn partition_epoch(&self, partition: &PartitionKey) -> Option<u64> {
        let _ = partition;
        None
    }

    /// The error the engine reports for a batch this sequencer answered with
    /// [`CommitOutcome::Rejected`]. `epoch` is the fence epoch the batch was
    /// produced at.
    ///
    /// Classifying a rejection is the sequencer's policy; the engine only
    /// forwards the result. The default reports
    /// [`ObjectLogError::Sequencer`] with the rejection reason.
    fn rejection_error(
        &self,
        partition: &PartitionKey,
        epoch: u64,
        reason: String,
    ) -> ObjectLogError {
        let _ = (partition, epoch);
        ObjectLogError::Sequencer(reason)
    }

    /// Acquire a fresh durable writer epoch and the matching bounds from this
    /// call's successful CAS. This grants no ongoing writer exclusivity.
    /// An error may follow a durable epoch advance; never infer no effect.
    /// Unsupported implementations fail without legacy fence fallback.
    fn acquire_authoritative_epoch(
        &self,
        partition: &PartitionKey,
    ) -> Result<AuthoritativePartitionState, ObjectLogError> {
        let _ = partition;
        Err(ObjectLogError::InvalidConfig(
            "native epoch acquisition is unsupported".into(),
        ))
    }

    /// Read validated durable state directly; unsupported implementations fail
    /// without invoking legacy refresh or cached getters.
    fn authoritative_partition_state(
        &self,
        partition: &PartitionKey,
    ) -> Result<AuthoritativePartitionState, ObjectLogError> {
        let _ = partition;
        Err(ObjectLogError::InvalidConfig(
            "native authoritative partition state is unsupported".into(),
        ))
    }

    /// Reload `partition`'s index from durable storage, so this handle sees
    /// commits made by other writers since it opened.
    ///
    /// The default does nothing: a sequencer with a single writer is already
    /// current.
    fn refresh_partition(&self, partition: &PartitionKey) -> Result<(), ObjectLogError> {
        let _ = partition;
        Ok(())
    }
}

/// Per-partition in-memory index state.
#[derive(Default)]
struct PartState {
    next_offset: i64,
    log_start: i64,
    entries: Vec<IndexEntry>,
}

/// A single-node, in-memory [`Sequencer`] (`Meta = ()`): assigns dense offsets and
/// keeps the offset→location index in memory.
///
/// The log *bytes* live durably in the `BlobStore`, but this index does not
/// survive a restart — use it for tests and single-process work, or persist the
/// index yourself for a crash-durable standalone log.
#[derive(Default)]
pub struct InMemorySequencer {
    state: Mutex<HashMap<PartitionKey, PartState>>,
}

impl InMemorySequencer {
    /// Create an empty sequencer.
    pub fn new() -> Self {
        Self::default()
    }

    /// Object ids referenced by any live index entry (for orphan reaping).
    pub fn live_object_ids(&self) -> HashSet<String> {
        let state = self.state.lock().expect("poisoned");
        let mut live = HashSet::new();
        for ps in state.values() {
            for e in &ps.entries {
                live.insert(e.location.object_id.clone());
            }
        }
        live
    }
}

impl Sequencer for InMemorySequencer {
    type Meta = ();

    fn supports_multi_object_commit(&self) -> bool {
        true
    }

    fn commit(
        &self,
        batches: &[CommitBatch<'_, Self::Meta>],
    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
        let mut state = self.state.lock().expect("poisoned");
        // Validate the entire input before changing any partition.
        let mut nexts: HashMap<&PartitionKey, i64> = HashMap::new();
        for batch in batches {
            let next = nexts.entry(&batch.partition).or_insert_with(|| {
                state
                    .get(&batch.partition)
                    .map_or(0, |part| part.next_offset)
            });
            if batch.record_count <= 0 || batch.location.object_id.is_empty() {
                return Err(ObjectLogError::InvalidBatch("invalid index batch".into()));
            }
            *next = next
                .checked_add(i64::from(batch.record_count))
                .ok_or_else(|| ObjectLogError::Sequencer("index offset overflow".into()))?;
        }
        let mut outcomes = Vec::with_capacity(batches.len());
        for cb in batches {
            let ps = state.entry(cb.partition.clone()).or_default();
            let base = ps.next_offset;
            ps.entries.push(IndexEntry {
                location: cb.location.clone(),
                base_offset: base,
                record_count: cb.record_count,
            });
            ps.next_offset = base + cb.record_count as i64;
            outcomes.push(CommitOutcome::Assigned {
                base_offset: base,
                record_count: cb.record_count,
            });
        }
        Ok(outcomes)
    }

    fn lookup(
        &self,
        partition: &PartitionKey,
        fetch_offset: i64,
    ) -> Result<Vec<IndexEntry>, ObjectLogError> {
        let state = self.state.lock().expect("poisoned");
        let Some(ps) = state.get(partition) else {
            return Ok(Vec::new());
        };
        Ok(ps
            .entries
            .iter()
            .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
            .cloned()
            .collect())
    }

    fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
        Ok(self
            .state
            .lock()
            .expect("poisoned")
            .get(partition)
            .map_or(0, |ps| ps.next_offset))
    }

    fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
        Ok(self
            .state
            .lock()
            .expect("poisoned")
            .get(partition)
            .map_or(0, |ps| ps.log_start))
    }

    fn truncate_before(
        &self,
        partition: &PartitionKey,
        offset: i64,
    ) -> Result<Vec<String>, ObjectLogError> {
        let mut state = self.state.lock().expect("poisoned");
        let mut dropped: Vec<String> = Vec::new();
        match state.get_mut(partition) {
            Some(ps) => {
                for e in ps.entries.iter() {
                    if e.base_offset + e.record_count as i64 <= offset {
                        dropped.push(e.location.object_id.clone());
                    }
                }
                ps.entries
                    .retain(|e| e.base_offset + e.record_count as i64 > offset);
                if offset > ps.log_start {
                    ps.log_start = offset.min(ps.next_offset);
                }
            }
            None => return Ok(Vec::new()),
        }
        // An object is reclaimable only when no entry in ANY partition references it.
        let mut live: HashSet<String> = HashSet::new();
        for ps in state.values() {
            for e in &ps.entries {
                live.insert(e.location.object_id.clone());
            }
        }
        let mut dead: Vec<String> = Vec::new();
        let mut seen: HashSet<String> = HashSet::new();
        for oid in dropped {
            if !live.contains(&oid) && seen.insert(oid.clone()) {
                dead.push(oid);
            }
        }
        Ok(dead)
    }
}

impl crate::BoundedSequencer for InMemorySequencer {
    fn lookup_page(
        &self,
        partition: &PartitionKey,
        offset: i64,
        max_entries: usize,
        max_bytes: usize,
    ) -> Result<Vec<IndexEntry>, ObjectLogError> {
        let state = self.state.lock().expect("poisoned");
        if offset < state.get(partition).map_or(0, |part| part.log_start) {
            return Err(ObjectLogError::RangeOutOfBounds(
                "lookup precedes retained history".into(),
            ));
        }
        let entries = state
            .get(partition)
            .map_or(&[][..], |part| part.entries.as_slice());
        crate::bounded_lookup::page(entries, offset, max_entries, max_bytes)
    }
}