Skip to main content

object_log/
manifest_sequencer.rs

1//! A `BlobStore`-persisted [`Sequencer`]: the offset→location index survives a
2//! restart, so a standalone object-log is crash-durable end to end.
3//!
4//! Each `commit` durably writes one manifest object (≤1 PUT per group-commit,
5//! co-amortized with the data object) recording the batches it assigned. On
6//! [`open`](ManifestSequencer::open) the index is rebuilt by replaying the
7//! manifest objects in order.
8
9use crate::{
10    BlobStore, CasOutcome, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey,
11    Sequencer,
12};
13use bytes::Bytes;
14use serde::{Deserialize, Serialize};
15use std::collections::{HashMap, HashSet};
16use std::sync::{Arc, Mutex};
17use tokio::runtime::{Builder, Runtime};
18
19#[derive(Serialize, Deserialize)]
20struct ManifestRecord {
21    entries: Vec<(PartitionKey, IndexEntry)>,
22}
23
24#[derive(Clone, Serialize, Deserialize)]
25struct PartitionIndexDoc {
26    epoch: u64,
27    next_offset: i64,
28    log_start: i64,
29    entries: Vec<IndexEntry>,
30}
31
32#[derive(Clone, Serialize, Deserialize, Default)]
33struct IndexCatalog {
34    partitions: Vec<String>,
35}
36
37#[derive(Default)]
38struct Part {
39    next_offset: i64,
40    log_start: i64,
41    epoch: u64,
42    entries: Vec<IndexEntry>,
43    /// Exact bytes last read or written for this partition's index object.
44    observed: Option<Bytes>,
45}
46
47struct Inner {
48    parts: HashMap<PartitionKey, Part>,
49    counter: u64,
50    catalog: Option<Bytes>,
51}
52
53/// A [`Sequencer`] (`Meta = ()`) that persists its index to a [`BlobStore`].
54pub struct ManifestSequencer {
55    blob: Arc<dyn BlobStore>,
56    prefix: String,
57    // `Option` so `Drop` can `shutdown_background()` — dropping a `Runtime` inside
58    // an async context (e.g. an `Arc` last-ref drop in a `#[tokio::test]`) panics.
59    rt: Option<Runtime>,
60    // Serialize mutations across publication without blocking committed reads.
61    commit_order: Mutex<()>,
62    inner: Mutex<Inner>,
63}
64
65impl Drop for ManifestSequencer {
66    fn drop(&mut self) {
67        if let Some(rt) = self.rt.take() {
68            rt.shutdown_background();
69        }
70    }
71}
72
73impl ManifestSequencer {
74    /// Open (or recover) a persisted sequencer. Manifest objects live under
75    /// `manifest_prefix` (keep it disjoint from the engine's data-object prefix).
76    /// Existing manifests are replayed to rebuild the index.
77    pub async fn open(
78        blob: Arc<dyn BlobStore>,
79        manifest_prefix: impl Into<String>,
80    ) -> Result<Self, ObjectLogError> {
81        let prefix = manifest_prefix.into();
82        let catalog_key = format!("{prefix}catalog");
83        let (parts, catalog, counter) = if let Some(catalog_bytes) = blob.get(&catalog_key).await? {
84            let catalog: IndexCatalog = serde_json::from_slice(&catalog_bytes)
85                .map_err(|e| ObjectLogError::Sequencer(format!("index catalog: {e}")))?;
86            let mut parts = HashMap::new();
87            for name in &catalog.partitions {
88                let key = index_object_key(&prefix, name);
89                let Some(doc_bytes) = blob.get(&key).await? else {
90                    parts.insert(PartitionKey(name.clone()), Part::default());
91                    continue;
92                };
93                let doc: PartitionIndexDoc = serde_json::from_slice(&doc_bytes).map_err(|e| {
94                    ObjectLogError::Sequencer(format!("partition index {name}: {e}"))
95                })?;
96                parts.insert(PartitionKey(name.clone()), Part::from_doc(doc, doc_bytes));
97            }
98            let counter = parts.len() as u64;
99            (parts, Some(catalog_bytes), counter)
100        } else {
101            let mut keys = blob.list(&prefix).await?;
102            keys.sort();
103            let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
104            for key in &keys {
105                if key == &catalog_key || key.starts_with(&format!("{prefix}index/")) {
106                    continue;
107                }
108                let Some(bytes) = blob.get(key).await? else {
109                    continue;
110                };
111                let Ok(rec) = serde_json::from_slice::<ManifestRecord>(&bytes) else {
112                    continue;
113                };
114                for (pkey, entry) in rec.entries {
115                    let p = parts.entry(pkey).or_default();
116                    let end = entry.base_offset + entry.record_count as i64;
117                    if end > p.next_offset {
118                        p.next_offset = end;
119                    }
120                    p.entries.push(entry);
121                }
122            }
123            (parts, None, keys.len() as u64)
124        };
125        let rt = Builder::new_current_thread()
126            .enable_all()
127            .build()
128            .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
129        Ok(Self {
130            blob,
131            prefix,
132            rt: Some(rt),
133            commit_order: Mutex::new(()),
134            inner: Mutex::new(Inner {
135                parts,
136                counter,
137                catalog,
138            }),
139        })
140    }
141
142    fn index_key(&self, partition: &PartitionKey) -> String {
143        index_object_key(&self.prefix, partition.as_str())
144    }
145
146    fn catalog_key(&self) -> String {
147        format!("{}catalog", self.prefix)
148    }
149
150    fn block_on<T>(&self, future: impl std::future::Future<Output = T>) -> T {
151        self.rt
152            .as_ref()
153            .expect("runtime present until drop")
154            .block_on(future)
155    }
156
157    fn read_object(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
158        self.block_on(self.blob.get(key))
159    }
160
161    fn cas(
162        &self,
163        key: &str,
164        expected: Option<Bytes>,
165        new_value: Bytes,
166    ) -> Result<CasOutcome, ObjectLogError> {
167        self.block_on(self.blob.compare_and_swap(key, expected, new_value))
168    }
169
170    fn ensure_catalog(&self, partitions: &[PartitionKey]) -> Result<(), ObjectLogError> {
171        let key = self.catalog_key();
172        for _ in 0..8 {
173            let current = self.read_object(&key)?;
174            let mut names = if let Some(bytes) = &current {
175                serde_json::from_slice::<IndexCatalog>(bytes)
176                    .map_err(|error| ObjectLogError::Sequencer(format!("index catalog: {error}")))?
177                    .partitions
178            } else {
179                Vec::new()
180            };
181            let mut changed = false;
182            for partition in partitions {
183                if !names.iter().any(|name| name == partition.as_str()) {
184                    names.push(partition.0.clone());
185                    changed = true;
186                }
187            }
188            if !changed {
189                self.inner.lock().expect("poisoned").catalog = current;
190                return Ok(());
191            }
192            names.sort();
193            names.dedup();
194            let new_bytes = Bytes::from(
195                serde_json::to_vec(&IndexCatalog { partitions: names })
196                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
197            );
198            match self.cas(&key, current, new_bytes.clone())? {
199                CasOutcome::Stored => {
200                    self.inner.lock().expect("poisoned").catalog = Some(new_bytes);
201                    return Ok(());
202                }
203                CasOutcome::Conflict { .. } => continue,
204            }
205        }
206        Err(ObjectLogError::Sequencer(
207            "catalog compare-and-swap did not land".into(),
208        ))
209    }
210
211    fn publish_partition(
212        &self,
213        partition: &PartitionKey,
214        batches: &[CommitBatch<'_, ()>],
215        indexes: &[usize],
216    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
217        let epoch = batches[indexes[0]].epoch;
218        if indexes.iter().any(|index| batches[*index].epoch != epoch) {
219            return Ok(reject_all(
220                indexes.len(),
221                "one partition commit mixed fence epochs",
222            ));
223        }
224        let (observed, mut doc) = {
225            let inner = self.inner.lock().expect("poisoned");
226            match inner
227                .parts
228                .get(partition)
229                .and_then(|part| part.observed.clone())
230            {
231                Some(bytes) => {
232                    let doc = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
233                        .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
234                    (Some(bytes), doc)
235                }
236                None => {
237                    let part = inner.parts.get(partition);
238                    (
239                        None,
240                        PartitionIndexDoc {
241                            epoch,
242                            next_offset: part.map_or(0, |part| part.next_offset),
243                            log_start: part.map_or(0, |part| part.log_start),
244                            entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
245                        },
246                    )
247                }
248            }
249        };
250        if observed.is_some() && doc.epoch != epoch {
251            return Ok(reject_all(
252                indexes.len(),
253                format!(
254                    "partition {} epoch is {}, not {epoch}",
255                    partition.as_str(),
256                    doc.epoch
257                ),
258            ));
259        }
260        let mut outcomes = Vec::with_capacity(indexes.len());
261        let mut planned = Vec::with_capacity(indexes.len());
262        for index in indexes {
263            let batch = &batches[*index];
264            let entry = IndexEntry {
265                location: batch.location.clone(),
266                base_offset: doc.next_offset,
267                record_count: batch.record_count,
268            };
269            outcomes.push(CommitOutcome::Assigned {
270                base_offset: entry.base_offset,
271                record_count: entry.record_count,
272            });
273            doc.next_offset += i64::from(batch.record_count);
274            planned.push(entry.clone());
275            doc.entries.push(entry);
276        }
277        let new_bytes = Bytes::from(
278            serde_json::to_vec(&doc)
279                .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
280        );
281        match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
282            CasOutcome::Stored => {
283                self.install_doc(partition, &doc, new_bytes);
284                Ok(outcomes)
285            }
286            CasOutcome::Conflict { current } => {
287                if let Some(bytes) = current
288                    && let Ok(found) = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
289                {
290                    let already = planned
291                        .iter()
292                        .all(|entry| found.entries.iter().any(|have| have == entry));
293                    self.install_doc(partition, &found, bytes);
294                    if already {
295                        return Ok(outcomes);
296                    }
297                }
298                Ok(reject_all(
299                    indexes.len(),
300                    format!("partition {} tail moved", partition.as_str()),
301                ))
302            }
303        }
304    }
305
306    /// Install `partition`'s durable index, if any, and return it.
307    fn reload_partition(
308        &self,
309        partition: &PartitionKey,
310    ) -> Result<Option<(PartitionIndexDoc, Bytes)>, ObjectLogError> {
311        let Some(bytes) = self.read_object(&self.index_key(partition))? else {
312            return Ok(None);
313        };
314        let doc: PartitionIndexDoc = serde_json::from_slice(&bytes)
315            .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
316        self.install_doc(partition, &doc, bytes.clone());
317        Ok(Some((doc, bytes)))
318    }
319
320    fn install_doc(&self, partition: &PartitionKey, doc: &PartitionIndexDoc, observed: Bytes) {
321        let mut inner = self.inner.lock().expect("poisoned");
322        inner.parts.insert(
323            partition.clone(),
324            Part {
325                next_offset: doc.next_offset,
326                log_start: doc.log_start,
327                epoch: doc.epoch,
328                entries: doc.entries.clone(),
329                observed: Some(observed),
330            },
331        );
332    }
333
334    /// Object ids referenced by any live index entry (for orphan reaping).
335    /// Does not include manifest object keys themselves.
336    pub fn live_object_ids(&self) -> HashSet<String> {
337        let st = self.inner.lock().expect("poisoned");
338        let mut live = HashSet::new();
339        for p in st.parts.values() {
340            for e in &p.entries {
341                live.insert(e.location.object_id.clone());
342            }
343        }
344        live
345    }
346
347    /// Point-in-time view of the rebuilt index (diagnostics / CLI).
348    pub fn snapshot(&self) -> IndexSnapshot {
349        let st = self.inner.lock().expect("poisoned");
350        let mut partitions: Vec<PartitionSnapshot> = st
351            .parts
352            .iter()
353            .map(|(k, p)| PartitionSnapshot {
354                partition: k.0.clone(),
355                log_start: p.log_start,
356                high_watermark: p.next_offset,
357                entry_count: p.entries.len(),
358                entries: p.entries.clone(),
359            })
360            .collect();
361        partitions.sort_by(|a, b| a.partition.cmp(&b.partition));
362        IndexSnapshot {
363            manifest_prefix: self.prefix.clone(),
364            manifest_count: st.counter,
365            partitions,
366        }
367    }
368}
369
370/// Full index view from [`ManifestSequencer::snapshot`].
371#[derive(Clone, Debug, Serialize, Deserialize)]
372pub struct IndexSnapshot {
373    /// Prefix used for manifest objects.
374    pub manifest_prefix: String,
375    /// Number of manifest objects replayed (commit counter).
376    pub manifest_count: u64,
377    /// Per-partition index state.
378    pub partitions: Vec<PartitionSnapshot>,
379}
380
381/// One partition's index bounds and entries.
382#[derive(Clone, Debug, Serialize, Deserialize)]
383pub struct PartitionSnapshot {
384    /// Partition key string.
385    pub partition: String,
386    /// First readable offset.
387    pub log_start: i64,
388    /// Next offset to assign (high watermark).
389    pub high_watermark: i64,
390    /// Number of index entries.
391    pub entry_count: usize,
392    /// Ordered index entries (may be large; CLI can omit via flag).
393    pub entries: Vec<IndexEntry>,
394}
395
396impl Sequencer for ManifestSequencer {
397    type Meta = ();
398
399    fn supports_multi_object_commit(&self) -> bool {
400        true
401    }
402
403    fn commit(
404        &self,
405        batches: &[CommitBatch<'_, ()>],
406    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
407        let _commit = self.commit_order.lock().expect("poisoned");
408        if batches.is_empty() {
409            return Ok(Vec::new());
410        }
411        let mut order: Vec<PartitionKey> = Vec::new();
412        let mut groups: HashMap<PartitionKey, Vec<usize>> = HashMap::new();
413        for (index, batch) in batches.iter().enumerate() {
414            if !groups.contains_key(&batch.partition) {
415                order.push(batch.partition.clone());
416            }
417            groups
418                .entry(batch.partition.clone())
419                .or_default()
420                .push(index);
421        }
422        self.ensure_catalog(&order)?;
423        let mut outcomes = vec![None; batches.len()];
424        let mut published = false;
425        for partition in order {
426            let indexes = &groups[&partition];
427            match self.publish_partition(&partition, batches, indexes) {
428                Ok(partition_outcomes) => {
429                    for (index, outcome) in indexes.iter().zip(partition_outcomes) {
430                        if matches!(
431                            outcome,
432                            CommitOutcome::Assigned { .. } | CommitOutcome::Duplicate { .. }
433                        ) {
434                            published = true;
435                        }
436                        outcomes[*index] = Some(outcome);
437                    }
438                }
439                Err(error) => {
440                    for index in indexes {
441                        outcomes[*index] = Some(CommitOutcome::Rejected {
442                            reason: error.to_string(),
443                        });
444                    }
445                }
446            }
447        }
448        if published {
449            let mut inner = self.inner.lock().expect("poisoned");
450            inner.counter = inner.counter.saturating_add(1);
451        }
452        Ok(outcomes
453            .into_iter()
454            .map(|outcome| outcome.expect("every batch has an outcome"))
455            .collect())
456    }
457
458    fn fence_epoch(
459        &self,
460        partition: &PartitionKey,
461        expected: u64,
462        new_epoch: u64,
463    ) -> Result<(), ObjectLogError> {
464        let _commit = self.commit_order.lock().expect("poisoned");
465        if expected == new_epoch {
466            return Ok(());
467        }
468        // Fence the durable index, not this handle's view of it: another writer
469        // may have created or advanced the index since this handle opened, and a
470        // fence that only compared cached bytes would silently store nothing.
471        self.ensure_catalog(std::slice::from_ref(partition))?;
472        for _ in 0..8 {
473            let (mut doc, observed) = match self.reload_partition(partition)? {
474                Some((current, bytes)) => {
475                    if current.epoch == new_epoch {
476                        return Ok(());
477                    }
478                    if current.epoch != expected {
479                        return Err(ObjectLogError::Sequencer(format!(
480                            "partition {} epoch is {}, not {expected}",
481                            partition.as_str(),
482                            current.epoch
483                        )));
484                    }
485                    (current, Some(bytes))
486                }
487                None => {
488                    // No index yet: create it at the new epoch, so a writer that
489                    // never saw one cannot publish under an older epoch.
490                    let inner = self.inner.lock().expect("poisoned");
491                    let part = inner.parts.get(partition);
492                    let doc = PartitionIndexDoc {
493                        epoch: new_epoch,
494                        next_offset: part.map_or(0, |part| part.next_offset),
495                        log_start: part.map_or(0, |part| part.log_start),
496                        entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
497                    };
498                    (doc, None)
499                }
500            };
501            doc.epoch = new_epoch;
502            let new_bytes = Bytes::from(
503                serde_json::to_vec(&doc)
504                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
505            );
506            match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
507                CasOutcome::Stored => {
508                    self.install_doc(partition, &doc, new_bytes);
509                    return Ok(());
510                }
511                // A concurrent commit or fence moved the index; re-read and retry.
512                CasOutcome::Conflict { .. } => continue,
513            }
514        }
515        Err(ObjectLogError::Sequencer(format!(
516            "partition {} epoch fence did not land",
517            partition.as_str()
518        )))
519    }
520
521    fn partition_epoch(&self, partition: &PartitionKey) -> Option<u64> {
522        let inner = self.inner.lock().expect("poisoned");
523        inner
524            .parts
525            .get(partition)
526            .filter(|part| part.observed.is_some())
527            .map(|part| part.epoch)
528    }
529
530    fn refresh_partition(&self, partition: &PartitionKey) -> Result<(), ObjectLogError> {
531        let _commit = self.commit_order.lock().expect("poisoned");
532        self.reload_partition(partition)?;
533        Ok(())
534    }
535
536    fn lookup(
537        &self,
538        partition: &PartitionKey,
539        fetch_offset: i64,
540    ) -> Result<Vec<IndexEntry>, ObjectLogError> {
541        let st = self.inner.lock().expect("poisoned");
542        let Some(p) = st.parts.get(partition) else {
543            return Ok(Vec::new());
544        };
545        Ok(p.entries
546            .iter()
547            .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
548            .cloned()
549            .collect())
550    }
551
552    fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
553        Ok(self
554            .inner
555            .lock()
556            .expect("poisoned")
557            .parts
558            .get(partition)
559            .map_or(0, |p| p.next_offset))
560    }
561
562    fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
563        Ok(self
564            .inner
565            .lock()
566            .expect("poisoned")
567            .parts
568            .get(partition)
569            .map_or(0, |p| p.log_start))
570    }
571
572    fn truncate_before(
573        &self,
574        partition: &PartitionKey,
575        offset: i64,
576    ) -> Result<Vec<String>, ObjectLogError> {
577        let _commit = self.commit_order.lock().expect("poisoned");
578        let snapshot = {
579            let st = self.inner.lock().expect("poisoned");
580            let Some(part) = st.parts.get(partition) else {
581                return Ok(Vec::new());
582            };
583            (
584                part.observed.clone(),
585                part.entries.clone(),
586                part.next_offset,
587                part.epoch,
588                part.log_start,
589            )
590        };
591        let (observed, entries, next_offset, epoch, mut log_start) = snapshot;
592        let mut dropped = Vec::new();
593        let mut kept = Vec::new();
594        for entry in entries {
595            if entry.base_offset + i64::from(entry.record_count) <= offset {
596                dropped.push(entry.location.object_id.clone());
597            } else {
598                kept.push(entry);
599            }
600        }
601        if offset > log_start {
602            log_start = offset.min(next_offset);
603        }
604        let doc = PartitionIndexDoc {
605            epoch,
606            next_offset,
607            log_start,
608            entries: kept,
609        };
610        if observed.is_some() {
611            let new_bytes = Bytes::from(
612                serde_json::to_vec(&doc)
613                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
614            );
615            match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
616                CasOutcome::Stored => self.install_doc(partition, &doc, new_bytes),
617                CasOutcome::Conflict { .. } => {
618                    return Err(ObjectLogError::Sequencer(format!(
619                        "partition {} truncate lost the compare-and-swap",
620                        partition.as_str()
621                    )));
622                }
623            }
624        } else {
625            let mut st = self.inner.lock().expect("poisoned");
626            if let Some(part) = st.parts.get_mut(partition) {
627                part.entries = doc.entries.clone();
628                part.log_start = doc.log_start;
629            }
630        }
631        let live = {
632            let st = self.inner.lock().expect("poisoned");
633            let mut live = HashSet::new();
634            for part in st.parts.values() {
635                for entry in &part.entries {
636                    live.insert(entry.location.object_id.clone());
637                }
638            }
639            live
640        };
641        let mut dead = Vec::new();
642        let mut seen = HashSet::new();
643        for object_id in dropped {
644            if !live.contains(&object_id) && seen.insert(object_id.clone()) {
645                dead.push(object_id);
646            }
647        }
648        Ok(dead)
649    }
650}
651
652impl Part {
653    fn from_doc(doc: PartitionIndexDoc, observed: Bytes) -> Self {
654        Self {
655            next_offset: doc.next_offset,
656            log_start: doc.log_start,
657            epoch: doc.epoch,
658            entries: doc.entries,
659            observed: Some(observed),
660        }
661    }
662}
663
664fn index_object_key(prefix: &str, partition: &str) -> String {
665    let mut encoded = String::with_capacity(partition.len() * 2);
666    for byte in partition.as_bytes() {
667        encoded.push_str(&format!("{byte:02x}"));
668    }
669    format!("{prefix}index/{encoded}")
670}
671
672fn reject_all(count: usize, reason: impl Into<String>) -> Vec<CommitOutcome> {
673    let reason = reason.into();
674    vec![CommitOutcome::Rejected { reason }; count]
675}
676
677#[cfg(test)]
678mod index_tests {
679    use super::*;
680    use crate::{
681        BatchLocation, BlobStore, CommitBatch, CommitOutcome, Durability, FlushConfig,
682        InMemorySequencer, LogEngine, MemoryBlobStore, ObjectLogError, PartitionKey,
683    };
684    use async_trait::async_trait;
685    use bytes::Bytes;
686    use std::sync::Arc;
687    use std::sync::atomic::{AtomicUsize, Ordering};
688
689    fn location(object: &str) -> BatchLocation {
690        BatchLocation {
691            object_id: object.to_string(),
692            byte_start: 0,
693            byte_len: 1,
694        }
695    }
696
697    fn run_commit(
698        seq: Arc<ManifestSequencer>,
699        partitions: Vec<(&str, &str, u64)>,
700    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
701        let owned: Vec<(String, String, u64)> = partitions
702            .into_iter()
703            .map(|(partition, object, epoch)| (partition.to_string(), object.to_string(), epoch))
704            .collect();
705        std::thread::spawn(move || {
706            let meta = ();
707            let batches: Vec<CommitBatch<'_, ()>> = owned
708                .iter()
709                .map(|(partition, object, epoch)| CommitBatch {
710                    partition: PartitionKey(partition.clone()),
711                    record_count: 1,
712                    location: location(object),
713                    meta: &meta,
714                    epoch: *epoch,
715                })
716                .collect();
717            seq.commit(&batches)
718        })
719        .join()
720        .expect("commit thread")
721    }
722
723    /// Run a blocking sequencer call off the async test thread.
724    fn on_thread<T: Send + 'static>(
725        seq: &Arc<ManifestSequencer>,
726        call: impl FnOnce(&ManifestSequencer) -> T + Send + 'static,
727    ) -> T {
728        let seq = Arc::clone(seq);
729        std::thread::spawn(move || call(&seq))
730            .join()
731            .expect("sequencer thread")
732    }
733
734    struct ListProbe {
735        inner: MemoryBlobStore,
736        lists: Arc<AtomicUsize>,
737    }
738
739    #[async_trait]
740    impl BlobStore for ListProbe {
741        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
742            self.inner.put(key, value).await
743        }
744        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
745            self.inner.get(key).await
746        }
747        async fn get_range(
748            &self,
749            key: &str,
750            range: std::ops::Range<u64>,
751        ) -> Result<Option<Bytes>, ObjectLogError> {
752            self.inner.get_range(key, range).await
753        }
754        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
755            self.lists.fetch_add(1, Ordering::SeqCst);
756            self.inner.list(prefix).await
757        }
758        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
759            self.inner.delete(key).await
760        }
761        async fn compare_and_swap(
762            &self,
763            key: &str,
764            expected: Option<Bytes>,
765            new_value: Bytes,
766        ) -> Result<crate::CasOutcome, ObjectLogError> {
767            self.inner.compare_and_swap(key, expected, new_value).await
768        }
769    }
770
771    #[tokio::test]
772    async fn two_writers_cannot_commit_the_same_partition_tail() {
773        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
774        let first = Arc::new(
775            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
776                .await
777                .unwrap(),
778        );
779        let second = Arc::new(
780            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
781                .await
782                .unwrap(),
783        );
784        let won = run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
785        assert!(matches!(
786            won[0],
787            CommitOutcome::Assigned {
788                base_offset: 0,
789                record_count: 1
790            }
791        ));
792        let lost = run_commit(second, vec![("p", "obj-b", 1)]).unwrap();
793        assert!(
794            matches!(lost[0], CommitOutcome::Rejected { .. }),
795            "stale tail must fail closed, got {lost:?}"
796        );
797        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
798        let partition = PartitionKey("p".into());
799        assert_eq!(reopened.high_watermark(&partition).unwrap(), 1);
800        let entries = reopened.lookup(&partition, 0).unwrap();
801        assert_eq!(entries.len(), 1);
802        assert_eq!(entries[0].location.object_id, "obj-a");
803    }
804
805    #[tokio::test]
806    async fn mixed_seal_keeps_the_partition_that_won_and_rejects_the_moved_tail() {
807        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
808        let first = Arc::new(
809            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
810                .await
811                .unwrap(),
812        );
813        let second = Arc::new(
814            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
815                .await
816                .unwrap(),
817        );
818        run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
819        let mixed = run_commit(second, vec![("p", "obj-b", 1), ("q", "obj-b", 1)]).unwrap();
820        assert!(matches!(mixed[0], CommitOutcome::Rejected { .. }));
821        assert!(matches!(
822            mixed[1],
823            CommitOutcome::Assigned { base_offset: 0, .. }
824        ));
825        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
826        assert_eq!(
827            reopened.lookup(&PartitionKey("p".into()), 0).unwrap().len(),
828            1
829        );
830        assert_eq!(
831            reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
832            1,
833            "the rejected partition must not skip an offset"
834        );
835        assert_eq!(
836            reopened.lookup(&PartitionKey("q".into()), 0).unwrap()[0]
837                .location
838                .object_id,
839            "obj-b"
840        );
841    }
842
843    #[tokio::test]
844    async fn open_does_not_list_the_manifest_prefix_when_an_index_exists() {
845        let memory = MemoryBlobStore::new();
846        let blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
847        let seq = Arc::new(
848            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
849                .await
850                .unwrap(),
851        );
852        run_commit(seq, vec![("p", "obj-a", 4)]).unwrap();
853        let lists = Arc::new(AtomicUsize::new(0));
854        let probe = Arc::new(ListProbe {
855            inner: memory,
856            lists: Arc::clone(&lists),
857        });
858        let reopened = ManifestSequencer::open(probe, "manifest/").await.unwrap();
859        assert_eq!(
860            lists.load(Ordering::SeqCst),
861            0,
862            "a current index catalog must be read by key, not by listing manifests"
863        );
864        assert_eq!(
865            reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
866            1
867        );
868    }
869
870    #[tokio::test]
871    async fn distinct_writers_do_not_share_a_data_object_key() {
872        let blob = Arc::new(MemoryBlobStore::new());
873        let config = FlushConfig {
874            linger: std::time::Duration::ZERO,
875            ..FlushConfig::default()
876        };
877        let left = LogEngine::new_with_writer(
878            Arc::clone(&blob) as Arc<dyn BlobStore>,
879            Arc::new(InMemorySequencer::new()),
880            config,
881            "data/",
882            "writer-a",
883        );
884        let right = LogEngine::new_with_writer(
885            Arc::clone(&blob) as Arc<dyn BlobStore>,
886            Arc::new(InMemorySequencer::new()),
887            config,
888            "data/",
889            "writer-b",
890        );
891        left.produce_at_epoch(
892            PartitionKey("p".into()),
893            Bytes::from_static(b"left"),
894            1,
895            (),
896            Durability::Sequenced,
897            3,
898        )
899        .await
900        .unwrap();
901        right
902            .produce_at_epoch(
903                PartitionKey("p".into()),
904                Bytes::from_static(b"right"),
905                1,
906                (),
907                Durability::Sequenced,
908                3,
909            )
910            .await
911            .unwrap();
912        let keys = blob.list("data/").await.unwrap();
913        assert_eq!(
914            keys.len(),
915            2,
916            "writers must not overwrite one object: {keys:?}"
917        );
918        assert!(keys.iter().any(|key| key.contains("writer-a")));
919        assert!(keys.iter().any(|key| key.contains("writer-b")));
920        assert!(
921            keys.iter()
922                .all(|key| key.contains("/00000000000000000003/"))
923        );
924    }
925
926    #[tokio::test]
927    async fn fence_from_a_handle_that_never_saw_the_index_rejects_the_stale_writer() {
928        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
929        let owner = Arc::new(
930            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
931                .await
932                .unwrap(),
933        );
934        // The standby opens before the owner's first commit, so it holds no index.
935        let standby = Arc::new(
936            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
937                .await
938                .unwrap(),
939        );
940        run_commit(Arc::clone(&owner), vec![("p", "obj-a", 1)]).unwrap();
941        let partition = PartitionKey("p".into());
942        let fence_partition = partition.clone();
943        on_thread(&standby, move |seq| seq.fence_epoch(&fence_partition, 1, 2)).unwrap();
944
945        let stale = run_commit(Arc::clone(&owner), vec![("p", "obj-stale", 1)]).unwrap();
946        assert!(
947            matches!(stale[0], CommitOutcome::Rejected { .. }),
948            "a commit at the fenced epoch must fail closed, got {stale:?}"
949        );
950        let fresh = run_commit(standby, vec![("p", "obj-b", 2)]).unwrap();
951        assert!(matches!(
952            fresh[0],
953            CommitOutcome::Assigned { base_offset: 1, .. }
954        ));
955        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
956        let objects: Vec<_> = reopened
957            .lookup(&partition, 0)
958            .unwrap()
959            .into_iter()
960            .map(|entry| entry.location.object_id)
961            .collect();
962        assert_eq!(objects, ["obj-a", "obj-b"]);
963    }
964
965    #[tokio::test]
966    async fn fence_before_any_commit_creates_the_index_at_the_new_epoch() {
967        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
968        let stale = Arc::new(
969            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
970                .await
971                .unwrap(),
972        );
973        let fencer = Arc::new(
974            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
975                .await
976                .unwrap(),
977        );
978        on_thread(&fencer, |seq| {
979            seq.fence_epoch(&PartitionKey("p".into()), 0, 2)
980        })
981        .unwrap();
982        let lost = run_commit(stale, vec![("p", "obj-stale", 1)]).unwrap();
983        assert!(
984            matches!(lost[0], CommitOutcome::Rejected { .. }),
985            "a writer that never saw the fenced index must not commit, got {lost:?}"
986        );
987        let won = run_commit(fencer, vec![("p", "obj-b", 2)]).unwrap();
988        assert!(matches!(
989            won[0],
990            CommitOutcome::Assigned { base_offset: 0, .. }
991        ));
992        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
993        assert_eq!(
994            reopened.snapshot().partitions[0].entries[0]
995                .location
996                .object_id,
997            "obj-b"
998        );
999    }
1000
1001    #[tokio::test]
1002    async fn refresh_partition_reveals_another_writers_commits() {
1003        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1004        let writer = Arc::new(
1005            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1006                .await
1007                .unwrap(),
1008        );
1009        let reader = Arc::new(
1010            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1011                .await
1012                .unwrap(),
1013        );
1014        run_commit(writer, vec![("p", "obj-a", 1), ("p", "obj-b", 1)]).unwrap();
1015        let partition = PartitionKey("p".into());
1016        assert_eq!(reader.high_watermark(&partition).unwrap(), 0);
1017        let refreshed = partition.clone();
1018        on_thread(&reader, move |seq| seq.refresh_partition(&refreshed)).unwrap();
1019        assert_eq!(reader.high_watermark(&partition).unwrap(), 2);
1020        assert_eq!(reader.lookup(&partition, 0).unwrap().len(), 2);
1021    }
1022
1023    #[tokio::test]
1024    async fn a_fenced_writer_gets_a_typed_fenced_error() {
1025        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1026        let config = FlushConfig {
1027            linger: std::time::Duration::ZERO,
1028            ..FlushConfig::default()
1029        };
1030        let owner_sequencer = Arc::new(
1031            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1032                .await
1033                .unwrap(),
1034        );
1035        let standby = Arc::new(
1036            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1037                .await
1038                .unwrap(),
1039        );
1040        let owner = LogEngine::new_with_writer(
1041            Arc::clone(&blob),
1042            owner_sequencer,
1043            config,
1044            "data/",
1045            "writer-a",
1046        );
1047        let partition = PartitionKey("p".into());
1048        owner
1049            .produce_at_epoch(
1050                partition.clone(),
1051                Bytes::from_static(b"before"),
1052                1,
1053                (),
1054                Durability::Sequenced,
1055                1,
1056            )
1057            .await
1058            .unwrap();
1059        let fenced = partition.clone();
1060        on_thread(&standby, move |seq| seq.fence_epoch(&fenced, 1, 2)).unwrap();
1061
1062        let stale = owner
1063            .produce_at_epoch(
1064                partition.clone(),
1065                Bytes::from_static(b"stale"),
1066                1,
1067                (),
1068                Durability::Sequenced,
1069                1,
1070            )
1071            .await;
1072        match stale {
1073            Err(ObjectLogError::Fenced {
1074                partition: name,
1075                epoch: 1,
1076                current: 2,
1077            }) => assert_eq!(name, "p"),
1078            other => panic!("a write behind the fence must be Fenced, got {other:?}"),
1079        }
1080    }
1081}