Skip to main content

object_log/
manifest_sequencer.rs

1//! A `BlobStore`-persisted [`Sequencer`]: the offset→location index survives a
2//! restart, so a standalone object-log is crash-durable end to end.
3//!
4//! Each `commit` durably writes one manifest object (≤1 PUT per group-commit,
5//! co-amortized with the data object) recording the batches it assigned. On
6//! [`open`](ManifestSequencer::open) the index is rebuilt by replaying the
7//! manifest objects in order.
8
9use crate::{
10    BlobStore, CasOutcome, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey,
11    Sequencer,
12};
13use bytes::Bytes;
14use serde::{Deserialize, Serialize};
15use std::collections::{HashMap, HashSet};
16use std::sync::{Arc, Mutex};
17use tokio::runtime::{Builder, Runtime};
18
19#[derive(Serialize, Deserialize)]
20struct ManifestRecord {
21    entries: Vec<(PartitionKey, IndexEntry)>,
22}
23
24#[derive(Clone, Serialize, Deserialize)]
25struct PartitionIndexDoc {
26    epoch: u64,
27    next_offset: i64,
28    log_start: i64,
29    entries: Vec<IndexEntry>,
30}
31
32#[derive(Clone, Serialize, Deserialize, Default)]
33struct IndexCatalog {
34    partitions: Vec<String>,
35}
36
37#[derive(Default)]
38struct Part {
39    next_offset: i64,
40    log_start: i64,
41    epoch: u64,
42    entries: Vec<IndexEntry>,
43    /// Exact bytes last read or written for this partition's index object.
44    observed: Option<Bytes>,
45}
46
47struct Inner {
48    parts: HashMap<PartitionKey, Part>,
49    counter: u64,
50    catalog: Option<Bytes>,
51}
52
53/// A [`Sequencer`] (`Meta = ()`) that persists its index to a [`BlobStore`].
54pub struct ManifestSequencer {
55    blob: Arc<dyn BlobStore>,
56    prefix: String,
57    // `Option` so `Drop` can `shutdown_background()` — dropping a `Runtime` inside
58    // an async context (e.g. an `Arc` last-ref drop in a `#[tokio::test]`) panics.
59    rt: Option<Runtime>,
60    // Serialize mutations across publication without blocking committed reads.
61    commit_order: Mutex<()>,
62    inner: Mutex<Inner>,
63}
64
65impl Drop for ManifestSequencer {
66    fn drop(&mut self) {
67        if let Some(rt) = self.rt.take() {
68            rt.shutdown_background();
69        }
70    }
71}
72
73impl ManifestSequencer {
74    /// Open (or recover) a persisted sequencer. Manifest objects live under
75    /// `manifest_prefix` (keep it disjoint from the engine's data-object prefix).
76    /// Existing manifests are replayed to rebuild the index.
77    pub async fn open(
78        blob: Arc<dyn BlobStore>,
79        manifest_prefix: impl Into<String>,
80    ) -> Result<Self, ObjectLogError> {
81        let prefix = manifest_prefix.into();
82        let catalog_key = format!("{prefix}catalog");
83        let (parts, catalog, counter) = if let Some(catalog_bytes) = blob.get(&catalog_key).await? {
84            let catalog: IndexCatalog = serde_json::from_slice(&catalog_bytes)
85                .map_err(|e| ObjectLogError::Sequencer(format!("index catalog: {e}")))?;
86            let mut parts = HashMap::new();
87            for name in &catalog.partitions {
88                let key = index_object_key(&prefix, name);
89                let Some(doc_bytes) = blob.get(&key).await? else {
90                    parts.insert(PartitionKey(name.clone()), Part::default());
91                    continue;
92                };
93                let doc: PartitionIndexDoc = serde_json::from_slice(&doc_bytes).map_err(|e| {
94                    ObjectLogError::Sequencer(format!("partition index {name}: {e}"))
95                })?;
96                parts.insert(PartitionKey(name.clone()), Part::from_doc(doc, doc_bytes));
97            }
98            let counter = parts.len() as u64;
99            (parts, Some(catalog_bytes), counter)
100        } else {
101            let mut keys = blob.list(&prefix).await?;
102            keys.sort();
103            let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
104            for key in &keys {
105                if key == &catalog_key || key.starts_with(&format!("{prefix}index/")) {
106                    continue;
107                }
108                let Some(bytes) = blob.get(key).await? else {
109                    continue;
110                };
111                let Ok(rec) = serde_json::from_slice::<ManifestRecord>(&bytes) else {
112                    continue;
113                };
114                for (pkey, entry) in rec.entries {
115                    let p = parts.entry(pkey).or_default();
116                    let end = entry.base_offset + entry.record_count as i64;
117                    if end > p.next_offset {
118                        p.next_offset = end;
119                    }
120                    p.entries.push(entry);
121                }
122            }
123            (parts, None, keys.len() as u64)
124        };
125        let rt = Builder::new_current_thread()
126            .enable_all()
127            .build()
128            .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
129        Ok(Self {
130            blob,
131            prefix,
132            rt: Some(rt),
133            commit_order: Mutex::new(()),
134            inner: Mutex::new(Inner {
135                parts,
136                counter,
137                catalog,
138            }),
139        })
140    }
141
142    fn index_key(&self, partition: &PartitionKey) -> String {
143        index_object_key(&self.prefix, partition.as_str())
144    }
145
146    fn catalog_key(&self) -> String {
147        format!("{}catalog", self.prefix)
148    }
149
150    fn block_on<T>(&self, future: impl std::future::Future<Output = T>) -> T {
151        self.rt
152            .as_ref()
153            .expect("runtime present until drop")
154            .block_on(future)
155    }
156
157    fn read_object(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
158        self.block_on(self.blob.get(key))
159    }
160
161    fn cas(
162        &self,
163        key: &str,
164        expected: Option<Bytes>,
165        new_value: Bytes,
166    ) -> Result<CasOutcome, ObjectLogError> {
167        self.block_on(self.blob.compare_and_swap(key, expected, new_value))
168    }
169
170    fn ensure_catalog(&self, partitions: &[PartitionKey]) -> Result<(), ObjectLogError> {
171        let key = self.catalog_key();
172        for _ in 0..8 {
173            let current = self.read_object(&key)?;
174            let mut names = if let Some(bytes) = &current {
175                serde_json::from_slice::<IndexCatalog>(bytes)
176                    .map_err(|error| ObjectLogError::Sequencer(format!("index catalog: {error}")))?
177                    .partitions
178            } else {
179                Vec::new()
180            };
181            let mut changed = false;
182            for partition in partitions {
183                if !names.iter().any(|name| name == partition.as_str()) {
184                    names.push(partition.0.clone());
185                    changed = true;
186                }
187            }
188            if !changed {
189                self.inner.lock().expect("poisoned").catalog = current;
190                return Ok(());
191            }
192            names.sort();
193            names.dedup();
194            let new_bytes = Bytes::from(
195                serde_json::to_vec(&IndexCatalog { partitions: names })
196                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
197            );
198            match self.cas(&key, current, new_bytes.clone())? {
199                CasOutcome::Stored => {
200                    self.inner.lock().expect("poisoned").catalog = Some(new_bytes);
201                    return Ok(());
202                }
203                CasOutcome::Conflict { .. } => continue,
204            }
205        }
206        Err(ObjectLogError::Sequencer(
207            "catalog compare-and-swap did not land".into(),
208        ))
209    }
210
211    fn publish_partition(
212        &self,
213        partition: &PartitionKey,
214        batches: &[CommitBatch<'_, ()>],
215        indexes: &[usize],
216    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
217        let epoch = batches[indexes[0]].epoch;
218        if indexes.iter().any(|index| batches[*index].epoch != epoch) {
219            return Ok(reject_all(
220                indexes.len(),
221                "one partition commit mixed fence epochs",
222            ));
223        }
224        let (observed, mut doc) = {
225            let inner = self.inner.lock().expect("poisoned");
226            match inner
227                .parts
228                .get(partition)
229                .and_then(|part| part.observed.clone())
230            {
231                Some(bytes) => {
232                    let doc = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
233                        .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
234                    (Some(bytes), doc)
235                }
236                None => {
237                    let part = inner.parts.get(partition);
238                    (
239                        None,
240                        PartitionIndexDoc {
241                            epoch,
242                            next_offset: part.map_or(0, |part| part.next_offset),
243                            log_start: part.map_or(0, |part| part.log_start),
244                            entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
245                        },
246                    )
247                }
248            }
249        };
250        if observed.is_some() && doc.epoch != epoch {
251            return Ok(reject_all(
252                indexes.len(),
253                format!(
254                    "partition {} epoch is {}, not {epoch}",
255                    partition.as_str(),
256                    doc.epoch
257                ),
258            ));
259        }
260        let mut outcomes = Vec::with_capacity(indexes.len());
261        let mut planned = Vec::with_capacity(indexes.len());
262        for index in indexes {
263            let batch = &batches[*index];
264            let entry = IndexEntry {
265                location: batch.location.clone(),
266                base_offset: doc.next_offset,
267                record_count: batch.record_count,
268            };
269            outcomes.push(CommitOutcome::Assigned {
270                base_offset: entry.base_offset,
271                record_count: entry.record_count,
272            });
273            doc.next_offset += i64::from(batch.record_count);
274            planned.push(entry.clone());
275            doc.entries.push(entry);
276        }
277        let new_bytes = Bytes::from(
278            serde_json::to_vec(&doc)
279                .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
280        );
281        match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
282            CasOutcome::Stored => {
283                self.install_doc(partition, &doc, new_bytes);
284                Ok(outcomes)
285            }
286            CasOutcome::Conflict { current } => {
287                if let Some(bytes) = current {
288                    if let Ok(found) = serde_json::from_slice::<PartitionIndexDoc>(&bytes) {
289                        let already = planned
290                            .iter()
291                            .all(|entry| found.entries.iter().any(|have| have == entry));
292                        self.install_doc(partition, &found, bytes);
293                        if already {
294                            return Ok(outcomes);
295                        }
296                    }
297                }
298                Ok(reject_all(
299                    indexes.len(),
300                    format!("partition {} tail moved", partition.as_str()),
301                ))
302            }
303        }
304    }
305
306    fn install_doc(&self, partition: &PartitionKey, doc: &PartitionIndexDoc, observed: Bytes) {
307        let mut inner = self.inner.lock().expect("poisoned");
308        inner.parts.insert(
309            partition.clone(),
310            Part {
311                next_offset: doc.next_offset,
312                log_start: doc.log_start,
313                epoch: doc.epoch,
314                entries: doc.entries.clone(),
315                observed: Some(observed),
316            },
317        );
318    }
319
320    /// Object ids referenced by any live index entry (for orphan reaping).
321    /// Does not include manifest object keys themselves.
322    pub fn live_object_ids(&self) -> HashSet<String> {
323        let st = self.inner.lock().expect("poisoned");
324        let mut live = HashSet::new();
325        for p in st.parts.values() {
326            for e in &p.entries {
327                live.insert(e.location.object_id.clone());
328            }
329        }
330        live
331    }
332
333    /// Point-in-time view of the rebuilt index (diagnostics / CLI).
334    pub fn snapshot(&self) -> IndexSnapshot {
335        let st = self.inner.lock().expect("poisoned");
336        let mut partitions: Vec<PartitionSnapshot> = st
337            .parts
338            .iter()
339            .map(|(k, p)| PartitionSnapshot {
340                partition: k.0.clone(),
341                log_start: p.log_start,
342                high_watermark: p.next_offset,
343                entry_count: p.entries.len(),
344                entries: p.entries.clone(),
345            })
346            .collect();
347        partitions.sort_by(|a, b| a.partition.cmp(&b.partition));
348        IndexSnapshot {
349            manifest_prefix: self.prefix.clone(),
350            manifest_count: st.counter,
351            partitions,
352        }
353    }
354}
355
356/// Full index view from [`ManifestSequencer::snapshot`].
357#[derive(Clone, Debug, Serialize, Deserialize)]
358pub struct IndexSnapshot {
359    /// Prefix used for manifest objects.
360    pub manifest_prefix: String,
361    /// Number of manifest objects replayed (commit counter).
362    pub manifest_count: u64,
363    /// Per-partition index state.
364    pub partitions: Vec<PartitionSnapshot>,
365}
366
367/// One partition's index bounds and entries.
368#[derive(Clone, Debug, Serialize, Deserialize)]
369pub struct PartitionSnapshot {
370    /// Partition key string.
371    pub partition: String,
372    /// First readable offset.
373    pub log_start: i64,
374    /// Next offset to assign (high watermark).
375    pub high_watermark: i64,
376    /// Number of index entries.
377    pub entry_count: usize,
378    /// Ordered index entries (may be large; CLI can omit via flag).
379    pub entries: Vec<IndexEntry>,
380}
381
382impl Sequencer for ManifestSequencer {
383    type Meta = ();
384
385    fn supports_multi_object_commit(&self) -> bool {
386        true
387    }
388
389    fn commit(
390        &self,
391        batches: &[CommitBatch<'_, ()>],
392    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
393        let _commit = self.commit_order.lock().expect("poisoned");
394        if batches.is_empty() {
395            return Ok(Vec::new());
396        }
397        let mut order: Vec<PartitionKey> = Vec::new();
398        let mut groups: HashMap<PartitionKey, Vec<usize>> = HashMap::new();
399        for (index, batch) in batches.iter().enumerate() {
400            if !groups.contains_key(&batch.partition) {
401                order.push(batch.partition.clone());
402            }
403            groups
404                .entry(batch.partition.clone())
405                .or_default()
406                .push(index);
407        }
408        self.ensure_catalog(&order)?;
409        let mut outcomes = vec![None; batches.len()];
410        let mut published = false;
411        for partition in order {
412            let indexes = &groups[&partition];
413            match self.publish_partition(&partition, batches, indexes) {
414                Ok(partition_outcomes) => {
415                    for (index, outcome) in indexes.iter().zip(partition_outcomes) {
416                        if matches!(
417                            outcome,
418                            CommitOutcome::Assigned { .. } | CommitOutcome::Duplicate { .. }
419                        ) {
420                            published = true;
421                        }
422                        outcomes[*index] = Some(outcome);
423                    }
424                }
425                Err(error) => {
426                    for index in indexes {
427                        outcomes[*index] = Some(CommitOutcome::Rejected {
428                            reason: error.to_string(),
429                        });
430                    }
431                }
432            }
433        }
434        if published {
435            let mut inner = self.inner.lock().expect("poisoned");
436            inner.counter = inner.counter.saturating_add(1);
437        }
438        Ok(outcomes
439            .into_iter()
440            .map(|outcome| outcome.expect("every batch has an outcome"))
441            .collect())
442    }
443
444    fn fence_epoch(
445        &self,
446        partition: &PartitionKey,
447        expected: u64,
448        new_epoch: u64,
449    ) -> Result<(), ObjectLogError> {
450        let _commit = self.commit_order.lock().expect("poisoned");
451        if expected == new_epoch {
452            return Ok(());
453        }
454        let observed = {
455            let inner = self.inner.lock().expect("poisoned");
456            match inner.parts.get(partition) {
457                Some(part) => part.observed.clone(),
458                None => return Ok(()),
459            }
460        };
461        let Some(current_bytes) = observed else {
462            return Ok(());
463        };
464        let current: PartitionIndexDoc = serde_json::from_slice(&current_bytes)
465            .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
466        if current.epoch == new_epoch {
467            return Ok(());
468        }
469        if current.epoch != expected {
470            return Err(ObjectLogError::Sequencer(format!(
471                "partition {} epoch is {}, not {expected}",
472                partition.as_str(),
473                current.epoch
474            )));
475        }
476        let mut updated = current;
477        updated.epoch = new_epoch;
478        let new_bytes = Bytes::from(
479            serde_json::to_vec(&updated)
480                .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
481        );
482        match self.cas(
483            &self.index_key(partition),
484            Some(current_bytes),
485            new_bytes.clone(),
486        )? {
487            CasOutcome::Stored => {
488                let mut inner = self.inner.lock().expect("poisoned");
489                if let Some(part) = inner.parts.get_mut(partition) {
490                    part.epoch = new_epoch;
491                    part.observed = Some(new_bytes);
492                }
493                Ok(())
494            }
495            CasOutcome::Conflict { current } => Err(ObjectLogError::Sequencer(format!(
496                "partition {} epoch fence lost the compare-and-swap ({})",
497                partition.as_str(),
498                current.is_some()
499            ))),
500        }
501    }
502
503    fn lookup(
504        &self,
505        partition: &PartitionKey,
506        fetch_offset: i64,
507    ) -> Result<Vec<IndexEntry>, ObjectLogError> {
508        let st = self.inner.lock().expect("poisoned");
509        let Some(p) = st.parts.get(partition) else {
510            return Ok(Vec::new());
511        };
512        Ok(p.entries
513            .iter()
514            .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
515            .cloned()
516            .collect())
517    }
518
519    fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
520        Ok(self
521            .inner
522            .lock()
523            .expect("poisoned")
524            .parts
525            .get(partition)
526            .map_or(0, |p| p.next_offset))
527    }
528
529    fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
530        Ok(self
531            .inner
532            .lock()
533            .expect("poisoned")
534            .parts
535            .get(partition)
536            .map_or(0, |p| p.log_start))
537    }
538
539    fn truncate_before(
540        &self,
541        partition: &PartitionKey,
542        offset: i64,
543    ) -> Result<Vec<String>, ObjectLogError> {
544        let _commit = self.commit_order.lock().expect("poisoned");
545        let snapshot = {
546            let st = self.inner.lock().expect("poisoned");
547            let Some(part) = st.parts.get(partition) else {
548                return Ok(Vec::new());
549            };
550            (
551                part.observed.clone(),
552                part.entries.clone(),
553                part.next_offset,
554                part.epoch,
555                part.log_start,
556            )
557        };
558        let (observed, entries, next_offset, epoch, mut log_start) = snapshot;
559        let mut dropped = Vec::new();
560        let mut kept = Vec::new();
561        for entry in entries {
562            if entry.base_offset + i64::from(entry.record_count) <= offset {
563                dropped.push(entry.location.object_id.clone());
564            } else {
565                kept.push(entry);
566            }
567        }
568        if offset > log_start {
569            log_start = offset.min(next_offset);
570        }
571        let doc = PartitionIndexDoc {
572            epoch,
573            next_offset,
574            log_start,
575            entries: kept,
576        };
577        if observed.is_some() {
578            let new_bytes = Bytes::from(
579                serde_json::to_vec(&doc)
580                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
581            );
582            match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
583                CasOutcome::Stored => self.install_doc(partition, &doc, new_bytes),
584                CasOutcome::Conflict { .. } => {
585                    return Err(ObjectLogError::Sequencer(format!(
586                        "partition {} truncate lost the compare-and-swap",
587                        partition.as_str()
588                    )));
589                }
590            }
591        } else {
592            let mut st = self.inner.lock().expect("poisoned");
593            if let Some(part) = st.parts.get_mut(partition) {
594                part.entries = doc.entries.clone();
595                part.log_start = doc.log_start;
596            }
597        }
598        let live = {
599            let st = self.inner.lock().expect("poisoned");
600            let mut live = HashSet::new();
601            for part in st.parts.values() {
602                for entry in &part.entries {
603                    live.insert(entry.location.object_id.clone());
604                }
605            }
606            live
607        };
608        let mut dead = Vec::new();
609        let mut seen = HashSet::new();
610        for object_id in dropped {
611            if !live.contains(&object_id) && seen.insert(object_id.clone()) {
612                dead.push(object_id);
613            }
614        }
615        Ok(dead)
616    }
617}
618
619impl Part {
620    fn from_doc(doc: PartitionIndexDoc, observed: Bytes) -> Self {
621        Self {
622            next_offset: doc.next_offset,
623            log_start: doc.log_start,
624            epoch: doc.epoch,
625            entries: doc.entries,
626            observed: Some(observed),
627        }
628    }
629}
630
631fn index_object_key(prefix: &str, partition: &str) -> String {
632    let mut encoded = String::with_capacity(partition.len() * 2);
633    for byte in partition.as_bytes() {
634        encoded.push_str(&format!("{byte:02x}"));
635    }
636    format!("{prefix}index/{encoded}")
637}
638
639fn reject_all(count: usize, reason: impl Into<String>) -> Vec<CommitOutcome> {
640    let reason = reason.into();
641    vec![CommitOutcome::Rejected { reason }; count]
642}
643
644#[cfg(test)]
645mod index_tests {
646    use super::*;
647    use crate::{
648        BatchLocation, BlobStore, CommitBatch, CommitOutcome, Durability, FlushConfig,
649        InMemorySequencer, LogEngine, MemoryBlobStore, ObjectLogError, PartitionKey,
650    };
651    use async_trait::async_trait;
652    use bytes::Bytes;
653    use std::sync::Arc;
654    use std::sync::atomic::{AtomicUsize, Ordering};
655
656    fn location(object: &str) -> BatchLocation {
657        BatchLocation {
658            object_id: object.to_string(),
659            byte_start: 0,
660            byte_len: 1,
661        }
662    }
663
664    fn run_commit(
665        seq: Arc<ManifestSequencer>,
666        partitions: Vec<(&str, &str, u64)>,
667    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
668        let owned: Vec<(String, String, u64)> = partitions
669            .into_iter()
670            .map(|(partition, object, epoch)| (partition.to_string(), object.to_string(), epoch))
671            .collect();
672        std::thread::spawn(move || {
673            let meta = ();
674            let batches: Vec<CommitBatch<'_, ()>> = owned
675                .iter()
676                .map(|(partition, object, epoch)| CommitBatch {
677                    partition: PartitionKey(partition.clone()),
678                    record_count: 1,
679                    location: location(object),
680                    meta: &meta,
681                    epoch: *epoch,
682                })
683                .collect();
684            seq.commit(&batches)
685        })
686        .join()
687        .expect("commit thread")
688    }
689
690    struct ListProbe {
691        inner: MemoryBlobStore,
692        lists: Arc<AtomicUsize>,
693    }
694
695    #[async_trait]
696    impl BlobStore for ListProbe {
697        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
698            self.inner.put(key, value).await
699        }
700        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
701            self.inner.get(key).await
702        }
703        async fn get_range(
704            &self,
705            key: &str,
706            range: std::ops::Range<u64>,
707        ) -> Result<Option<Bytes>, ObjectLogError> {
708            self.inner.get_range(key, range).await
709        }
710        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
711            self.lists.fetch_add(1, Ordering::SeqCst);
712            self.inner.list(prefix).await
713        }
714        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
715            self.inner.delete(key).await
716        }
717        async fn compare_and_swap(
718            &self,
719            key: &str,
720            expected: Option<Bytes>,
721            new_value: Bytes,
722        ) -> Result<crate::CasOutcome, ObjectLogError> {
723            self.inner.compare_and_swap(key, expected, new_value).await
724        }
725    }
726
727    #[tokio::test]
728    async fn two_writers_cannot_commit_the_same_partition_tail() {
729        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
730        let first = Arc::new(
731            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
732                .await
733                .unwrap(),
734        );
735        let second = Arc::new(
736            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
737                .await
738                .unwrap(),
739        );
740        let won = run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
741        assert!(matches!(
742            won[0],
743            CommitOutcome::Assigned {
744                base_offset: 0,
745                record_count: 1
746            }
747        ));
748        let lost = run_commit(second, vec![("p", "obj-b", 1)]).unwrap();
749        assert!(
750            matches!(lost[0], CommitOutcome::Rejected { .. }),
751            "stale tail must fail closed, got {lost:?}"
752        );
753        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
754        let partition = PartitionKey("p".into());
755        assert_eq!(reopened.high_watermark(&partition).unwrap(), 1);
756        let entries = reopened.lookup(&partition, 0).unwrap();
757        assert_eq!(entries.len(), 1);
758        assert_eq!(entries[0].location.object_id, "obj-a");
759    }
760
761    #[tokio::test]
762    async fn mixed_seal_keeps_the_partition_that_won_and_rejects_the_moved_tail() {
763        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
764        let first = Arc::new(
765            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
766                .await
767                .unwrap(),
768        );
769        let second = Arc::new(
770            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
771                .await
772                .unwrap(),
773        );
774        run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
775        let mixed = run_commit(second, vec![("p", "obj-b", 1), ("q", "obj-b", 1)]).unwrap();
776        assert!(matches!(mixed[0], CommitOutcome::Rejected { .. }));
777        assert!(matches!(
778            mixed[1],
779            CommitOutcome::Assigned { base_offset: 0, .. }
780        ));
781        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
782        assert_eq!(
783            reopened.lookup(&PartitionKey("p".into()), 0).unwrap().len(),
784            1
785        );
786        assert_eq!(
787            reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
788            1,
789            "the rejected partition must not skip an offset"
790        );
791        assert_eq!(
792            reopened.lookup(&PartitionKey("q".into()), 0).unwrap()[0]
793                .location
794                .object_id,
795            "obj-b"
796        );
797    }
798
799    #[tokio::test]
800    async fn open_does_not_list_the_manifest_prefix_when_an_index_exists() {
801        let memory = MemoryBlobStore::new();
802        let blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
803        let seq = Arc::new(
804            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
805                .await
806                .unwrap(),
807        );
808        run_commit(seq, vec![("p", "obj-a", 4)]).unwrap();
809        let lists = Arc::new(AtomicUsize::new(0));
810        let probe = Arc::new(ListProbe {
811            inner: memory,
812            lists: Arc::clone(&lists),
813        });
814        let reopened = ManifestSequencer::open(probe, "manifest/").await.unwrap();
815        assert_eq!(
816            lists.load(Ordering::SeqCst),
817            0,
818            "a current index catalog must be read by key, not by listing manifests"
819        );
820        assert_eq!(
821            reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
822            1
823        );
824    }
825
826    #[tokio::test]
827    async fn distinct_writers_do_not_share_a_data_object_key() {
828        let blob = Arc::new(MemoryBlobStore::new());
829        let config = FlushConfig {
830            linger: std::time::Duration::ZERO,
831            ..FlushConfig::default()
832        };
833        let left = LogEngine::new_with_writer(
834            Arc::clone(&blob) as Arc<dyn BlobStore>,
835            Arc::new(InMemorySequencer::new()),
836            config,
837            "data/",
838            "writer-a",
839        );
840        let right = LogEngine::new_with_writer(
841            Arc::clone(&blob) as Arc<dyn BlobStore>,
842            Arc::new(InMemorySequencer::new()),
843            config,
844            "data/",
845            "writer-b",
846        );
847        left.produce_at_epoch(
848            PartitionKey("p".into()),
849            Bytes::from_static(b"left"),
850            1,
851            (),
852            Durability::Sequenced,
853            3,
854        )
855        .await
856        .unwrap();
857        right
858            .produce_at_epoch(
859                PartitionKey("p".into()),
860                Bytes::from_static(b"right"),
861                1,
862                (),
863                Durability::Sequenced,
864                3,
865            )
866            .await
867            .unwrap();
868        let keys = blob.list("data/").await.unwrap();
869        assert_eq!(
870            keys.len(),
871            2,
872            "writers must not overwrite one object: {keys:?}"
873        );
874        assert!(keys.iter().any(|key| key.contains("writer-a")));
875        assert!(keys.iter().any(|key| key.contains("writer-b")));
876        assert!(
877            keys.iter()
878                .all(|key| key.contains("/00000000000000000003/"))
879        );
880    }
881}