Skip to main content

object_log/
manifest_sequencer.rs

1//! A `BlobStore`-persisted [`Sequencer`]: the offset→location index survives a
2//! restart, so a standalone object-log is crash-durable end to end.
3//!
4//! Each `commit` durably writes one manifest object (≤1 PUT per group-commit,
5//! co-amortized with the data object) recording the batches it assigned. On
6//! [`open`](ManifestSequencer::open) the index is rebuilt by replaying the
7//! manifest objects in order.
8
9use crate::{
10    BlobStore, CasOutcome, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey,
11    Sequencer,
12};
13use bytes::Bytes;
14use serde::{Deserialize, Serialize};
15use std::collections::{HashMap, HashSet};
16use std::sync::{Arc, Mutex};
17use tokio::runtime::{Builder, Runtime};
18
19#[derive(Serialize, Deserialize)]
20struct ManifestRecord {
21    entries: Vec<(PartitionKey, IndexEntry)>,
22}
23
24#[derive(Clone, Serialize, Deserialize)]
25struct PartitionIndexDoc {
26    epoch: u64,
27    next_offset: i64,
28    log_start: i64,
29    entries: Vec<IndexEntry>,
30}
31
32#[derive(Clone, Serialize, Deserialize, Default)]
33struct IndexCatalog {
34    partitions: Vec<String>,
35}
36
37#[derive(Default)]
38struct Part {
39    next_offset: i64,
40    log_start: i64,
41    epoch: u64,
42    entries: Vec<IndexEntry>,
43    /// Exact bytes last read or written for this partition's index object.
44    observed: Option<Bytes>,
45}
46
47struct Inner {
48    parts: HashMap<PartitionKey, Part>,
49    counter: u64,
50    catalog: Option<Bytes>,
51}
52
53/// A [`Sequencer`] (`Meta = ()`) that persists its index to a [`BlobStore`].
54pub struct ManifestSequencer {
55    blob: Arc<dyn BlobStore>,
56    prefix: String,
57    // `Option` so `Drop` can `shutdown_background()` — dropping a `Runtime` inside
58    // an async context (e.g. an `Arc` last-ref drop in a `#[tokio::test]`) panics.
59    rt: Option<Runtime>,
60    // Serialize mutations across publication without blocking committed reads.
61    commit_order: Mutex<()>,
62    inner: Mutex<Inner>,
63}
64
65impl Drop for ManifestSequencer {
66    fn drop(&mut self) {
67        if let Some(rt) = self.rt.take() {
68            rt.shutdown_background();
69        }
70    }
71}
72
73impl ManifestSequencer {
74    /// Open (or recover) a persisted sequencer. Manifest objects live under
75    /// `manifest_prefix` (keep it disjoint from the engine's data-object prefix).
76    /// Existing manifests are replayed to rebuild the index.
77    pub async fn open(
78        blob: Arc<dyn BlobStore>,
79        manifest_prefix: impl Into<String>,
80    ) -> Result<Self, ObjectLogError> {
81        let prefix = manifest_prefix.into();
82        let catalog_key = format!("{prefix}catalog");
83        let (parts, catalog, counter) = if let Some(catalog_bytes) = blob.get(&catalog_key).await? {
84            let catalog: IndexCatalog = serde_json::from_slice(&catalog_bytes)
85                .map_err(|e| ObjectLogError::Sequencer(format!("index catalog: {e}")))?;
86            let mut parts = HashMap::new();
87            for name in &catalog.partitions {
88                let key = index_object_key(&prefix, name);
89                let Some(doc_bytes) = blob.get(&key).await? else {
90                    parts.insert(PartitionKey(name.clone()), Part::default());
91                    continue;
92                };
93                let doc: PartitionIndexDoc = serde_json::from_slice(&doc_bytes).map_err(|e| {
94                    ObjectLogError::Sequencer(format!("partition index {name}: {e}"))
95                })?;
96                parts.insert(PartitionKey(name.clone()), Part::from_doc(doc, doc_bytes));
97            }
98            let counter = parts.len() as u64;
99            (parts, Some(catalog_bytes), counter)
100        } else {
101            let mut keys = blob.list(&prefix).await?;
102            keys.sort();
103            let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
104            for key in &keys {
105                if key == &catalog_key || key.starts_with(&format!("{prefix}index/")) {
106                    continue;
107                }
108                let Some(bytes) = blob.get(key).await? else {
109                    continue;
110                };
111                let Ok(rec) = serde_json::from_slice::<ManifestRecord>(&bytes) else {
112                    continue;
113                };
114                for (pkey, entry) in rec.entries {
115                    let p = parts.entry(pkey).or_default();
116                    let end = entry.base_offset + entry.record_count as i64;
117                    if end > p.next_offset {
118                        p.next_offset = end;
119                    }
120                    p.entries.push(entry);
121                }
122            }
123            (parts, None, keys.len() as u64)
124        };
125        let rt = Builder::new_current_thread()
126            .enable_all()
127            .build()
128            .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
129        Ok(Self {
130            blob,
131            prefix,
132            rt: Some(rt),
133            commit_order: Mutex::new(()),
134            inner: Mutex::new(Inner {
135                parts,
136                counter,
137                catalog,
138            }),
139        })
140    }
141
142    fn index_key(&self, partition: &PartitionKey) -> String {
143        index_object_key(&self.prefix, partition.as_str())
144    }
145
146    fn catalog_key(&self) -> String {
147        format!("{}catalog", self.prefix)
148    }
149
150    fn block_on<T>(&self, future: impl std::future::Future<Output = T>) -> T {
151        self.rt
152            .as_ref()
153            .expect("runtime present until drop")
154            .block_on(future)
155    }
156
157    fn read_object(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
158        self.block_on(self.blob.get(key))
159    }
160
161    fn cas(
162        &self,
163        key: &str,
164        expected: Option<Bytes>,
165        new_value: Bytes,
166    ) -> Result<CasOutcome, ObjectLogError> {
167        self.block_on(self.blob.compare_and_swap(key, expected, new_value))
168    }
169
170    fn ensure_catalog(&self, partitions: &[PartitionKey]) -> Result<(), ObjectLogError> {
171        let key = self.catalog_key();
172        for _ in 0..8 {
173            let current = self.read_object(&key)?;
174            let mut names = if let Some(bytes) = &current {
175                serde_json::from_slice::<IndexCatalog>(bytes)
176                    .map_err(|error| ObjectLogError::Sequencer(format!("index catalog: {error}")))?
177                    .partitions
178            } else {
179                Vec::new()
180            };
181            let mut changed = false;
182            for partition in partitions {
183                if !names.iter().any(|name| name == partition.as_str()) {
184                    names.push(partition.0.clone());
185                    changed = true;
186                }
187            }
188            if !changed {
189                self.inner.lock().expect("poisoned").catalog = current;
190                return Ok(());
191            }
192            names.sort();
193            names.dedup();
194            let new_bytes = Bytes::from(
195                serde_json::to_vec(&IndexCatalog { partitions: names })
196                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
197            );
198            match self.cas(&key, current, new_bytes.clone())? {
199                CasOutcome::Stored => {
200                    self.inner.lock().expect("poisoned").catalog = Some(new_bytes);
201                    return Ok(());
202                }
203                CasOutcome::Conflict { .. } => continue,
204            }
205        }
206        Err(ObjectLogError::Sequencer(
207            "catalog compare-and-swap did not land".into(),
208        ))
209    }
210
211    fn publish_partition(
212        &self,
213        partition: &PartitionKey,
214        batches: &[CommitBatch<'_, ()>],
215        indexes: &[usize],
216    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
217        let epoch = batches[indexes[0]].epoch;
218        if indexes.iter().any(|index| batches[*index].epoch != epoch) {
219            return Ok(reject_all(
220                indexes.len(),
221                "one partition commit mixed fence epochs",
222            ));
223        }
224        let (observed, mut doc) = {
225            let inner = self.inner.lock().expect("poisoned");
226            match inner
227                .parts
228                .get(partition)
229                .and_then(|part| part.observed.clone())
230            {
231                Some(bytes) => {
232                    let doc = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
233                        .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
234                    (Some(bytes), doc)
235                }
236                None => {
237                    let part = inner.parts.get(partition);
238                    (
239                        None,
240                        PartitionIndexDoc {
241                            epoch,
242                            next_offset: part.map_or(0, |part| part.next_offset),
243                            log_start: part.map_or(0, |part| part.log_start),
244                            entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
245                        },
246                    )
247                }
248            }
249        };
250        if observed.is_some() && doc.epoch != epoch {
251            return Ok(reject_all(
252                indexes.len(),
253                format!(
254                    "partition {} epoch is {}, not {epoch}",
255                    partition.as_str(),
256                    doc.epoch
257                ),
258            ));
259        }
260        let mut outcomes = Vec::with_capacity(indexes.len());
261        let mut planned = Vec::with_capacity(indexes.len());
262        for index in indexes {
263            let batch = &batches[*index];
264            let entry = IndexEntry {
265                location: batch.location.clone(),
266                base_offset: doc.next_offset,
267                record_count: batch.record_count,
268            };
269            outcomes.push(CommitOutcome::Assigned {
270                base_offset: entry.base_offset,
271                record_count: entry.record_count,
272            });
273            doc.next_offset += i64::from(batch.record_count);
274            planned.push(entry.clone());
275            doc.entries.push(entry);
276        }
277        let new_bytes = Bytes::from(
278            serde_json::to_vec(&doc)
279                .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
280        );
281        match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
282            CasOutcome::Stored => {
283                self.install_doc(partition, &doc, new_bytes);
284                Ok(outcomes)
285            }
286            CasOutcome::Conflict { current } => {
287                if let Some(bytes) = current
288                    && let Ok(found) = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
289                {
290                    let already = planned
291                        .iter()
292                        .all(|entry| found.entries.iter().any(|have| have == entry));
293                    self.install_doc(partition, &found, bytes);
294                    if already {
295                        return Ok(outcomes);
296                    }
297                }
298                Ok(reject_all(
299                    indexes.len(),
300                    format!("partition {} tail moved", partition.as_str()),
301                ))
302            }
303        }
304    }
305
306    /// Install `partition`'s durable index, if any, and return it.
307    fn reload_partition(
308        &self,
309        partition: &PartitionKey,
310    ) -> Result<Option<(PartitionIndexDoc, Bytes)>, ObjectLogError> {
311        let Some(bytes) = self.read_object(&self.index_key(partition))? else {
312            return Ok(None);
313        };
314        let doc: PartitionIndexDoc = serde_json::from_slice(&bytes)
315            .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
316        self.install_doc(partition, &doc, bytes.clone());
317        Ok(Some((doc, bytes)))
318    }
319
320    fn install_doc(&self, partition: &PartitionKey, doc: &PartitionIndexDoc, observed: Bytes) {
321        let mut inner = self.inner.lock().expect("poisoned");
322        inner.parts.insert(
323            partition.clone(),
324            Part {
325                next_offset: doc.next_offset,
326                log_start: doc.log_start,
327                epoch: doc.epoch,
328                entries: doc.entries.clone(),
329                observed: Some(observed),
330            },
331        );
332    }
333
334    /// Object ids referenced by any live index entry (for orphan reaping).
335    /// Does not include manifest object keys themselves.
336    pub fn live_object_ids(&self) -> HashSet<String> {
337        let st = self.inner.lock().expect("poisoned");
338        let mut live = HashSet::new();
339        for p in st.parts.values() {
340            for e in &p.entries {
341                live.insert(e.location.object_id.clone());
342            }
343        }
344        live
345    }
346
347    /// Point-in-time view of the rebuilt index (diagnostics / CLI).
348    pub fn snapshot(&self) -> IndexSnapshot {
349        let st = self.inner.lock().expect("poisoned");
350        let mut partitions: Vec<PartitionSnapshot> = st
351            .parts
352            .iter()
353            .map(|(k, p)| PartitionSnapshot {
354                partition: k.0.clone(),
355                log_start: p.log_start,
356                high_watermark: p.next_offset,
357                entry_count: p.entries.len(),
358                entries: p.entries.clone(),
359            })
360            .collect();
361        partitions.sort_by(|a, b| a.partition.cmp(&b.partition));
362        IndexSnapshot {
363            manifest_prefix: self.prefix.clone(),
364            manifest_count: st.counter,
365            partitions,
366        }
367    }
368}
369
370/// Full index view from [`ManifestSequencer::snapshot`].
371#[derive(Clone, Debug, Serialize, Deserialize)]
372pub struct IndexSnapshot {
373    /// Prefix used for manifest objects.
374    pub manifest_prefix: String,
375    /// Number of manifest objects replayed (commit counter).
376    pub manifest_count: u64,
377    /// Per-partition index state.
378    pub partitions: Vec<PartitionSnapshot>,
379}
380
381/// One partition's index bounds and entries.
382#[derive(Clone, Debug, Serialize, Deserialize)]
383pub struct PartitionSnapshot {
384    /// Partition key string.
385    pub partition: String,
386    /// First readable offset.
387    pub log_start: i64,
388    /// Next offset to assign (high watermark).
389    pub high_watermark: i64,
390    /// Number of index entries.
391    pub entry_count: usize,
392    /// Ordered index entries (may be large; CLI can omit via flag).
393    pub entries: Vec<IndexEntry>,
394}
395
396impl Sequencer for ManifestSequencer {
397    type Meta = ();
398
399    fn supports_multi_object_commit(&self) -> bool {
400        true
401    }
402
403    fn commit(
404        &self,
405        batches: &[CommitBatch<'_, ()>],
406    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
407        let _commit = self.commit_order.lock().expect("poisoned");
408        if batches.is_empty() {
409            return Ok(Vec::new());
410        }
411        let mut order: Vec<PartitionKey> = Vec::new();
412        let mut groups: HashMap<PartitionKey, Vec<usize>> = HashMap::new();
413        for (index, batch) in batches.iter().enumerate() {
414            if !groups.contains_key(&batch.partition) {
415                order.push(batch.partition.clone());
416            }
417            groups
418                .entry(batch.partition.clone())
419                .or_default()
420                .push(index);
421        }
422        self.ensure_catalog(&order)?;
423        let mut outcomes = vec![None; batches.len()];
424        let mut published = false;
425        for partition in order {
426            let indexes = &groups[&partition];
427            match self.publish_partition(&partition, batches, indexes) {
428                Ok(partition_outcomes) => {
429                    for (index, outcome) in indexes.iter().zip(partition_outcomes) {
430                        if matches!(
431                            outcome,
432                            CommitOutcome::Assigned { .. } | CommitOutcome::Duplicate { .. }
433                        ) {
434                            published = true;
435                        }
436                        outcomes[*index] = Some(outcome);
437                    }
438                }
439                Err(error) => {
440                    for index in indexes {
441                        outcomes[*index] = Some(CommitOutcome::Rejected {
442                            reason: error.to_string(),
443                        });
444                    }
445                }
446            }
447        }
448        if published {
449            let mut inner = self.inner.lock().expect("poisoned");
450            inner.counter = inner.counter.saturating_add(1);
451        }
452        Ok(outcomes
453            .into_iter()
454            .map(|outcome| outcome.expect("every batch has an outcome"))
455            .collect())
456    }
457
458    fn fence_epoch(
459        &self,
460        partition: &PartitionKey,
461        expected: u64,
462        new_epoch: u64,
463    ) -> Result<(), ObjectLogError> {
464        let _commit = self.commit_order.lock().expect("poisoned");
465        if expected == new_epoch {
466            return Ok(());
467        }
468        // Fence the durable index, not this handle's view of it: another writer
469        // may have created or advanced the index since this handle opened, and a
470        // fence that only compared cached bytes would silently store nothing.
471        self.ensure_catalog(std::slice::from_ref(partition))?;
472        for _ in 0..8 {
473            let (mut doc, observed) = match self.reload_partition(partition)? {
474                Some((current, bytes)) => {
475                    if current.epoch == new_epoch {
476                        return Ok(());
477                    }
478                    if current.epoch != expected {
479                        return Err(ObjectLogError::Sequencer(format!(
480                            "partition {} epoch is {}, not {expected}",
481                            partition.as_str(),
482                            current.epoch
483                        )));
484                    }
485                    (current, Some(bytes))
486                }
487                None => {
488                    // No index yet: create it at the new epoch, so a writer that
489                    // never saw one cannot publish under an older epoch.
490                    let inner = self.inner.lock().expect("poisoned");
491                    let part = inner.parts.get(partition);
492                    let doc = PartitionIndexDoc {
493                        epoch: new_epoch,
494                        next_offset: part.map_or(0, |part| part.next_offset),
495                        log_start: part.map_or(0, |part| part.log_start),
496                        entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
497                    };
498                    (doc, None)
499                }
500            };
501            doc.epoch = new_epoch;
502            let new_bytes = Bytes::from(
503                serde_json::to_vec(&doc)
504                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
505            );
506            match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
507                CasOutcome::Stored => {
508                    self.install_doc(partition, &doc, new_bytes);
509                    return Ok(());
510                }
511                // A concurrent commit or fence moved the index; re-read and retry.
512                CasOutcome::Conflict { .. } => continue,
513            }
514        }
515        Err(ObjectLogError::Sequencer(format!(
516            "partition {} epoch fence did not land",
517            partition.as_str()
518        )))
519    }
520
521    fn refresh_partition(&self, partition: &PartitionKey) -> Result<(), ObjectLogError> {
522        let _commit = self.commit_order.lock().expect("poisoned");
523        self.reload_partition(partition)?;
524        Ok(())
525    }
526
527    fn lookup(
528        &self,
529        partition: &PartitionKey,
530        fetch_offset: i64,
531    ) -> Result<Vec<IndexEntry>, ObjectLogError> {
532        let st = self.inner.lock().expect("poisoned");
533        let Some(p) = st.parts.get(partition) else {
534            return Ok(Vec::new());
535        };
536        Ok(p.entries
537            .iter()
538            .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
539            .cloned()
540            .collect())
541    }
542
543    fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
544        Ok(self
545            .inner
546            .lock()
547            .expect("poisoned")
548            .parts
549            .get(partition)
550            .map_or(0, |p| p.next_offset))
551    }
552
553    fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
554        Ok(self
555            .inner
556            .lock()
557            .expect("poisoned")
558            .parts
559            .get(partition)
560            .map_or(0, |p| p.log_start))
561    }
562
563    fn truncate_before(
564        &self,
565        partition: &PartitionKey,
566        offset: i64,
567    ) -> Result<Vec<String>, ObjectLogError> {
568        let _commit = self.commit_order.lock().expect("poisoned");
569        let snapshot = {
570            let st = self.inner.lock().expect("poisoned");
571            let Some(part) = st.parts.get(partition) else {
572                return Ok(Vec::new());
573            };
574            (
575                part.observed.clone(),
576                part.entries.clone(),
577                part.next_offset,
578                part.epoch,
579                part.log_start,
580            )
581        };
582        let (observed, entries, next_offset, epoch, mut log_start) = snapshot;
583        let mut dropped = Vec::new();
584        let mut kept = Vec::new();
585        for entry in entries {
586            if entry.base_offset + i64::from(entry.record_count) <= offset {
587                dropped.push(entry.location.object_id.clone());
588            } else {
589                kept.push(entry);
590            }
591        }
592        if offset > log_start {
593            log_start = offset.min(next_offset);
594        }
595        let doc = PartitionIndexDoc {
596            epoch,
597            next_offset,
598            log_start,
599            entries: kept,
600        };
601        if observed.is_some() {
602            let new_bytes = Bytes::from(
603                serde_json::to_vec(&doc)
604                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
605            );
606            match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
607                CasOutcome::Stored => self.install_doc(partition, &doc, new_bytes),
608                CasOutcome::Conflict { .. } => {
609                    return Err(ObjectLogError::Sequencer(format!(
610                        "partition {} truncate lost the compare-and-swap",
611                        partition.as_str()
612                    )));
613                }
614            }
615        } else {
616            let mut st = self.inner.lock().expect("poisoned");
617            if let Some(part) = st.parts.get_mut(partition) {
618                part.entries = doc.entries.clone();
619                part.log_start = doc.log_start;
620            }
621        }
622        let live = {
623            let st = self.inner.lock().expect("poisoned");
624            let mut live = HashSet::new();
625            for part in st.parts.values() {
626                for entry in &part.entries {
627                    live.insert(entry.location.object_id.clone());
628                }
629            }
630            live
631        };
632        let mut dead = Vec::new();
633        let mut seen = HashSet::new();
634        for object_id in dropped {
635            if !live.contains(&object_id) && seen.insert(object_id.clone()) {
636                dead.push(object_id);
637            }
638        }
639        Ok(dead)
640    }
641}
642
643impl Part {
644    fn from_doc(doc: PartitionIndexDoc, observed: Bytes) -> Self {
645        Self {
646            next_offset: doc.next_offset,
647            log_start: doc.log_start,
648            epoch: doc.epoch,
649            entries: doc.entries,
650            observed: Some(observed),
651        }
652    }
653}
654
655fn index_object_key(prefix: &str, partition: &str) -> String {
656    let mut encoded = String::with_capacity(partition.len() * 2);
657    for byte in partition.as_bytes() {
658        encoded.push_str(&format!("{byte:02x}"));
659    }
660    format!("{prefix}index/{encoded}")
661}
662
663fn reject_all(count: usize, reason: impl Into<String>) -> Vec<CommitOutcome> {
664    let reason = reason.into();
665    vec![CommitOutcome::Rejected { reason }; count]
666}
667
668#[cfg(test)]
669mod index_tests {
670    use super::*;
671    use crate::{
672        BatchLocation, BlobStore, CommitBatch, CommitOutcome, Durability, FlushConfig,
673        InMemorySequencer, LogEngine, MemoryBlobStore, ObjectLogError, PartitionKey,
674    };
675    use async_trait::async_trait;
676    use bytes::Bytes;
677    use std::sync::Arc;
678    use std::sync::atomic::{AtomicUsize, Ordering};
679
680    fn location(object: &str) -> BatchLocation {
681        BatchLocation {
682            object_id: object.to_string(),
683            byte_start: 0,
684            byte_len: 1,
685        }
686    }
687
688    fn run_commit(
689        seq: Arc<ManifestSequencer>,
690        partitions: Vec<(&str, &str, u64)>,
691    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
692        let owned: Vec<(String, String, u64)> = partitions
693            .into_iter()
694            .map(|(partition, object, epoch)| (partition.to_string(), object.to_string(), epoch))
695            .collect();
696        std::thread::spawn(move || {
697            let meta = ();
698            let batches: Vec<CommitBatch<'_, ()>> = owned
699                .iter()
700                .map(|(partition, object, epoch)| CommitBatch {
701                    partition: PartitionKey(partition.clone()),
702                    record_count: 1,
703                    location: location(object),
704                    meta: &meta,
705                    epoch: *epoch,
706                })
707                .collect();
708            seq.commit(&batches)
709        })
710        .join()
711        .expect("commit thread")
712    }
713
714    /// Run a blocking sequencer call off the async test thread.
715    fn on_thread<T: Send + 'static>(
716        seq: &Arc<ManifestSequencer>,
717        call: impl FnOnce(&ManifestSequencer) -> T + Send + 'static,
718    ) -> T {
719        let seq = Arc::clone(seq);
720        std::thread::spawn(move || call(&seq))
721            .join()
722            .expect("sequencer thread")
723    }
724
725    struct ListProbe {
726        inner: MemoryBlobStore,
727        lists: Arc<AtomicUsize>,
728    }
729
730    #[async_trait]
731    impl BlobStore for ListProbe {
732        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
733            self.inner.put(key, value).await
734        }
735        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
736            self.inner.get(key).await
737        }
738        async fn get_range(
739            &self,
740            key: &str,
741            range: std::ops::Range<u64>,
742        ) -> Result<Option<Bytes>, ObjectLogError> {
743            self.inner.get_range(key, range).await
744        }
745        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
746            self.lists.fetch_add(1, Ordering::SeqCst);
747            self.inner.list(prefix).await
748        }
749        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
750            self.inner.delete(key).await
751        }
752        async fn compare_and_swap(
753            &self,
754            key: &str,
755            expected: Option<Bytes>,
756            new_value: Bytes,
757        ) -> Result<crate::CasOutcome, ObjectLogError> {
758            self.inner.compare_and_swap(key, expected, new_value).await
759        }
760    }
761
762    #[tokio::test]
763    async fn two_writers_cannot_commit_the_same_partition_tail() {
764        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
765        let first = Arc::new(
766            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
767                .await
768                .unwrap(),
769        );
770        let second = Arc::new(
771            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
772                .await
773                .unwrap(),
774        );
775        let won = run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
776        assert!(matches!(
777            won[0],
778            CommitOutcome::Assigned {
779                base_offset: 0,
780                record_count: 1
781            }
782        ));
783        let lost = run_commit(second, vec![("p", "obj-b", 1)]).unwrap();
784        assert!(
785            matches!(lost[0], CommitOutcome::Rejected { .. }),
786            "stale tail must fail closed, got {lost:?}"
787        );
788        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
789        let partition = PartitionKey("p".into());
790        assert_eq!(reopened.high_watermark(&partition).unwrap(), 1);
791        let entries = reopened.lookup(&partition, 0).unwrap();
792        assert_eq!(entries.len(), 1);
793        assert_eq!(entries[0].location.object_id, "obj-a");
794    }
795
796    #[tokio::test]
797    async fn mixed_seal_keeps_the_partition_that_won_and_rejects_the_moved_tail() {
798        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
799        let first = Arc::new(
800            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
801                .await
802                .unwrap(),
803        );
804        let second = Arc::new(
805            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
806                .await
807                .unwrap(),
808        );
809        run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
810        let mixed = run_commit(second, vec![("p", "obj-b", 1), ("q", "obj-b", 1)]).unwrap();
811        assert!(matches!(mixed[0], CommitOutcome::Rejected { .. }));
812        assert!(matches!(
813            mixed[1],
814            CommitOutcome::Assigned { base_offset: 0, .. }
815        ));
816        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
817        assert_eq!(
818            reopened.lookup(&PartitionKey("p".into()), 0).unwrap().len(),
819            1
820        );
821        assert_eq!(
822            reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
823            1,
824            "the rejected partition must not skip an offset"
825        );
826        assert_eq!(
827            reopened.lookup(&PartitionKey("q".into()), 0).unwrap()[0]
828                .location
829                .object_id,
830            "obj-b"
831        );
832    }
833
834    #[tokio::test]
835    async fn open_does_not_list_the_manifest_prefix_when_an_index_exists() {
836        let memory = MemoryBlobStore::new();
837        let blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
838        let seq = Arc::new(
839            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
840                .await
841                .unwrap(),
842        );
843        run_commit(seq, vec![("p", "obj-a", 4)]).unwrap();
844        let lists = Arc::new(AtomicUsize::new(0));
845        let probe = Arc::new(ListProbe {
846            inner: memory,
847            lists: Arc::clone(&lists),
848        });
849        let reopened = ManifestSequencer::open(probe, "manifest/").await.unwrap();
850        assert_eq!(
851            lists.load(Ordering::SeqCst),
852            0,
853            "a current index catalog must be read by key, not by listing manifests"
854        );
855        assert_eq!(
856            reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
857            1
858        );
859    }
860
861    #[tokio::test]
862    async fn distinct_writers_do_not_share_a_data_object_key() {
863        let blob = Arc::new(MemoryBlobStore::new());
864        let config = FlushConfig {
865            linger: std::time::Duration::ZERO,
866            ..FlushConfig::default()
867        };
868        let left = LogEngine::new_with_writer(
869            Arc::clone(&blob) as Arc<dyn BlobStore>,
870            Arc::new(InMemorySequencer::new()),
871            config,
872            "data/",
873            "writer-a",
874        );
875        let right = LogEngine::new_with_writer(
876            Arc::clone(&blob) as Arc<dyn BlobStore>,
877            Arc::new(InMemorySequencer::new()),
878            config,
879            "data/",
880            "writer-b",
881        );
882        left.produce_at_epoch(
883            PartitionKey("p".into()),
884            Bytes::from_static(b"left"),
885            1,
886            (),
887            Durability::Sequenced,
888            3,
889        )
890        .await
891        .unwrap();
892        right
893            .produce_at_epoch(
894                PartitionKey("p".into()),
895                Bytes::from_static(b"right"),
896                1,
897                (),
898                Durability::Sequenced,
899                3,
900            )
901            .await
902            .unwrap();
903        let keys = blob.list("data/").await.unwrap();
904        assert_eq!(
905            keys.len(),
906            2,
907            "writers must not overwrite one object: {keys:?}"
908        );
909        assert!(keys.iter().any(|key| key.contains("writer-a")));
910        assert!(keys.iter().any(|key| key.contains("writer-b")));
911        assert!(
912            keys.iter()
913                .all(|key| key.contains("/00000000000000000003/"))
914        );
915    }
916
917    #[tokio::test]
918    async fn fence_from_a_handle_that_never_saw_the_index_rejects_the_stale_writer() {
919        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
920        let owner = Arc::new(
921            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
922                .await
923                .unwrap(),
924        );
925        // The standby opens before the owner's first commit, so it holds no index.
926        let standby = Arc::new(
927            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
928                .await
929                .unwrap(),
930        );
931        run_commit(Arc::clone(&owner), vec![("p", "obj-a", 1)]).unwrap();
932        let partition = PartitionKey("p".into());
933        let fence_partition = partition.clone();
934        on_thread(&standby, move |seq| seq.fence_epoch(&fence_partition, 1, 2)).unwrap();
935
936        let stale = run_commit(Arc::clone(&owner), vec![("p", "obj-stale", 1)]).unwrap();
937        assert!(
938            matches!(stale[0], CommitOutcome::Rejected { .. }),
939            "a commit at the fenced epoch must fail closed, got {stale:?}"
940        );
941        let fresh = run_commit(standby, vec![("p", "obj-b", 2)]).unwrap();
942        assert!(matches!(
943            fresh[0],
944            CommitOutcome::Assigned { base_offset: 1, .. }
945        ));
946        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
947        let objects: Vec<_> = reopened
948            .lookup(&partition, 0)
949            .unwrap()
950            .into_iter()
951            .map(|entry| entry.location.object_id)
952            .collect();
953        assert_eq!(objects, ["obj-a", "obj-b"]);
954    }
955
956    #[tokio::test]
957    async fn fence_before_any_commit_creates_the_index_at_the_new_epoch() {
958        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
959        let stale = Arc::new(
960            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
961                .await
962                .unwrap(),
963        );
964        let fencer = Arc::new(
965            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
966                .await
967                .unwrap(),
968        );
969        on_thread(&fencer, |seq| {
970            seq.fence_epoch(&PartitionKey("p".into()), 0, 2)
971        })
972        .unwrap();
973        let lost = run_commit(stale, vec![("p", "obj-stale", 1)]).unwrap();
974        assert!(
975            matches!(lost[0], CommitOutcome::Rejected { .. }),
976            "a writer that never saw the fenced index must not commit, got {lost:?}"
977        );
978        let won = run_commit(fencer, vec![("p", "obj-b", 2)]).unwrap();
979        assert!(matches!(
980            won[0],
981            CommitOutcome::Assigned { base_offset: 0, .. }
982        ));
983        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
984        assert_eq!(
985            reopened.snapshot().partitions[0].entries[0]
986                .location
987                .object_id,
988            "obj-b"
989        );
990    }
991
992    #[tokio::test]
993    async fn refresh_partition_reveals_another_writers_commits() {
994        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
995        let writer = Arc::new(
996            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
997                .await
998                .unwrap(),
999        );
1000        let reader = Arc::new(
1001            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1002                .await
1003                .unwrap(),
1004        );
1005        run_commit(writer, vec![("p", "obj-a", 1), ("p", "obj-b", 1)]).unwrap();
1006        let partition = PartitionKey("p".into());
1007        assert_eq!(reader.high_watermark(&partition).unwrap(), 0);
1008        let refreshed = partition.clone();
1009        on_thread(&reader, move |seq| seq.refresh_partition(&refreshed)).unwrap();
1010        assert_eq!(reader.high_watermark(&partition).unwrap(), 2);
1011        assert_eq!(reader.lookup(&partition, 0).unwrap().len(), 2);
1012    }
1013}