Skip to main content

object_log/
manifest_sequencer.rs

1//! A `BlobStore`-persisted [`Sequencer`]: the offset→location index survives a
2//! restart, so a standalone object-log is crash-durable end to end.
3//!
4//! Each `commit` durably writes one manifest object (≤1 PUT per group-commit,
5//! co-amortized with the data object) recording the batches it assigned. On
6//! [`open`](ManifestSequencer::open) the index is rebuilt by replaying the
7//! manifest objects in order.
8
9use crate::{
10    AuthoritativePartitionState, BlobStore, CasOutcome, CommitBatch, CommitOutcome,
11    ConditionalCommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
12};
13use bytes::Bytes;
14use serde::{Deserialize, Serialize};
15use std::collections::{HashMap, HashSet};
16use std::sync::{Arc, Mutex};
17use tokio::runtime::{Builder, Runtime};
18
19#[derive(Serialize, Deserialize)]
20struct ManifestRecord {
21    entries: Vec<(PartitionKey, IndexEntry)>,
22}
23
24#[derive(Clone, Serialize, Deserialize)]
25struct PartitionIndexDoc {
26    epoch: u64,
27    next_offset: i64,
28    log_start: i64,
29    entries: Vec<IndexEntry>,
30}
31
32#[derive(Clone, Serialize, Deserialize, Default)]
33struct IndexCatalog {
34    partitions: Vec<String>,
35}
36
37#[derive(Default)]
38struct Part {
39    next_offset: i64,
40    log_start: i64,
41    epoch: u64,
42    entries: Vec<IndexEntry>,
43    /// Exact bytes last read or written for this partition's index object.
44    observed: Option<Bytes>,
45}
46
47struct Inner {
48    uncertain_epochs: HashSet<PartitionKey>,
49    parts: HashMap<PartitionKey, Part>,
50    counter: u64,
51    catalog: Option<Bytes>,
52}
53
54/// A [`Sequencer`] (`Meta = ()`) that persists its index to a [`BlobStore`].
55pub struct ManifestSequencer {
56    blob: Arc<dyn BlobStore>,
57    prefix: String,
58    // `Option` so `Drop` can `shutdown_background()` — dropping a `Runtime` inside
59    // an async context (e.g. an `Arc` last-ref drop in a `#[tokio::test]`) panics.
60    rt: Option<Runtime>,
61    // Serialize mutations across publication without blocking committed reads.
62    commit_order: Mutex<()>,
63    inner: Mutex<Inner>,
64}
65
66impl Drop for ManifestSequencer {
67    fn drop(&mut self) {
68        if let Some(rt) = self.rt.take() {
69            rt.shutdown_background();
70        }
71    }
72}
73
74impl ManifestSequencer {
75    /// Open (or recover) a persisted sequencer. Manifest objects live under
76    /// `manifest_prefix` (keep it disjoint from the engine's data-object prefix).
77    /// Existing manifests are replayed to rebuild the index.
78    pub async fn open(
79        blob: Arc<dyn BlobStore>,
80        manifest_prefix: impl Into<String>,
81    ) -> Result<Self, ObjectLogError> {
82        let prefix = manifest_prefix.into();
83        let catalog_key = format!("{prefix}catalog");
84        let (parts, catalog, counter) = if let Some(catalog_bytes) = blob.get(&catalog_key).await? {
85            let catalog: IndexCatalog = serde_json::from_slice(&catalog_bytes)
86                .map_err(|e| ObjectLogError::Sequencer(format!("index catalog: {e}")))?;
87            let mut parts = HashMap::new();
88            for name in &catalog.partitions {
89                let key = index_object_key(&prefix, name);
90                let Some(doc_bytes) = blob.get(&key).await? else {
91                    parts.insert(PartitionKey(name.clone()), Part::default());
92                    continue;
93                };
94                let doc: PartitionIndexDoc = serde_json::from_slice(&doc_bytes).map_err(|e| {
95                    ObjectLogError::Sequencer(format!("partition index {name}: {e}"))
96                })?;
97                parts.insert(PartitionKey(name.clone()), Part::from_doc(doc, doc_bytes)?);
98            }
99            let counter = parts.len() as u64;
100            (parts, Some(catalog_bytes), counter)
101        } else {
102            let mut keys = blob.list(&prefix).await?;
103            keys.sort();
104            let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
105            for key in &keys {
106                if key == &catalog_key || key.starts_with(&format!("{prefix}index/")) {
107                    continue;
108                }
109                let Some(bytes) = blob.get(key).await? else {
110                    continue;
111                };
112                let Ok(rec) = serde_json::from_slice::<ManifestRecord>(&bytes) else {
113                    continue;
114                };
115                for (pkey, entry) in rec.entries {
116                    let p = parts.entry(pkey).or_default();
117                    let end = crate::bounded_lookup::entry_end(&entry)?;
118                    if end > p.next_offset {
119                        p.next_offset = end;
120                    }
121                    p.entries.push(entry);
122                }
123            }
124            (parts, None, keys.len() as u64)
125        };
126        for part in parts.values() {
127            crate::bounded_lookup::validate_index(&part.entries, part.log_start, part.next_offset)?;
128        }
129        // The same blob client is also used by the engine's upload runtime.
130        // Keep connection tasks and their I/O reactor driven between synchronous
131        // calls; a current-thread runtime strands pooled connection drivers.
132        let rt = Builder::new_multi_thread()
133            .worker_threads(1)
134            .enable_all()
135            .build()
136            .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
137        Ok(Self {
138            blob,
139            prefix,
140            rt: Some(rt),
141            commit_order: Mutex::new(()),
142            inner: Mutex::new(Inner {
143                uncertain_epochs: parts.keys().cloned().collect(),
144                parts,
145                counter,
146                catalog,
147            }),
148        })
149    }
150
151    fn index_key(&self, partition: &PartitionKey) -> String {
152        index_object_key(&self.prefix, partition.as_str())
153    }
154
155    fn catalog_key(&self) -> String {
156        format!("{}catalog", self.prefix)
157    }
158
159    fn block_on<T>(&self, future: impl std::future::Future<Output = T>) -> T {
160        self.rt
161            .as_ref()
162            .expect("runtime present until drop")
163            .block_on(future)
164    }
165
166    fn read_object(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
167        self.block_on(self.blob.get(key))
168    }
169
170    fn cas(
171        &self,
172        key: &str,
173        expected: Option<Bytes>,
174        new_value: Bytes,
175    ) -> Result<CasOutcome, ObjectLogError> {
176        self.block_on(self.blob.compare_and_swap(key, expected, new_value))
177    }
178
179    fn ensure_catalog(&self, partitions: &[PartitionKey]) -> Result<bool, ObjectLogError> {
180        let key = self.catalog_key();
181        for _ in 0..8 {
182            let current = self.read_object(&key)?;
183            let mut names = if let Some(bytes) = &current {
184                serde_json::from_slice::<IndexCatalog>(bytes)
185                    .map_err(|error| ObjectLogError::Sequencer(format!("index catalog: {error}")))?
186                    .partitions
187            } else {
188                Vec::new()
189            };
190            // Catalog observations remain known even if a later snapshot
191            // loses membership or the catalog itself is deleted.
192            self.inner
193                .lock()
194                .expect("poisoned")
195                .uncertain_epochs
196                .extend(names.iter().cloned().map(PartitionKey));
197            let mut changed = false;
198            for partition in partitions {
199                if !names.iter().any(|name| name == partition.as_str()) {
200                    names.push(partition.0.clone());
201                    changed = true;
202                }
203            }
204            if !changed {
205                self.inner.lock().expect("poisoned").catalog = current;
206                return Ok(false);
207            }
208            names.sort();
209            names.dedup();
210            let new_bytes = Bytes::from(
211                serde_json::to_vec(&IndexCatalog { partitions: names })
212                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
213            );
214            self.inner
215                .lock()
216                .expect("poisoned")
217                .uncertain_epochs
218                .extend(partitions.iter().cloned());
219            match self
220                .cas(&key, current, new_bytes.clone())
221                .inspect_err(|_| {
222                    self.inner
223                        .lock()
224                        .expect("poisoned")
225                        .uncertain_epochs
226                        .extend(partitions.iter().cloned());
227                })? {
228                CasOutcome::Stored => {
229                    let mut inner = self.inner.lock().expect("poisoned");
230                    inner.uncertain_epochs.extend(partitions.iter().cloned());
231                    inner.catalog = Some(new_bytes);
232                    return Ok(true);
233                }
234                CasOutcome::Conflict {
235                    current: Some(bytes),
236                } => {
237                    let catalog: IndexCatalog =
238                        serde_json::from_slice(&bytes).map_err(|error| {
239                            ObjectLogError::Sequencer(format!("conflicting catalog: {error}"))
240                        })?;
241                    let registered = partitions.iter().all(|partition| {
242                        catalog
243                            .partitions
244                            .iter()
245                            .any(|name| name == partition.as_str())
246                    });
247                    let mut inner = self.inner.lock().expect("poisoned");
248                    inner
249                        .uncertain_epochs
250                        .extend(catalog.partitions.into_iter().map(PartitionKey));
251                    inner.catalog = Some(bytes);
252                    if registered {
253                        return Ok(false);
254                    }
255                }
256                CasOutcome::Conflict { current: None } => {
257                    self.inner
258                        .lock()
259                        .expect("poisoned")
260                        .uncertain_epochs
261                        .extend(partitions.iter().cloned());
262                    return Err(ObjectLogError::StorageUnavailable(
263                        "catalog conflict lost authority".into(),
264                    ));
265                }
266            }
267        }
268        Err(ObjectLogError::Sequencer(
269            "catalog compare-and-swap did not land".into(),
270        ))
271    }
272
273    fn publish_partition(
274        &self,
275        partition: &PartitionKey,
276        batches: &[CommitBatch<'_, ()>],
277        indexes: &[usize],
278    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
279        let epoch = batches[indexes[0]].epoch;
280        if indexes.iter().any(|index| batches[*index].epoch != epoch) {
281            return Ok(reject_all(
282                indexes.len(),
283                "one partition commit mixed fence epochs",
284            ));
285        }
286        let (observed, mut doc) = {
287            let inner = self.inner.lock().expect("poisoned");
288            match inner
289                .parts
290                .get(partition)
291                .and_then(|part| part.observed.clone())
292            {
293                Some(bytes) => {
294                    let doc = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
295                        .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
296                    (Some(bytes), doc)
297                }
298                None => {
299                    let part = inner.parts.get(partition);
300                    (
301                        None,
302                        PartitionIndexDoc {
303                            epoch,
304                            next_offset: part.map_or(0, |part| part.next_offset),
305                            log_start: part.map_or(0, |part| part.log_start),
306                            entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
307                        },
308                    )
309                }
310            }
311        };
312        if observed.is_some() && doc.epoch != epoch {
313            return Ok(reject_all(
314                indexes.len(),
315                format!(
316                    "partition {} epoch is {}, not {epoch}",
317                    partition.as_str(),
318                    doc.epoch
319                ),
320            ));
321        }
322        let mut outcomes = Vec::with_capacity(indexes.len());
323        let mut planned = Vec::with_capacity(indexes.len());
324        for index in indexes {
325            let batch = &batches[*index];
326            let entry = IndexEntry {
327                location: batch.location.clone(),
328                base_offset: doc.next_offset,
329                record_count: batch.record_count,
330            };
331            outcomes.push(CommitOutcome::Assigned {
332                base_offset: entry.base_offset,
333                record_count: entry.record_count,
334            });
335            if batch.record_count <= 0 {
336                return Err(ObjectLogError::InvalidBatch(
337                    "invalid index record count".into(),
338                ));
339            }
340            doc.next_offset = doc
341                .next_offset
342                .checked_add(i64::from(batch.record_count))
343                .ok_or_else(|| ObjectLogError::Sequencer("index offset overflow".into()))?;
344            planned.push(entry.clone());
345            doc.entries.push(entry);
346        }
347        crate::bounded_lookup::validate_index(&doc.entries, doc.log_start, doc.next_offset)?;
348        let new_bytes = Bytes::from(
349            serde_json::to_vec(&doc)
350                .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
351        );
352        match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
353            CasOutcome::Stored => {
354                self.install_doc(partition, &doc, new_bytes)?;
355                Ok(outcomes)
356            }
357            CasOutcome::Conflict { current } => {
358                if let Some(bytes) = current
359                    && let Ok(found) = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
360                {
361                    let already = planned
362                        .iter()
363                        .all(|entry| found.entries.iter().any(|have| have == entry));
364                    self.install_doc(partition, &found, bytes)?;
365                    if already {
366                        return Ok(outcomes);
367                    }
368                }
369                Ok(reject_all(
370                    indexes.len(),
371                    format!("partition {} tail moved", partition.as_str()),
372                ))
373            }
374        }
375    }
376
377    fn publish_partition_conditionally(
378        &self,
379        partition: &PartitionKey,
380        batches: &[CommitBatch<'_, ()>],
381        indexes: &[usize],
382        heads: &[Option<i64>],
383    ) -> Result<Vec<ConditionalCommitOutcome>, ObjectLogError> {
384        for _ in 0..8 {
385            let (mut doc, observed) = match self.reload_partition(partition)? {
386                Some((doc, bytes)) => (doc, Some(bytes)),
387                None => {
388                    if self.has_known_history(partition)
389                        || self.has_durable_catalog_entry(partition)?
390                    {
391                        return Err(ObjectLogError::Sequencer("previously observed conditional index vanished or requires legacy migration".into()));
392                    }
393                    (
394                        PartitionIndexDoc {
395                            epoch: batches[indexes[0]].epoch,
396                            next_offset: 0,
397                            log_start: 0,
398                            entries: Vec::new(),
399                        },
400                        None,
401                    )
402                }
403            };
404            if observed.is_some() {
405                self.ensure_catalog(std::slice::from_ref(partition))?;
406            }
407            let mut outcomes = Vec::with_capacity(indexes.len());
408            let mut planned = Vec::with_capacity(indexes.len());
409            for index in indexes {
410                let batch = &batches[*index];
411                let mut matching = doc
412                    .entries
413                    .iter()
414                    .filter(|entry| entry.location == batch.location);
415                if let Some(entry) = matching.next() {
416                    if matching.next().is_some()
417                        || entry.record_count != batch.record_count
418                        || heads[*index].is_some_and(|head| head != entry.base_offset)
419                    {
420                        return Err(ObjectLogError::Sequencer(
421                            "conditional location identity mismatch".into(),
422                        ));
423                    }
424                    outcomes.push(ConditionalCommitOutcome::Committed(
425                        CommitOutcome::Duplicate {
426                            base_offset: entry.base_offset,
427                        },
428                    ));
429                    continue;
430                }
431                if heads[*index].is_some_and(|head| head < doc.log_start) {
432                    return Err(ObjectLogError::Sequencer(
433                        "conditional expected range is no longer inspectable".into(),
434                    ));
435                }
436                if doc.epoch != batch.epoch {
437                    outcomes.push(ConditionalCommitOutcome::Committed(
438                        CommitOutcome::Rejected {
439                            reason: format!(
440                                "partition {} epoch is {}, not {}",
441                                partition.as_str(),
442                                doc.epoch,
443                                batch.epoch
444                            ),
445                        },
446                    ));
447                    continue;
448                }
449                if let Some(expected) = heads[*index]
450                    && expected != doc.next_offset
451                {
452                    outcomes.push(ConditionalCommitOutcome::HeadRejected {
453                        expected,
454                        current: doc.next_offset,
455                    });
456                    continue;
457                }
458                let entry = IndexEntry {
459                    location: batch.location.clone(),
460                    base_offset: doc.next_offset,
461                    record_count: batch.record_count,
462                };
463                doc.next_offset = doc
464                    .next_offset
465                    .checked_add(i64::from(batch.record_count))
466                    .ok_or_else(|| {
467                        ObjectLogError::Sequencer("conditional offset overflow".into())
468                    })?;
469                outcomes.push(ConditionalCommitOutcome::Committed(
470                    CommitOutcome::Assigned {
471                        base_offset: entry.base_offset,
472                        record_count: entry.record_count,
473                    },
474                ));
475                planned.push(entry.clone());
476                doc.entries.push(entry);
477            }
478            if planned.is_empty() {
479                return Ok(outcomes);
480            }
481            // A rejected first request must not register a catalog-only queue.
482            if observed.is_none() && !self.ensure_catalog(std::slice::from_ref(partition))? {
483                if self.reload_partition(partition)?.is_some() {
484                    continue;
485                }
486                return Err(ObjectLogError::Sequencer(
487                    "catalog-known conditional index is missing".into(),
488                ));
489            }
490            crate::bounded_lookup::validate_index(&doc.entries, doc.log_start, doc.next_offset)?;
491            let bytes = Bytes::from(
492                serde_json::to_vec(&doc)
493                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
494            );
495            match self.cas(&self.index_key(partition), observed.clone(), bytes.clone())? {
496                CasOutcome::Stored => {
497                    self.install_doc(partition, &doc, bytes)?;
498                    return Ok(outcomes);
499                }
500                CasOutcome::Conflict { current } => {
501                    let (found, current) = match current {
502                        Some(bytes) => {
503                            let found: PartitionIndexDoc = serde_json::from_slice(&bytes)
504                                .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
505                            self.install_doc(partition, &found, bytes.clone())?;
506                            (found, bytes)
507                        }
508                        None => self.reload_partition(partition)?.ok_or_else(|| {
509                            ObjectLogError::Sequencer(
510                                "conditional conflict lost durable index".into(),
511                            )
512                        })?,
513                    };
514                    let _ = current;
515                    let present = planned
516                        .iter()
517                        .filter(|entry| found.entries.iter().any(|have| have == *entry))
518                        .count();
519                    if present == planned.len() {
520                        // Also prove that rows rejected in this attempt were not
521                        // independently indexed under the same physical location.
522                        for (index, outcome) in indexes.iter().zip(&outcomes) {
523                            if matches!(
524                                outcome,
525                                ConditionalCommitOutcome::HeadRejected { .. }
526                                    | ConditionalCommitOutcome::Committed(
527                                        CommitOutcome::Rejected { .. }
528                                    )
529                            ) && found
530                                .entries
531                                .iter()
532                                .any(|entry| entry.location == batches[*index].location)
533                            {
534                                return Err(ObjectLogError::Sequencer(
535                                    "rejected conditional location already indexed".into(),
536                                ));
537                            }
538                        }
539                        return Ok(outcomes);
540                    }
541                    if planned.iter().any(|entry| {
542                        entry.base_offset < found.log_start
543                            && !found.entries.iter().any(|have| have == entry)
544                    }) {
545                        return Err(ObjectLogError::Sequencer(
546                            "conditional planned range is no longer inspectable".into(),
547                        ));
548                    }
549                    if present != 0 {
550                        return Err(ObjectLogError::Sequencer(
551                            "partial conditional publication reconciliation".into(),
552                        ));
553                    }
554                    // Re-read and re-evaluate original conditions, never a new
555                    // expected head inferred from this conflicting snapshot.
556                }
557            }
558        }
559        Err(ObjectLogError::StorageUnavailable(
560            "conditional index CAS retry exhausted".into(),
561        ))
562    }
563
564    /// Cached durable observation or legacy history forbids treating absence as new.
565    fn has_known_history(&self, partition: &PartitionKey) -> bool {
566        let inner = self.inner.lock().expect("poisoned");
567        inner.uncertain_epochs.contains(partition)
568            || inner.parts.get(partition).is_some_and(|part| {
569                part.observed.is_some()
570                    || part.next_offset != 0
571                    || part.log_start != 0
572                    || !part.entries.is_empty()
573            })
574    }
575
576    /// A missing index is new only when the fresh durable catalog also omits it.
577    /// Catalog publication precedes index publication, so interrupted creation
578    /// must fail closed rather than manufacture replacement authority.
579    fn has_durable_catalog_entry(&self, partition: &PartitionKey) -> Result<bool, ObjectLogError> {
580        let Some(bytes) = self.read_object(&self.catalog_key())? else {
581            return Ok(false);
582        };
583        let catalog: IndexCatalog = serde_json::from_slice(&bytes)
584            .map_err(|error| ObjectLogError::Sequencer(format!("index catalog: {error}")))?;
585        let known = catalog
586            .partitions
587            .iter()
588            .any(|name| name == partition.as_str());
589        self.inner
590            .lock()
591            .expect("poisoned")
592            .uncertain_epochs
593            .extend(catalog.partitions.into_iter().map(PartitionKey));
594        Ok(known)
595    }
596
597    /// Install `partition`'s validated durable index, if any, and return it.
598    fn reload_partition(
599        &self,
600        partition: &PartitionKey,
601    ) -> Result<Option<(PartitionIndexDoc, Bytes)>, ObjectLogError> {
602        let Some(bytes) = self.read_object(&self.index_key(partition))? else {
603            return Ok(None);
604        };
605        self.inner
606            .lock()
607            .expect("poisoned")
608            .uncertain_epochs
609            .insert(partition.clone());
610        let doc: PartitionIndexDoc = serde_json::from_slice(&bytes)
611            .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
612        self.install_doc(partition, &doc, bytes.clone())?;
613        Ok(Some((doc, bytes)))
614    }
615
616    fn install_doc(
617        &self,
618        partition: &PartitionKey,
619        doc: &PartitionIndexDoc,
620        observed: Bytes,
621    ) -> Result<(), ObjectLogError> {
622        crate::bounded_lookup::validate_index(&doc.entries, doc.log_start, doc.next_offset)?;
623        let mut inner = self.inner.lock().expect("poisoned");
624        inner.parts.insert(
625            partition.clone(),
626            Part {
627                next_offset: doc.next_offset,
628                log_start: doc.log_start,
629                epoch: doc.epoch,
630                entries: doc.entries.clone(),
631                observed: Some(observed),
632            },
633        );
634        Ok(())
635    }
636
637    /// Object ids referenced by any live index entry (for orphan reaping).
638    /// Does not include manifest object keys themselves.
639    pub fn live_object_ids(&self) -> HashSet<String> {
640        let st = self.inner.lock().expect("poisoned");
641        let mut live = HashSet::new();
642        for p in st.parts.values() {
643            for e in &p.entries {
644                live.insert(e.location.object_id.clone());
645            }
646        }
647        live
648    }
649
650    /// Point-in-time view of the rebuilt index (diagnostics / CLI).
651    pub fn snapshot(&self) -> IndexSnapshot {
652        let st = self.inner.lock().expect("poisoned");
653        let mut partitions: Vec<PartitionSnapshot> = st
654            .parts
655            .iter()
656            .map(|(k, p)| PartitionSnapshot {
657                partition: k.0.clone(),
658                log_start: p.log_start,
659                high_watermark: p.next_offset,
660                entry_count: p.entries.len(),
661                entries: p.entries.clone(),
662            })
663            .collect();
664        partitions.sort_by(|a, b| a.partition.cmp(&b.partition));
665        IndexSnapshot {
666            manifest_prefix: self.prefix.clone(),
667            manifest_count: st.counter,
668            partitions,
669        }
670    }
671}
672
673/// Full index view from [`ManifestSequencer::snapshot`].
674#[derive(Clone, Debug, Serialize, Deserialize)]
675pub struct IndexSnapshot {
676    /// Prefix used for manifest objects.
677    pub manifest_prefix: String,
678    /// Number of manifest objects replayed (commit counter).
679    pub manifest_count: u64,
680    /// Per-partition index state.
681    pub partitions: Vec<PartitionSnapshot>,
682}
683
684/// One partition's index bounds and entries.
685#[derive(Clone, Debug, Serialize, Deserialize)]
686pub struct PartitionSnapshot {
687    /// Partition key string.
688    pub partition: String,
689    /// First readable offset.
690    pub log_start: i64,
691    /// Next offset to assign (high watermark).
692    pub high_watermark: i64,
693    /// Number of index entries.
694    pub entry_count: usize,
695    /// Ordered index entries (may be large; CLI can omit via flag).
696    pub entries: Vec<IndexEntry>,
697}
698
699impl Sequencer for ManifestSequencer {
700    type Meta = ();
701
702    fn supports_multi_object_commit(&self) -> bool {
703        true
704    }
705
706    fn supports_conditional_commit(&self) -> bool {
707        true
708    }
709
710    fn commit_conditionally(
711        &self,
712        batches: &[CommitBatch<'_, ()>],
713        expected_heads: &[Option<i64>],
714    ) -> Result<Vec<ConditionalCommitOutcome>, ObjectLogError> {
715        if batches.len() != expected_heads.len()
716            || expected_heads.iter().flatten().any(|head| *head < 0)
717            || batches
718                .iter()
719                .any(|batch| batch.record_count <= 0 || batch.location.object_id.is_empty())
720        {
721            return Err(ObjectLogError::InvalidBatch(
722                "invalid conditional commit inputs".into(),
723            ));
724        }
725        if expected_heads.iter().all(Option::is_none) {
726            return self.commit(batches).map(|outcomes| {
727                outcomes
728                    .into_iter()
729                    .map(ConditionalCommitOutcome::Committed)
730                    .collect()
731            });
732        }
733        let _commit = self.commit_order.lock().expect("poisoned");
734        let mut order = Vec::new();
735        let mut groups: HashMap<PartitionKey, Vec<usize>> = HashMap::new();
736        for (index, batch) in batches.iter().enumerate() {
737            if !groups.contains_key(&batch.partition) {
738                order.push(batch.partition.clone());
739            }
740            groups
741                .entry(batch.partition.clone())
742                .or_default()
743                .push(index);
744        }
745        let mut outcomes = vec![None; batches.len()];
746        let mut published = false;
747        for partition in order {
748            let indexes = &groups[&partition];
749            match self.publish_partition_conditionally(&partition, batches, indexes, expected_heads)
750            {
751                Ok(rows) => {
752                    for (index, outcome) in indexes.iter().zip(rows) {
753                        if matches!(
754                            outcome,
755                            ConditionalCommitOutcome::Committed(
756                                CommitOutcome::Assigned { .. } | CommitOutcome::Duplicate { .. }
757                            )
758                        ) {
759                            published = true;
760                        }
761                        outcomes[*index] = Some(outcome);
762                    }
763                }
764                Err(error) => {
765                    for index in indexes {
766                        outcomes[*index] = Some(ConditionalCommitOutcome::Unknown(error.clone()));
767                    }
768                }
769            }
770        }
771        if published {
772            let mut inner = self.inner.lock().expect("poisoned");
773            inner.counter = inner.counter.saturating_add(1);
774        }
775        Ok(outcomes
776            .into_iter()
777            .map(|outcome| outcome.expect("every conditional batch has an outcome"))
778            .collect())
779    }
780
781    fn commit(
782        &self,
783        batches: &[CommitBatch<'_, ()>],
784    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
785        let _commit = self.commit_order.lock().expect("poisoned");
786        if batches.is_empty() {
787            return Ok(Vec::new());
788        }
789        if batches
790            .iter()
791            .any(|batch| batch.record_count <= 0 || batch.location.object_id.is_empty())
792        {
793            return Err(ObjectLogError::InvalidBatch("invalid index batch".into()));
794        }
795        let mut order: Vec<PartitionKey> = Vec::new();
796        let mut groups: HashMap<PartitionKey, Vec<usize>> = HashMap::new();
797        for (index, batch) in batches.iter().enumerate() {
798            if !groups.contains_key(&batch.partition) {
799                order.push(batch.partition.clone());
800            }
801            groups
802                .entry(batch.partition.clone())
803                .or_default()
804                .push(index);
805        }
806        self.ensure_catalog(&order)?;
807        let mut outcomes = vec![None; batches.len()];
808        let mut published = false;
809        for partition in order {
810            let indexes = &groups[&partition];
811            match self.publish_partition(&partition, batches, indexes) {
812                Ok(partition_outcomes) => {
813                    for (index, outcome) in indexes.iter().zip(partition_outcomes) {
814                        if matches!(
815                            outcome,
816                            CommitOutcome::Assigned { .. } | CommitOutcome::Duplicate { .. }
817                        ) {
818                            published = true;
819                        }
820                        outcomes[*index] = Some(outcome);
821                    }
822                }
823                Err(error) => {
824                    for index in indexes {
825                        outcomes[*index] = Some(CommitOutcome::Rejected {
826                            reason: error.to_string(),
827                        });
828                    }
829                }
830            }
831        }
832        if published {
833            let mut inner = self.inner.lock().expect("poisoned");
834            inner.counter = inner.counter.saturating_add(1);
835        }
836        Ok(outcomes
837            .into_iter()
838            .map(|outcome| outcome.expect("every batch has an outcome"))
839            .collect())
840    }
841
842    fn fence_epoch(
843        &self,
844        partition: &PartitionKey,
845        expected: u64,
846        new_epoch: u64,
847    ) -> Result<(), ObjectLogError> {
848        let _commit = self.commit_order.lock().expect("poisoned");
849        if expected == new_epoch {
850            return Ok(());
851        }
852        // Fence the durable index, not this handle's view of it: another writer
853        // may have created or advanced the index since this handle opened, and a
854        // fence that only compared cached bytes would silently store nothing.
855        self.ensure_catalog(std::slice::from_ref(partition))?;
856        for _ in 0..8 {
857            let (mut doc, observed) = match self.reload_partition(partition)? {
858                Some((current, bytes)) => {
859                    if current.epoch == new_epoch {
860                        return Ok(());
861                    }
862                    if current.epoch != expected {
863                        return Err(ObjectLogError::Sequencer(format!(
864                            "partition {} epoch is {}, not {expected}",
865                            partition.as_str(),
866                            current.epoch
867                        )));
868                    }
869                    (current, Some(bytes))
870                }
871                None => {
872                    // No index yet: create it at the new epoch, so a writer that
873                    // never saw one cannot publish under an older epoch.
874                    let inner = self.inner.lock().expect("poisoned");
875                    let part = inner.parts.get(partition);
876                    let doc = PartitionIndexDoc {
877                        epoch: new_epoch,
878                        next_offset: part.map_or(0, |part| part.next_offset),
879                        log_start: part.map_or(0, |part| part.log_start),
880                        entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
881                    };
882                    (doc, None)
883                }
884            };
885            doc.epoch = new_epoch;
886            let new_bytes = Bytes::from(
887                serde_json::to_vec(&doc)
888                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
889            );
890            match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
891                CasOutcome::Stored => {
892                    self.install_doc(partition, &doc, new_bytes)?;
893                    return Ok(());
894                }
895                // A concurrent commit or fence moved the index; re-read and retry.
896                CasOutcome::Conflict { .. } => continue,
897            }
898        }
899        Err(ObjectLogError::Sequencer(format!(
900            "partition {} epoch fence did not land",
901            partition.as_str()
902        )))
903    }
904
905    fn partition_epoch(&self, partition: &PartitionKey) -> Option<u64> {
906        let inner = self.inner.lock().expect("poisoned");
907        inner
908            .parts
909            .get(partition)
910            .filter(|part| part.observed.is_some())
911            .map(|part| part.epoch)
912    }
913
914    /// A batch rejected because this partition's index was fenced past its epoch
915    /// can never commit: report [`ObjectLogError::Fenced`]. Other rejections stay
916    /// [`ObjectLogError::Sequencer`]. The index read during the rejected commit
917    /// is the one consulted.
918    fn rejection_error(
919        &self,
920        partition: &PartitionKey,
921        epoch: u64,
922        reason: String,
923    ) -> ObjectLogError {
924        #[allow(deprecated)]
925        match self.partition_epoch(partition) {
926            Some(current) if current > epoch => ObjectLogError::Fenced {
927                partition: partition.as_str().to_owned(),
928                epoch,
929                current,
930            },
931            _ => ObjectLogError::Sequencer(reason),
932        }
933    }
934
935    fn acquire_authoritative_epoch(
936        &self,
937        partition: &PartitionKey,
938    ) -> Result<AuthoritativePartitionState, ObjectLogError> {
939        let _commit = self.commit_order.lock().expect("poisoned");
940        for _ in 0..8 {
941            let (mut doc, observed) = match self.reload_partition(partition)? {
942                Some((doc, bytes)) => (doc, Some(bytes)),
943                None if self.has_known_history(partition)
944                    || self.has_durable_catalog_entry(partition)? =>
945                {
946                    return Err(ObjectLogError::Sequencer(
947                        "previously observed epoch index vanished or requires legacy migration"
948                            .into(),
949                    ));
950                }
951                None => (
952                    PartitionIndexDoc {
953                        epoch: 0,
954                        next_offset: 0,
955                        log_start: 0,
956                        entries: Vec::new(),
957                    },
958                    None,
959                ),
960            };
961            doc.epoch = doc
962                .epoch
963                .checked_add(1)
964                .ok_or_else(|| ObjectLogError::Sequencer("writer epoch exhausted".into()))?;
965            let registered = self.ensure_catalog(std::slice::from_ref(partition))?;
966            if observed.is_none() && !registered {
967                // Another creator registered after our absence check. Its
968                // surviving index may be acquired on the next iteration;
969                // missing authority must never be replaced by CAS(None).
970                if self.reload_partition(partition)?.is_some() {
971                    continue;
972                }
973                return Err(ObjectLogError::Sequencer(
974                    "catalog-known epoch index is missing".into(),
975                ));
976            }
977            let bytes = Bytes::from(
978                serde_json::to_vec(&doc)
979                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
980            );
981            match self
982                .cas(&self.index_key(partition), observed, bytes.clone())
983                .inspect_err(|_error| {
984                    self.inner
985                        .lock()
986                        .expect("poisoned")
987                        .uncertain_epochs
988                        .insert(partition.clone());
989                })? {
990                CasOutcome::Stored => {
991                    self.install_doc(partition, &doc, bytes)?;
992                    return Ok(AuthoritativePartitionState {
993                        epoch: doc.epoch,
994                        log_start: doc.log_start,
995                        next_offset: doc.next_offset,
996                    });
997                }
998                CasOutcome::Conflict {
999                    current: Some(bytes),
1000                } => {
1001                    self.inner
1002                        .lock()
1003                        .expect("poisoned")
1004                        .uncertain_epochs
1005                        .insert(partition.clone());
1006                    let current: PartitionIndexDoc = serde_json::from_slice(&bytes)
1007                        .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
1008                    self.install_doc(partition, &current, bytes)?;
1009                }
1010                CasOutcome::Conflict { current: None } => {
1011                    self.inner
1012                        .lock()
1013                        .expect("poisoned")
1014                        .uncertain_epochs
1015                        .insert(partition.clone());
1016                    return Err(ObjectLogError::StorageUnavailable(
1017                        "epoch acquisition conflicted without surviving authority".into(),
1018                    ));
1019                }
1020            }
1021        }
1022        Err(ObjectLogError::StorageUnavailable(
1023            "authoritative epoch CAS retry exhausted".into(),
1024        ))
1025    }
1026
1027    fn authoritative_partition_state(
1028        &self,
1029        partition: &PartitionKey,
1030    ) -> Result<AuthoritativePartitionState, ObjectLogError> {
1031        let _commit = self.commit_order.lock().expect("poisoned");
1032        match self.reload_partition(partition)? {
1033            Some((doc, _)) => Ok(AuthoritativePartitionState {
1034                epoch: doc.epoch,
1035                log_start: doc.log_start,
1036                next_offset: doc.next_offset,
1037            }),
1038            None if self.has_known_history(partition)
1039                || self.has_durable_catalog_entry(partition)? =>
1040            {
1041                Err(ObjectLogError::Sequencer(
1042                    "previously observed authoritative index vanished or requires legacy migration"
1043                        .into(),
1044                ))
1045            }
1046            None => Ok(AuthoritativePartitionState {
1047                epoch: 0,
1048                log_start: 0,
1049                next_offset: 0,
1050            }),
1051        }
1052    }
1053
1054    fn refresh_partition(&self, partition: &PartitionKey) -> Result<(), ObjectLogError> {
1055        let _commit = self.commit_order.lock().expect("poisoned");
1056        self.reload_partition(partition)?;
1057        Ok(())
1058    }
1059
1060    fn lookup(
1061        &self,
1062        partition: &PartitionKey,
1063        fetch_offset: i64,
1064    ) -> Result<Vec<IndexEntry>, ObjectLogError> {
1065        let st = self.inner.lock().expect("poisoned");
1066        let Some(p) = st.parts.get(partition) else {
1067            return Ok(Vec::new());
1068        };
1069        Ok(p.entries
1070            .iter()
1071            .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
1072            .cloned()
1073            .collect())
1074    }
1075
1076    fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
1077        Ok(self
1078            .inner
1079            .lock()
1080            .expect("poisoned")
1081            .parts
1082            .get(partition)
1083            .map_or(0, |p| p.next_offset))
1084    }
1085
1086    fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
1087        Ok(self
1088            .inner
1089            .lock()
1090            .expect("poisoned")
1091            .parts
1092            .get(partition)
1093            .map_or(0, |p| p.log_start))
1094    }
1095
1096    fn truncate_before(
1097        &self,
1098        partition: &PartitionKey,
1099        offset: i64,
1100    ) -> Result<Vec<String>, ObjectLogError> {
1101        let _commit = self.commit_order.lock().expect("poisoned");
1102        let snapshot = {
1103            let st = self.inner.lock().expect("poisoned");
1104            let Some(part) = st.parts.get(partition) else {
1105                return Ok(Vec::new());
1106            };
1107            (
1108                part.observed.clone(),
1109                part.entries.clone(),
1110                part.next_offset,
1111                part.epoch,
1112                part.log_start,
1113            )
1114        };
1115        let (observed, entries, next_offset, epoch, mut log_start) = snapshot;
1116        let mut dropped = Vec::new();
1117        let mut kept = Vec::new();
1118        for entry in entries {
1119            if entry.base_offset + i64::from(entry.record_count) <= offset {
1120                dropped.push(entry.location.object_id.clone());
1121            } else {
1122                kept.push(entry);
1123            }
1124        }
1125        if offset > log_start {
1126            log_start = offset.min(next_offset);
1127        }
1128        let doc = PartitionIndexDoc {
1129            epoch,
1130            next_offset,
1131            log_start,
1132            entries: kept,
1133        };
1134        if observed.is_some() {
1135            let new_bytes = Bytes::from(
1136                serde_json::to_vec(&doc)
1137                    .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
1138            );
1139            match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
1140                CasOutcome::Stored => self.install_doc(partition, &doc, new_bytes)?,
1141                CasOutcome::Conflict { .. } => {
1142                    return Err(ObjectLogError::Sequencer(format!(
1143                        "partition {} truncate lost the compare-and-swap",
1144                        partition.as_str()
1145                    )));
1146                }
1147            }
1148        } else {
1149            let mut st = self.inner.lock().expect("poisoned");
1150            if let Some(part) = st.parts.get_mut(partition) {
1151                part.entries = doc.entries.clone();
1152                part.log_start = doc.log_start;
1153            }
1154        }
1155        let live = {
1156            let st = self.inner.lock().expect("poisoned");
1157            let mut live = HashSet::new();
1158            for part in st.parts.values() {
1159                for entry in &part.entries {
1160                    live.insert(entry.location.object_id.clone());
1161                }
1162            }
1163            live
1164        };
1165        let mut dead = Vec::new();
1166        let mut seen = HashSet::new();
1167        for object_id in dropped {
1168            if !live.contains(&object_id) && seen.insert(object_id.clone()) {
1169                dead.push(object_id);
1170            }
1171        }
1172        Ok(dead)
1173    }
1174}
1175
1176impl Part {
1177    fn from_doc(doc: PartitionIndexDoc, observed: Bytes) -> Result<Self, ObjectLogError> {
1178        crate::bounded_lookup::validate_index(&doc.entries, doc.log_start, doc.next_offset)?;
1179        Ok(Self {
1180            next_offset: doc.next_offset,
1181            log_start: doc.log_start,
1182            epoch: doc.epoch,
1183            entries: doc.entries,
1184            observed: Some(observed),
1185        })
1186    }
1187}
1188
1189fn index_object_key(prefix: &str, partition: &str) -> String {
1190    let mut encoded = String::with_capacity(partition.len() * 2);
1191    for byte in partition.as_bytes() {
1192        encoded.push_str(&format!("{byte:02x}"));
1193    }
1194    format!("{prefix}index/{encoded}")
1195}
1196
1197fn reject_all(count: usize, reason: impl Into<String>) -> Vec<CommitOutcome> {
1198    let reason = reason.into();
1199    vec![CommitOutcome::Rejected { reason }; count]
1200}
1201
1202impl crate::BoundedSequencer for ManifestSequencer {
1203    fn lookup_page(
1204        &self,
1205        partition: &PartitionKey,
1206        offset: i64,
1207        max_entries: usize,
1208        max_bytes: usize,
1209    ) -> Result<Vec<IndexEntry>, ObjectLogError> {
1210        let state = self.inner.lock().expect("poisoned");
1211        if offset < state.parts.get(partition).map_or(0, |part| part.log_start) {
1212            return Err(ObjectLogError::RangeOutOfBounds(
1213                "lookup precedes retained history".into(),
1214            ));
1215        }
1216        let entries = state
1217            .parts
1218            .get(partition)
1219            .map_or(&[][..], |part| part.entries.as_slice());
1220        crate::bounded_lookup::page(entries, offset, max_entries, max_bytes)
1221    }
1222}
1223
1224#[cfg(test)]
1225mod index_tests {
1226    use super::*;
1227    use crate::{
1228        BatchLocation, BlobStore, CommitBatch, CommitOutcome, Durability, FlushConfig,
1229        InMemorySequencer, LogEngine, MemoryBlobStore, ObjectLogError, PartitionKey,
1230    };
1231    use async_trait::async_trait;
1232    use bytes::Bytes;
1233    use std::sync::Arc;
1234    use std::sync::atomic::{AtomicUsize, Ordering};
1235
1236    fn location(object: &str) -> BatchLocation {
1237        BatchLocation {
1238            object_id: object.to_string(),
1239            byte_start: 0,
1240            byte_len: 1,
1241        }
1242    }
1243
1244    fn run_commit(
1245        seq: Arc<ManifestSequencer>,
1246        partitions: Vec<(&str, &str, u64)>,
1247    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
1248        let owned: Vec<(String, String, u64)> = partitions
1249            .into_iter()
1250            .map(|(partition, object, epoch)| (partition.to_string(), object.to_string(), epoch))
1251            .collect();
1252        std::thread::spawn(move || {
1253            let meta = ();
1254            let batches: Vec<CommitBatch<'_, ()>> = owned
1255                .iter()
1256                .map(|(partition, object, epoch)| CommitBatch {
1257                    partition: PartitionKey(partition.clone()),
1258                    record_count: 1,
1259                    location: location(object),
1260                    meta: &meta,
1261                    epoch: *epoch,
1262                })
1263                .collect();
1264            seq.commit(&batches)
1265        })
1266        .join()
1267        .expect("commit thread")
1268    }
1269
1270    /// Run a blocking sequencer call off the async test thread.
1271    fn on_thread<T: Send + 'static>(
1272        seq: &Arc<ManifestSequencer>,
1273        call: impl FnOnce(&ManifestSequencer) -> T + Send + 'static,
1274    ) -> T {
1275        let seq = Arc::clone(seq);
1276        std::thread::spawn(move || call(&seq))
1277            .join()
1278            .expect("sequencer thread")
1279    }
1280
1281    struct ListProbe {
1282        inner: MemoryBlobStore,
1283        lists: Arc<AtomicUsize>,
1284    }
1285
1286    #[async_trait]
1287    impl BlobStore for ListProbe {
1288        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1289            self.inner.put(key, value).await
1290        }
1291        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1292            self.inner.get(key).await
1293        }
1294        async fn get_range(
1295            &self,
1296            key: &str,
1297            range: std::ops::Range<u64>,
1298        ) -> Result<Option<Bytes>, ObjectLogError> {
1299            self.inner.get_range(key, range).await
1300        }
1301        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1302            self.lists.fetch_add(1, Ordering::SeqCst);
1303            self.inner.list(prefix).await
1304        }
1305        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1306            self.inner.delete(key).await
1307        }
1308        async fn compare_and_swap(
1309            &self,
1310            key: &str,
1311            expected: Option<Bytes>,
1312            new_value: Bytes,
1313        ) -> Result<crate::CasOutcome, ObjectLogError> {
1314            self.inner.compare_and_swap(key, expected, new_value).await
1315        }
1316    }
1317
1318    struct DriverRetired(std::sync::mpsc::Sender<()>);
1319    impl Drop for DriverRetired {
1320        fn drop(&mut self) {
1321            let _ = self.0.send(());
1322        }
1323    }
1324    struct DriverProbe {
1325        inner: MemoryBlobStore,
1326        armed: std::sync::atomic::AtomicBool,
1327        release: Arc<tokio::sync::Notify>,
1328        completed: Mutex<Option<std::sync::mpsc::Sender<()>>>,
1329        retired: Mutex<Option<std::sync::mpsc::Sender<()>>>,
1330    }
1331    #[async_trait]
1332    impl BlobStore for DriverProbe {
1333        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1334            self.inner.put(key, value).await
1335        }
1336        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1337            if self.armed.swap(false, Ordering::AcqRel) {
1338                let release = self.release.clone();
1339                let completed = self.completed.lock().unwrap().take().unwrap();
1340                let retired = DriverRetired(self.retired.lock().unwrap().take().unwrap());
1341                // Like a pooled HTTP connection, this task belongs to the
1342                // runtime driving the request that first creates it.
1343                tokio::spawn(async move {
1344                    let _retired = retired;
1345                    release.notified().await;
1346                    tokio::time::sleep(std::time::Duration::from_millis(1)).await;
1347                    let _ = completed.send(());
1348                    std::future::pending::<()>().await;
1349                });
1350            }
1351            self.inner.get(key).await
1352        }
1353        async fn get_range(
1354            &self,
1355            key: &str,
1356            range: std::ops::Range<u64>,
1357        ) -> Result<Option<Bytes>, ObjectLogError> {
1358            self.inner.get_range(key, range).await
1359        }
1360        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1361            self.inner.list(prefix).await
1362        }
1363        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1364            self.inner.delete(key).await
1365        }
1366        async fn compare_and_swap(
1367            &self,
1368            key: &str,
1369            expected: Option<Bytes>,
1370            value: Bytes,
1371        ) -> Result<CasOutcome, ObjectLogError> {
1372            self.inner.compare_and_swap(key, expected, value).await
1373        }
1374    }
1375    #[test]
1376    fn synchronous_sequencer_keeps_connection_driver_progress_between_calls() {
1377        let caller = Builder::new_current_thread().enable_all().build().unwrap();
1378        let (completed, received) = std::sync::mpsc::channel();
1379        let (retired, cleanup) = std::sync::mpsc::channel();
1380        let release = Arc::new(tokio::sync::Notify::new());
1381        let blob = Arc::new(DriverProbe {
1382            inner: MemoryBlobStore::new(),
1383            armed: std::sync::atomic::AtomicBool::new(false),
1384            release: release.clone(),
1385            completed: Mutex::new(Some(completed)),
1386            retired: Mutex::new(Some(retired)),
1387        });
1388        let store: Arc<dyn BlobStore> = blob.clone();
1389        let sequencer = Arc::new(
1390            caller
1391                .block_on(ManifestSequencer::open(store, "manifest/"))
1392                .unwrap(),
1393        );
1394        // Initial open I/O belongs to the caller. Arm only the later owned
1395        // synchronous sequencer runtime whose idle progress is under test.
1396        blob.armed.store(true, Ordering::Release);
1397        let state = on_thread(&sequencer, |seq| {
1398            seq.authoritative_partition_state(&PartitionKey("new".into()))
1399        })
1400        .unwrap();
1401        assert_eq!(
1402            state,
1403            AuthoritativePartitionState {
1404                epoch: 0,
1405                log_start: 0,
1406                next_offset: 0
1407            }
1408        );
1409        assert!(!blob.armed.load(Ordering::Acquire));
1410        assert!(matches!(
1411            received.try_recv(),
1412            Err(std::sync::mpsc::TryRecvError::Empty)
1413        ));
1414        // The synchronous call has returned. Never enter either runtime again
1415        // while waiting: the owned worker/reactor must drive the gated task.
1416        release.notify_one();
1417        received
1418            .recv_timeout(std::time::Duration::from_secs(5))
1419            .expect("owned connection driver must progress between sequencer calls");
1420        assert!(matches!(
1421            cleanup.try_recv(),
1422            Err(std::sync::mpsc::TryRecvError::Empty)
1423        ));
1424        drop(sequencer);
1425        // shutdown_background is asynchronous; observe actual task retirement
1426        // rather than treating Drop return as a worker join.
1427        cleanup
1428            .recv_timeout(std::time::Duration::from_secs(5))
1429            .expect("owned connection driver retires after sequencer destruction");
1430        drop(caller);
1431    }
1432
1433    #[tokio::test]
1434    async fn two_writers_cannot_commit_the_same_partition_tail() {
1435        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1436        let first = Arc::new(
1437            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1438                .await
1439                .unwrap(),
1440        );
1441        let second = Arc::new(
1442            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1443                .await
1444                .unwrap(),
1445        );
1446        let won = run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
1447        assert!(matches!(
1448            won[0],
1449            CommitOutcome::Assigned {
1450                base_offset: 0,
1451                record_count: 1
1452            }
1453        ));
1454        let lost = run_commit(second, vec![("p", "obj-b", 1)]).unwrap();
1455        assert!(
1456            matches!(lost[0], CommitOutcome::Rejected { .. }),
1457            "stale tail must fail closed, got {lost:?}"
1458        );
1459        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
1460        let partition = PartitionKey("p".into());
1461        assert_eq!(reopened.high_watermark(&partition).unwrap(), 1);
1462        let entries = reopened.lookup(&partition, 0).unwrap();
1463        assert_eq!(entries.len(), 1);
1464        assert_eq!(entries[0].location.object_id, "obj-a");
1465    }
1466
1467    #[tokio::test]
1468    async fn mixed_seal_keeps_the_partition_that_won_and_rejects_the_moved_tail() {
1469        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1470        let first = Arc::new(
1471            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1472                .await
1473                .unwrap(),
1474        );
1475        let second = Arc::new(
1476            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1477                .await
1478                .unwrap(),
1479        );
1480        run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
1481        let mixed = run_commit(second, vec![("p", "obj-b", 1), ("q", "obj-b", 1)]).unwrap();
1482        assert!(matches!(mixed[0], CommitOutcome::Rejected { .. }));
1483        assert!(matches!(
1484            mixed[1],
1485            CommitOutcome::Assigned { base_offset: 0, .. }
1486        ));
1487        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
1488        assert_eq!(
1489            reopened.lookup(&PartitionKey("p".into()), 0).unwrap().len(),
1490            1
1491        );
1492        assert_eq!(
1493            reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
1494            1,
1495            "the rejected partition must not skip an offset"
1496        );
1497        assert_eq!(
1498            reopened.lookup(&PartitionKey("q".into()), 0).unwrap()[0]
1499                .location
1500                .object_id,
1501            "obj-b"
1502        );
1503    }
1504
1505    #[tokio::test]
1506    async fn open_does_not_list_the_manifest_prefix_when_an_index_exists() {
1507        let memory = MemoryBlobStore::new();
1508        let blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
1509        let seq = Arc::new(
1510            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1511                .await
1512                .unwrap(),
1513        );
1514        run_commit(seq, vec![("p", "obj-a", 4)]).unwrap();
1515        let lists = Arc::new(AtomicUsize::new(0));
1516        let probe = Arc::new(ListProbe {
1517            inner: memory,
1518            lists: Arc::clone(&lists),
1519        });
1520        let reopened = ManifestSequencer::open(probe, "manifest/").await.unwrap();
1521        assert_eq!(
1522            lists.load(Ordering::SeqCst),
1523            0,
1524            "a current index catalog must be read by key, not by listing manifests"
1525        );
1526        assert_eq!(
1527            reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
1528            1
1529        );
1530    }
1531
1532    #[tokio::test]
1533    async fn distinct_writers_do_not_share_a_data_object_key() {
1534        let blob = Arc::new(MemoryBlobStore::new());
1535        let config = FlushConfig {
1536            linger: std::time::Duration::ZERO,
1537            ..FlushConfig::default()
1538        };
1539        let left = LogEngine::new_with_writer(
1540            Arc::clone(&blob) as Arc<dyn BlobStore>,
1541            Arc::new(InMemorySequencer::new()),
1542            config,
1543            "data/",
1544            "writer-a",
1545        );
1546        let right = LogEngine::new_with_writer(
1547            Arc::clone(&blob) as Arc<dyn BlobStore>,
1548            Arc::new(InMemorySequencer::new()),
1549            config,
1550            "data/",
1551            "writer-b",
1552        );
1553        left.produce_at_epoch(
1554            PartitionKey("p".into()),
1555            Bytes::from_static(b"left"),
1556            1,
1557            (),
1558            Durability::Sequenced,
1559            3,
1560        )
1561        .await
1562        .unwrap();
1563        right
1564            .produce_at_epoch(
1565                PartitionKey("p".into()),
1566                Bytes::from_static(b"right"),
1567                1,
1568                (),
1569                Durability::Sequenced,
1570                3,
1571            )
1572            .await
1573            .unwrap();
1574        let keys = blob.list("data/").await.unwrap();
1575        assert_eq!(
1576            keys.len(),
1577            2,
1578            "writers must not overwrite one object: {keys:?}"
1579        );
1580        assert!(keys.iter().any(|key| key.contains("writer-a")));
1581        assert!(keys.iter().any(|key| key.contains("writer-b")));
1582        assert!(
1583            keys.iter()
1584                .all(|key| key.contains("/00000000000000000003/"))
1585        );
1586    }
1587
1588    #[tokio::test]
1589    async fn fence_from_a_handle_that_never_saw_the_index_rejects_the_stale_writer() {
1590        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1591        let owner = Arc::new(
1592            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1593                .await
1594                .unwrap(),
1595        );
1596        // The standby opens before the owner's first commit, so it holds no index.
1597        let standby = Arc::new(
1598            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1599                .await
1600                .unwrap(),
1601        );
1602        run_commit(Arc::clone(&owner), vec![("p", "obj-a", 1)]).unwrap();
1603        let partition = PartitionKey("p".into());
1604        let fence_partition = partition.clone();
1605        on_thread(&standby, move |seq| seq.fence_epoch(&fence_partition, 1, 2)).unwrap();
1606
1607        let stale = run_commit(Arc::clone(&owner), vec![("p", "obj-stale", 1)]).unwrap();
1608        assert!(
1609            matches!(stale[0], CommitOutcome::Rejected { .. }),
1610            "a commit at the fenced epoch must fail closed, got {stale:?}"
1611        );
1612        let fresh = run_commit(standby, vec![("p", "obj-b", 2)]).unwrap();
1613        assert!(matches!(
1614            fresh[0],
1615            CommitOutcome::Assigned { base_offset: 1, .. }
1616        ));
1617        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
1618        let objects: Vec<_> = reopened
1619            .lookup(&partition, 0)
1620            .unwrap()
1621            .into_iter()
1622            .map(|entry| entry.location.object_id)
1623            .collect();
1624        assert_eq!(objects, ["obj-a", "obj-b"]);
1625    }
1626
1627    #[tokio::test]
1628    async fn fence_before_any_commit_creates_the_index_at_the_new_epoch() {
1629        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1630        let stale = Arc::new(
1631            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1632                .await
1633                .unwrap(),
1634        );
1635        let fencer = Arc::new(
1636            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1637                .await
1638                .unwrap(),
1639        );
1640        on_thread(&fencer, |seq| {
1641            seq.fence_epoch(&PartitionKey("p".into()), 0, 2)
1642        })
1643        .unwrap();
1644        let lost = run_commit(stale, vec![("p", "obj-stale", 1)]).unwrap();
1645        assert!(
1646            matches!(lost[0], CommitOutcome::Rejected { .. }),
1647            "a writer that never saw the fenced index must not commit, got {lost:?}"
1648        );
1649        let won = run_commit(fencer, vec![("p", "obj-b", 2)]).unwrap();
1650        assert!(matches!(
1651            won[0],
1652            CommitOutcome::Assigned { base_offset: 0, .. }
1653        ));
1654        let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
1655        assert_eq!(
1656            reopened.snapshot().partitions[0].entries[0]
1657                .location
1658                .object_id,
1659            "obj-b"
1660        );
1661    }
1662
1663    #[tokio::test]
1664    async fn refresh_partition_reveals_another_writers_commits() {
1665        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1666        let writer = Arc::new(
1667            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1668                .await
1669                .unwrap(),
1670        );
1671        let reader = Arc::new(
1672            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1673                .await
1674                .unwrap(),
1675        );
1676        run_commit(writer, vec![("p", "obj-a", 1), ("p", "obj-b", 1)]).unwrap();
1677        let partition = PartitionKey("p".into());
1678        assert_eq!(reader.high_watermark(&partition).unwrap(), 0);
1679        let refreshed = partition.clone();
1680        on_thread(&reader, move |seq| seq.refresh_partition(&refreshed)).unwrap();
1681        assert_eq!(reader.high_watermark(&partition).unwrap(), 2);
1682        assert_eq!(reader.lookup(&partition, 0).unwrap().len(), 2);
1683    }
1684
1685    #[tokio::test]
1686    async fn a_fenced_writer_gets_a_typed_fenced_error() {
1687        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1688        let config = FlushConfig {
1689            linger: std::time::Duration::ZERO,
1690            ..FlushConfig::default()
1691        };
1692        let owner_sequencer = Arc::new(
1693            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1694                .await
1695                .unwrap(),
1696        );
1697        let standby = Arc::new(
1698            ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1699                .await
1700                .unwrap(),
1701        );
1702        let owner = LogEngine::new_with_writer(
1703            Arc::clone(&blob),
1704            owner_sequencer,
1705            config,
1706            "data/",
1707            "writer-a",
1708        );
1709        let partition = PartitionKey("p".into());
1710        owner
1711            .produce_at_epoch(
1712                partition.clone(),
1713                Bytes::from_static(b"before"),
1714                1,
1715                (),
1716                Durability::Sequenced,
1717                1,
1718            )
1719            .await
1720            .unwrap();
1721        let fenced = partition.clone();
1722        on_thread(&standby, move |seq| seq.fence_epoch(&fenced, 1, 2)).unwrap();
1723
1724        let stale = owner
1725            .produce_at_epoch(
1726                partition.clone(),
1727                Bytes::from_static(b"stale"),
1728                1,
1729                (),
1730                Durability::Sequenced,
1731                1,
1732            )
1733            .await;
1734        match stale {
1735            Err(ObjectLogError::Fenced {
1736                partition: name,
1737                epoch: 1,
1738                current: 2,
1739            }) => assert_eq!(name, "p"),
1740            other => panic!("a write behind the fence must be Fenced, got {other:?}"),
1741        }
1742    }
1743
1744    #[tokio::test]
1745    async fn default_rejection_error_is_a_sequencer_error() {
1746        let seq = crate::InMemorySequencer::new();
1747        let error = seq.rejection_error(&PartitionKey("p".into()), 1, "tail moved".into());
1748        assert!(matches!(error, ObjectLogError::Sequencer(reason) if reason == "tail moved"));
1749    }
1750}