1use crate::{
10 BlobStore, CasOutcome, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey,
11 Sequencer,
12};
13use bytes::Bytes;
14use serde::{Deserialize, Serialize};
15use std::collections::{HashMap, HashSet};
16use std::sync::{Arc, Mutex};
17use tokio::runtime::{Builder, Runtime};
18
19#[derive(Serialize, Deserialize)]
20struct ManifestRecord {
21 entries: Vec<(PartitionKey, IndexEntry)>,
22}
23
24#[derive(Clone, Serialize, Deserialize)]
25struct PartitionIndexDoc {
26 epoch: u64,
27 next_offset: i64,
28 log_start: i64,
29 entries: Vec<IndexEntry>,
30}
31
32#[derive(Clone, Serialize, Deserialize, Default)]
33struct IndexCatalog {
34 partitions: Vec<String>,
35}
36
37#[derive(Default)]
38struct Part {
39 next_offset: i64,
40 log_start: i64,
41 epoch: u64,
42 entries: Vec<IndexEntry>,
43 observed: Option<Bytes>,
45}
46
47struct Inner {
48 parts: HashMap<PartitionKey, Part>,
49 counter: u64,
50 catalog: Option<Bytes>,
51}
52
53pub struct ManifestSequencer {
55 blob: Arc<dyn BlobStore>,
56 prefix: String,
57 rt: Option<Runtime>,
60 commit_order: Mutex<()>,
62 inner: Mutex<Inner>,
63}
64
65impl Drop for ManifestSequencer {
66 fn drop(&mut self) {
67 if let Some(rt) = self.rt.take() {
68 rt.shutdown_background();
69 }
70 }
71}
72
73impl ManifestSequencer {
74 pub async fn open(
78 blob: Arc<dyn BlobStore>,
79 manifest_prefix: impl Into<String>,
80 ) -> Result<Self, ObjectLogError> {
81 let prefix = manifest_prefix.into();
82 let catalog_key = format!("{prefix}catalog");
83 let (parts, catalog, counter) = if let Some(catalog_bytes) = blob.get(&catalog_key).await? {
84 let catalog: IndexCatalog = serde_json::from_slice(&catalog_bytes)
85 .map_err(|e| ObjectLogError::Sequencer(format!("index catalog: {e}")))?;
86 let mut parts = HashMap::new();
87 for name in &catalog.partitions {
88 let key = index_object_key(&prefix, name);
89 let Some(doc_bytes) = blob.get(&key).await? else {
90 parts.insert(PartitionKey(name.clone()), Part::default());
91 continue;
92 };
93 let doc: PartitionIndexDoc = serde_json::from_slice(&doc_bytes).map_err(|e| {
94 ObjectLogError::Sequencer(format!("partition index {name}: {e}"))
95 })?;
96 parts.insert(PartitionKey(name.clone()), Part::from_doc(doc, doc_bytes));
97 }
98 let counter = parts.len() as u64;
99 (parts, Some(catalog_bytes), counter)
100 } else {
101 let mut keys = blob.list(&prefix).await?;
102 keys.sort();
103 let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
104 for key in &keys {
105 if key == &catalog_key || key.starts_with(&format!("{prefix}index/")) {
106 continue;
107 }
108 let Some(bytes) = blob.get(key).await? else {
109 continue;
110 };
111 let Ok(rec) = serde_json::from_slice::<ManifestRecord>(&bytes) else {
112 continue;
113 };
114 for (pkey, entry) in rec.entries {
115 let p = parts.entry(pkey).or_default();
116 let end = entry.base_offset + entry.record_count as i64;
117 if end > p.next_offset {
118 p.next_offset = end;
119 }
120 p.entries.push(entry);
121 }
122 }
123 (parts, None, keys.len() as u64)
124 };
125 let rt = Builder::new_current_thread()
126 .enable_all()
127 .build()
128 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
129 Ok(Self {
130 blob,
131 prefix,
132 rt: Some(rt),
133 commit_order: Mutex::new(()),
134 inner: Mutex::new(Inner {
135 parts,
136 counter,
137 catalog,
138 }),
139 })
140 }
141
142 fn index_key(&self, partition: &PartitionKey) -> String {
143 index_object_key(&self.prefix, partition.as_str())
144 }
145
146 fn catalog_key(&self) -> String {
147 format!("{}catalog", self.prefix)
148 }
149
150 fn block_on<T>(&self, future: impl std::future::Future<Output = T>) -> T {
151 self.rt
152 .as_ref()
153 .expect("runtime present until drop")
154 .block_on(future)
155 }
156
157 fn read_object(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
158 self.block_on(self.blob.get(key))
159 }
160
161 fn cas(
162 &self,
163 key: &str,
164 expected: Option<Bytes>,
165 new_value: Bytes,
166 ) -> Result<CasOutcome, ObjectLogError> {
167 self.block_on(self.blob.compare_and_swap(key, expected, new_value))
168 }
169
170 fn ensure_catalog(&self, partitions: &[PartitionKey]) -> Result<(), ObjectLogError> {
171 let key = self.catalog_key();
172 for _ in 0..8 {
173 let current = self.read_object(&key)?;
174 let mut names = if let Some(bytes) = ¤t {
175 serde_json::from_slice::<IndexCatalog>(bytes)
176 .map_err(|error| ObjectLogError::Sequencer(format!("index catalog: {error}")))?
177 .partitions
178 } else {
179 Vec::new()
180 };
181 let mut changed = false;
182 for partition in partitions {
183 if !names.iter().any(|name| name == partition.as_str()) {
184 names.push(partition.0.clone());
185 changed = true;
186 }
187 }
188 if !changed {
189 self.inner.lock().expect("poisoned").catalog = current;
190 return Ok(());
191 }
192 names.sort();
193 names.dedup();
194 let new_bytes = Bytes::from(
195 serde_json::to_vec(&IndexCatalog { partitions: names })
196 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
197 );
198 match self.cas(&key, current, new_bytes.clone())? {
199 CasOutcome::Stored => {
200 self.inner.lock().expect("poisoned").catalog = Some(new_bytes);
201 return Ok(());
202 }
203 CasOutcome::Conflict { .. } => continue,
204 }
205 }
206 Err(ObjectLogError::Sequencer(
207 "catalog compare-and-swap did not land".into(),
208 ))
209 }
210
211 fn publish_partition(
212 &self,
213 partition: &PartitionKey,
214 batches: &[CommitBatch<'_, ()>],
215 indexes: &[usize],
216 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
217 let epoch = batches[indexes[0]].epoch;
218 if indexes.iter().any(|index| batches[*index].epoch != epoch) {
219 return Ok(reject_all(
220 indexes.len(),
221 "one partition commit mixed fence epochs",
222 ));
223 }
224 let (observed, mut doc) = {
225 let inner = self.inner.lock().expect("poisoned");
226 match inner
227 .parts
228 .get(partition)
229 .and_then(|part| part.observed.clone())
230 {
231 Some(bytes) => {
232 let doc = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
233 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
234 (Some(bytes), doc)
235 }
236 None => {
237 let part = inner.parts.get(partition);
238 (
239 None,
240 PartitionIndexDoc {
241 epoch,
242 next_offset: part.map_or(0, |part| part.next_offset),
243 log_start: part.map_or(0, |part| part.log_start),
244 entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
245 },
246 )
247 }
248 }
249 };
250 if observed.is_some() && doc.epoch != epoch {
251 return Ok(reject_all(
252 indexes.len(),
253 format!(
254 "partition {} epoch is {}, not {epoch}",
255 partition.as_str(),
256 doc.epoch
257 ),
258 ));
259 }
260 let mut outcomes = Vec::with_capacity(indexes.len());
261 let mut planned = Vec::with_capacity(indexes.len());
262 for index in indexes {
263 let batch = &batches[*index];
264 let entry = IndexEntry {
265 location: batch.location.clone(),
266 base_offset: doc.next_offset,
267 record_count: batch.record_count,
268 };
269 outcomes.push(CommitOutcome::Assigned {
270 base_offset: entry.base_offset,
271 record_count: entry.record_count,
272 });
273 doc.next_offset += i64::from(batch.record_count);
274 planned.push(entry.clone());
275 doc.entries.push(entry);
276 }
277 let new_bytes = Bytes::from(
278 serde_json::to_vec(&doc)
279 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
280 );
281 match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
282 CasOutcome::Stored => {
283 self.install_doc(partition, &doc, new_bytes);
284 Ok(outcomes)
285 }
286 CasOutcome::Conflict { current } => {
287 if let Some(bytes) = current
288 && let Ok(found) = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
289 {
290 let already = planned
291 .iter()
292 .all(|entry| found.entries.iter().any(|have| have == entry));
293 self.install_doc(partition, &found, bytes);
294 if already {
295 return Ok(outcomes);
296 }
297 }
298 Ok(reject_all(
299 indexes.len(),
300 format!("partition {} tail moved", partition.as_str()),
301 ))
302 }
303 }
304 }
305
306 fn reload_partition(
308 &self,
309 partition: &PartitionKey,
310 ) -> Result<Option<(PartitionIndexDoc, Bytes)>, ObjectLogError> {
311 let Some(bytes) = self.read_object(&self.index_key(partition))? else {
312 return Ok(None);
313 };
314 let doc: PartitionIndexDoc = serde_json::from_slice(&bytes)
315 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
316 self.install_doc(partition, &doc, bytes.clone());
317 Ok(Some((doc, bytes)))
318 }
319
320 fn install_doc(&self, partition: &PartitionKey, doc: &PartitionIndexDoc, observed: Bytes) {
321 let mut inner = self.inner.lock().expect("poisoned");
322 inner.parts.insert(
323 partition.clone(),
324 Part {
325 next_offset: doc.next_offset,
326 log_start: doc.log_start,
327 epoch: doc.epoch,
328 entries: doc.entries.clone(),
329 observed: Some(observed),
330 },
331 );
332 }
333
334 pub fn live_object_ids(&self) -> HashSet<String> {
337 let st = self.inner.lock().expect("poisoned");
338 let mut live = HashSet::new();
339 for p in st.parts.values() {
340 for e in &p.entries {
341 live.insert(e.location.object_id.clone());
342 }
343 }
344 live
345 }
346
347 pub fn snapshot(&self) -> IndexSnapshot {
349 let st = self.inner.lock().expect("poisoned");
350 let mut partitions: Vec<PartitionSnapshot> = st
351 .parts
352 .iter()
353 .map(|(k, p)| PartitionSnapshot {
354 partition: k.0.clone(),
355 log_start: p.log_start,
356 high_watermark: p.next_offset,
357 entry_count: p.entries.len(),
358 entries: p.entries.clone(),
359 })
360 .collect();
361 partitions.sort_by(|a, b| a.partition.cmp(&b.partition));
362 IndexSnapshot {
363 manifest_prefix: self.prefix.clone(),
364 manifest_count: st.counter,
365 partitions,
366 }
367 }
368}
369
370#[derive(Clone, Debug, Serialize, Deserialize)]
372pub struct IndexSnapshot {
373 pub manifest_prefix: String,
375 pub manifest_count: u64,
377 pub partitions: Vec<PartitionSnapshot>,
379}
380
381#[derive(Clone, Debug, Serialize, Deserialize)]
383pub struct PartitionSnapshot {
384 pub partition: String,
386 pub log_start: i64,
388 pub high_watermark: i64,
390 pub entry_count: usize,
392 pub entries: Vec<IndexEntry>,
394}
395
396impl Sequencer for ManifestSequencer {
397 type Meta = ();
398
399 fn supports_multi_object_commit(&self) -> bool {
400 true
401 }
402
403 fn commit(
404 &self,
405 batches: &[CommitBatch<'_, ()>],
406 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
407 let _commit = self.commit_order.lock().expect("poisoned");
408 if batches.is_empty() {
409 return Ok(Vec::new());
410 }
411 let mut order: Vec<PartitionKey> = Vec::new();
412 let mut groups: HashMap<PartitionKey, Vec<usize>> = HashMap::new();
413 for (index, batch) in batches.iter().enumerate() {
414 if !groups.contains_key(&batch.partition) {
415 order.push(batch.partition.clone());
416 }
417 groups
418 .entry(batch.partition.clone())
419 .or_default()
420 .push(index);
421 }
422 self.ensure_catalog(&order)?;
423 let mut outcomes = vec![None; batches.len()];
424 let mut published = false;
425 for partition in order {
426 let indexes = &groups[&partition];
427 match self.publish_partition(&partition, batches, indexes) {
428 Ok(partition_outcomes) => {
429 for (index, outcome) in indexes.iter().zip(partition_outcomes) {
430 if matches!(
431 outcome,
432 CommitOutcome::Assigned { .. } | CommitOutcome::Duplicate { .. }
433 ) {
434 published = true;
435 }
436 outcomes[*index] = Some(outcome);
437 }
438 }
439 Err(error) => {
440 for index in indexes {
441 outcomes[*index] = Some(CommitOutcome::Rejected {
442 reason: error.to_string(),
443 });
444 }
445 }
446 }
447 }
448 if published {
449 let mut inner = self.inner.lock().expect("poisoned");
450 inner.counter = inner.counter.saturating_add(1);
451 }
452 Ok(outcomes
453 .into_iter()
454 .map(|outcome| outcome.expect("every batch has an outcome"))
455 .collect())
456 }
457
458 fn fence_epoch(
459 &self,
460 partition: &PartitionKey,
461 expected: u64,
462 new_epoch: u64,
463 ) -> Result<(), ObjectLogError> {
464 let _commit = self.commit_order.lock().expect("poisoned");
465 if expected == new_epoch {
466 return Ok(());
467 }
468 self.ensure_catalog(std::slice::from_ref(partition))?;
472 for _ in 0..8 {
473 let (mut doc, observed) = match self.reload_partition(partition)? {
474 Some((current, bytes)) => {
475 if current.epoch == new_epoch {
476 return Ok(());
477 }
478 if current.epoch != expected {
479 return Err(ObjectLogError::Sequencer(format!(
480 "partition {} epoch is {}, not {expected}",
481 partition.as_str(),
482 current.epoch
483 )));
484 }
485 (current, Some(bytes))
486 }
487 None => {
488 let inner = self.inner.lock().expect("poisoned");
491 let part = inner.parts.get(partition);
492 let doc = PartitionIndexDoc {
493 epoch: new_epoch,
494 next_offset: part.map_or(0, |part| part.next_offset),
495 log_start: part.map_or(0, |part| part.log_start),
496 entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
497 };
498 (doc, None)
499 }
500 };
501 doc.epoch = new_epoch;
502 let new_bytes = Bytes::from(
503 serde_json::to_vec(&doc)
504 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
505 );
506 match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
507 CasOutcome::Stored => {
508 self.install_doc(partition, &doc, new_bytes);
509 return Ok(());
510 }
511 CasOutcome::Conflict { .. } => continue,
513 }
514 }
515 Err(ObjectLogError::Sequencer(format!(
516 "partition {} epoch fence did not land",
517 partition.as_str()
518 )))
519 }
520
521 fn refresh_partition(&self, partition: &PartitionKey) -> Result<(), ObjectLogError> {
522 let _commit = self.commit_order.lock().expect("poisoned");
523 self.reload_partition(partition)?;
524 Ok(())
525 }
526
527 fn lookup(
528 &self,
529 partition: &PartitionKey,
530 fetch_offset: i64,
531 ) -> Result<Vec<IndexEntry>, ObjectLogError> {
532 let st = self.inner.lock().expect("poisoned");
533 let Some(p) = st.parts.get(partition) else {
534 return Ok(Vec::new());
535 };
536 Ok(p.entries
537 .iter()
538 .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
539 .cloned()
540 .collect())
541 }
542
543 fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
544 Ok(self
545 .inner
546 .lock()
547 .expect("poisoned")
548 .parts
549 .get(partition)
550 .map_or(0, |p| p.next_offset))
551 }
552
553 fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
554 Ok(self
555 .inner
556 .lock()
557 .expect("poisoned")
558 .parts
559 .get(partition)
560 .map_or(0, |p| p.log_start))
561 }
562
563 fn truncate_before(
564 &self,
565 partition: &PartitionKey,
566 offset: i64,
567 ) -> Result<Vec<String>, ObjectLogError> {
568 let _commit = self.commit_order.lock().expect("poisoned");
569 let snapshot = {
570 let st = self.inner.lock().expect("poisoned");
571 let Some(part) = st.parts.get(partition) else {
572 return Ok(Vec::new());
573 };
574 (
575 part.observed.clone(),
576 part.entries.clone(),
577 part.next_offset,
578 part.epoch,
579 part.log_start,
580 )
581 };
582 let (observed, entries, next_offset, epoch, mut log_start) = snapshot;
583 let mut dropped = Vec::new();
584 let mut kept = Vec::new();
585 for entry in entries {
586 if entry.base_offset + i64::from(entry.record_count) <= offset {
587 dropped.push(entry.location.object_id.clone());
588 } else {
589 kept.push(entry);
590 }
591 }
592 if offset > log_start {
593 log_start = offset.min(next_offset);
594 }
595 let doc = PartitionIndexDoc {
596 epoch,
597 next_offset,
598 log_start,
599 entries: kept,
600 };
601 if observed.is_some() {
602 let new_bytes = Bytes::from(
603 serde_json::to_vec(&doc)
604 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
605 );
606 match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
607 CasOutcome::Stored => self.install_doc(partition, &doc, new_bytes),
608 CasOutcome::Conflict { .. } => {
609 return Err(ObjectLogError::Sequencer(format!(
610 "partition {} truncate lost the compare-and-swap",
611 partition.as_str()
612 )));
613 }
614 }
615 } else {
616 let mut st = self.inner.lock().expect("poisoned");
617 if let Some(part) = st.parts.get_mut(partition) {
618 part.entries = doc.entries.clone();
619 part.log_start = doc.log_start;
620 }
621 }
622 let live = {
623 let st = self.inner.lock().expect("poisoned");
624 let mut live = HashSet::new();
625 for part in st.parts.values() {
626 for entry in &part.entries {
627 live.insert(entry.location.object_id.clone());
628 }
629 }
630 live
631 };
632 let mut dead = Vec::new();
633 let mut seen = HashSet::new();
634 for object_id in dropped {
635 if !live.contains(&object_id) && seen.insert(object_id.clone()) {
636 dead.push(object_id);
637 }
638 }
639 Ok(dead)
640 }
641}
642
643impl Part {
644 fn from_doc(doc: PartitionIndexDoc, observed: Bytes) -> Self {
645 Self {
646 next_offset: doc.next_offset,
647 log_start: doc.log_start,
648 epoch: doc.epoch,
649 entries: doc.entries,
650 observed: Some(observed),
651 }
652 }
653}
654
655fn index_object_key(prefix: &str, partition: &str) -> String {
656 let mut encoded = String::with_capacity(partition.len() * 2);
657 for byte in partition.as_bytes() {
658 encoded.push_str(&format!("{byte:02x}"));
659 }
660 format!("{prefix}index/{encoded}")
661}
662
663fn reject_all(count: usize, reason: impl Into<String>) -> Vec<CommitOutcome> {
664 let reason = reason.into();
665 vec![CommitOutcome::Rejected { reason }; count]
666}
667
668#[cfg(test)]
669mod index_tests {
670 use super::*;
671 use crate::{
672 BatchLocation, BlobStore, CommitBatch, CommitOutcome, Durability, FlushConfig,
673 InMemorySequencer, LogEngine, MemoryBlobStore, ObjectLogError, PartitionKey,
674 };
675 use async_trait::async_trait;
676 use bytes::Bytes;
677 use std::sync::Arc;
678 use std::sync::atomic::{AtomicUsize, Ordering};
679
680 fn location(object: &str) -> BatchLocation {
681 BatchLocation {
682 object_id: object.to_string(),
683 byte_start: 0,
684 byte_len: 1,
685 }
686 }
687
688 fn run_commit(
689 seq: Arc<ManifestSequencer>,
690 partitions: Vec<(&str, &str, u64)>,
691 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
692 let owned: Vec<(String, String, u64)> = partitions
693 .into_iter()
694 .map(|(partition, object, epoch)| (partition.to_string(), object.to_string(), epoch))
695 .collect();
696 std::thread::spawn(move || {
697 let meta = ();
698 let batches: Vec<CommitBatch<'_, ()>> = owned
699 .iter()
700 .map(|(partition, object, epoch)| CommitBatch {
701 partition: PartitionKey(partition.clone()),
702 record_count: 1,
703 location: location(object),
704 meta: &meta,
705 epoch: *epoch,
706 })
707 .collect();
708 seq.commit(&batches)
709 })
710 .join()
711 .expect("commit thread")
712 }
713
714 fn on_thread<T: Send + 'static>(
716 seq: &Arc<ManifestSequencer>,
717 call: impl FnOnce(&ManifestSequencer) -> T + Send + 'static,
718 ) -> T {
719 let seq = Arc::clone(seq);
720 std::thread::spawn(move || call(&seq))
721 .join()
722 .expect("sequencer thread")
723 }
724
725 struct ListProbe {
726 inner: MemoryBlobStore,
727 lists: Arc<AtomicUsize>,
728 }
729
730 #[async_trait]
731 impl BlobStore for ListProbe {
732 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
733 self.inner.put(key, value).await
734 }
735 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
736 self.inner.get(key).await
737 }
738 async fn get_range(
739 &self,
740 key: &str,
741 range: std::ops::Range<u64>,
742 ) -> Result<Option<Bytes>, ObjectLogError> {
743 self.inner.get_range(key, range).await
744 }
745 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
746 self.lists.fetch_add(1, Ordering::SeqCst);
747 self.inner.list(prefix).await
748 }
749 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
750 self.inner.delete(key).await
751 }
752 async fn compare_and_swap(
753 &self,
754 key: &str,
755 expected: Option<Bytes>,
756 new_value: Bytes,
757 ) -> Result<crate::CasOutcome, ObjectLogError> {
758 self.inner.compare_and_swap(key, expected, new_value).await
759 }
760 }
761
762 #[tokio::test]
763 async fn two_writers_cannot_commit_the_same_partition_tail() {
764 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
765 let first = Arc::new(
766 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
767 .await
768 .unwrap(),
769 );
770 let second = Arc::new(
771 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
772 .await
773 .unwrap(),
774 );
775 let won = run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
776 assert!(matches!(
777 won[0],
778 CommitOutcome::Assigned {
779 base_offset: 0,
780 record_count: 1
781 }
782 ));
783 let lost = run_commit(second, vec![("p", "obj-b", 1)]).unwrap();
784 assert!(
785 matches!(lost[0], CommitOutcome::Rejected { .. }),
786 "stale tail must fail closed, got {lost:?}"
787 );
788 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
789 let partition = PartitionKey("p".into());
790 assert_eq!(reopened.high_watermark(&partition).unwrap(), 1);
791 let entries = reopened.lookup(&partition, 0).unwrap();
792 assert_eq!(entries.len(), 1);
793 assert_eq!(entries[0].location.object_id, "obj-a");
794 }
795
796 #[tokio::test]
797 async fn mixed_seal_keeps_the_partition_that_won_and_rejects_the_moved_tail() {
798 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
799 let first = Arc::new(
800 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
801 .await
802 .unwrap(),
803 );
804 let second = Arc::new(
805 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
806 .await
807 .unwrap(),
808 );
809 run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
810 let mixed = run_commit(second, vec![("p", "obj-b", 1), ("q", "obj-b", 1)]).unwrap();
811 assert!(matches!(mixed[0], CommitOutcome::Rejected { .. }));
812 assert!(matches!(
813 mixed[1],
814 CommitOutcome::Assigned { base_offset: 0, .. }
815 ));
816 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
817 assert_eq!(
818 reopened.lookup(&PartitionKey("p".into()), 0).unwrap().len(),
819 1
820 );
821 assert_eq!(
822 reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
823 1,
824 "the rejected partition must not skip an offset"
825 );
826 assert_eq!(
827 reopened.lookup(&PartitionKey("q".into()), 0).unwrap()[0]
828 .location
829 .object_id,
830 "obj-b"
831 );
832 }
833
834 #[tokio::test]
835 async fn open_does_not_list_the_manifest_prefix_when_an_index_exists() {
836 let memory = MemoryBlobStore::new();
837 let blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
838 let seq = Arc::new(
839 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
840 .await
841 .unwrap(),
842 );
843 run_commit(seq, vec![("p", "obj-a", 4)]).unwrap();
844 let lists = Arc::new(AtomicUsize::new(0));
845 let probe = Arc::new(ListProbe {
846 inner: memory,
847 lists: Arc::clone(&lists),
848 });
849 let reopened = ManifestSequencer::open(probe, "manifest/").await.unwrap();
850 assert_eq!(
851 lists.load(Ordering::SeqCst),
852 0,
853 "a current index catalog must be read by key, not by listing manifests"
854 );
855 assert_eq!(
856 reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
857 1
858 );
859 }
860
861 #[tokio::test]
862 async fn distinct_writers_do_not_share_a_data_object_key() {
863 let blob = Arc::new(MemoryBlobStore::new());
864 let config = FlushConfig {
865 linger: std::time::Duration::ZERO,
866 ..FlushConfig::default()
867 };
868 let left = LogEngine::new_with_writer(
869 Arc::clone(&blob) as Arc<dyn BlobStore>,
870 Arc::new(InMemorySequencer::new()),
871 config,
872 "data/",
873 "writer-a",
874 );
875 let right = LogEngine::new_with_writer(
876 Arc::clone(&blob) as Arc<dyn BlobStore>,
877 Arc::new(InMemorySequencer::new()),
878 config,
879 "data/",
880 "writer-b",
881 );
882 left.produce_at_epoch(
883 PartitionKey("p".into()),
884 Bytes::from_static(b"left"),
885 1,
886 (),
887 Durability::Sequenced,
888 3,
889 )
890 .await
891 .unwrap();
892 right
893 .produce_at_epoch(
894 PartitionKey("p".into()),
895 Bytes::from_static(b"right"),
896 1,
897 (),
898 Durability::Sequenced,
899 3,
900 )
901 .await
902 .unwrap();
903 let keys = blob.list("data/").await.unwrap();
904 assert_eq!(
905 keys.len(),
906 2,
907 "writers must not overwrite one object: {keys:?}"
908 );
909 assert!(keys.iter().any(|key| key.contains("writer-a")));
910 assert!(keys.iter().any(|key| key.contains("writer-b")));
911 assert!(
912 keys.iter()
913 .all(|key| key.contains("/00000000000000000003/"))
914 );
915 }
916
917 #[tokio::test]
918 async fn fence_from_a_handle_that_never_saw_the_index_rejects_the_stale_writer() {
919 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
920 let owner = Arc::new(
921 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
922 .await
923 .unwrap(),
924 );
925 let standby = Arc::new(
927 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
928 .await
929 .unwrap(),
930 );
931 run_commit(Arc::clone(&owner), vec![("p", "obj-a", 1)]).unwrap();
932 let partition = PartitionKey("p".into());
933 let fence_partition = partition.clone();
934 on_thread(&standby, move |seq| seq.fence_epoch(&fence_partition, 1, 2)).unwrap();
935
936 let stale = run_commit(Arc::clone(&owner), vec![("p", "obj-stale", 1)]).unwrap();
937 assert!(
938 matches!(stale[0], CommitOutcome::Rejected { .. }),
939 "a commit at the fenced epoch must fail closed, got {stale:?}"
940 );
941 let fresh = run_commit(standby, vec![("p", "obj-b", 2)]).unwrap();
942 assert!(matches!(
943 fresh[0],
944 CommitOutcome::Assigned { base_offset: 1, .. }
945 ));
946 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
947 let objects: Vec<_> = reopened
948 .lookup(&partition, 0)
949 .unwrap()
950 .into_iter()
951 .map(|entry| entry.location.object_id)
952 .collect();
953 assert_eq!(objects, ["obj-a", "obj-b"]);
954 }
955
956 #[tokio::test]
957 async fn fence_before_any_commit_creates_the_index_at_the_new_epoch() {
958 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
959 let stale = Arc::new(
960 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
961 .await
962 .unwrap(),
963 );
964 let fencer = Arc::new(
965 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
966 .await
967 .unwrap(),
968 );
969 on_thread(&fencer, |seq| {
970 seq.fence_epoch(&PartitionKey("p".into()), 0, 2)
971 })
972 .unwrap();
973 let lost = run_commit(stale, vec![("p", "obj-stale", 1)]).unwrap();
974 assert!(
975 matches!(lost[0], CommitOutcome::Rejected { .. }),
976 "a writer that never saw the fenced index must not commit, got {lost:?}"
977 );
978 let won = run_commit(fencer, vec![("p", "obj-b", 2)]).unwrap();
979 assert!(matches!(
980 won[0],
981 CommitOutcome::Assigned { base_offset: 0, .. }
982 ));
983 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
984 assert_eq!(
985 reopened.snapshot().partitions[0].entries[0]
986 .location
987 .object_id,
988 "obj-b"
989 );
990 }
991
992 #[tokio::test]
993 async fn refresh_partition_reveals_another_writers_commits() {
994 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
995 let writer = Arc::new(
996 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
997 .await
998 .unwrap(),
999 );
1000 let reader = Arc::new(
1001 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1002 .await
1003 .unwrap(),
1004 );
1005 run_commit(writer, vec![("p", "obj-a", 1), ("p", "obj-b", 1)]).unwrap();
1006 let partition = PartitionKey("p".into());
1007 assert_eq!(reader.high_watermark(&partition).unwrap(), 0);
1008 let refreshed = partition.clone();
1009 on_thread(&reader, move |seq| seq.refresh_partition(&refreshed)).unwrap();
1010 assert_eq!(reader.high_watermark(&partition).unwrap(), 2);
1011 assert_eq!(reader.lookup(&partition, 0).unwrap().len(), 2);
1012 }
1013}