1use crate::{
10 BlobStore, CasOutcome, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey,
11 Sequencer,
12};
13use bytes::Bytes;
14use serde::{Deserialize, Serialize};
15use std::collections::{HashMap, HashSet};
16use std::sync::{Arc, Mutex};
17use tokio::runtime::{Builder, Runtime};
18
19#[derive(Serialize, Deserialize)]
20struct ManifestRecord {
21 entries: Vec<(PartitionKey, IndexEntry)>,
22}
23
24#[derive(Clone, Serialize, Deserialize)]
25struct PartitionIndexDoc {
26 epoch: u64,
27 next_offset: i64,
28 log_start: i64,
29 entries: Vec<IndexEntry>,
30}
31
32#[derive(Clone, Serialize, Deserialize, Default)]
33struct IndexCatalog {
34 partitions: Vec<String>,
35}
36
37#[derive(Default)]
38struct Part {
39 next_offset: i64,
40 log_start: i64,
41 epoch: u64,
42 entries: Vec<IndexEntry>,
43 observed: Option<Bytes>,
45}
46
47struct Inner {
48 parts: HashMap<PartitionKey, Part>,
49 counter: u64,
50 catalog: Option<Bytes>,
51}
52
53pub struct ManifestSequencer {
55 blob: Arc<dyn BlobStore>,
56 prefix: String,
57 rt: Option<Runtime>,
60 commit_order: Mutex<()>,
62 inner: Mutex<Inner>,
63}
64
65impl Drop for ManifestSequencer {
66 fn drop(&mut self) {
67 if let Some(rt) = self.rt.take() {
68 rt.shutdown_background();
69 }
70 }
71}
72
73impl ManifestSequencer {
74 pub async fn open(
78 blob: Arc<dyn BlobStore>,
79 manifest_prefix: impl Into<String>,
80 ) -> Result<Self, ObjectLogError> {
81 let prefix = manifest_prefix.into();
82 let catalog_key = format!("{prefix}catalog");
83 let (parts, catalog, counter) = if let Some(catalog_bytes) = blob.get(&catalog_key).await? {
84 let catalog: IndexCatalog = serde_json::from_slice(&catalog_bytes)
85 .map_err(|e| ObjectLogError::Sequencer(format!("index catalog: {e}")))?;
86 let mut parts = HashMap::new();
87 for name in &catalog.partitions {
88 let key = index_object_key(&prefix, name);
89 let Some(doc_bytes) = blob.get(&key).await? else {
90 parts.insert(PartitionKey(name.clone()), Part::default());
91 continue;
92 };
93 let doc: PartitionIndexDoc = serde_json::from_slice(&doc_bytes).map_err(|e| {
94 ObjectLogError::Sequencer(format!("partition index {name}: {e}"))
95 })?;
96 parts.insert(PartitionKey(name.clone()), Part::from_doc(doc, doc_bytes));
97 }
98 let counter = parts.len() as u64;
99 (parts, Some(catalog_bytes), counter)
100 } else {
101 let mut keys = blob.list(&prefix).await?;
102 keys.sort();
103 let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
104 for key in &keys {
105 if key == &catalog_key || key.starts_with(&format!("{prefix}index/")) {
106 continue;
107 }
108 let Some(bytes) = blob.get(key).await? else {
109 continue;
110 };
111 let Ok(rec) = serde_json::from_slice::<ManifestRecord>(&bytes) else {
112 continue;
113 };
114 for (pkey, entry) in rec.entries {
115 let p = parts.entry(pkey).or_default();
116 let end = entry.base_offset + entry.record_count as i64;
117 if end > p.next_offset {
118 p.next_offset = end;
119 }
120 p.entries.push(entry);
121 }
122 }
123 (parts, None, keys.len() as u64)
124 };
125 let rt = Builder::new_current_thread()
126 .enable_all()
127 .build()
128 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
129 Ok(Self {
130 blob,
131 prefix,
132 rt: Some(rt),
133 commit_order: Mutex::new(()),
134 inner: Mutex::new(Inner {
135 parts,
136 counter,
137 catalog,
138 }),
139 })
140 }
141
142 fn index_key(&self, partition: &PartitionKey) -> String {
143 index_object_key(&self.prefix, partition.as_str())
144 }
145
146 fn catalog_key(&self) -> String {
147 format!("{}catalog", self.prefix)
148 }
149
150 fn block_on<T>(&self, future: impl std::future::Future<Output = T>) -> T {
151 self.rt
152 .as_ref()
153 .expect("runtime present until drop")
154 .block_on(future)
155 }
156
157 fn read_object(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
158 self.block_on(self.blob.get(key))
159 }
160
161 fn cas(
162 &self,
163 key: &str,
164 expected: Option<Bytes>,
165 new_value: Bytes,
166 ) -> Result<CasOutcome, ObjectLogError> {
167 self.block_on(self.blob.compare_and_swap(key, expected, new_value))
168 }
169
170 fn ensure_catalog(&self, partitions: &[PartitionKey]) -> Result<(), ObjectLogError> {
171 let key = self.catalog_key();
172 for _ in 0..8 {
173 let current = self.read_object(&key)?;
174 let mut names = if let Some(bytes) = ¤t {
175 serde_json::from_slice::<IndexCatalog>(bytes)
176 .map_err(|error| ObjectLogError::Sequencer(format!("index catalog: {error}")))?
177 .partitions
178 } else {
179 Vec::new()
180 };
181 let mut changed = false;
182 for partition in partitions {
183 if !names.iter().any(|name| name == partition.as_str()) {
184 names.push(partition.0.clone());
185 changed = true;
186 }
187 }
188 if !changed {
189 self.inner.lock().expect("poisoned").catalog = current;
190 return Ok(());
191 }
192 names.sort();
193 names.dedup();
194 let new_bytes = Bytes::from(
195 serde_json::to_vec(&IndexCatalog { partitions: names })
196 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
197 );
198 match self.cas(&key, current, new_bytes.clone())? {
199 CasOutcome::Stored => {
200 self.inner.lock().expect("poisoned").catalog = Some(new_bytes);
201 return Ok(());
202 }
203 CasOutcome::Conflict { .. } => continue,
204 }
205 }
206 Err(ObjectLogError::Sequencer(
207 "catalog compare-and-swap did not land".into(),
208 ))
209 }
210
211 fn publish_partition(
212 &self,
213 partition: &PartitionKey,
214 batches: &[CommitBatch<'_, ()>],
215 indexes: &[usize],
216 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
217 let epoch = batches[indexes[0]].epoch;
218 if indexes.iter().any(|index| batches[*index].epoch != epoch) {
219 return Ok(reject_all(
220 indexes.len(),
221 "one partition commit mixed fence epochs",
222 ));
223 }
224 let (observed, mut doc) = {
225 let inner = self.inner.lock().expect("poisoned");
226 match inner
227 .parts
228 .get(partition)
229 .and_then(|part| part.observed.clone())
230 {
231 Some(bytes) => {
232 let doc = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
233 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
234 (Some(bytes), doc)
235 }
236 None => {
237 let part = inner.parts.get(partition);
238 (
239 None,
240 PartitionIndexDoc {
241 epoch,
242 next_offset: part.map_or(0, |part| part.next_offset),
243 log_start: part.map_or(0, |part| part.log_start),
244 entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
245 },
246 )
247 }
248 }
249 };
250 if observed.is_some() && doc.epoch != epoch {
251 return Ok(reject_all(
252 indexes.len(),
253 format!(
254 "partition {} epoch is {}, not {epoch}",
255 partition.as_str(),
256 doc.epoch
257 ),
258 ));
259 }
260 let mut outcomes = Vec::with_capacity(indexes.len());
261 let mut planned = Vec::with_capacity(indexes.len());
262 for index in indexes {
263 let batch = &batches[*index];
264 let entry = IndexEntry {
265 location: batch.location.clone(),
266 base_offset: doc.next_offset,
267 record_count: batch.record_count,
268 };
269 outcomes.push(CommitOutcome::Assigned {
270 base_offset: entry.base_offset,
271 record_count: entry.record_count,
272 });
273 doc.next_offset += i64::from(batch.record_count);
274 planned.push(entry.clone());
275 doc.entries.push(entry);
276 }
277 let new_bytes = Bytes::from(
278 serde_json::to_vec(&doc)
279 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
280 );
281 match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
282 CasOutcome::Stored => {
283 self.install_doc(partition, &doc, new_bytes);
284 Ok(outcomes)
285 }
286 CasOutcome::Conflict { current } => {
287 if let Some(bytes) = current
288 && let Ok(found) = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
289 {
290 let already = planned
291 .iter()
292 .all(|entry| found.entries.iter().any(|have| have == entry));
293 self.install_doc(partition, &found, bytes);
294 if already {
295 return Ok(outcomes);
296 }
297 }
298 Ok(reject_all(
299 indexes.len(),
300 format!("partition {} tail moved", partition.as_str()),
301 ))
302 }
303 }
304 }
305
306 fn reload_partition(
308 &self,
309 partition: &PartitionKey,
310 ) -> Result<Option<(PartitionIndexDoc, Bytes)>, ObjectLogError> {
311 let Some(bytes) = self.read_object(&self.index_key(partition))? else {
312 return Ok(None);
313 };
314 let doc: PartitionIndexDoc = serde_json::from_slice(&bytes)
315 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
316 self.install_doc(partition, &doc, bytes.clone());
317 Ok(Some((doc, bytes)))
318 }
319
320 fn install_doc(&self, partition: &PartitionKey, doc: &PartitionIndexDoc, observed: Bytes) {
321 let mut inner = self.inner.lock().expect("poisoned");
322 inner.parts.insert(
323 partition.clone(),
324 Part {
325 next_offset: doc.next_offset,
326 log_start: doc.log_start,
327 epoch: doc.epoch,
328 entries: doc.entries.clone(),
329 observed: Some(observed),
330 },
331 );
332 }
333
334 pub fn live_object_ids(&self) -> HashSet<String> {
337 let st = self.inner.lock().expect("poisoned");
338 let mut live = HashSet::new();
339 for p in st.parts.values() {
340 for e in &p.entries {
341 live.insert(e.location.object_id.clone());
342 }
343 }
344 live
345 }
346
347 pub fn snapshot(&self) -> IndexSnapshot {
349 let st = self.inner.lock().expect("poisoned");
350 let mut partitions: Vec<PartitionSnapshot> = st
351 .parts
352 .iter()
353 .map(|(k, p)| PartitionSnapshot {
354 partition: k.0.clone(),
355 log_start: p.log_start,
356 high_watermark: p.next_offset,
357 entry_count: p.entries.len(),
358 entries: p.entries.clone(),
359 })
360 .collect();
361 partitions.sort_by(|a, b| a.partition.cmp(&b.partition));
362 IndexSnapshot {
363 manifest_prefix: self.prefix.clone(),
364 manifest_count: st.counter,
365 partitions,
366 }
367 }
368}
369
370#[derive(Clone, Debug, Serialize, Deserialize)]
372pub struct IndexSnapshot {
373 pub manifest_prefix: String,
375 pub manifest_count: u64,
377 pub partitions: Vec<PartitionSnapshot>,
379}
380
381#[derive(Clone, Debug, Serialize, Deserialize)]
383pub struct PartitionSnapshot {
384 pub partition: String,
386 pub log_start: i64,
388 pub high_watermark: i64,
390 pub entry_count: usize,
392 pub entries: Vec<IndexEntry>,
394}
395
396impl Sequencer for ManifestSequencer {
397 type Meta = ();
398
399 fn supports_multi_object_commit(&self) -> bool {
400 true
401 }
402
403 fn commit(
404 &self,
405 batches: &[CommitBatch<'_, ()>],
406 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
407 let _commit = self.commit_order.lock().expect("poisoned");
408 if batches.is_empty() {
409 return Ok(Vec::new());
410 }
411 let mut order: Vec<PartitionKey> = Vec::new();
412 let mut groups: HashMap<PartitionKey, Vec<usize>> = HashMap::new();
413 for (index, batch) in batches.iter().enumerate() {
414 if !groups.contains_key(&batch.partition) {
415 order.push(batch.partition.clone());
416 }
417 groups
418 .entry(batch.partition.clone())
419 .or_default()
420 .push(index);
421 }
422 self.ensure_catalog(&order)?;
423 let mut outcomes = vec![None; batches.len()];
424 let mut published = false;
425 for partition in order {
426 let indexes = &groups[&partition];
427 match self.publish_partition(&partition, batches, indexes) {
428 Ok(partition_outcomes) => {
429 for (index, outcome) in indexes.iter().zip(partition_outcomes) {
430 if matches!(
431 outcome,
432 CommitOutcome::Assigned { .. } | CommitOutcome::Duplicate { .. }
433 ) {
434 published = true;
435 }
436 outcomes[*index] = Some(outcome);
437 }
438 }
439 Err(error) => {
440 for index in indexes {
441 outcomes[*index] = Some(CommitOutcome::Rejected {
442 reason: error.to_string(),
443 });
444 }
445 }
446 }
447 }
448 if published {
449 let mut inner = self.inner.lock().expect("poisoned");
450 inner.counter = inner.counter.saturating_add(1);
451 }
452 Ok(outcomes
453 .into_iter()
454 .map(|outcome| outcome.expect("every batch has an outcome"))
455 .collect())
456 }
457
458 fn fence_epoch(
459 &self,
460 partition: &PartitionKey,
461 expected: u64,
462 new_epoch: u64,
463 ) -> Result<(), ObjectLogError> {
464 let _commit = self.commit_order.lock().expect("poisoned");
465 if expected == new_epoch {
466 return Ok(());
467 }
468 self.ensure_catalog(std::slice::from_ref(partition))?;
472 for _ in 0..8 {
473 let (mut doc, observed) = match self.reload_partition(partition)? {
474 Some((current, bytes)) => {
475 if current.epoch == new_epoch {
476 return Ok(());
477 }
478 if current.epoch != expected {
479 return Err(ObjectLogError::Sequencer(format!(
480 "partition {} epoch is {}, not {expected}",
481 partition.as_str(),
482 current.epoch
483 )));
484 }
485 (current, Some(bytes))
486 }
487 None => {
488 let inner = self.inner.lock().expect("poisoned");
491 let part = inner.parts.get(partition);
492 let doc = PartitionIndexDoc {
493 epoch: new_epoch,
494 next_offset: part.map_or(0, |part| part.next_offset),
495 log_start: part.map_or(0, |part| part.log_start),
496 entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
497 };
498 (doc, None)
499 }
500 };
501 doc.epoch = new_epoch;
502 let new_bytes = Bytes::from(
503 serde_json::to_vec(&doc)
504 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
505 );
506 match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
507 CasOutcome::Stored => {
508 self.install_doc(partition, &doc, new_bytes);
509 return Ok(());
510 }
511 CasOutcome::Conflict { .. } => continue,
513 }
514 }
515 Err(ObjectLogError::Sequencer(format!(
516 "partition {} epoch fence did not land",
517 partition.as_str()
518 )))
519 }
520
521 fn partition_epoch(&self, partition: &PartitionKey) -> Option<u64> {
522 let inner = self.inner.lock().expect("poisoned");
523 inner
524 .parts
525 .get(partition)
526 .filter(|part| part.observed.is_some())
527 .map(|part| part.epoch)
528 }
529
530 fn refresh_partition(&self, partition: &PartitionKey) -> Result<(), ObjectLogError> {
531 let _commit = self.commit_order.lock().expect("poisoned");
532 self.reload_partition(partition)?;
533 Ok(())
534 }
535
536 fn lookup(
537 &self,
538 partition: &PartitionKey,
539 fetch_offset: i64,
540 ) -> Result<Vec<IndexEntry>, ObjectLogError> {
541 let st = self.inner.lock().expect("poisoned");
542 let Some(p) = st.parts.get(partition) else {
543 return Ok(Vec::new());
544 };
545 Ok(p.entries
546 .iter()
547 .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
548 .cloned()
549 .collect())
550 }
551
552 fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
553 Ok(self
554 .inner
555 .lock()
556 .expect("poisoned")
557 .parts
558 .get(partition)
559 .map_or(0, |p| p.next_offset))
560 }
561
562 fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
563 Ok(self
564 .inner
565 .lock()
566 .expect("poisoned")
567 .parts
568 .get(partition)
569 .map_or(0, |p| p.log_start))
570 }
571
572 fn truncate_before(
573 &self,
574 partition: &PartitionKey,
575 offset: i64,
576 ) -> Result<Vec<String>, ObjectLogError> {
577 let _commit = self.commit_order.lock().expect("poisoned");
578 let snapshot = {
579 let st = self.inner.lock().expect("poisoned");
580 let Some(part) = st.parts.get(partition) else {
581 return Ok(Vec::new());
582 };
583 (
584 part.observed.clone(),
585 part.entries.clone(),
586 part.next_offset,
587 part.epoch,
588 part.log_start,
589 )
590 };
591 let (observed, entries, next_offset, epoch, mut log_start) = snapshot;
592 let mut dropped = Vec::new();
593 let mut kept = Vec::new();
594 for entry in entries {
595 if entry.base_offset + i64::from(entry.record_count) <= offset {
596 dropped.push(entry.location.object_id.clone());
597 } else {
598 kept.push(entry);
599 }
600 }
601 if offset > log_start {
602 log_start = offset.min(next_offset);
603 }
604 let doc = PartitionIndexDoc {
605 epoch,
606 next_offset,
607 log_start,
608 entries: kept,
609 };
610 if observed.is_some() {
611 let new_bytes = Bytes::from(
612 serde_json::to_vec(&doc)
613 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
614 );
615 match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
616 CasOutcome::Stored => self.install_doc(partition, &doc, new_bytes),
617 CasOutcome::Conflict { .. } => {
618 return Err(ObjectLogError::Sequencer(format!(
619 "partition {} truncate lost the compare-and-swap",
620 partition.as_str()
621 )));
622 }
623 }
624 } else {
625 let mut st = self.inner.lock().expect("poisoned");
626 if let Some(part) = st.parts.get_mut(partition) {
627 part.entries = doc.entries.clone();
628 part.log_start = doc.log_start;
629 }
630 }
631 let live = {
632 let st = self.inner.lock().expect("poisoned");
633 let mut live = HashSet::new();
634 for part in st.parts.values() {
635 for entry in &part.entries {
636 live.insert(entry.location.object_id.clone());
637 }
638 }
639 live
640 };
641 let mut dead = Vec::new();
642 let mut seen = HashSet::new();
643 for object_id in dropped {
644 if !live.contains(&object_id) && seen.insert(object_id.clone()) {
645 dead.push(object_id);
646 }
647 }
648 Ok(dead)
649 }
650}
651
652impl Part {
653 fn from_doc(doc: PartitionIndexDoc, observed: Bytes) -> Self {
654 Self {
655 next_offset: doc.next_offset,
656 log_start: doc.log_start,
657 epoch: doc.epoch,
658 entries: doc.entries,
659 observed: Some(observed),
660 }
661 }
662}
663
664fn index_object_key(prefix: &str, partition: &str) -> String {
665 let mut encoded = String::with_capacity(partition.len() * 2);
666 for byte in partition.as_bytes() {
667 encoded.push_str(&format!("{byte:02x}"));
668 }
669 format!("{prefix}index/{encoded}")
670}
671
672fn reject_all(count: usize, reason: impl Into<String>) -> Vec<CommitOutcome> {
673 let reason = reason.into();
674 vec![CommitOutcome::Rejected { reason }; count]
675}
676
677#[cfg(test)]
678mod index_tests {
679 use super::*;
680 use crate::{
681 BatchLocation, BlobStore, CommitBatch, CommitOutcome, Durability, FlushConfig,
682 InMemorySequencer, LogEngine, MemoryBlobStore, ObjectLogError, PartitionKey,
683 };
684 use async_trait::async_trait;
685 use bytes::Bytes;
686 use std::sync::Arc;
687 use std::sync::atomic::{AtomicUsize, Ordering};
688
689 fn location(object: &str) -> BatchLocation {
690 BatchLocation {
691 object_id: object.to_string(),
692 byte_start: 0,
693 byte_len: 1,
694 }
695 }
696
697 fn run_commit(
698 seq: Arc<ManifestSequencer>,
699 partitions: Vec<(&str, &str, u64)>,
700 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
701 let owned: Vec<(String, String, u64)> = partitions
702 .into_iter()
703 .map(|(partition, object, epoch)| (partition.to_string(), object.to_string(), epoch))
704 .collect();
705 std::thread::spawn(move || {
706 let meta = ();
707 let batches: Vec<CommitBatch<'_, ()>> = owned
708 .iter()
709 .map(|(partition, object, epoch)| CommitBatch {
710 partition: PartitionKey(partition.clone()),
711 record_count: 1,
712 location: location(object),
713 meta: &meta,
714 epoch: *epoch,
715 })
716 .collect();
717 seq.commit(&batches)
718 })
719 .join()
720 .expect("commit thread")
721 }
722
723 fn on_thread<T: Send + 'static>(
725 seq: &Arc<ManifestSequencer>,
726 call: impl FnOnce(&ManifestSequencer) -> T + Send + 'static,
727 ) -> T {
728 let seq = Arc::clone(seq);
729 std::thread::spawn(move || call(&seq))
730 .join()
731 .expect("sequencer thread")
732 }
733
734 struct ListProbe {
735 inner: MemoryBlobStore,
736 lists: Arc<AtomicUsize>,
737 }
738
739 #[async_trait]
740 impl BlobStore for ListProbe {
741 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
742 self.inner.put(key, value).await
743 }
744 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
745 self.inner.get(key).await
746 }
747 async fn get_range(
748 &self,
749 key: &str,
750 range: std::ops::Range<u64>,
751 ) -> Result<Option<Bytes>, ObjectLogError> {
752 self.inner.get_range(key, range).await
753 }
754 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
755 self.lists.fetch_add(1, Ordering::SeqCst);
756 self.inner.list(prefix).await
757 }
758 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
759 self.inner.delete(key).await
760 }
761 async fn compare_and_swap(
762 &self,
763 key: &str,
764 expected: Option<Bytes>,
765 new_value: Bytes,
766 ) -> Result<crate::CasOutcome, ObjectLogError> {
767 self.inner.compare_and_swap(key, expected, new_value).await
768 }
769 }
770
771 #[tokio::test]
772 async fn two_writers_cannot_commit_the_same_partition_tail() {
773 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
774 let first = Arc::new(
775 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
776 .await
777 .unwrap(),
778 );
779 let second = Arc::new(
780 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
781 .await
782 .unwrap(),
783 );
784 let won = run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
785 assert!(matches!(
786 won[0],
787 CommitOutcome::Assigned {
788 base_offset: 0,
789 record_count: 1
790 }
791 ));
792 let lost = run_commit(second, vec![("p", "obj-b", 1)]).unwrap();
793 assert!(
794 matches!(lost[0], CommitOutcome::Rejected { .. }),
795 "stale tail must fail closed, got {lost:?}"
796 );
797 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
798 let partition = PartitionKey("p".into());
799 assert_eq!(reopened.high_watermark(&partition).unwrap(), 1);
800 let entries = reopened.lookup(&partition, 0).unwrap();
801 assert_eq!(entries.len(), 1);
802 assert_eq!(entries[0].location.object_id, "obj-a");
803 }
804
805 #[tokio::test]
806 async fn mixed_seal_keeps_the_partition_that_won_and_rejects_the_moved_tail() {
807 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
808 let first = Arc::new(
809 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
810 .await
811 .unwrap(),
812 );
813 let second = Arc::new(
814 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
815 .await
816 .unwrap(),
817 );
818 run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
819 let mixed = run_commit(second, vec![("p", "obj-b", 1), ("q", "obj-b", 1)]).unwrap();
820 assert!(matches!(mixed[0], CommitOutcome::Rejected { .. }));
821 assert!(matches!(
822 mixed[1],
823 CommitOutcome::Assigned { base_offset: 0, .. }
824 ));
825 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
826 assert_eq!(
827 reopened.lookup(&PartitionKey("p".into()), 0).unwrap().len(),
828 1
829 );
830 assert_eq!(
831 reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
832 1,
833 "the rejected partition must not skip an offset"
834 );
835 assert_eq!(
836 reopened.lookup(&PartitionKey("q".into()), 0).unwrap()[0]
837 .location
838 .object_id,
839 "obj-b"
840 );
841 }
842
843 #[tokio::test]
844 async fn open_does_not_list_the_manifest_prefix_when_an_index_exists() {
845 let memory = MemoryBlobStore::new();
846 let blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
847 let seq = Arc::new(
848 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
849 .await
850 .unwrap(),
851 );
852 run_commit(seq, vec![("p", "obj-a", 4)]).unwrap();
853 let lists = Arc::new(AtomicUsize::new(0));
854 let probe = Arc::new(ListProbe {
855 inner: memory,
856 lists: Arc::clone(&lists),
857 });
858 let reopened = ManifestSequencer::open(probe, "manifest/").await.unwrap();
859 assert_eq!(
860 lists.load(Ordering::SeqCst),
861 0,
862 "a current index catalog must be read by key, not by listing manifests"
863 );
864 assert_eq!(
865 reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
866 1
867 );
868 }
869
870 #[tokio::test]
871 async fn distinct_writers_do_not_share_a_data_object_key() {
872 let blob = Arc::new(MemoryBlobStore::new());
873 let config = FlushConfig {
874 linger: std::time::Duration::ZERO,
875 ..FlushConfig::default()
876 };
877 let left = LogEngine::new_with_writer(
878 Arc::clone(&blob) as Arc<dyn BlobStore>,
879 Arc::new(InMemorySequencer::new()),
880 config,
881 "data/",
882 "writer-a",
883 );
884 let right = LogEngine::new_with_writer(
885 Arc::clone(&blob) as Arc<dyn BlobStore>,
886 Arc::new(InMemorySequencer::new()),
887 config,
888 "data/",
889 "writer-b",
890 );
891 left.produce_at_epoch(
892 PartitionKey("p".into()),
893 Bytes::from_static(b"left"),
894 1,
895 (),
896 Durability::Sequenced,
897 3,
898 )
899 .await
900 .unwrap();
901 right
902 .produce_at_epoch(
903 PartitionKey("p".into()),
904 Bytes::from_static(b"right"),
905 1,
906 (),
907 Durability::Sequenced,
908 3,
909 )
910 .await
911 .unwrap();
912 let keys = blob.list("data/").await.unwrap();
913 assert_eq!(
914 keys.len(),
915 2,
916 "writers must not overwrite one object: {keys:?}"
917 );
918 assert!(keys.iter().any(|key| key.contains("writer-a")));
919 assert!(keys.iter().any(|key| key.contains("writer-b")));
920 assert!(
921 keys.iter()
922 .all(|key| key.contains("/00000000000000000003/"))
923 );
924 }
925
926 #[tokio::test]
927 async fn fence_from_a_handle_that_never_saw_the_index_rejects_the_stale_writer() {
928 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
929 let owner = Arc::new(
930 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
931 .await
932 .unwrap(),
933 );
934 let standby = Arc::new(
936 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
937 .await
938 .unwrap(),
939 );
940 run_commit(Arc::clone(&owner), vec![("p", "obj-a", 1)]).unwrap();
941 let partition = PartitionKey("p".into());
942 let fence_partition = partition.clone();
943 on_thread(&standby, move |seq| seq.fence_epoch(&fence_partition, 1, 2)).unwrap();
944
945 let stale = run_commit(Arc::clone(&owner), vec![("p", "obj-stale", 1)]).unwrap();
946 assert!(
947 matches!(stale[0], CommitOutcome::Rejected { .. }),
948 "a commit at the fenced epoch must fail closed, got {stale:?}"
949 );
950 let fresh = run_commit(standby, vec![("p", "obj-b", 2)]).unwrap();
951 assert!(matches!(
952 fresh[0],
953 CommitOutcome::Assigned { base_offset: 1, .. }
954 ));
955 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
956 let objects: Vec<_> = reopened
957 .lookup(&partition, 0)
958 .unwrap()
959 .into_iter()
960 .map(|entry| entry.location.object_id)
961 .collect();
962 assert_eq!(objects, ["obj-a", "obj-b"]);
963 }
964
965 #[tokio::test]
966 async fn fence_before_any_commit_creates_the_index_at_the_new_epoch() {
967 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
968 let stale = Arc::new(
969 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
970 .await
971 .unwrap(),
972 );
973 let fencer = Arc::new(
974 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
975 .await
976 .unwrap(),
977 );
978 on_thread(&fencer, |seq| {
979 seq.fence_epoch(&PartitionKey("p".into()), 0, 2)
980 })
981 .unwrap();
982 let lost = run_commit(stale, vec![("p", "obj-stale", 1)]).unwrap();
983 assert!(
984 matches!(lost[0], CommitOutcome::Rejected { .. }),
985 "a writer that never saw the fenced index must not commit, got {lost:?}"
986 );
987 let won = run_commit(fencer, vec![("p", "obj-b", 2)]).unwrap();
988 assert!(matches!(
989 won[0],
990 CommitOutcome::Assigned { base_offset: 0, .. }
991 ));
992 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
993 assert_eq!(
994 reopened.snapshot().partitions[0].entries[0]
995 .location
996 .object_id,
997 "obj-b"
998 );
999 }
1000
1001 #[tokio::test]
1002 async fn refresh_partition_reveals_another_writers_commits() {
1003 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1004 let writer = Arc::new(
1005 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1006 .await
1007 .unwrap(),
1008 );
1009 let reader = Arc::new(
1010 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1011 .await
1012 .unwrap(),
1013 );
1014 run_commit(writer, vec![("p", "obj-a", 1), ("p", "obj-b", 1)]).unwrap();
1015 let partition = PartitionKey("p".into());
1016 assert_eq!(reader.high_watermark(&partition).unwrap(), 0);
1017 let refreshed = partition.clone();
1018 on_thread(&reader, move |seq| seq.refresh_partition(&refreshed)).unwrap();
1019 assert_eq!(reader.high_watermark(&partition).unwrap(), 2);
1020 assert_eq!(reader.lookup(&partition, 0).unwrap().len(), 2);
1021 }
1022
1023 #[tokio::test]
1024 async fn a_fenced_writer_gets_a_typed_fenced_error() {
1025 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1026 let config = FlushConfig {
1027 linger: std::time::Duration::ZERO,
1028 ..FlushConfig::default()
1029 };
1030 let owner_sequencer = Arc::new(
1031 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1032 .await
1033 .unwrap(),
1034 );
1035 let standby = Arc::new(
1036 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1037 .await
1038 .unwrap(),
1039 );
1040 let owner = LogEngine::new_with_writer(
1041 Arc::clone(&blob),
1042 owner_sequencer,
1043 config,
1044 "data/",
1045 "writer-a",
1046 );
1047 let partition = PartitionKey("p".into());
1048 owner
1049 .produce_at_epoch(
1050 partition.clone(),
1051 Bytes::from_static(b"before"),
1052 1,
1053 (),
1054 Durability::Sequenced,
1055 1,
1056 )
1057 .await
1058 .unwrap();
1059 let fenced = partition.clone();
1060 on_thread(&standby, move |seq| seq.fence_epoch(&fenced, 1, 2)).unwrap();
1061
1062 let stale = owner
1063 .produce_at_epoch(
1064 partition.clone(),
1065 Bytes::from_static(b"stale"),
1066 1,
1067 (),
1068 Durability::Sequenced,
1069 1,
1070 )
1071 .await;
1072 match stale {
1073 Err(ObjectLogError::Fenced {
1074 partition: name,
1075 epoch: 1,
1076 current: 2,
1077 }) => assert_eq!(name, "p"),
1078 other => panic!("a write behind the fence must be Fenced, got {other:?}"),
1079 }
1080 }
1081}