1use crate::{
10 AuthoritativePartitionState, BlobStore, CasOutcome, CommitBatch, CommitOutcome,
11 ConditionalCommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
12};
13use bytes::Bytes;
14use serde::{Deserialize, Serialize};
15use std::collections::{HashMap, HashSet};
16use std::sync::{Arc, Mutex};
17use tokio::runtime::{Builder, Runtime};
18
19#[derive(Serialize, Deserialize)]
20struct ManifestRecord {
21 entries: Vec<(PartitionKey, IndexEntry)>,
22}
23
24#[derive(Clone, Serialize, Deserialize)]
25struct PartitionIndexDoc {
26 epoch: u64,
27 next_offset: i64,
28 log_start: i64,
29 entries: Vec<IndexEntry>,
30}
31
32#[derive(Clone, Serialize, Deserialize, Default)]
33struct IndexCatalog {
34 partitions: Vec<String>,
35}
36
37#[derive(Default)]
38struct Part {
39 next_offset: i64,
40 log_start: i64,
41 epoch: u64,
42 entries: Vec<IndexEntry>,
43 observed: Option<Bytes>,
45}
46
47struct Inner {
48 uncertain_epochs: HashSet<PartitionKey>,
49 parts: HashMap<PartitionKey, Part>,
50 counter: u64,
51 catalog: Option<Bytes>,
52}
53
54pub struct ManifestSequencer {
56 blob: Arc<dyn BlobStore>,
57 prefix: String,
58 rt: Option<Runtime>,
61 commit_order: Mutex<()>,
63 inner: Mutex<Inner>,
64}
65
66impl Drop for ManifestSequencer {
67 fn drop(&mut self) {
68 if let Some(rt) = self.rt.take() {
69 rt.shutdown_background();
70 }
71 }
72}
73
74impl ManifestSequencer {
75 pub async fn open(
79 blob: Arc<dyn BlobStore>,
80 manifest_prefix: impl Into<String>,
81 ) -> Result<Self, ObjectLogError> {
82 let prefix = manifest_prefix.into();
83 let catalog_key = format!("{prefix}catalog");
84 let (parts, catalog, counter) = if let Some(catalog_bytes) = blob.get(&catalog_key).await? {
85 let catalog: IndexCatalog = serde_json::from_slice(&catalog_bytes)
86 .map_err(|e| ObjectLogError::Sequencer(format!("index catalog: {e}")))?;
87 let mut parts = HashMap::new();
88 for name in &catalog.partitions {
89 let key = index_object_key(&prefix, name);
90 let Some(doc_bytes) = blob.get(&key).await? else {
91 parts.insert(PartitionKey(name.clone()), Part::default());
92 continue;
93 };
94 let doc: PartitionIndexDoc = serde_json::from_slice(&doc_bytes).map_err(|e| {
95 ObjectLogError::Sequencer(format!("partition index {name}: {e}"))
96 })?;
97 parts.insert(PartitionKey(name.clone()), Part::from_doc(doc, doc_bytes)?);
98 }
99 let counter = parts.len() as u64;
100 (parts, Some(catalog_bytes), counter)
101 } else {
102 let mut keys = blob.list(&prefix).await?;
103 keys.sort();
104 let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
105 for key in &keys {
106 if key == &catalog_key || key.starts_with(&format!("{prefix}index/")) {
107 continue;
108 }
109 let Some(bytes) = blob.get(key).await? else {
110 continue;
111 };
112 let Ok(rec) = serde_json::from_slice::<ManifestRecord>(&bytes) else {
113 continue;
114 };
115 for (pkey, entry) in rec.entries {
116 let p = parts.entry(pkey).or_default();
117 let end = crate::bounded_lookup::entry_end(&entry)?;
118 if end > p.next_offset {
119 p.next_offset = end;
120 }
121 p.entries.push(entry);
122 }
123 }
124 (parts, None, keys.len() as u64)
125 };
126 for part in parts.values() {
127 crate::bounded_lookup::validate_index(&part.entries, part.log_start, part.next_offset)?;
128 }
129 let rt = Builder::new_multi_thread()
133 .worker_threads(1)
134 .enable_all()
135 .build()
136 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
137 Ok(Self {
138 blob,
139 prefix,
140 rt: Some(rt),
141 commit_order: Mutex::new(()),
142 inner: Mutex::new(Inner {
143 uncertain_epochs: parts.keys().cloned().collect(),
144 parts,
145 counter,
146 catalog,
147 }),
148 })
149 }
150
151 fn index_key(&self, partition: &PartitionKey) -> String {
152 index_object_key(&self.prefix, partition.as_str())
153 }
154
155 fn catalog_key(&self) -> String {
156 format!("{}catalog", self.prefix)
157 }
158
159 fn block_on<T>(&self, future: impl std::future::Future<Output = T>) -> T {
160 self.rt
161 .as_ref()
162 .expect("runtime present until drop")
163 .block_on(future)
164 }
165
166 fn read_object(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
167 self.block_on(self.blob.get(key))
168 }
169
170 fn cas(
171 &self,
172 key: &str,
173 expected: Option<Bytes>,
174 new_value: Bytes,
175 ) -> Result<CasOutcome, ObjectLogError> {
176 self.block_on(self.blob.compare_and_swap(key, expected, new_value))
177 }
178
179 fn ensure_catalog(&self, partitions: &[PartitionKey]) -> Result<bool, ObjectLogError> {
180 let key = self.catalog_key();
181 for _ in 0..8 {
182 let current = self.read_object(&key)?;
183 let mut names = if let Some(bytes) = ¤t {
184 serde_json::from_slice::<IndexCatalog>(bytes)
185 .map_err(|error| ObjectLogError::Sequencer(format!("index catalog: {error}")))?
186 .partitions
187 } else {
188 Vec::new()
189 };
190 self.inner
193 .lock()
194 .expect("poisoned")
195 .uncertain_epochs
196 .extend(names.iter().cloned().map(PartitionKey));
197 let mut changed = false;
198 for partition in partitions {
199 if !names.iter().any(|name| name == partition.as_str()) {
200 names.push(partition.0.clone());
201 changed = true;
202 }
203 }
204 if !changed {
205 self.inner.lock().expect("poisoned").catalog = current;
206 return Ok(false);
207 }
208 names.sort();
209 names.dedup();
210 let new_bytes = Bytes::from(
211 serde_json::to_vec(&IndexCatalog { partitions: names })
212 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
213 );
214 self.inner
215 .lock()
216 .expect("poisoned")
217 .uncertain_epochs
218 .extend(partitions.iter().cloned());
219 match self
220 .cas(&key, current, new_bytes.clone())
221 .inspect_err(|_| {
222 self.inner
223 .lock()
224 .expect("poisoned")
225 .uncertain_epochs
226 .extend(partitions.iter().cloned());
227 })? {
228 CasOutcome::Stored => {
229 let mut inner = self.inner.lock().expect("poisoned");
230 inner.uncertain_epochs.extend(partitions.iter().cloned());
231 inner.catalog = Some(new_bytes);
232 return Ok(true);
233 }
234 CasOutcome::Conflict {
235 current: Some(bytes),
236 } => {
237 let catalog: IndexCatalog =
238 serde_json::from_slice(&bytes).map_err(|error| {
239 ObjectLogError::Sequencer(format!("conflicting catalog: {error}"))
240 })?;
241 let registered = partitions.iter().all(|partition| {
242 catalog
243 .partitions
244 .iter()
245 .any(|name| name == partition.as_str())
246 });
247 let mut inner = self.inner.lock().expect("poisoned");
248 inner
249 .uncertain_epochs
250 .extend(catalog.partitions.into_iter().map(PartitionKey));
251 inner.catalog = Some(bytes);
252 if registered {
253 return Ok(false);
254 }
255 }
256 CasOutcome::Conflict { current: None } => {
257 self.inner
258 .lock()
259 .expect("poisoned")
260 .uncertain_epochs
261 .extend(partitions.iter().cloned());
262 return Err(ObjectLogError::StorageUnavailable(
263 "catalog conflict lost authority".into(),
264 ));
265 }
266 }
267 }
268 Err(ObjectLogError::Sequencer(
269 "catalog compare-and-swap did not land".into(),
270 ))
271 }
272
273 fn publish_partition(
274 &self,
275 partition: &PartitionKey,
276 batches: &[CommitBatch<'_, ()>],
277 indexes: &[usize],
278 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
279 let epoch = batches[indexes[0]].epoch;
280 if indexes.iter().any(|index| batches[*index].epoch != epoch) {
281 return Ok(reject_all(
282 indexes.len(),
283 "one partition commit mixed fence epochs",
284 ));
285 }
286 let (observed, mut doc) = {
287 let inner = self.inner.lock().expect("poisoned");
288 match inner
289 .parts
290 .get(partition)
291 .and_then(|part| part.observed.clone())
292 {
293 Some(bytes) => {
294 let doc = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
295 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
296 (Some(bytes), doc)
297 }
298 None => {
299 let part = inner.parts.get(partition);
300 (
301 None,
302 PartitionIndexDoc {
303 epoch,
304 next_offset: part.map_or(0, |part| part.next_offset),
305 log_start: part.map_or(0, |part| part.log_start),
306 entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
307 },
308 )
309 }
310 }
311 };
312 if observed.is_some() && doc.epoch != epoch {
313 return Ok(reject_all(
314 indexes.len(),
315 format!(
316 "partition {} epoch is {}, not {epoch}",
317 partition.as_str(),
318 doc.epoch
319 ),
320 ));
321 }
322 let mut outcomes = Vec::with_capacity(indexes.len());
323 let mut planned = Vec::with_capacity(indexes.len());
324 for index in indexes {
325 let batch = &batches[*index];
326 let entry = IndexEntry {
327 location: batch.location.clone(),
328 base_offset: doc.next_offset,
329 record_count: batch.record_count,
330 };
331 outcomes.push(CommitOutcome::Assigned {
332 base_offset: entry.base_offset,
333 record_count: entry.record_count,
334 });
335 if batch.record_count <= 0 {
336 return Err(ObjectLogError::InvalidBatch(
337 "invalid index record count".into(),
338 ));
339 }
340 doc.next_offset = doc
341 .next_offset
342 .checked_add(i64::from(batch.record_count))
343 .ok_or_else(|| ObjectLogError::Sequencer("index offset overflow".into()))?;
344 planned.push(entry.clone());
345 doc.entries.push(entry);
346 }
347 crate::bounded_lookup::validate_index(&doc.entries, doc.log_start, doc.next_offset)?;
348 let new_bytes = Bytes::from(
349 serde_json::to_vec(&doc)
350 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
351 );
352 match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
353 CasOutcome::Stored => {
354 self.install_doc(partition, &doc, new_bytes)?;
355 Ok(outcomes)
356 }
357 CasOutcome::Conflict { current } => {
358 if let Some(bytes) = current
359 && let Ok(found) = serde_json::from_slice::<PartitionIndexDoc>(&bytes)
360 {
361 let already = planned
362 .iter()
363 .all(|entry| found.entries.iter().any(|have| have == entry));
364 self.install_doc(partition, &found, bytes)?;
365 if already {
366 return Ok(outcomes);
367 }
368 }
369 Ok(reject_all(
370 indexes.len(),
371 format!("partition {} tail moved", partition.as_str()),
372 ))
373 }
374 }
375 }
376
377 fn publish_partition_conditionally(
378 &self,
379 partition: &PartitionKey,
380 batches: &[CommitBatch<'_, ()>],
381 indexes: &[usize],
382 heads: &[Option<i64>],
383 ) -> Result<Vec<ConditionalCommitOutcome>, ObjectLogError> {
384 for _ in 0..8 {
385 let (mut doc, observed) = match self.reload_partition(partition)? {
386 Some((doc, bytes)) => (doc, Some(bytes)),
387 None => {
388 if self.has_known_history(partition)
389 || self.has_durable_catalog_entry(partition)?
390 {
391 return Err(ObjectLogError::Sequencer("previously observed conditional index vanished or requires legacy migration".into()));
392 }
393 (
394 PartitionIndexDoc {
395 epoch: batches[indexes[0]].epoch,
396 next_offset: 0,
397 log_start: 0,
398 entries: Vec::new(),
399 },
400 None,
401 )
402 }
403 };
404 if observed.is_some() {
405 self.ensure_catalog(std::slice::from_ref(partition))?;
406 }
407 let mut outcomes = Vec::with_capacity(indexes.len());
408 let mut planned = Vec::with_capacity(indexes.len());
409 for index in indexes {
410 let batch = &batches[*index];
411 let mut matching = doc
412 .entries
413 .iter()
414 .filter(|entry| entry.location == batch.location);
415 if let Some(entry) = matching.next() {
416 if matching.next().is_some()
417 || entry.record_count != batch.record_count
418 || heads[*index].is_some_and(|head| head != entry.base_offset)
419 {
420 return Err(ObjectLogError::Sequencer(
421 "conditional location identity mismatch".into(),
422 ));
423 }
424 outcomes.push(ConditionalCommitOutcome::Committed(
425 CommitOutcome::Duplicate {
426 base_offset: entry.base_offset,
427 },
428 ));
429 continue;
430 }
431 if heads[*index].is_some_and(|head| head < doc.log_start) {
432 return Err(ObjectLogError::Sequencer(
433 "conditional expected range is no longer inspectable".into(),
434 ));
435 }
436 if doc.epoch != batch.epoch {
437 outcomes.push(ConditionalCommitOutcome::Committed(
438 CommitOutcome::Rejected {
439 reason: format!(
440 "partition {} epoch is {}, not {}",
441 partition.as_str(),
442 doc.epoch,
443 batch.epoch
444 ),
445 },
446 ));
447 continue;
448 }
449 if let Some(expected) = heads[*index]
450 && expected != doc.next_offset
451 {
452 outcomes.push(ConditionalCommitOutcome::HeadRejected {
453 expected,
454 current: doc.next_offset,
455 });
456 continue;
457 }
458 let entry = IndexEntry {
459 location: batch.location.clone(),
460 base_offset: doc.next_offset,
461 record_count: batch.record_count,
462 };
463 doc.next_offset = doc
464 .next_offset
465 .checked_add(i64::from(batch.record_count))
466 .ok_or_else(|| {
467 ObjectLogError::Sequencer("conditional offset overflow".into())
468 })?;
469 outcomes.push(ConditionalCommitOutcome::Committed(
470 CommitOutcome::Assigned {
471 base_offset: entry.base_offset,
472 record_count: entry.record_count,
473 },
474 ));
475 planned.push(entry.clone());
476 doc.entries.push(entry);
477 }
478 if planned.is_empty() {
479 return Ok(outcomes);
480 }
481 if observed.is_none() && !self.ensure_catalog(std::slice::from_ref(partition))? {
483 if self.reload_partition(partition)?.is_some() {
484 continue;
485 }
486 return Err(ObjectLogError::Sequencer(
487 "catalog-known conditional index is missing".into(),
488 ));
489 }
490 crate::bounded_lookup::validate_index(&doc.entries, doc.log_start, doc.next_offset)?;
491 let bytes = Bytes::from(
492 serde_json::to_vec(&doc)
493 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
494 );
495 match self.cas(&self.index_key(partition), observed.clone(), bytes.clone())? {
496 CasOutcome::Stored => {
497 self.install_doc(partition, &doc, bytes)?;
498 return Ok(outcomes);
499 }
500 CasOutcome::Conflict { current } => {
501 let (found, current) = match current {
502 Some(bytes) => {
503 let found: PartitionIndexDoc = serde_json::from_slice(&bytes)
504 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
505 self.install_doc(partition, &found, bytes.clone())?;
506 (found, bytes)
507 }
508 None => self.reload_partition(partition)?.ok_or_else(|| {
509 ObjectLogError::Sequencer(
510 "conditional conflict lost durable index".into(),
511 )
512 })?,
513 };
514 let _ = current;
515 let present = planned
516 .iter()
517 .filter(|entry| found.entries.iter().any(|have| have == *entry))
518 .count();
519 if present == planned.len() {
520 for (index, outcome) in indexes.iter().zip(&outcomes) {
523 if matches!(
524 outcome,
525 ConditionalCommitOutcome::HeadRejected { .. }
526 | ConditionalCommitOutcome::Committed(
527 CommitOutcome::Rejected { .. }
528 )
529 ) && found
530 .entries
531 .iter()
532 .any(|entry| entry.location == batches[*index].location)
533 {
534 return Err(ObjectLogError::Sequencer(
535 "rejected conditional location already indexed".into(),
536 ));
537 }
538 }
539 return Ok(outcomes);
540 }
541 if planned.iter().any(|entry| {
542 entry.base_offset < found.log_start
543 && !found.entries.iter().any(|have| have == entry)
544 }) {
545 return Err(ObjectLogError::Sequencer(
546 "conditional planned range is no longer inspectable".into(),
547 ));
548 }
549 if present != 0 {
550 return Err(ObjectLogError::Sequencer(
551 "partial conditional publication reconciliation".into(),
552 ));
553 }
554 }
557 }
558 }
559 Err(ObjectLogError::StorageUnavailable(
560 "conditional index CAS retry exhausted".into(),
561 ))
562 }
563
564 fn has_known_history(&self, partition: &PartitionKey) -> bool {
566 let inner = self.inner.lock().expect("poisoned");
567 inner.uncertain_epochs.contains(partition)
568 || inner.parts.get(partition).is_some_and(|part| {
569 part.observed.is_some()
570 || part.next_offset != 0
571 || part.log_start != 0
572 || !part.entries.is_empty()
573 })
574 }
575
576 fn has_durable_catalog_entry(&self, partition: &PartitionKey) -> Result<bool, ObjectLogError> {
580 let Some(bytes) = self.read_object(&self.catalog_key())? else {
581 return Ok(false);
582 };
583 let catalog: IndexCatalog = serde_json::from_slice(&bytes)
584 .map_err(|error| ObjectLogError::Sequencer(format!("index catalog: {error}")))?;
585 let known = catalog
586 .partitions
587 .iter()
588 .any(|name| name == partition.as_str());
589 self.inner
590 .lock()
591 .expect("poisoned")
592 .uncertain_epochs
593 .extend(catalog.partitions.into_iter().map(PartitionKey));
594 Ok(known)
595 }
596
597 fn reload_partition(
599 &self,
600 partition: &PartitionKey,
601 ) -> Result<Option<(PartitionIndexDoc, Bytes)>, ObjectLogError> {
602 let Some(bytes) = self.read_object(&self.index_key(partition))? else {
603 return Ok(None);
604 };
605 self.inner
606 .lock()
607 .expect("poisoned")
608 .uncertain_epochs
609 .insert(partition.clone());
610 let doc: PartitionIndexDoc = serde_json::from_slice(&bytes)
611 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
612 self.install_doc(partition, &doc, bytes.clone())?;
613 Ok(Some((doc, bytes)))
614 }
615
616 fn install_doc(
617 &self,
618 partition: &PartitionKey,
619 doc: &PartitionIndexDoc,
620 observed: Bytes,
621 ) -> Result<(), ObjectLogError> {
622 crate::bounded_lookup::validate_index(&doc.entries, doc.log_start, doc.next_offset)?;
623 let mut inner = self.inner.lock().expect("poisoned");
624 inner.parts.insert(
625 partition.clone(),
626 Part {
627 next_offset: doc.next_offset,
628 log_start: doc.log_start,
629 epoch: doc.epoch,
630 entries: doc.entries.clone(),
631 observed: Some(observed),
632 },
633 );
634 Ok(())
635 }
636
637 pub fn live_object_ids(&self) -> HashSet<String> {
640 let st = self.inner.lock().expect("poisoned");
641 let mut live = HashSet::new();
642 for p in st.parts.values() {
643 for e in &p.entries {
644 live.insert(e.location.object_id.clone());
645 }
646 }
647 live
648 }
649
650 pub fn snapshot(&self) -> IndexSnapshot {
652 let st = self.inner.lock().expect("poisoned");
653 let mut partitions: Vec<PartitionSnapshot> = st
654 .parts
655 .iter()
656 .map(|(k, p)| PartitionSnapshot {
657 partition: k.0.clone(),
658 log_start: p.log_start,
659 high_watermark: p.next_offset,
660 entry_count: p.entries.len(),
661 entries: p.entries.clone(),
662 })
663 .collect();
664 partitions.sort_by(|a, b| a.partition.cmp(&b.partition));
665 IndexSnapshot {
666 manifest_prefix: self.prefix.clone(),
667 manifest_count: st.counter,
668 partitions,
669 }
670 }
671}
672
673#[derive(Clone, Debug, Serialize, Deserialize)]
675pub struct IndexSnapshot {
676 pub manifest_prefix: String,
678 pub manifest_count: u64,
680 pub partitions: Vec<PartitionSnapshot>,
682}
683
684#[derive(Clone, Debug, Serialize, Deserialize)]
686pub struct PartitionSnapshot {
687 pub partition: String,
689 pub log_start: i64,
691 pub high_watermark: i64,
693 pub entry_count: usize,
695 pub entries: Vec<IndexEntry>,
697}
698
699impl Sequencer for ManifestSequencer {
700 type Meta = ();
701
702 fn supports_multi_object_commit(&self) -> bool {
703 true
704 }
705
706 fn supports_conditional_commit(&self) -> bool {
707 true
708 }
709
710 fn commit_conditionally(
711 &self,
712 batches: &[CommitBatch<'_, ()>],
713 expected_heads: &[Option<i64>],
714 ) -> Result<Vec<ConditionalCommitOutcome>, ObjectLogError> {
715 if batches.len() != expected_heads.len()
716 || expected_heads.iter().flatten().any(|head| *head < 0)
717 || batches
718 .iter()
719 .any(|batch| batch.record_count <= 0 || batch.location.object_id.is_empty())
720 {
721 return Err(ObjectLogError::InvalidBatch(
722 "invalid conditional commit inputs".into(),
723 ));
724 }
725 if expected_heads.iter().all(Option::is_none) {
726 return self.commit(batches).map(|outcomes| {
727 outcomes
728 .into_iter()
729 .map(ConditionalCommitOutcome::Committed)
730 .collect()
731 });
732 }
733 let _commit = self.commit_order.lock().expect("poisoned");
734 let mut order = Vec::new();
735 let mut groups: HashMap<PartitionKey, Vec<usize>> = HashMap::new();
736 for (index, batch) in batches.iter().enumerate() {
737 if !groups.contains_key(&batch.partition) {
738 order.push(batch.partition.clone());
739 }
740 groups
741 .entry(batch.partition.clone())
742 .or_default()
743 .push(index);
744 }
745 let mut outcomes = vec![None; batches.len()];
746 let mut published = false;
747 for partition in order {
748 let indexes = &groups[&partition];
749 match self.publish_partition_conditionally(&partition, batches, indexes, expected_heads)
750 {
751 Ok(rows) => {
752 for (index, outcome) in indexes.iter().zip(rows) {
753 if matches!(
754 outcome,
755 ConditionalCommitOutcome::Committed(
756 CommitOutcome::Assigned { .. } | CommitOutcome::Duplicate { .. }
757 )
758 ) {
759 published = true;
760 }
761 outcomes[*index] = Some(outcome);
762 }
763 }
764 Err(error) => {
765 for index in indexes {
766 outcomes[*index] = Some(ConditionalCommitOutcome::Unknown(error.clone()));
767 }
768 }
769 }
770 }
771 if published {
772 let mut inner = self.inner.lock().expect("poisoned");
773 inner.counter = inner.counter.saturating_add(1);
774 }
775 Ok(outcomes
776 .into_iter()
777 .map(|outcome| outcome.expect("every conditional batch has an outcome"))
778 .collect())
779 }
780
781 fn commit(
782 &self,
783 batches: &[CommitBatch<'_, ()>],
784 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
785 let _commit = self.commit_order.lock().expect("poisoned");
786 if batches.is_empty() {
787 return Ok(Vec::new());
788 }
789 if batches
790 .iter()
791 .any(|batch| batch.record_count <= 0 || batch.location.object_id.is_empty())
792 {
793 return Err(ObjectLogError::InvalidBatch("invalid index batch".into()));
794 }
795 let mut order: Vec<PartitionKey> = Vec::new();
796 let mut groups: HashMap<PartitionKey, Vec<usize>> = HashMap::new();
797 for (index, batch) in batches.iter().enumerate() {
798 if !groups.contains_key(&batch.partition) {
799 order.push(batch.partition.clone());
800 }
801 groups
802 .entry(batch.partition.clone())
803 .or_default()
804 .push(index);
805 }
806 self.ensure_catalog(&order)?;
807 let mut outcomes = vec![None; batches.len()];
808 let mut published = false;
809 for partition in order {
810 let indexes = &groups[&partition];
811 match self.publish_partition(&partition, batches, indexes) {
812 Ok(partition_outcomes) => {
813 for (index, outcome) in indexes.iter().zip(partition_outcomes) {
814 if matches!(
815 outcome,
816 CommitOutcome::Assigned { .. } | CommitOutcome::Duplicate { .. }
817 ) {
818 published = true;
819 }
820 outcomes[*index] = Some(outcome);
821 }
822 }
823 Err(error) => {
824 for index in indexes {
825 outcomes[*index] = Some(CommitOutcome::Rejected {
826 reason: error.to_string(),
827 });
828 }
829 }
830 }
831 }
832 if published {
833 let mut inner = self.inner.lock().expect("poisoned");
834 inner.counter = inner.counter.saturating_add(1);
835 }
836 Ok(outcomes
837 .into_iter()
838 .map(|outcome| outcome.expect("every batch has an outcome"))
839 .collect())
840 }
841
842 fn fence_epoch(
843 &self,
844 partition: &PartitionKey,
845 expected: u64,
846 new_epoch: u64,
847 ) -> Result<(), ObjectLogError> {
848 let _commit = self.commit_order.lock().expect("poisoned");
849 if expected == new_epoch {
850 return Ok(());
851 }
852 self.ensure_catalog(std::slice::from_ref(partition))?;
856 for _ in 0..8 {
857 let (mut doc, observed) = match self.reload_partition(partition)? {
858 Some((current, bytes)) => {
859 if current.epoch == new_epoch {
860 return Ok(());
861 }
862 if current.epoch != expected {
863 return Err(ObjectLogError::Sequencer(format!(
864 "partition {} epoch is {}, not {expected}",
865 partition.as_str(),
866 current.epoch
867 )));
868 }
869 (current, Some(bytes))
870 }
871 None => {
872 let inner = self.inner.lock().expect("poisoned");
875 let part = inner.parts.get(partition);
876 let doc = PartitionIndexDoc {
877 epoch: new_epoch,
878 next_offset: part.map_or(0, |part| part.next_offset),
879 log_start: part.map_or(0, |part| part.log_start),
880 entries: part.map(|part| part.entries.clone()).unwrap_or_default(),
881 };
882 (doc, None)
883 }
884 };
885 doc.epoch = new_epoch;
886 let new_bytes = Bytes::from(
887 serde_json::to_vec(&doc)
888 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
889 );
890 match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
891 CasOutcome::Stored => {
892 self.install_doc(partition, &doc, new_bytes)?;
893 return Ok(());
894 }
895 CasOutcome::Conflict { .. } => continue,
897 }
898 }
899 Err(ObjectLogError::Sequencer(format!(
900 "partition {} epoch fence did not land",
901 partition.as_str()
902 )))
903 }
904
905 fn partition_epoch(&self, partition: &PartitionKey) -> Option<u64> {
906 let inner = self.inner.lock().expect("poisoned");
907 inner
908 .parts
909 .get(partition)
910 .filter(|part| part.observed.is_some())
911 .map(|part| part.epoch)
912 }
913
914 fn rejection_error(
919 &self,
920 partition: &PartitionKey,
921 epoch: u64,
922 reason: String,
923 ) -> ObjectLogError {
924 #[allow(deprecated)]
925 match self.partition_epoch(partition) {
926 Some(current) if current > epoch => ObjectLogError::Fenced {
927 partition: partition.as_str().to_owned(),
928 epoch,
929 current,
930 },
931 _ => ObjectLogError::Sequencer(reason),
932 }
933 }
934
935 fn acquire_authoritative_epoch(
936 &self,
937 partition: &PartitionKey,
938 ) -> Result<AuthoritativePartitionState, ObjectLogError> {
939 let _commit = self.commit_order.lock().expect("poisoned");
940 for _ in 0..8 {
941 let (mut doc, observed) = match self.reload_partition(partition)? {
942 Some((doc, bytes)) => (doc, Some(bytes)),
943 None if self.has_known_history(partition)
944 || self.has_durable_catalog_entry(partition)? =>
945 {
946 return Err(ObjectLogError::Sequencer(
947 "previously observed epoch index vanished or requires legacy migration"
948 .into(),
949 ));
950 }
951 None => (
952 PartitionIndexDoc {
953 epoch: 0,
954 next_offset: 0,
955 log_start: 0,
956 entries: Vec::new(),
957 },
958 None,
959 ),
960 };
961 doc.epoch = doc
962 .epoch
963 .checked_add(1)
964 .ok_or_else(|| ObjectLogError::Sequencer("writer epoch exhausted".into()))?;
965 let registered = self.ensure_catalog(std::slice::from_ref(partition))?;
966 if observed.is_none() && !registered {
967 if self.reload_partition(partition)?.is_some() {
971 continue;
972 }
973 return Err(ObjectLogError::Sequencer(
974 "catalog-known epoch index is missing".into(),
975 ));
976 }
977 let bytes = Bytes::from(
978 serde_json::to_vec(&doc)
979 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
980 );
981 match self
982 .cas(&self.index_key(partition), observed, bytes.clone())
983 .inspect_err(|_error| {
984 self.inner
985 .lock()
986 .expect("poisoned")
987 .uncertain_epochs
988 .insert(partition.clone());
989 })? {
990 CasOutcome::Stored => {
991 self.install_doc(partition, &doc, bytes)?;
992 return Ok(AuthoritativePartitionState {
993 epoch: doc.epoch,
994 log_start: doc.log_start,
995 next_offset: doc.next_offset,
996 });
997 }
998 CasOutcome::Conflict {
999 current: Some(bytes),
1000 } => {
1001 self.inner
1002 .lock()
1003 .expect("poisoned")
1004 .uncertain_epochs
1005 .insert(partition.clone());
1006 let current: PartitionIndexDoc = serde_json::from_slice(&bytes)
1007 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?;
1008 self.install_doc(partition, ¤t, bytes)?;
1009 }
1010 CasOutcome::Conflict { current: None } => {
1011 self.inner
1012 .lock()
1013 .expect("poisoned")
1014 .uncertain_epochs
1015 .insert(partition.clone());
1016 return Err(ObjectLogError::StorageUnavailable(
1017 "epoch acquisition conflicted without surviving authority".into(),
1018 ));
1019 }
1020 }
1021 }
1022 Err(ObjectLogError::StorageUnavailable(
1023 "authoritative epoch CAS retry exhausted".into(),
1024 ))
1025 }
1026
1027 fn authoritative_partition_state(
1028 &self,
1029 partition: &PartitionKey,
1030 ) -> Result<AuthoritativePartitionState, ObjectLogError> {
1031 let _commit = self.commit_order.lock().expect("poisoned");
1032 match self.reload_partition(partition)? {
1033 Some((doc, _)) => Ok(AuthoritativePartitionState {
1034 epoch: doc.epoch,
1035 log_start: doc.log_start,
1036 next_offset: doc.next_offset,
1037 }),
1038 None if self.has_known_history(partition)
1039 || self.has_durable_catalog_entry(partition)? =>
1040 {
1041 Err(ObjectLogError::Sequencer(
1042 "previously observed authoritative index vanished or requires legacy migration"
1043 .into(),
1044 ))
1045 }
1046 None => Ok(AuthoritativePartitionState {
1047 epoch: 0,
1048 log_start: 0,
1049 next_offset: 0,
1050 }),
1051 }
1052 }
1053
1054 fn refresh_partition(&self, partition: &PartitionKey) -> Result<(), ObjectLogError> {
1055 let _commit = self.commit_order.lock().expect("poisoned");
1056 self.reload_partition(partition)?;
1057 Ok(())
1058 }
1059
1060 fn lookup(
1061 &self,
1062 partition: &PartitionKey,
1063 fetch_offset: i64,
1064 ) -> Result<Vec<IndexEntry>, ObjectLogError> {
1065 let st = self.inner.lock().expect("poisoned");
1066 let Some(p) = st.parts.get(partition) else {
1067 return Ok(Vec::new());
1068 };
1069 Ok(p.entries
1070 .iter()
1071 .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
1072 .cloned()
1073 .collect())
1074 }
1075
1076 fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
1077 Ok(self
1078 .inner
1079 .lock()
1080 .expect("poisoned")
1081 .parts
1082 .get(partition)
1083 .map_or(0, |p| p.next_offset))
1084 }
1085
1086 fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
1087 Ok(self
1088 .inner
1089 .lock()
1090 .expect("poisoned")
1091 .parts
1092 .get(partition)
1093 .map_or(0, |p| p.log_start))
1094 }
1095
1096 fn truncate_before(
1097 &self,
1098 partition: &PartitionKey,
1099 offset: i64,
1100 ) -> Result<Vec<String>, ObjectLogError> {
1101 let _commit = self.commit_order.lock().expect("poisoned");
1102 let snapshot = {
1103 let st = self.inner.lock().expect("poisoned");
1104 let Some(part) = st.parts.get(partition) else {
1105 return Ok(Vec::new());
1106 };
1107 (
1108 part.observed.clone(),
1109 part.entries.clone(),
1110 part.next_offset,
1111 part.epoch,
1112 part.log_start,
1113 )
1114 };
1115 let (observed, entries, next_offset, epoch, mut log_start) = snapshot;
1116 let mut dropped = Vec::new();
1117 let mut kept = Vec::new();
1118 for entry in entries {
1119 if entry.base_offset + i64::from(entry.record_count) <= offset {
1120 dropped.push(entry.location.object_id.clone());
1121 } else {
1122 kept.push(entry);
1123 }
1124 }
1125 if offset > log_start {
1126 log_start = offset.min(next_offset);
1127 }
1128 let doc = PartitionIndexDoc {
1129 epoch,
1130 next_offset,
1131 log_start,
1132 entries: kept,
1133 };
1134 if observed.is_some() {
1135 let new_bytes = Bytes::from(
1136 serde_json::to_vec(&doc)
1137 .map_err(|error| ObjectLogError::Sequencer(error.to_string()))?,
1138 );
1139 match self.cas(&self.index_key(partition), observed, new_bytes.clone())? {
1140 CasOutcome::Stored => self.install_doc(partition, &doc, new_bytes)?,
1141 CasOutcome::Conflict { .. } => {
1142 return Err(ObjectLogError::Sequencer(format!(
1143 "partition {} truncate lost the compare-and-swap",
1144 partition.as_str()
1145 )));
1146 }
1147 }
1148 } else {
1149 let mut st = self.inner.lock().expect("poisoned");
1150 if let Some(part) = st.parts.get_mut(partition) {
1151 part.entries = doc.entries.clone();
1152 part.log_start = doc.log_start;
1153 }
1154 }
1155 let live = {
1156 let st = self.inner.lock().expect("poisoned");
1157 let mut live = HashSet::new();
1158 for part in st.parts.values() {
1159 for entry in &part.entries {
1160 live.insert(entry.location.object_id.clone());
1161 }
1162 }
1163 live
1164 };
1165 let mut dead = Vec::new();
1166 let mut seen = HashSet::new();
1167 for object_id in dropped {
1168 if !live.contains(&object_id) && seen.insert(object_id.clone()) {
1169 dead.push(object_id);
1170 }
1171 }
1172 Ok(dead)
1173 }
1174}
1175
1176impl Part {
1177 fn from_doc(doc: PartitionIndexDoc, observed: Bytes) -> Result<Self, ObjectLogError> {
1178 crate::bounded_lookup::validate_index(&doc.entries, doc.log_start, doc.next_offset)?;
1179 Ok(Self {
1180 next_offset: doc.next_offset,
1181 log_start: doc.log_start,
1182 epoch: doc.epoch,
1183 entries: doc.entries,
1184 observed: Some(observed),
1185 })
1186 }
1187}
1188
1189fn index_object_key(prefix: &str, partition: &str) -> String {
1190 let mut encoded = String::with_capacity(partition.len() * 2);
1191 for byte in partition.as_bytes() {
1192 encoded.push_str(&format!("{byte:02x}"));
1193 }
1194 format!("{prefix}index/{encoded}")
1195}
1196
1197fn reject_all(count: usize, reason: impl Into<String>) -> Vec<CommitOutcome> {
1198 let reason = reason.into();
1199 vec![CommitOutcome::Rejected { reason }; count]
1200}
1201
1202impl crate::BoundedSequencer for ManifestSequencer {
1203 fn lookup_page(
1204 &self,
1205 partition: &PartitionKey,
1206 offset: i64,
1207 max_entries: usize,
1208 max_bytes: usize,
1209 ) -> Result<Vec<IndexEntry>, ObjectLogError> {
1210 let state = self.inner.lock().expect("poisoned");
1211 if offset < state.parts.get(partition).map_or(0, |part| part.log_start) {
1212 return Err(ObjectLogError::RangeOutOfBounds(
1213 "lookup precedes retained history".into(),
1214 ));
1215 }
1216 let entries = state
1217 .parts
1218 .get(partition)
1219 .map_or(&[][..], |part| part.entries.as_slice());
1220 crate::bounded_lookup::page(entries, offset, max_entries, max_bytes)
1221 }
1222}
1223
1224#[cfg(test)]
1225mod index_tests {
1226 use super::*;
1227 use crate::{
1228 BatchLocation, BlobStore, CommitBatch, CommitOutcome, Durability, FlushConfig,
1229 InMemorySequencer, LogEngine, MemoryBlobStore, ObjectLogError, PartitionKey,
1230 };
1231 use async_trait::async_trait;
1232 use bytes::Bytes;
1233 use std::sync::Arc;
1234 use std::sync::atomic::{AtomicUsize, Ordering};
1235
1236 fn location(object: &str) -> BatchLocation {
1237 BatchLocation {
1238 object_id: object.to_string(),
1239 byte_start: 0,
1240 byte_len: 1,
1241 }
1242 }
1243
1244 fn run_commit(
1245 seq: Arc<ManifestSequencer>,
1246 partitions: Vec<(&str, &str, u64)>,
1247 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
1248 let owned: Vec<(String, String, u64)> = partitions
1249 .into_iter()
1250 .map(|(partition, object, epoch)| (partition.to_string(), object.to_string(), epoch))
1251 .collect();
1252 std::thread::spawn(move || {
1253 let meta = ();
1254 let batches: Vec<CommitBatch<'_, ()>> = owned
1255 .iter()
1256 .map(|(partition, object, epoch)| CommitBatch {
1257 partition: PartitionKey(partition.clone()),
1258 record_count: 1,
1259 location: location(object),
1260 meta: &meta,
1261 epoch: *epoch,
1262 })
1263 .collect();
1264 seq.commit(&batches)
1265 })
1266 .join()
1267 .expect("commit thread")
1268 }
1269
1270 fn on_thread<T: Send + 'static>(
1272 seq: &Arc<ManifestSequencer>,
1273 call: impl FnOnce(&ManifestSequencer) -> T + Send + 'static,
1274 ) -> T {
1275 let seq = Arc::clone(seq);
1276 std::thread::spawn(move || call(&seq))
1277 .join()
1278 .expect("sequencer thread")
1279 }
1280
1281 struct ListProbe {
1282 inner: MemoryBlobStore,
1283 lists: Arc<AtomicUsize>,
1284 }
1285
1286 #[async_trait]
1287 impl BlobStore for ListProbe {
1288 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1289 self.inner.put(key, value).await
1290 }
1291 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1292 self.inner.get(key).await
1293 }
1294 async fn get_range(
1295 &self,
1296 key: &str,
1297 range: std::ops::Range<u64>,
1298 ) -> Result<Option<Bytes>, ObjectLogError> {
1299 self.inner.get_range(key, range).await
1300 }
1301 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1302 self.lists.fetch_add(1, Ordering::SeqCst);
1303 self.inner.list(prefix).await
1304 }
1305 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1306 self.inner.delete(key).await
1307 }
1308 async fn compare_and_swap(
1309 &self,
1310 key: &str,
1311 expected: Option<Bytes>,
1312 new_value: Bytes,
1313 ) -> Result<crate::CasOutcome, ObjectLogError> {
1314 self.inner.compare_and_swap(key, expected, new_value).await
1315 }
1316 }
1317
1318 struct DriverRetired(std::sync::mpsc::Sender<()>);
1319 impl Drop for DriverRetired {
1320 fn drop(&mut self) {
1321 let _ = self.0.send(());
1322 }
1323 }
1324 struct DriverProbe {
1325 inner: MemoryBlobStore,
1326 armed: std::sync::atomic::AtomicBool,
1327 release: Arc<tokio::sync::Notify>,
1328 completed: Mutex<Option<std::sync::mpsc::Sender<()>>>,
1329 retired: Mutex<Option<std::sync::mpsc::Sender<()>>>,
1330 }
1331 #[async_trait]
1332 impl BlobStore for DriverProbe {
1333 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1334 self.inner.put(key, value).await
1335 }
1336 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1337 if self.armed.swap(false, Ordering::AcqRel) {
1338 let release = self.release.clone();
1339 let completed = self.completed.lock().unwrap().take().unwrap();
1340 let retired = DriverRetired(self.retired.lock().unwrap().take().unwrap());
1341 tokio::spawn(async move {
1344 let _retired = retired;
1345 release.notified().await;
1346 tokio::time::sleep(std::time::Duration::from_millis(1)).await;
1347 let _ = completed.send(());
1348 std::future::pending::<()>().await;
1349 });
1350 }
1351 self.inner.get(key).await
1352 }
1353 async fn get_range(
1354 &self,
1355 key: &str,
1356 range: std::ops::Range<u64>,
1357 ) -> Result<Option<Bytes>, ObjectLogError> {
1358 self.inner.get_range(key, range).await
1359 }
1360 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1361 self.inner.list(prefix).await
1362 }
1363 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1364 self.inner.delete(key).await
1365 }
1366 async fn compare_and_swap(
1367 &self,
1368 key: &str,
1369 expected: Option<Bytes>,
1370 value: Bytes,
1371 ) -> Result<CasOutcome, ObjectLogError> {
1372 self.inner.compare_and_swap(key, expected, value).await
1373 }
1374 }
1375 #[test]
1376 fn synchronous_sequencer_keeps_connection_driver_progress_between_calls() {
1377 let caller = Builder::new_current_thread().enable_all().build().unwrap();
1378 let (completed, received) = std::sync::mpsc::channel();
1379 let (retired, cleanup) = std::sync::mpsc::channel();
1380 let release = Arc::new(tokio::sync::Notify::new());
1381 let blob = Arc::new(DriverProbe {
1382 inner: MemoryBlobStore::new(),
1383 armed: std::sync::atomic::AtomicBool::new(false),
1384 release: release.clone(),
1385 completed: Mutex::new(Some(completed)),
1386 retired: Mutex::new(Some(retired)),
1387 });
1388 let store: Arc<dyn BlobStore> = blob.clone();
1389 let sequencer = Arc::new(
1390 caller
1391 .block_on(ManifestSequencer::open(store, "manifest/"))
1392 .unwrap(),
1393 );
1394 blob.armed.store(true, Ordering::Release);
1397 let state = on_thread(&sequencer, |seq| {
1398 seq.authoritative_partition_state(&PartitionKey("new".into()))
1399 })
1400 .unwrap();
1401 assert_eq!(
1402 state,
1403 AuthoritativePartitionState {
1404 epoch: 0,
1405 log_start: 0,
1406 next_offset: 0
1407 }
1408 );
1409 assert!(!blob.armed.load(Ordering::Acquire));
1410 assert!(matches!(
1411 received.try_recv(),
1412 Err(std::sync::mpsc::TryRecvError::Empty)
1413 ));
1414 release.notify_one();
1417 received
1418 .recv_timeout(std::time::Duration::from_secs(5))
1419 .expect("owned connection driver must progress between sequencer calls");
1420 assert!(matches!(
1421 cleanup.try_recv(),
1422 Err(std::sync::mpsc::TryRecvError::Empty)
1423 ));
1424 drop(sequencer);
1425 cleanup
1428 .recv_timeout(std::time::Duration::from_secs(5))
1429 .expect("owned connection driver retires after sequencer destruction");
1430 drop(caller);
1431 }
1432
1433 #[tokio::test]
1434 async fn two_writers_cannot_commit_the_same_partition_tail() {
1435 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1436 let first = Arc::new(
1437 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1438 .await
1439 .unwrap(),
1440 );
1441 let second = Arc::new(
1442 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1443 .await
1444 .unwrap(),
1445 );
1446 let won = run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
1447 assert!(matches!(
1448 won[0],
1449 CommitOutcome::Assigned {
1450 base_offset: 0,
1451 record_count: 1
1452 }
1453 ));
1454 let lost = run_commit(second, vec![("p", "obj-b", 1)]).unwrap();
1455 assert!(
1456 matches!(lost[0], CommitOutcome::Rejected { .. }),
1457 "stale tail must fail closed, got {lost:?}"
1458 );
1459 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
1460 let partition = PartitionKey("p".into());
1461 assert_eq!(reopened.high_watermark(&partition).unwrap(), 1);
1462 let entries = reopened.lookup(&partition, 0).unwrap();
1463 assert_eq!(entries.len(), 1);
1464 assert_eq!(entries[0].location.object_id, "obj-a");
1465 }
1466
1467 #[tokio::test]
1468 async fn mixed_seal_keeps_the_partition_that_won_and_rejects_the_moved_tail() {
1469 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1470 let first = Arc::new(
1471 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1472 .await
1473 .unwrap(),
1474 );
1475 let second = Arc::new(
1476 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1477 .await
1478 .unwrap(),
1479 );
1480 run_commit(first, vec![("p", "obj-a", 1)]).unwrap();
1481 let mixed = run_commit(second, vec![("p", "obj-b", 1), ("q", "obj-b", 1)]).unwrap();
1482 assert!(matches!(mixed[0], CommitOutcome::Rejected { .. }));
1483 assert!(matches!(
1484 mixed[1],
1485 CommitOutcome::Assigned { base_offset: 0, .. }
1486 ));
1487 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
1488 assert_eq!(
1489 reopened.lookup(&PartitionKey("p".into()), 0).unwrap().len(),
1490 1
1491 );
1492 assert_eq!(
1493 reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
1494 1,
1495 "the rejected partition must not skip an offset"
1496 );
1497 assert_eq!(
1498 reopened.lookup(&PartitionKey("q".into()), 0).unwrap()[0]
1499 .location
1500 .object_id,
1501 "obj-b"
1502 );
1503 }
1504
1505 #[tokio::test]
1506 async fn open_does_not_list_the_manifest_prefix_when_an_index_exists() {
1507 let memory = MemoryBlobStore::new();
1508 let blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
1509 let seq = Arc::new(
1510 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1511 .await
1512 .unwrap(),
1513 );
1514 run_commit(seq, vec![("p", "obj-a", 4)]).unwrap();
1515 let lists = Arc::new(AtomicUsize::new(0));
1516 let probe = Arc::new(ListProbe {
1517 inner: memory,
1518 lists: Arc::clone(&lists),
1519 });
1520 let reopened = ManifestSequencer::open(probe, "manifest/").await.unwrap();
1521 assert_eq!(
1522 lists.load(Ordering::SeqCst),
1523 0,
1524 "a current index catalog must be read by key, not by listing manifests"
1525 );
1526 assert_eq!(
1527 reopened.high_watermark(&PartitionKey("p".into())).unwrap(),
1528 1
1529 );
1530 }
1531
1532 #[tokio::test]
1533 async fn distinct_writers_do_not_share_a_data_object_key() {
1534 let blob = Arc::new(MemoryBlobStore::new());
1535 let config = FlushConfig {
1536 linger: std::time::Duration::ZERO,
1537 ..FlushConfig::default()
1538 };
1539 let left = LogEngine::new_with_writer(
1540 Arc::clone(&blob) as Arc<dyn BlobStore>,
1541 Arc::new(InMemorySequencer::new()),
1542 config,
1543 "data/",
1544 "writer-a",
1545 );
1546 let right = LogEngine::new_with_writer(
1547 Arc::clone(&blob) as Arc<dyn BlobStore>,
1548 Arc::new(InMemorySequencer::new()),
1549 config,
1550 "data/",
1551 "writer-b",
1552 );
1553 left.produce_at_epoch(
1554 PartitionKey("p".into()),
1555 Bytes::from_static(b"left"),
1556 1,
1557 (),
1558 Durability::Sequenced,
1559 3,
1560 )
1561 .await
1562 .unwrap();
1563 right
1564 .produce_at_epoch(
1565 PartitionKey("p".into()),
1566 Bytes::from_static(b"right"),
1567 1,
1568 (),
1569 Durability::Sequenced,
1570 3,
1571 )
1572 .await
1573 .unwrap();
1574 let keys = blob.list("data/").await.unwrap();
1575 assert_eq!(
1576 keys.len(),
1577 2,
1578 "writers must not overwrite one object: {keys:?}"
1579 );
1580 assert!(keys.iter().any(|key| key.contains("writer-a")));
1581 assert!(keys.iter().any(|key| key.contains("writer-b")));
1582 assert!(
1583 keys.iter()
1584 .all(|key| key.contains("/00000000000000000003/"))
1585 );
1586 }
1587
1588 #[tokio::test]
1589 async fn fence_from_a_handle_that_never_saw_the_index_rejects_the_stale_writer() {
1590 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1591 let owner = Arc::new(
1592 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1593 .await
1594 .unwrap(),
1595 );
1596 let standby = Arc::new(
1598 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1599 .await
1600 .unwrap(),
1601 );
1602 run_commit(Arc::clone(&owner), vec![("p", "obj-a", 1)]).unwrap();
1603 let partition = PartitionKey("p".into());
1604 let fence_partition = partition.clone();
1605 on_thread(&standby, move |seq| seq.fence_epoch(&fence_partition, 1, 2)).unwrap();
1606
1607 let stale = run_commit(Arc::clone(&owner), vec![("p", "obj-stale", 1)]).unwrap();
1608 assert!(
1609 matches!(stale[0], CommitOutcome::Rejected { .. }),
1610 "a commit at the fenced epoch must fail closed, got {stale:?}"
1611 );
1612 let fresh = run_commit(standby, vec![("p", "obj-b", 2)]).unwrap();
1613 assert!(matches!(
1614 fresh[0],
1615 CommitOutcome::Assigned { base_offset: 1, .. }
1616 ));
1617 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
1618 let objects: Vec<_> = reopened
1619 .lookup(&partition, 0)
1620 .unwrap()
1621 .into_iter()
1622 .map(|entry| entry.location.object_id)
1623 .collect();
1624 assert_eq!(objects, ["obj-a", "obj-b"]);
1625 }
1626
1627 #[tokio::test]
1628 async fn fence_before_any_commit_creates_the_index_at_the_new_epoch() {
1629 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1630 let stale = Arc::new(
1631 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1632 .await
1633 .unwrap(),
1634 );
1635 let fencer = Arc::new(
1636 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1637 .await
1638 .unwrap(),
1639 );
1640 on_thread(&fencer, |seq| {
1641 seq.fence_epoch(&PartitionKey("p".into()), 0, 2)
1642 })
1643 .unwrap();
1644 let lost = run_commit(stale, vec![("p", "obj-stale", 1)]).unwrap();
1645 assert!(
1646 matches!(lost[0], CommitOutcome::Rejected { .. }),
1647 "a writer that never saw the fenced index must not commit, got {lost:?}"
1648 );
1649 let won = run_commit(fencer, vec![("p", "obj-b", 2)]).unwrap();
1650 assert!(matches!(
1651 won[0],
1652 CommitOutcome::Assigned { base_offset: 0, .. }
1653 ));
1654 let reopened = ManifestSequencer::open(blob, "manifest/").await.unwrap();
1655 assert_eq!(
1656 reopened.snapshot().partitions[0].entries[0]
1657 .location
1658 .object_id,
1659 "obj-b"
1660 );
1661 }
1662
1663 #[tokio::test]
1664 async fn refresh_partition_reveals_another_writers_commits() {
1665 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1666 let writer = Arc::new(
1667 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1668 .await
1669 .unwrap(),
1670 );
1671 let reader = Arc::new(
1672 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1673 .await
1674 .unwrap(),
1675 );
1676 run_commit(writer, vec![("p", "obj-a", 1), ("p", "obj-b", 1)]).unwrap();
1677 let partition = PartitionKey("p".into());
1678 assert_eq!(reader.high_watermark(&partition).unwrap(), 0);
1679 let refreshed = partition.clone();
1680 on_thread(&reader, move |seq| seq.refresh_partition(&refreshed)).unwrap();
1681 assert_eq!(reader.high_watermark(&partition).unwrap(), 2);
1682 assert_eq!(reader.lookup(&partition, 0).unwrap().len(), 2);
1683 }
1684
1685 #[tokio::test]
1686 async fn a_fenced_writer_gets_a_typed_fenced_error() {
1687 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1688 let config = FlushConfig {
1689 linger: std::time::Duration::ZERO,
1690 ..FlushConfig::default()
1691 };
1692 let owner_sequencer = Arc::new(
1693 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1694 .await
1695 .unwrap(),
1696 );
1697 let standby = Arc::new(
1698 ManifestSequencer::open(Arc::clone(&blob), "manifest/")
1699 .await
1700 .unwrap(),
1701 );
1702 let owner = LogEngine::new_with_writer(
1703 Arc::clone(&blob),
1704 owner_sequencer,
1705 config,
1706 "data/",
1707 "writer-a",
1708 );
1709 let partition = PartitionKey("p".into());
1710 owner
1711 .produce_at_epoch(
1712 partition.clone(),
1713 Bytes::from_static(b"before"),
1714 1,
1715 (),
1716 Durability::Sequenced,
1717 1,
1718 )
1719 .await
1720 .unwrap();
1721 let fenced = partition.clone();
1722 on_thread(&standby, move |seq| seq.fence_epoch(&fenced, 1, 2)).unwrap();
1723
1724 let stale = owner
1725 .produce_at_epoch(
1726 partition.clone(),
1727 Bytes::from_static(b"stale"),
1728 1,
1729 (),
1730 Durability::Sequenced,
1731 1,
1732 )
1733 .await;
1734 match stale {
1735 Err(ObjectLogError::Fenced {
1736 partition: name,
1737 epoch: 1,
1738 current: 2,
1739 }) => assert_eq!(name, "p"),
1740 other => panic!("a write behind the fence must be Fenced, got {other:?}"),
1741 }
1742 }
1743
1744 #[tokio::test]
1745 async fn default_rejection_error_is_a_sequencer_error() {
1746 let seq = crate::InMemorySequencer::new();
1747 let error = seq.rejection_error(&PartitionKey("p".into()), 1, "tail moved".into());
1748 assert!(matches!(error, ObjectLogError::Sequencer(reason) if reason == "tail moved"));
1749 }
1750}