Skip to main content

object_log/
manifest_sequencer.rs

1//! A `BlobStore`-persisted [`Sequencer`]: the offset→location index survives a
2//! restart, so a standalone object-log is crash-durable end to end.
3//!
4//! Each `commit` durably writes one manifest object (≤1 PUT per group-commit,
5//! co-amortized with the data object) recording the batches it assigned. On
6//! [`open`](ManifestSequencer::open) the index is rebuilt by replaying the
7//! manifest objects in order.
8
9use crate::{
10    BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
11};
12use bytes::Bytes;
13use serde::{Deserialize, Serialize};
14use std::collections::{HashMap, HashSet};
15use std::sync::{Arc, Mutex};
16use tokio::runtime::{Builder, Runtime};
17
18#[derive(Serialize, Deserialize)]
19struct ManifestRecord {
20    entries: Vec<(PartitionKey, IndexEntry)>,
21}
22
23#[derive(Default)]
24struct Part {
25    next_offset: i64,
26    log_start: i64,
27    entries: Vec<IndexEntry>,
28}
29
30struct Inner {
31    parts: HashMap<PartitionKey, Part>,
32    counter: u64,
33}
34
35/// A [`Sequencer`] (`Meta = ()`) that persists its index to a [`BlobStore`].
36pub struct ManifestSequencer {
37    blob: Arc<dyn BlobStore>,
38    prefix: String,
39    // `Option` so `Drop` can `shutdown_background()` — dropping a `Runtime` inside
40    // an async context (e.g. an `Arc` last-ref drop in a `#[tokio::test]`) panics.
41    rt: Option<Runtime>,
42    inner: Mutex<Inner>,
43}
44
45impl Drop for ManifestSequencer {
46    fn drop(&mut self) {
47        if let Some(rt) = self.rt.take() {
48            rt.shutdown_background();
49        }
50    }
51}
52
53impl ManifestSequencer {
54    /// Open (or recover) a persisted sequencer. Manifest objects live under
55    /// `manifest_prefix` (keep it disjoint from the engine's data-object prefix).
56    /// Existing manifests are replayed to rebuild the index.
57    pub async fn open(
58        blob: Arc<dyn BlobStore>,
59        manifest_prefix: impl Into<String>,
60    ) -> Result<Self, ObjectLogError> {
61        let prefix = manifest_prefix.into();
62        let mut keys = blob.list(&prefix).await?;
63        keys.sort();
64        let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
65        for key in &keys {
66            let bytes = blob
67                .get(key)
68                .await?
69                .ok_or_else(|| ObjectLogError::MissingObject(key.clone()))?;
70            let rec: ManifestRecord = serde_json::from_slice(&bytes)
71                .map_err(|e| ObjectLogError::Sequencer(format!("manifest {key}: {e}")))?;
72            for (pkey, entry) in rec.entries {
73                let p = parts.entry(pkey).or_default();
74                let end = entry.base_offset + entry.record_count as i64;
75                if end > p.next_offset {
76                    p.next_offset = end;
77                }
78                p.entries.push(entry);
79            }
80        }
81        let rt = Builder::new_current_thread()
82            .enable_all()
83            .build()
84            .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
85        Ok(Self {
86            blob,
87            prefix,
88            rt: Some(rt),
89            inner: Mutex::new(Inner {
90                parts,
91                counter: keys.len() as u64,
92            }),
93        })
94    }
95}
96
97impl Sequencer for ManifestSequencer {
98    type Meta = ();
99
100    fn commit(
101        &self,
102        batches: &[CommitBatch<'_, ()>],
103    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
104        let mut st = self.inner.lock().expect("poisoned");
105        // Plan the assignment without mutating the index yet.
106        let mut local_next: HashMap<PartitionKey, i64> = HashMap::new();
107        let mut planned: Vec<(PartitionKey, IndexEntry)> = Vec::with_capacity(batches.len());
108        let mut outcomes = Vec::with_capacity(batches.len());
109        for cb in batches {
110            let base = *local_next
111                .entry(cb.partition.clone())
112                .or_insert_with(|| st.parts.get(&cb.partition).map_or(0, |p| p.next_offset));
113            let entry = IndexEntry {
114                location: cb.location.clone(),
115                base_offset: base,
116                record_count: cb.record_count,
117            };
118            planned.push((cb.partition.clone(), entry));
119            *local_next.get_mut(&cb.partition).expect("inserted above") =
120                base + cb.record_count as i64;
121            outcomes.push(CommitOutcome::Assigned {
122                base_offset: base,
123                record_count: cb.record_count,
124            });
125        }
126
127        // Persist the manifest durably BEFORE the index becomes visible.
128        let counter = st.counter + 1;
129        let key = format!("{}{:020}", self.prefix, counter);
130        let bytes = serde_json::to_vec(&ManifestRecord {
131            entries: planned.clone(),
132        })
133        .map_err(|e| ObjectLogError::Sequencer(e.to_string()))?;
134        self.rt
135            .as_ref()
136            .expect("runtime present until drop")
137            .block_on(self.blob.put(&key, Bytes::from(bytes)))?;
138
139        // Apply.
140        for (pkey, entry) in planned {
141            let p = st.parts.entry(pkey).or_default();
142            let end = entry.base_offset + entry.record_count as i64;
143            if end > p.next_offset {
144                p.next_offset = end;
145            }
146            p.entries.push(entry);
147        }
148        st.counter = counter;
149        Ok(outcomes)
150    }
151
152    fn lookup(
153        &self,
154        partition: &PartitionKey,
155        fetch_offset: i64,
156    ) -> Result<Vec<IndexEntry>, ObjectLogError> {
157        let st = self.inner.lock().expect("poisoned");
158        let Some(p) = st.parts.get(partition) else {
159            return Ok(Vec::new());
160        };
161        Ok(p.entries
162            .iter()
163            .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
164            .cloned()
165            .collect())
166    }
167
168    fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
169        Ok(self
170            .inner
171            .lock()
172            .expect("poisoned")
173            .parts
174            .get(partition)
175            .map_or(0, |p| p.next_offset))
176    }
177
178    fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
179        Ok(self
180            .inner
181            .lock()
182            .expect("poisoned")
183            .parts
184            .get(partition)
185            .map_or(0, |p| p.log_start))
186    }
187
188    fn truncate_before(
189        &self,
190        partition: &PartitionKey,
191        offset: i64,
192    ) -> Result<Vec<String>, ObjectLogError> {
193        let mut st = self.inner.lock().expect("poisoned");
194        let mut dropped: Vec<String> = Vec::new();
195        match st.parts.get_mut(partition) {
196            Some(p) => {
197                for e in p.entries.iter() {
198                    if e.base_offset + e.record_count as i64 <= offset {
199                        dropped.push(e.location.object_id.clone());
200                    }
201                }
202                p.entries
203                    .retain(|e| e.base_offset + e.record_count as i64 > offset);
204                if offset > p.log_start {
205                    p.log_start = offset.min(p.next_offset);
206                }
207            }
208            None => return Ok(Vec::new()),
209        }
210        let mut live: HashSet<String> = HashSet::new();
211        for p in st.parts.values() {
212            for e in &p.entries {
213                live.insert(e.location.object_id.clone());
214            }
215        }
216        let mut dead: Vec<String> = Vec::new();
217        let mut seen: HashSet<String> = HashSet::new();
218        for oid in dropped {
219            if !live.contains(&oid) && seen.insert(oid.clone()) {
220                dead.push(oid);
221            }
222        }
223        Ok(dead)
224    }
225}