Skip to main content

object_log/
manifest_sequencer.rs

1//! A `BlobStore`-persisted [`Sequencer`]: the offset→location index survives a
2//! restart, so a standalone object-log is crash-durable end to end.
3//!
4//! Each `commit` durably writes one manifest object (≤1 PUT per group-commit,
5//! co-amortized with the data object) recording the batches it assigned. On
6//! [`open`](ManifestSequencer::open) the index is rebuilt by replaying the
7//! manifest objects in order.
8
9use crate::{
10    BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
11};
12use bytes::Bytes;
13use serde::{Deserialize, Serialize};
14use std::collections::{HashMap, HashSet};
15use std::sync::{Arc, Mutex};
16use tokio::runtime::{Builder, Runtime};
17
18#[derive(Serialize, Deserialize)]
19struct ManifestRecord {
20    entries: Vec<(PartitionKey, IndexEntry)>,
21}
22
23#[derive(Default)]
24struct Part {
25    next_offset: i64,
26    log_start: i64,
27    entries: Vec<IndexEntry>,
28}
29
30struct Inner {
31    parts: HashMap<PartitionKey, Part>,
32    counter: u64,
33}
34
35/// A [`Sequencer`] (`Meta = ()`) that persists its index to a [`BlobStore`].
36pub struct ManifestSequencer {
37    blob: Arc<dyn BlobStore>,
38    prefix: String,
39    // `Option` so `Drop` can `shutdown_background()` — dropping a `Runtime` inside
40    // an async context (e.g. an `Arc` last-ref drop in a `#[tokio::test]`) panics.
41    rt: Option<Runtime>,
42    // Serialize mutations across publication without blocking committed reads.
43    commit_order: Mutex<()>,
44    inner: Mutex<Inner>,
45}
46
47impl Drop for ManifestSequencer {
48    fn drop(&mut self) {
49        if let Some(rt) = self.rt.take() {
50            rt.shutdown_background();
51        }
52    }
53}
54
55impl ManifestSequencer {
56    /// Open (or recover) a persisted sequencer. Manifest objects live under
57    /// `manifest_prefix` (keep it disjoint from the engine's data-object prefix).
58    /// Existing manifests are replayed to rebuild the index.
59    pub async fn open(
60        blob: Arc<dyn BlobStore>,
61        manifest_prefix: impl Into<String>,
62    ) -> Result<Self, ObjectLogError> {
63        let prefix = manifest_prefix.into();
64        let mut keys = blob.list(&prefix).await?;
65        keys.sort();
66        let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
67        for key in &keys {
68            let bytes = blob
69                .get(key)
70                .await?
71                .ok_or_else(|| ObjectLogError::MissingObject(key.clone()))?;
72            let rec: ManifestRecord = serde_json::from_slice(&bytes)
73                .map_err(|e| ObjectLogError::Sequencer(format!("manifest {key}: {e}")))?;
74            for (pkey, entry) in rec.entries {
75                let p = parts.entry(pkey).or_default();
76                let end = entry.base_offset + entry.record_count as i64;
77                if end > p.next_offset {
78                    p.next_offset = end;
79                }
80                p.entries.push(entry);
81            }
82        }
83        let rt = Builder::new_current_thread()
84            .enable_all()
85            .build()
86            .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
87        Ok(Self {
88            blob,
89            prefix,
90            rt: Some(rt),
91            commit_order: Mutex::new(()),
92            inner: Mutex::new(Inner {
93                parts,
94                counter: keys.len() as u64,
95            }),
96        })
97    }
98
99    /// Object ids referenced by any live index entry (for orphan reaping).
100    /// Does not include manifest object keys themselves.
101    pub fn live_object_ids(&self) -> HashSet<String> {
102        let st = self.inner.lock().expect("poisoned");
103        let mut live = HashSet::new();
104        for p in st.parts.values() {
105            for e in &p.entries {
106                live.insert(e.location.object_id.clone());
107            }
108        }
109        live
110    }
111
112    /// Point-in-time view of the rebuilt index (diagnostics / CLI).
113    pub fn snapshot(&self) -> IndexSnapshot {
114        let st = self.inner.lock().expect("poisoned");
115        let mut partitions: Vec<PartitionSnapshot> = st
116            .parts
117            .iter()
118            .map(|(k, p)| PartitionSnapshot {
119                partition: k.0.clone(),
120                log_start: p.log_start,
121                high_watermark: p.next_offset,
122                entry_count: p.entries.len(),
123                entries: p.entries.clone(),
124            })
125            .collect();
126        partitions.sort_by(|a, b| a.partition.cmp(&b.partition));
127        IndexSnapshot {
128            manifest_prefix: self.prefix.clone(),
129            manifest_count: st.counter,
130            partitions,
131        }
132    }
133}
134
135/// Full index view from [`ManifestSequencer::snapshot`].
136#[derive(Clone, Debug, Serialize, Deserialize)]
137pub struct IndexSnapshot {
138    /// Prefix used for manifest objects.
139    pub manifest_prefix: String,
140    /// Number of manifest objects replayed (commit counter).
141    pub manifest_count: u64,
142    /// Per-partition index state.
143    pub partitions: Vec<PartitionSnapshot>,
144}
145
146/// One partition's index bounds and entries.
147#[derive(Clone, Debug, Serialize, Deserialize)]
148pub struct PartitionSnapshot {
149    /// Partition key string.
150    pub partition: String,
151    /// First readable offset.
152    pub log_start: i64,
153    /// Next offset to assign (high watermark).
154    pub high_watermark: i64,
155    /// Number of index entries.
156    pub entry_count: usize,
157    /// Ordered index entries (may be large; CLI can omit via flag).
158    pub entries: Vec<IndexEntry>,
159}
160
161impl Sequencer for ManifestSequencer {
162    type Meta = ();
163
164    fn supports_multi_object_commit(&self) -> bool {
165        true
166    }
167
168    fn commit(
169        &self,
170        batches: &[CommitBatch<'_, ()>],
171    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
172        let _commit = self.commit_order.lock().expect("poisoned");
173        let st = self.inner.lock().expect("poisoned");
174        // Plan the assignment without mutating the index yet.
175        let mut local_next: HashMap<PartitionKey, i64> = HashMap::new();
176        let mut planned: Vec<(PartitionKey, IndexEntry)> = Vec::with_capacity(batches.len());
177        let mut outcomes = Vec::with_capacity(batches.len());
178        for cb in batches {
179            let base = *local_next
180                .entry(cb.partition.clone())
181                .or_insert_with(|| st.parts.get(&cb.partition).map_or(0, |p| p.next_offset));
182            let entry = IndexEntry {
183                location: cb.location.clone(),
184                base_offset: base,
185                record_count: cb.record_count,
186            };
187            planned.push((cb.partition.clone(), entry));
188            *local_next.get_mut(&cb.partition).expect("inserted above") =
189                base + cb.record_count as i64;
190            outcomes.push(CommitOutcome::Assigned {
191                base_offset: base,
192                record_count: cb.record_count,
193            });
194        }
195
196        // Persist the manifest durably BEFORE the index becomes visible.
197        let counter = st.counter + 1;
198        // Readers keep seeing the last durable index while this publication
199        // waits. The commit-order guard prevents another mutation invalidating
200        // the planned offsets or manifest key.
201        drop(st);
202        let key = format!("{}{:020}", self.prefix, counter);
203        let bytes = serde_json::to_vec(&ManifestRecord {
204            entries: planned.clone(),
205        })
206        .map_err(|e| ObjectLogError::Sequencer(e.to_string()))?;
207        self.rt
208            .as_ref()
209            .expect("runtime present until drop")
210            .block_on(self.blob.put(&key, Bytes::from(bytes)))?;
211
212        // Publish the complete index change only after durable success.
213        let mut st = self.inner.lock().expect("poisoned");
214        for (pkey, entry) in planned {
215            let p = st.parts.entry(pkey).or_default();
216            let end = entry.base_offset + entry.record_count as i64;
217            if end > p.next_offset {
218                p.next_offset = end;
219            }
220            p.entries.push(entry);
221        }
222        st.counter = counter;
223        Ok(outcomes)
224    }
225
226    fn lookup(
227        &self,
228        partition: &PartitionKey,
229        fetch_offset: i64,
230    ) -> Result<Vec<IndexEntry>, ObjectLogError> {
231        let st = self.inner.lock().expect("poisoned");
232        let Some(p) = st.parts.get(partition) else {
233            return Ok(Vec::new());
234        };
235        Ok(p.entries
236            .iter()
237            .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
238            .cloned()
239            .collect())
240    }
241
242    fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
243        Ok(self
244            .inner
245            .lock()
246            .expect("poisoned")
247            .parts
248            .get(partition)
249            .map_or(0, |p| p.next_offset))
250    }
251
252    fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
253        Ok(self
254            .inner
255            .lock()
256            .expect("poisoned")
257            .parts
258            .get(partition)
259            .map_or(0, |p| p.log_start))
260    }
261
262    fn truncate_before(
263        &self,
264        partition: &PartitionKey,
265        offset: i64,
266    ) -> Result<Vec<String>, ObjectLogError> {
267        let _commit = self.commit_order.lock().expect("poisoned");
268        let mut st = self.inner.lock().expect("poisoned");
269        let mut dropped: Vec<String> = Vec::new();
270        match st.parts.get_mut(partition) {
271            Some(p) => {
272                for e in p.entries.iter() {
273                    if e.base_offset + e.record_count as i64 <= offset {
274                        dropped.push(e.location.object_id.clone());
275                    }
276                }
277                p.entries
278                    .retain(|e| e.base_offset + e.record_count as i64 > offset);
279                if offset > p.log_start {
280                    p.log_start = offset.min(p.next_offset);
281                }
282            }
283            None => return Ok(Vec::new()),
284        }
285        let mut live: HashSet<String> = HashSet::new();
286        for p in st.parts.values() {
287            for e in &p.entries {
288                live.insert(e.location.object_id.clone());
289            }
290        }
291        let mut dead: Vec<String> = Vec::new();
292        let mut seen: HashSet<String> = HashSet::new();
293        for oid in dropped {
294            if !live.contains(&oid) && seen.insert(oid.clone()) {
295                dead.push(oid);
296            }
297        }
298        Ok(dead)
299    }
300}