Skip to main content

object_log/
manifest_sequencer.rs

1//! A `BlobStore`-persisted [`Sequencer`]: the offset→location index survives a
2//! restart, so a standalone object-log is crash-durable end to end.
3//!
4//! Each `commit` durably writes one manifest object (≤1 PUT per group-commit,
5//! co-amortized with the data object) recording the batches it assigned. On
6//! [`open`](ManifestSequencer::open) the index is rebuilt by replaying the
7//! manifest objects in order.
8
9use crate::{
10    BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
11};
12use bytes::Bytes;
13use serde::{Deserialize, Serialize};
14use std::collections::{HashMap, HashSet};
15use std::sync::{Arc, Mutex};
16use tokio::runtime::{Builder, Runtime};
17
18#[derive(Serialize, Deserialize)]
19struct ManifestRecord {
20    entries: Vec<(PartitionKey, IndexEntry)>,
21}
22
23#[derive(Default)]
24struct Part {
25    next_offset: i64,
26    log_start: i64,
27    entries: Vec<IndexEntry>,
28}
29
30struct Inner {
31    parts: HashMap<PartitionKey, Part>,
32    counter: u64,
33}
34
35/// A [`Sequencer`] (`Meta = ()`) that persists its index to a [`BlobStore`].
36pub struct ManifestSequencer {
37    blob: Arc<dyn BlobStore>,
38    prefix: String,
39    // `Option` so `Drop` can `shutdown_background()` — dropping a `Runtime` inside
40    // an async context (e.g. an `Arc` last-ref drop in a `#[tokio::test]`) panics.
41    rt: Option<Runtime>,
42    inner: Mutex<Inner>,
43}
44
45impl Drop for ManifestSequencer {
46    fn drop(&mut self) {
47        if let Some(rt) = self.rt.take() {
48            rt.shutdown_background();
49        }
50    }
51}
52
53impl ManifestSequencer {
54    /// Open (or recover) a persisted sequencer. Manifest objects live under
55    /// `manifest_prefix` (keep it disjoint from the engine's data-object prefix).
56    /// Existing manifests are replayed to rebuild the index.
57    pub async fn open(
58        blob: Arc<dyn BlobStore>,
59        manifest_prefix: impl Into<String>,
60    ) -> Result<Self, ObjectLogError> {
61        let prefix = manifest_prefix.into();
62        let mut keys = blob.list(&prefix).await?;
63        keys.sort();
64        let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
65        for key in &keys {
66            let bytes = blob
67                .get(key)
68                .await?
69                .ok_or_else(|| ObjectLogError::MissingObject(key.clone()))?;
70            let rec: ManifestRecord = serde_json::from_slice(&bytes)
71                .map_err(|e| ObjectLogError::Sequencer(format!("manifest {key}: {e}")))?;
72            for (pkey, entry) in rec.entries {
73                let p = parts.entry(pkey).or_default();
74                let end = entry.base_offset + entry.record_count as i64;
75                if end > p.next_offset {
76                    p.next_offset = end;
77                }
78                p.entries.push(entry);
79            }
80        }
81        let rt = Builder::new_current_thread()
82            .enable_all()
83            .build()
84            .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
85        Ok(Self {
86            blob,
87            prefix,
88            rt: Some(rt),
89            inner: Mutex::new(Inner {
90                parts,
91                counter: keys.len() as u64,
92            }),
93        })
94    }
95
96    /// Object ids referenced by any live index entry (for orphan reaping).
97    /// Does not include manifest object keys themselves.
98    pub fn live_object_ids(&self) -> HashSet<String> {
99        let st = self.inner.lock().expect("poisoned");
100        let mut live = HashSet::new();
101        for p in st.parts.values() {
102            for e in &p.entries {
103                live.insert(e.location.object_id.clone());
104            }
105        }
106        live
107    }
108
109    /// Point-in-time view of the rebuilt index (diagnostics / CLI).
110    pub fn snapshot(&self) -> IndexSnapshot {
111        let st = self.inner.lock().expect("poisoned");
112        let mut partitions: Vec<PartitionSnapshot> = st
113            .parts
114            .iter()
115            .map(|(k, p)| PartitionSnapshot {
116                partition: k.0.clone(),
117                log_start: p.log_start,
118                high_watermark: p.next_offset,
119                entry_count: p.entries.len(),
120                entries: p.entries.clone(),
121            })
122            .collect();
123        partitions.sort_by(|a, b| a.partition.cmp(&b.partition));
124        IndexSnapshot {
125            manifest_prefix: self.prefix.clone(),
126            manifest_count: st.counter,
127            partitions,
128        }
129    }
130}
131
132/// Full index view from [`ManifestSequencer::snapshot`].
133#[derive(Clone, Debug, Serialize, Deserialize)]
134pub struct IndexSnapshot {
135    /// Prefix used for manifest objects.
136    pub manifest_prefix: String,
137    /// Number of manifest objects replayed (commit counter).
138    pub manifest_count: u64,
139    /// Per-partition index state.
140    pub partitions: Vec<PartitionSnapshot>,
141}
142
143/// One partition's index bounds and entries.
144#[derive(Clone, Debug, Serialize, Deserialize)]
145pub struct PartitionSnapshot {
146    /// Partition key string.
147    pub partition: String,
148    /// First readable offset.
149    pub log_start: i64,
150    /// Next offset to assign (high watermark).
151    pub high_watermark: i64,
152    /// Number of index entries.
153    pub entry_count: usize,
154    /// Ordered index entries (may be large; CLI can omit via flag).
155    pub entries: Vec<IndexEntry>,
156}
157
158impl Sequencer for ManifestSequencer {
159    type Meta = ();
160
161    fn commit(
162        &self,
163        batches: &[CommitBatch<'_, ()>],
164    ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
165        let mut st = self.inner.lock().expect("poisoned");
166        // Plan the assignment without mutating the index yet.
167        let mut local_next: HashMap<PartitionKey, i64> = HashMap::new();
168        let mut planned: Vec<(PartitionKey, IndexEntry)> = Vec::with_capacity(batches.len());
169        let mut outcomes = Vec::with_capacity(batches.len());
170        for cb in batches {
171            let base = *local_next
172                .entry(cb.partition.clone())
173                .or_insert_with(|| st.parts.get(&cb.partition).map_or(0, |p| p.next_offset));
174            let entry = IndexEntry {
175                location: cb.location.clone(),
176                base_offset: base,
177                record_count: cb.record_count,
178            };
179            planned.push((cb.partition.clone(), entry));
180            *local_next.get_mut(&cb.partition).expect("inserted above") =
181                base + cb.record_count as i64;
182            outcomes.push(CommitOutcome::Assigned {
183                base_offset: base,
184                record_count: cb.record_count,
185            });
186        }
187
188        // Persist the manifest durably BEFORE the index becomes visible.
189        let counter = st.counter + 1;
190        let key = format!("{}{:020}", self.prefix, counter);
191        let bytes = serde_json::to_vec(&ManifestRecord {
192            entries: planned.clone(),
193        })
194        .map_err(|e| ObjectLogError::Sequencer(e.to_string()))?;
195        self.rt
196            .as_ref()
197            .expect("runtime present until drop")
198            .block_on(self.blob.put(&key, Bytes::from(bytes)))?;
199
200        // Apply.
201        for (pkey, entry) in planned {
202            let p = st.parts.entry(pkey).or_default();
203            let end = entry.base_offset + entry.record_count as i64;
204            if end > p.next_offset {
205                p.next_offset = end;
206            }
207            p.entries.push(entry);
208        }
209        st.counter = counter;
210        Ok(outcomes)
211    }
212
213    fn lookup(
214        &self,
215        partition: &PartitionKey,
216        fetch_offset: i64,
217    ) -> Result<Vec<IndexEntry>, ObjectLogError> {
218        let st = self.inner.lock().expect("poisoned");
219        let Some(p) = st.parts.get(partition) else {
220            return Ok(Vec::new());
221        };
222        Ok(p.entries
223            .iter()
224            .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
225            .cloned()
226            .collect())
227    }
228
229    fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
230        Ok(self
231            .inner
232            .lock()
233            .expect("poisoned")
234            .parts
235            .get(partition)
236            .map_or(0, |p| p.next_offset))
237    }
238
239    fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
240        Ok(self
241            .inner
242            .lock()
243            .expect("poisoned")
244            .parts
245            .get(partition)
246            .map_or(0, |p| p.log_start))
247    }
248
249    fn truncate_before(
250        &self,
251        partition: &PartitionKey,
252        offset: i64,
253    ) -> Result<Vec<String>, ObjectLogError> {
254        let mut st = self.inner.lock().expect("poisoned");
255        let mut dropped: Vec<String> = Vec::new();
256        match st.parts.get_mut(partition) {
257            Some(p) => {
258                for e in p.entries.iter() {
259                    if e.base_offset + e.record_count as i64 <= offset {
260                        dropped.push(e.location.object_id.clone());
261                    }
262                }
263                p.entries
264                    .retain(|e| e.base_offset + e.record_count as i64 > offset);
265                if offset > p.log_start {
266                    p.log_start = offset.min(p.next_offset);
267                }
268            }
269            None => return Ok(Vec::new()),
270        }
271        let mut live: HashSet<String> = HashSet::new();
272        for p in st.parts.values() {
273            for e in &p.entries {
274                live.insert(e.location.object_id.clone());
275            }
276        }
277        let mut dead: Vec<String> = Vec::new();
278        let mut seen: HashSet<String> = HashSet::new();
279        for oid in dropped {
280            if !live.contains(&oid) && seen.insert(oid.clone()) {
281                dead.push(oid);
282            }
283        }
284        Ok(dead)
285    }
286}