1use crate::{
10 BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
11};
12use bytes::Bytes;
13use serde::{Deserialize, Serialize};
14use std::collections::{HashMap, HashSet};
15use std::sync::{Arc, Mutex};
16use tokio::runtime::{Builder, Runtime};
17
18#[derive(Serialize, Deserialize)]
19struct ManifestRecord {
20 entries: Vec<(PartitionKey, IndexEntry)>,
21}
22
23#[derive(Default)]
24struct Part {
25 next_offset: i64,
26 log_start: i64,
27 entries: Vec<IndexEntry>,
28}
29
30struct Inner {
31 parts: HashMap<PartitionKey, Part>,
32 counter: u64,
33}
34
35pub struct ManifestSequencer {
37 blob: Arc<dyn BlobStore>,
38 prefix: String,
39 rt: Option<Runtime>,
42 inner: Mutex<Inner>,
43}
44
45impl Drop for ManifestSequencer {
46 fn drop(&mut self) {
47 if let Some(rt) = self.rt.take() {
48 rt.shutdown_background();
49 }
50 }
51}
52
53impl ManifestSequencer {
54 pub async fn open(
58 blob: Arc<dyn BlobStore>,
59 manifest_prefix: impl Into<String>,
60 ) -> Result<Self, ObjectLogError> {
61 let prefix = manifest_prefix.into();
62 let mut keys = blob.list(&prefix).await?;
63 keys.sort();
64 let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
65 for key in &keys {
66 let bytes = blob
67 .get(key)
68 .await?
69 .ok_or_else(|| ObjectLogError::MissingObject(key.clone()))?;
70 let rec: ManifestRecord = serde_json::from_slice(&bytes)
71 .map_err(|e| ObjectLogError::Sequencer(format!("manifest {key}: {e}")))?;
72 for (pkey, entry) in rec.entries {
73 let p = parts.entry(pkey).or_default();
74 let end = entry.base_offset + entry.record_count as i64;
75 if end > p.next_offset {
76 p.next_offset = end;
77 }
78 p.entries.push(entry);
79 }
80 }
81 let rt = Builder::new_current_thread()
82 .enable_all()
83 .build()
84 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
85 Ok(Self {
86 blob,
87 prefix,
88 rt: Some(rt),
89 inner: Mutex::new(Inner {
90 parts,
91 counter: keys.len() as u64,
92 }),
93 })
94 }
95
96 pub fn live_object_ids(&self) -> HashSet<String> {
99 let st = self.inner.lock().expect("poisoned");
100 let mut live = HashSet::new();
101 for p in st.parts.values() {
102 for e in &p.entries {
103 live.insert(e.location.object_id.clone());
104 }
105 }
106 live
107 }
108
109 pub fn snapshot(&self) -> IndexSnapshot {
111 let st = self.inner.lock().expect("poisoned");
112 let mut partitions: Vec<PartitionSnapshot> = st
113 .parts
114 .iter()
115 .map(|(k, p)| PartitionSnapshot {
116 partition: k.0.clone(),
117 log_start: p.log_start,
118 high_watermark: p.next_offset,
119 entry_count: p.entries.len(),
120 entries: p.entries.clone(),
121 })
122 .collect();
123 partitions.sort_by(|a, b| a.partition.cmp(&b.partition));
124 IndexSnapshot {
125 manifest_prefix: self.prefix.clone(),
126 manifest_count: st.counter,
127 partitions,
128 }
129 }
130}
131
132#[derive(Clone, Debug, Serialize, Deserialize)]
134pub struct IndexSnapshot {
135 pub manifest_prefix: String,
137 pub manifest_count: u64,
139 pub partitions: Vec<PartitionSnapshot>,
141}
142
143#[derive(Clone, Debug, Serialize, Deserialize)]
145pub struct PartitionSnapshot {
146 pub partition: String,
148 pub log_start: i64,
150 pub high_watermark: i64,
152 pub entry_count: usize,
154 pub entries: Vec<IndexEntry>,
156}
157
158impl Sequencer for ManifestSequencer {
159 type Meta = ();
160
161 fn commit(
162 &self,
163 batches: &[CommitBatch<'_, ()>],
164 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
165 let mut st = self.inner.lock().expect("poisoned");
166 let mut local_next: HashMap<PartitionKey, i64> = HashMap::new();
168 let mut planned: Vec<(PartitionKey, IndexEntry)> = Vec::with_capacity(batches.len());
169 let mut outcomes = Vec::with_capacity(batches.len());
170 for cb in batches {
171 let base = *local_next
172 .entry(cb.partition.clone())
173 .or_insert_with(|| st.parts.get(&cb.partition).map_or(0, |p| p.next_offset));
174 let entry = IndexEntry {
175 location: cb.location.clone(),
176 base_offset: base,
177 record_count: cb.record_count,
178 };
179 planned.push((cb.partition.clone(), entry));
180 *local_next.get_mut(&cb.partition).expect("inserted above") =
181 base + cb.record_count as i64;
182 outcomes.push(CommitOutcome::Assigned {
183 base_offset: base,
184 record_count: cb.record_count,
185 });
186 }
187
188 let counter = st.counter + 1;
190 let key = format!("{}{:020}", self.prefix, counter);
191 let bytes = serde_json::to_vec(&ManifestRecord {
192 entries: planned.clone(),
193 })
194 .map_err(|e| ObjectLogError::Sequencer(e.to_string()))?;
195 self.rt
196 .as_ref()
197 .expect("runtime present until drop")
198 .block_on(self.blob.put(&key, Bytes::from(bytes)))?;
199
200 for (pkey, entry) in planned {
202 let p = st.parts.entry(pkey).or_default();
203 let end = entry.base_offset + entry.record_count as i64;
204 if end > p.next_offset {
205 p.next_offset = end;
206 }
207 p.entries.push(entry);
208 }
209 st.counter = counter;
210 Ok(outcomes)
211 }
212
213 fn lookup(
214 &self,
215 partition: &PartitionKey,
216 fetch_offset: i64,
217 ) -> Result<Vec<IndexEntry>, ObjectLogError> {
218 let st = self.inner.lock().expect("poisoned");
219 let Some(p) = st.parts.get(partition) else {
220 return Ok(Vec::new());
221 };
222 Ok(p.entries
223 .iter()
224 .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
225 .cloned()
226 .collect())
227 }
228
229 fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
230 Ok(self
231 .inner
232 .lock()
233 .expect("poisoned")
234 .parts
235 .get(partition)
236 .map_or(0, |p| p.next_offset))
237 }
238
239 fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
240 Ok(self
241 .inner
242 .lock()
243 .expect("poisoned")
244 .parts
245 .get(partition)
246 .map_or(0, |p| p.log_start))
247 }
248
249 fn truncate_before(
250 &self,
251 partition: &PartitionKey,
252 offset: i64,
253 ) -> Result<Vec<String>, ObjectLogError> {
254 let mut st = self.inner.lock().expect("poisoned");
255 let mut dropped: Vec<String> = Vec::new();
256 match st.parts.get_mut(partition) {
257 Some(p) => {
258 for e in p.entries.iter() {
259 if e.base_offset + e.record_count as i64 <= offset {
260 dropped.push(e.location.object_id.clone());
261 }
262 }
263 p.entries
264 .retain(|e| e.base_offset + e.record_count as i64 > offset);
265 if offset > p.log_start {
266 p.log_start = offset.min(p.next_offset);
267 }
268 }
269 None => return Ok(Vec::new()),
270 }
271 let mut live: HashSet<String> = HashSet::new();
272 for p in st.parts.values() {
273 for e in &p.entries {
274 live.insert(e.location.object_id.clone());
275 }
276 }
277 let mut dead: Vec<String> = Vec::new();
278 let mut seen: HashSet<String> = HashSet::new();
279 for oid in dropped {
280 if !live.contains(&oid) && seen.insert(oid.clone()) {
281 dead.push(oid);
282 }
283 }
284 Ok(dead)
285 }
286}