1use crate::{
10 BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
11};
12use bytes::Bytes;
13use serde::{Deserialize, Serialize};
14use std::collections::{HashMap, HashSet};
15use std::sync::{Arc, Mutex};
16use tokio::runtime::{Builder, Runtime};
17
18#[derive(Serialize, Deserialize)]
19struct ManifestRecord {
20 entries: Vec<(PartitionKey, IndexEntry)>,
21}
22
23#[derive(Default)]
24struct Part {
25 next_offset: i64,
26 log_start: i64,
27 entries: Vec<IndexEntry>,
28}
29
30struct Inner {
31 parts: HashMap<PartitionKey, Part>,
32 counter: u64,
33}
34
35pub struct ManifestSequencer {
37 blob: Arc<dyn BlobStore>,
38 prefix: String,
39 rt: Option<Runtime>,
42 commit_order: Mutex<()>,
44 inner: Mutex<Inner>,
45}
46
47impl Drop for ManifestSequencer {
48 fn drop(&mut self) {
49 if let Some(rt) = self.rt.take() {
50 rt.shutdown_background();
51 }
52 }
53}
54
55impl ManifestSequencer {
56 pub async fn open(
60 blob: Arc<dyn BlobStore>,
61 manifest_prefix: impl Into<String>,
62 ) -> Result<Self, ObjectLogError> {
63 let prefix = manifest_prefix.into();
64 let mut keys = blob.list(&prefix).await?;
65 keys.sort();
66 let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
67 for key in &keys {
68 let bytes = blob
69 .get(key)
70 .await?
71 .ok_or_else(|| ObjectLogError::MissingObject(key.clone()))?;
72 let rec: ManifestRecord = serde_json::from_slice(&bytes)
73 .map_err(|e| ObjectLogError::Sequencer(format!("manifest {key}: {e}")))?;
74 for (pkey, entry) in rec.entries {
75 let p = parts.entry(pkey).or_default();
76 let end = entry.base_offset + entry.record_count as i64;
77 if end > p.next_offset {
78 p.next_offset = end;
79 }
80 p.entries.push(entry);
81 }
82 }
83 let rt = Builder::new_current_thread()
84 .enable_all()
85 .build()
86 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
87 Ok(Self {
88 blob,
89 prefix,
90 rt: Some(rt),
91 commit_order: Mutex::new(()),
92 inner: Mutex::new(Inner {
93 parts,
94 counter: keys.len() as u64,
95 }),
96 })
97 }
98
99 pub fn live_object_ids(&self) -> HashSet<String> {
102 let st = self.inner.lock().expect("poisoned");
103 let mut live = HashSet::new();
104 for p in st.parts.values() {
105 for e in &p.entries {
106 live.insert(e.location.object_id.clone());
107 }
108 }
109 live
110 }
111
112 pub fn snapshot(&self) -> IndexSnapshot {
114 let st = self.inner.lock().expect("poisoned");
115 let mut partitions: Vec<PartitionSnapshot> = st
116 .parts
117 .iter()
118 .map(|(k, p)| PartitionSnapshot {
119 partition: k.0.clone(),
120 log_start: p.log_start,
121 high_watermark: p.next_offset,
122 entry_count: p.entries.len(),
123 entries: p.entries.clone(),
124 })
125 .collect();
126 partitions.sort_by(|a, b| a.partition.cmp(&b.partition));
127 IndexSnapshot {
128 manifest_prefix: self.prefix.clone(),
129 manifest_count: st.counter,
130 partitions,
131 }
132 }
133}
134
135#[derive(Clone, Debug, Serialize, Deserialize)]
137pub struct IndexSnapshot {
138 pub manifest_prefix: String,
140 pub manifest_count: u64,
142 pub partitions: Vec<PartitionSnapshot>,
144}
145
146#[derive(Clone, Debug, Serialize, Deserialize)]
148pub struct PartitionSnapshot {
149 pub partition: String,
151 pub log_start: i64,
153 pub high_watermark: i64,
155 pub entry_count: usize,
157 pub entries: Vec<IndexEntry>,
159}
160
161impl Sequencer for ManifestSequencer {
162 type Meta = ();
163
164 fn supports_multi_object_commit(&self) -> bool {
165 true
166 }
167
168 fn commit(
169 &self,
170 batches: &[CommitBatch<'_, ()>],
171 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
172 let _commit = self.commit_order.lock().expect("poisoned");
173 let st = self.inner.lock().expect("poisoned");
174 let mut local_next: HashMap<PartitionKey, i64> = HashMap::new();
176 let mut planned: Vec<(PartitionKey, IndexEntry)> = Vec::with_capacity(batches.len());
177 let mut outcomes = Vec::with_capacity(batches.len());
178 for cb in batches {
179 let base = *local_next
180 .entry(cb.partition.clone())
181 .or_insert_with(|| st.parts.get(&cb.partition).map_or(0, |p| p.next_offset));
182 let entry = IndexEntry {
183 location: cb.location.clone(),
184 base_offset: base,
185 record_count: cb.record_count,
186 };
187 planned.push((cb.partition.clone(), entry));
188 *local_next.get_mut(&cb.partition).expect("inserted above") =
189 base + cb.record_count as i64;
190 outcomes.push(CommitOutcome::Assigned {
191 base_offset: base,
192 record_count: cb.record_count,
193 });
194 }
195
196 let counter = st.counter + 1;
198 drop(st);
202 let key = format!("{}{:020}", self.prefix, counter);
203 let bytes = serde_json::to_vec(&ManifestRecord {
204 entries: planned.clone(),
205 })
206 .map_err(|e| ObjectLogError::Sequencer(e.to_string()))?;
207 self.rt
208 .as_ref()
209 .expect("runtime present until drop")
210 .block_on(self.blob.put(&key, Bytes::from(bytes)))?;
211
212 let mut st = self.inner.lock().expect("poisoned");
214 for (pkey, entry) in planned {
215 let p = st.parts.entry(pkey).or_default();
216 let end = entry.base_offset + entry.record_count as i64;
217 if end > p.next_offset {
218 p.next_offset = end;
219 }
220 p.entries.push(entry);
221 }
222 st.counter = counter;
223 Ok(outcomes)
224 }
225
226 fn lookup(
227 &self,
228 partition: &PartitionKey,
229 fetch_offset: i64,
230 ) -> Result<Vec<IndexEntry>, ObjectLogError> {
231 let st = self.inner.lock().expect("poisoned");
232 let Some(p) = st.parts.get(partition) else {
233 return Ok(Vec::new());
234 };
235 Ok(p.entries
236 .iter()
237 .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
238 .cloned()
239 .collect())
240 }
241
242 fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
243 Ok(self
244 .inner
245 .lock()
246 .expect("poisoned")
247 .parts
248 .get(partition)
249 .map_or(0, |p| p.next_offset))
250 }
251
252 fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
253 Ok(self
254 .inner
255 .lock()
256 .expect("poisoned")
257 .parts
258 .get(partition)
259 .map_or(0, |p| p.log_start))
260 }
261
262 fn truncate_before(
263 &self,
264 partition: &PartitionKey,
265 offset: i64,
266 ) -> Result<Vec<String>, ObjectLogError> {
267 let _commit = self.commit_order.lock().expect("poisoned");
268 let mut st = self.inner.lock().expect("poisoned");
269 let mut dropped: Vec<String> = Vec::new();
270 match st.parts.get_mut(partition) {
271 Some(p) => {
272 for e in p.entries.iter() {
273 if e.base_offset + e.record_count as i64 <= offset {
274 dropped.push(e.location.object_id.clone());
275 }
276 }
277 p.entries
278 .retain(|e| e.base_offset + e.record_count as i64 > offset);
279 if offset > p.log_start {
280 p.log_start = offset.min(p.next_offset);
281 }
282 }
283 None => return Ok(Vec::new()),
284 }
285 let mut live: HashSet<String> = HashSet::new();
286 for p in st.parts.values() {
287 for e in &p.entries {
288 live.insert(e.location.object_id.clone());
289 }
290 }
291 let mut dead: Vec<String> = Vec::new();
292 let mut seen: HashSet<String> = HashSet::new();
293 for oid in dropped {
294 if !live.contains(&oid) && seen.insert(oid.clone()) {
295 dead.push(oid);
296 }
297 }
298 Ok(dead)
299 }
300}