object_log/
manifest_sequencer.rs1use crate::{
10 BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
11};
12use bytes::Bytes;
13use serde::{Deserialize, Serialize};
14use std::collections::{HashMap, HashSet};
15use std::sync::{Arc, Mutex};
16use tokio::runtime::{Builder, Runtime};
17
18#[derive(Serialize, Deserialize)]
19struct ManifestRecord {
20 entries: Vec<(PartitionKey, IndexEntry)>,
21}
22
23#[derive(Default)]
24struct Part {
25 next_offset: i64,
26 log_start: i64,
27 entries: Vec<IndexEntry>,
28}
29
30struct Inner {
31 parts: HashMap<PartitionKey, Part>,
32 counter: u64,
33}
34
35pub struct ManifestSequencer {
37 blob: Arc<dyn BlobStore>,
38 prefix: String,
39 rt: Option<Runtime>,
42 inner: Mutex<Inner>,
43}
44
45impl Drop for ManifestSequencer {
46 fn drop(&mut self) {
47 if let Some(rt) = self.rt.take() {
48 rt.shutdown_background();
49 }
50 }
51}
52
53impl ManifestSequencer {
54 pub async fn open(
58 blob: Arc<dyn BlobStore>,
59 manifest_prefix: impl Into<String>,
60 ) -> Result<Self, ObjectLogError> {
61 let prefix = manifest_prefix.into();
62 let mut keys = blob.list(&prefix).await?;
63 keys.sort();
64 let mut parts: HashMap<PartitionKey, Part> = HashMap::new();
65 for key in &keys {
66 let bytes = blob
67 .get(key)
68 .await?
69 .ok_or_else(|| ObjectLogError::MissingObject(key.clone()))?;
70 let rec: ManifestRecord = serde_json::from_slice(&bytes)
71 .map_err(|e| ObjectLogError::Sequencer(format!("manifest {key}: {e}")))?;
72 for (pkey, entry) in rec.entries {
73 let p = parts.entry(pkey).or_default();
74 let end = entry.base_offset + entry.record_count as i64;
75 if end > p.next_offset {
76 p.next_offset = end;
77 }
78 p.entries.push(entry);
79 }
80 }
81 let rt = Builder::new_current_thread()
82 .enable_all()
83 .build()
84 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?;
85 Ok(Self {
86 blob,
87 prefix,
88 rt: Some(rt),
89 inner: Mutex::new(Inner {
90 parts,
91 counter: keys.len() as u64,
92 }),
93 })
94 }
95}
96
97impl Sequencer for ManifestSequencer {
98 type Meta = ();
99
100 fn commit(
101 &self,
102 batches: &[CommitBatch<'_, ()>],
103 ) -> Result<Vec<CommitOutcome>, ObjectLogError> {
104 let mut st = self.inner.lock().expect("poisoned");
105 let mut local_next: HashMap<PartitionKey, i64> = HashMap::new();
107 let mut planned: Vec<(PartitionKey, IndexEntry)> = Vec::with_capacity(batches.len());
108 let mut outcomes = Vec::with_capacity(batches.len());
109 for cb in batches {
110 let base = *local_next
111 .entry(cb.partition.clone())
112 .or_insert_with(|| st.parts.get(&cb.partition).map_or(0, |p| p.next_offset));
113 let entry = IndexEntry {
114 location: cb.location.clone(),
115 base_offset: base,
116 record_count: cb.record_count,
117 };
118 planned.push((cb.partition.clone(), entry));
119 *local_next.get_mut(&cb.partition).expect("inserted above") =
120 base + cb.record_count as i64;
121 outcomes.push(CommitOutcome::Assigned {
122 base_offset: base,
123 record_count: cb.record_count,
124 });
125 }
126
127 let counter = st.counter + 1;
129 let key = format!("{}{:020}", self.prefix, counter);
130 let bytes = serde_json::to_vec(&ManifestRecord {
131 entries: planned.clone(),
132 })
133 .map_err(|e| ObjectLogError::Sequencer(e.to_string()))?;
134 self.rt
135 .as_ref()
136 .expect("runtime present until drop")
137 .block_on(self.blob.put(&key, Bytes::from(bytes)))?;
138
139 for (pkey, entry) in planned {
141 let p = st.parts.entry(pkey).or_default();
142 let end = entry.base_offset + entry.record_count as i64;
143 if end > p.next_offset {
144 p.next_offset = end;
145 }
146 p.entries.push(entry);
147 }
148 st.counter = counter;
149 Ok(outcomes)
150 }
151
152 fn lookup(
153 &self,
154 partition: &PartitionKey,
155 fetch_offset: i64,
156 ) -> Result<Vec<IndexEntry>, ObjectLogError> {
157 let st = self.inner.lock().expect("poisoned");
158 let Some(p) = st.parts.get(partition) else {
159 return Ok(Vec::new());
160 };
161 Ok(p.entries
162 .iter()
163 .filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
164 .cloned()
165 .collect())
166 }
167
168 fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
169 Ok(self
170 .inner
171 .lock()
172 .expect("poisoned")
173 .parts
174 .get(partition)
175 .map_or(0, |p| p.next_offset))
176 }
177
178 fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
179 Ok(self
180 .inner
181 .lock()
182 .expect("poisoned")
183 .parts
184 .get(partition)
185 .map_or(0, |p| p.log_start))
186 }
187
188 fn truncate_before(
189 &self,
190 partition: &PartitionKey,
191 offset: i64,
192 ) -> Result<Vec<String>, ObjectLogError> {
193 let mut st = self.inner.lock().expect("poisoned");
194 let mut dropped: Vec<String> = Vec::new();
195 match st.parts.get_mut(partition) {
196 Some(p) => {
197 for e in p.entries.iter() {
198 if e.base_offset + e.record_count as i64 <= offset {
199 dropped.push(e.location.object_id.clone());
200 }
201 }
202 p.entries
203 .retain(|e| e.base_offset + e.record_count as i64 > offset);
204 if offset > p.log_start {
205 p.log_start = offset.min(p.next_offset);
206 }
207 }
208 None => return Ok(Vec::new()),
209 }
210 let mut live: HashSet<String> = HashSet::new();
211 for p in st.parts.values() {
212 for e in &p.entries {
213 live.insert(e.location.object_id.clone());
214 }
215 }
216 let mut dead: Vec<String> = Vec::new();
217 let mut seen: HashSet<String> = HashSet::new();
218 for oid in dropped {
219 if !live.contains(&oid) && seen.insert(oid.clone()) {
220 dead.push(oid);
221 }
222 }
223 Ok(dead)
224 }
225}