1use crate::ObjectLogError;
4use async_trait::async_trait;
5use bytes::{Bytes, BytesMut};
6use std::collections::BTreeMap;
7use std::io::{Read, Seek, SeekFrom, Write};
8use std::ops::Range;
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicU64, Ordering};
11use std::sync::{Arc, Mutex};
12
13#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
15pub struct MediaOpStats {
16 pub media_ops: u64,
18 pub bytes: u64,
20}
21
22const TMP_SUFFIX: &str = ".olog-tmp";
25
26#[async_trait]
37pub trait BlobStore: Send + Sync {
38 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError>;
42
43 async fn put_chunks(&self, key: &str, chunks: Vec<Bytes>) -> Result<(), ObjectLogError> {
49 match chunks.len() {
50 0 => self.put(key, Bytes::new()).await,
51 1 => self.put(key, chunks.into_iter().next().unwrap()).await,
52 _ => {
53 let total = chunks.iter().map(Bytes::len).sum();
54 let mut value = BytesMut::with_capacity(total);
55 for chunk in chunks {
56 value.extend_from_slice(&chunk);
57 }
58 self.put(key, value.freeze()).await
59 }
60 }
61 }
62
63 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError>;
65
66 async fn get_range(
71 &self,
72 key: &str,
73 range: Range<u64>,
74 ) -> Result<Option<Bytes>, ObjectLogError>;
75
76 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError>;
78
79 async fn delete(&self, key: &str) -> Result<(), ObjectLogError>;
81
82 fn take_media_op_stats(&self) -> Option<MediaOpStats> {
89 None
90 }
91}
92
93fn slice_range(bytes: &Bytes, range: Range<u64>) -> Result<Bytes, ObjectLogError> {
95 if range.start > range.end {
96 return Err(ObjectLogError::RangeOutOfBounds(format!(
97 "start {} > end {}",
98 range.start, range.end
99 )));
100 }
101 let len = bytes.len() as u64;
102 if range.end > len {
103 return Err(ObjectLogError::RangeOutOfBounds(format!(
104 "end {} > len {len}",
105 range.end
106 )));
107 }
108 Ok(bytes.slice(range.start as usize..range.end as usize))
109}
110
111#[derive(Clone, Default)]
114pub struct MemoryBlobStore {
115 objects: Arc<Mutex<BTreeMap<String, Bytes>>>,
116 bytes_written: Arc<AtomicU64>,
117}
118
119impl MemoryBlobStore {
120 pub fn new() -> Self {
122 Self::default()
123 }
124
125 pub fn object_count(&self) -> usize {
127 self.objects.lock().expect("poisoned").len()
128 }
129
130 pub fn total_bytes(&self) -> usize {
132 self.objects
133 .lock()
134 .expect("poisoned")
135 .values()
136 .map(|v| v.len())
137 .sum()
138 }
139}
140
141#[async_trait]
142impl BlobStore for MemoryBlobStore {
143 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
144 self.bytes_written
145 .fetch_add(value.len() as u64, Ordering::Relaxed);
146 self.objects
147 .lock()
148 .expect("poisoned")
149 .insert(key.to_string(), value);
150 Ok(())
151 }
152
153 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
154 Ok(self.objects.lock().expect("poisoned").get(key).cloned())
155 }
156
157 async fn get_range(
158 &self,
159 key: &str,
160 range: Range<u64>,
161 ) -> Result<Option<Bytes>, ObjectLogError> {
162 let object = self.objects.lock().expect("poisoned").get(key).cloned();
163 match object {
164 Some(bytes) => Ok(Some(slice_range(&bytes, range)?)),
165 None => Ok(None),
166 }
167 }
168
169 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
170 Ok(self
171 .objects
172 .lock()
173 .expect("poisoned")
174 .keys()
175 .filter(|k| k.starts_with(prefix))
176 .cloned()
177 .collect())
178 }
179
180 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
181 self.objects.lock().expect("poisoned").remove(key);
182 Ok(())
183 }
184
185 fn take_media_op_stats(&self) -> Option<MediaOpStats> {
186 Some(MediaOpStats {
188 media_ops: 0,
189 bytes: self.bytes_written.swap(0, Ordering::Relaxed),
190 })
191 }
192}
193
194#[derive(Clone)]
205pub struct LocalBlobStore {
206 root: Arc<PathBuf>,
207 media_ops: Arc<AtomicU64>,
208 bytes_written: Arc<AtomicU64>,
209}
210
211impl LocalBlobStore {
212 pub fn new(root: impl Into<PathBuf>) -> Self {
214 Self {
215 root: Arc::new(root.into()),
216 media_ops: Arc::new(AtomicU64::new(0)),
217 bytes_written: Arc::new(AtomicU64::new(0)),
218 }
219 }
220
221 fn path_for(&self, key: &str) -> Result<PathBuf, ObjectLogError> {
222 if key.is_empty() || key.contains('\0') || key.split('/').any(|p| p == "..") {
223 return Err(ObjectLogError::InvalidObjectKey(key.to_string()));
224 }
225 Ok(self.root.join(key))
226 }
227
228 fn durable_publish_chunks(path: PathBuf, chunks: Vec<Bytes>) -> std::io::Result<u64> {
231 let parent = path.parent().expect("object path has a parent");
232 std::fs::create_dir_all(parent)?;
233 let mut tmp = path.clone().into_os_string();
234 tmp.push(TMP_SUFFIX);
235 let tmp = PathBuf::from(tmp);
236 let mut byte_len = 0u64;
237 {
238 let mut f = std::fs::File::create(&tmp)?;
239 for chunk in &chunks {
240 f.write_all(chunk)?;
241 byte_len += chunk.len() as u64;
242 }
243 f.sync_data()?;
245 }
246 std::fs::rename(&tmp, &path)?;
247 std::fs::File::open(parent)?.sync_all()?;
248 Ok(byte_len)
249 }
250
251 fn record_put(&self, byte_len: u64) {
252 self.media_ops.fetch_add(2, Ordering::Relaxed);
254 self.bytes_written.fetch_add(byte_len, Ordering::Relaxed);
255 }
256
257 async fn publish_async(
258 &self,
259 path: PathBuf,
260 chunks: Vec<Bytes>,
261 ) -> Result<u64, ObjectLogError> {
262 let run = move || {
263 Self::durable_publish_chunks(path, chunks)
264 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))
265 };
266 match tokio::runtime::Handle::try_current() {
267 Ok(h) if h.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => {
269 tokio::task::block_in_place(run)
270 }
271 Ok(_) => tokio::task::spawn_blocking(run)
273 .await
274 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?,
275 Err(_) => run(),
276 }
277 }
278}
279
280#[async_trait]
281impl BlobStore for LocalBlobStore {
282 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
283 let path = self.path_for(key)?;
284 let byte_len = self.publish_async(path, vec![value]).await?;
285 self.record_put(byte_len);
286 Ok(())
287 }
288
289 async fn put_chunks(&self, key: &str, chunks: Vec<Bytes>) -> Result<(), ObjectLogError> {
290 match chunks.len() {
291 0 => self.put(key, Bytes::new()).await,
292 1 => self.put(key, chunks.into_iter().next().unwrap()).await,
293 _ => {
294 let path = self.path_for(key)?;
295 let byte_len = self.publish_async(path, chunks).await?;
296 self.record_put(byte_len);
297 Ok(())
298 }
299 }
300 }
301
302 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
303 let path = self.path_for(key)?;
304 match tokio::fs::read(&path).await {
305 Ok(v) => Ok(Some(Bytes::from(v))),
306 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
307 Err(e) => Err(e.into()),
308 }
309 }
310
311 async fn get_range(
312 &self,
313 key: &str,
314 range: Range<u64>,
315 ) -> Result<Option<Bytes>, ObjectLogError> {
316 if range.start > range.end {
317 return Err(ObjectLogError::RangeOutOfBounds(format!(
318 "start {} > end {}",
319 range.start, range.end
320 )));
321 }
322 let path = self.path_for(key)?;
323 tokio::task::spawn_blocking(move || -> Result<Option<Bytes>, ObjectLogError> {
324 let mut f = match std::fs::File::open(&path) {
325 Ok(f) => f,
326 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
327 Err(e) => return Err(e.into()),
328 };
329 let len = f.metadata()?.len();
330 if range.end > len {
331 return Err(ObjectLogError::RangeOutOfBounds(format!(
332 "end {} > len {len}",
333 range.end
334 )));
335 }
336 let count = (range.end - range.start) as usize;
337 let mut buf = vec![0u8; count];
338 f.seek(SeekFrom::Start(range.start))?;
339 f.read_exact(&mut buf)?;
340 Ok(Some(Bytes::from(buf)))
341 })
342 .await
343 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?
344 }
345
346 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
347 let root = self.root.clone();
348 let prefix = prefix.to_string();
349 let keys = tokio::task::spawn_blocking(move || -> std::io::Result<Vec<String>> {
350 let mut out = Vec::new();
351 collect_files(&root, &root, &mut out)?;
352 Ok(out)
353 })
354 .await
355 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))??;
356 Ok(keys
357 .into_iter()
358 .filter(|k| k.starts_with(&prefix))
359 .collect())
360 }
361
362 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
363 let path = self.path_for(key)?;
364 match tokio::fs::remove_file(&path).await {
365 Ok(()) => Ok(()),
366 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
367 Err(e) => Err(e.into()),
368 }
369 }
370
371 fn take_media_op_stats(&self) -> Option<MediaOpStats> {
372 Some(MediaOpStats {
373 media_ops: self.media_ops.swap(0, Ordering::Relaxed),
374 bytes: self.bytes_written.swap(0, Ordering::Relaxed),
375 })
376 }
377}
378
379fn collect_files(root: &Path, dir: &Path, out: &mut Vec<String>) -> std::io::Result<()> {
380 if !dir.exists() {
381 return Ok(());
382 }
383 for entry in std::fs::read_dir(dir)? {
384 let entry = entry?;
385 let path = entry.path();
386 if path.is_dir() {
387 collect_files(root, &path, out)?;
388 } else if !path.to_string_lossy().ends_with(TMP_SUFFIX)
389 && let Ok(rel) = path.strip_prefix(root)
390 {
391 out.push(rel.to_string_lossy().replace('\\', "/"));
392 }
393 }
394 Ok(())
395}