Skip to main content

object_log/
blob.rs

1//! The object-storage port and the bundled adapters.
2
3use crate::ObjectLogError;
4use async_trait::async_trait;
5use bytes::{Bytes, BytesMut};
6use std::collections::BTreeMap;
7use std::io::{Read, Seek, SeekFrom, Write};
8use std::ops::Range;
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicU64, Ordering};
11use std::sync::{Arc, Mutex, OnceLock};
12use std::time::{Instant, SystemTime, UNIX_EPOCH};
13
14/// Durable-media accounting for the flush budget controller (TD-004).
15#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
16pub struct MediaOpStats {
17    /// File/dir fsyncs or billable durable requests performed since last take.
18    pub media_ops: u64,
19    /// Bytes written through durable (or memory) put paths since last take.
20    pub bytes: u64,
21}
22
23/// Suffix for in-flight temp files written by [`LocalBlobStore`]; keys ending in
24/// it are skipped by [`list`](BlobStore::list).
25const TMP_SUFFIX: &str = ".olog-tmp";
26
27fn diagnostic_unix_us() -> u128 {
28    SystemTime::now()
29        .duration_since(UNIX_EPOCH)
30        .unwrap_or_default()
31        .as_micros()
32}
33
34fn object_class(key: &str) -> &'static str {
35    if key.starts_with("fwlog/") {
36        "segment"
37    } else if key.starts_with("fwmeta/manifest/") {
38        "manifest"
39    } else {
40        "metadata"
41    }
42}
43
44/// Physical store join id for opt-in publication traces.
45///
46/// Workload stores live at `shard-N/{log,projection.db}`. Object keys and
47/// queue ids are shared across those physical stores.
48fn diagnostic_store_tag(path: &Path) -> String {
49    let mut fallback = None;
50    for component in path.components() {
51        let name = component.as_os_str().to_string_lossy();
52        if let Some(rest) = name.strip_prefix("shard-")
53            && !rest.is_empty()
54            && rest.bytes().all(|b| b.is_ascii_digit())
55        {
56            return name.into_owned();
57        }
58        if !matches!(
59            name.as_ref(),
60            "log" | "fwlog" | "fwmeta" | "manifest" | "projection.db" | "/" | "."
61        ) {
62            fallback = Some(name.into_owned());
63        }
64    }
65    fallback
66        .filter(|name| !name.is_empty())
67        .unwrap_or_else(|| "unknown".into())
68}
69
70/// A minimal async object store over immutable, string-keyed blobs.
71///
72/// This is the storage *port* the log engine is built on. Implement it for your
73/// backend (e.g. S3) to store the log there. The engine writes each flushed
74/// object under a unique key, so no conditional writes are needed.
75///
76/// Durability: [`put`](BlobStore::put) is **durable-on-return** for crash-durable
77/// adapters ([`LocalBlobStore`], an S3 adapter) — once it resolves `Ok`, the bytes
78/// survive a crash, so a caller may treat `Ok` as a durability barrier.
79/// [`MemoryBlobStore`] is a test/dev backend and is **not** crash-durable.
80#[async_trait]
81pub trait BlobStore: Send + Sync {
82    /// Durably store `value` at `key` (see the trait-level durability note).
83    /// `value` may be arbitrarily large; a network adapter should chunk it
84    /// (e.g. S3 multipart) rather than rely on a single request.
85    async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError>;
86
87    /// Durably store a logical object assembled from immutable byte chunks.
88    ///
89    /// The default implementation concatenates chunks and delegates to
90    /// [`put`](BlobStore::put). Object-store adapters can override this to stream
91    /// or multipart-upload chunks without materializing the full object twice.
92    async fn put_chunks(&self, key: &str, chunks: Vec<Bytes>) -> Result<(), ObjectLogError> {
93        match chunks.len() {
94            0 => self.put(key, Bytes::new()).await,
95            1 => self.put(key, chunks.into_iter().next().unwrap()).await,
96            _ => {
97                let total = chunks.iter().map(Bytes::len).sum();
98                let mut value = BytesMut::with_capacity(total);
99                for chunk in chunks {
100                    value.extend_from_slice(&chunk);
101                }
102                self.put(key, value.freeze()).await
103            }
104        }
105    }
106
107    /// Fetch the whole object at `key`, or `None` if absent.
108    async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError>;
109
110    /// Read a byte sub-range of an object without fetching the whole thing.
111    /// `None` if the key is absent. `range.end > len` or `range.start > range.end`
112    /// is [`ObjectLogError::RangeOutOfBounds`]; an empty `n..n` returns
113    /// `Ok(Some(<empty>))`. No integrity check is performed — payloads are opaque.
114    async fn get_range(
115        &self,
116        key: &str,
117        range: Range<u64>,
118    ) -> Result<Option<Bytes>, ObjectLogError>;
119
120    /// List keys beginning with `prefix`.
121    async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError>;
122
123    /// Delete an object; deleting a missing key is a no-op success.
124    async fn delete(&self, key: &str) -> Result<(), ObjectLogError>;
125
126    /// Snapshot and reset media-op counters since the previous take.
127    ///
128    /// Adapters that can count exact durable units (fsyncs, S3 PUTs) should
129    /// implement this. The default returns [`None`]; the engine then treats each
130    /// successful [`put`](BlobStore::put) / [`put_chunks`](BlobStore::put_chunks)
131    /// as **one** media op (documented fallback).
132    fn take_media_op_stats(&self) -> Option<MediaOpStats> {
133        None
134    }
135}
136
137/// Slice `bytes` by `range`, applying the [`BlobStore::get_range`] bounds rules.
138fn slice_range(bytes: &Bytes, range: Range<u64>) -> Result<Bytes, ObjectLogError> {
139    if range.start > range.end {
140        return Err(ObjectLogError::RangeOutOfBounds(format!(
141            "start {} > end {}",
142            range.start, range.end
143        )));
144    }
145    let len = bytes.len() as u64;
146    if range.end > len {
147        return Err(ObjectLogError::RangeOutOfBounds(format!(
148            "end {} > len {len}",
149            range.end
150        )));
151    }
152    Ok(bytes.slice(range.start as usize..range.end as usize))
153}
154
155/// In-process [`BlobStore`] backed by a map. For tests and development; state is
156/// lost on drop and is **not** crash-durable.
157#[derive(Clone, Default)]
158pub struct MemoryBlobStore {
159    objects: Arc<Mutex<BTreeMap<String, Bytes>>>,
160    bytes_written: Arc<AtomicU64>,
161}
162
163impl MemoryBlobStore {
164    /// Create an empty store.
165    pub fn new() -> Self {
166        Self::default()
167    }
168
169    /// Number of stored objects (lets tests assert PUT-count / cost invariants).
170    pub fn object_count(&self) -> usize {
171        self.objects.lock().expect("poisoned").len()
172    }
173
174    /// Total stored bytes across all objects.
175    pub fn total_bytes(&self) -> usize {
176        self.objects
177            .lock()
178            .expect("poisoned")
179            .values()
180            .map(|v| v.len())
181            .sum()
182    }
183}
184
185#[async_trait]
186impl BlobStore for MemoryBlobStore {
187    async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
188        self.bytes_written
189            .fetch_add(value.len() as u64, Ordering::Relaxed);
190        self.objects
191            .lock()
192            .expect("poisoned")
193            .insert(key.to_string(), value);
194        Ok(())
195    }
196
197    async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
198        Ok(self.objects.lock().expect("poisoned").get(key).cloned())
199    }
200
201    async fn get_range(
202        &self,
203        key: &str,
204        range: Range<u64>,
205    ) -> Result<Option<Bytes>, ObjectLogError> {
206        let object = self.objects.lock().expect("poisoned").get(key).cloned();
207        match object {
208            Some(bytes) => Ok(Some(slice_range(&bytes, range)?)),
209            None => Ok(None),
210        }
211    }
212
213    async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
214        Ok(self
215            .objects
216            .lock()
217            .expect("poisoned")
218            .keys()
219            .filter(|k| k.starts_with(prefix))
220            .cloned()
221            .collect())
222    }
223
224    async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
225        self.objects.lock().expect("poisoned").remove(key);
226        Ok(())
227    }
228
229    fn take_media_op_stats(&self) -> Option<MediaOpStats> {
230        // Memory has no crash-durable media cost.
231        Some(MediaOpStats {
232            media_ops: 0,
233            bytes: self.bytes_written.swap(0, Ordering::Relaxed),
234        })
235    }
236}
237
238/// Filesystem-backed [`BlobStore`] rooted at a directory.
239///
240/// Single-node. Writes are **durable-on-return**: each `put` / `put_chunks`:
241/// 1. write a temp file in the same directory as the final key  
242/// 2. [`std::fs::File::sync_data`] (fdatasync on Unix) so payload bytes are durable  
243/// 3. `rename` into place  
244/// 4. `fsync` the parent directory so the directory entry survives power loss  
245///
246/// (On macOS, true device durability may need `F_FULLFSYNC`; Linux `sync_data` /
247/// dir `sync_all` is the intended contract.)
248#[derive(Clone)]
249pub struct LocalBlobStore {
250    root: Arc<PathBuf>,
251    media_ops: Arc<AtomicU64>,
252    bytes_written: Arc<AtomicU64>,
253}
254
255impl LocalBlobStore {
256    /// Create a store rooted at `root` (created lazily on first write).
257    pub fn new(root: impl Into<PathBuf>) -> Self {
258        Self {
259            root: Arc::new(root.into()),
260            media_ops: Arc::new(AtomicU64::new(0)),
261            bytes_written: Arc::new(AtomicU64::new(0)),
262        }
263    }
264
265    fn path_for(&self, key: &str) -> Result<PathBuf, ObjectLogError> {
266        if key.is_empty() || key.contains('\0') || key.split('/').any(|p| p == "..") {
267            return Err(ObjectLogError::InvalidObjectKey(key.to_string()));
268        }
269        Ok(self.root.join(key))
270    }
271
272    /// Durable publish: temp → sync_data → rename → dir fsync. No full pre-merge
273    /// of `chunks` (streams them to the temp file).
274    fn durable_publish_chunks(
275        path: PathBuf,
276        chunks: Vec<Bytes>,
277        key: &str,
278    ) -> std::io::Result<u64> {
279        // Opt-in diagnostic only: preserve the exact publication operations and
280        // ordering. No paths or payloads are emitted. Failed puts remain errors;
281        // phase records below describe successful publications only.
282        static TRACE: OnceLock<bool> = OnceLock::new();
283        let trace =
284            *TRACE.get_or_init(|| std::env::var_os("OBJECT_LOG_LOCAL_PUBLISH_TRACE").is_some());
285        let started = trace.then(Instant::now);
286        let started_unix_us = trace.then(diagnostic_unix_us);
287        let mut phase = started;
288        let mut elapsed = || {
289            phase.map_or(0, |before| {
290                let now = Instant::now();
291                phase = Some(now);
292                now.duration_since(before).as_micros()
293            })
294        };
295        let parent = path.parent().expect("object path has a parent");
296        std::fs::create_dir_all(parent)?;
297        let mkdir_us = elapsed();
298        let mut tmp = path.clone().into_os_string();
299        tmp.push(TMP_SUFFIX);
300        let tmp = PathBuf::from(tmp);
301        let mut byte_len = 0u64;
302        let (create_us, write_us, data_sync_us);
303        {
304            let mut f = std::fs::File::create(&tmp)?;
305            create_us = elapsed();
306            for chunk in &chunks {
307                f.write_all(chunk)?;
308                byte_len += chunk.len() as u64;
309            }
310            write_us = elapsed();
311            // fdatasync on Unix: data durable; dir fsync below covers the name.
312            f.sync_data()?;
313            data_sync_us = elapsed();
314        }
315        std::fs::rename(&tmp, &path)?;
316        let rename_us = elapsed(); // Includes closing the data file.
317        let directory = std::fs::File::open(parent)?;
318        let dir_open_us = elapsed();
319        directory.sync_all()?;
320        let dir_sync_us = elapsed();
321        if let (Some(started), Some(started_unix_us)) = (started, started_unix_us) {
322            eprintln!(
323                "local_publish bytes={byte_len} mkdir_us={mkdir_us} create_us={create_us} write_us={write_us} data_sync_us={data_sync_us} rename_us={rename_us} dir_open_us={dir_open_us} dir_sync_us={dir_sync_us} total_us={} class={} store={} started_unix_us={started_unix_us}",
324                started.elapsed().as_micros(),
325                object_class(key),
326                diagnostic_store_tag(&path),
327            );
328        }
329        Ok(byte_len)
330    }
331
332    fn record_put(&self, byte_len: u64) {
333        // Two media ops: file data sync + parent directory sync.
334        self.media_ops.fetch_add(2, Ordering::Relaxed);
335        self.bytes_written.fetch_add(byte_len, Ordering::Relaxed);
336    }
337
338    async fn publish_async(
339        &self,
340        path: PathBuf,
341        chunks: Vec<Bytes>,
342        key: String,
343    ) -> Result<u64, ObjectLogError> {
344        let run = move || {
345            Self::durable_publish_chunks(path, chunks, &key)
346                .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))
347        };
348        match tokio::runtime::Handle::try_current() {
349            // Multi-thread worker: avoid spawn_blocking queue (flush path uses this).
350            Ok(h) if h.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => {
351                tokio::task::block_in_place(run)
352            }
353            // Current-thread runtime (many unit tests): must not block_in_place.
354            Ok(_) => tokio::task::spawn_blocking(run)
355                .await
356                .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?,
357            Err(_) => run(),
358        }
359    }
360}
361
362#[async_trait]
363impl BlobStore for LocalBlobStore {
364    async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
365        let path = self.path_for(key)?;
366        let byte_len = self
367            .publish_async(path, vec![value], key.to_string())
368            .await?;
369        self.record_put(byte_len);
370        Ok(())
371    }
372
373    async fn put_chunks(&self, key: &str, chunks: Vec<Bytes>) -> Result<(), ObjectLogError> {
374        match chunks.len() {
375            0 => self.put(key, Bytes::new()).await,
376            1 => self.put(key, chunks.into_iter().next().unwrap()).await,
377            _ => {
378                let path = self.path_for(key)?;
379                let byte_len = self.publish_async(path, chunks, key.to_string()).await?;
380                self.record_put(byte_len);
381                Ok(())
382            }
383        }
384    }
385
386    async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
387        let path = self.path_for(key)?;
388        match tokio::fs::read(&path).await {
389            Ok(v) => Ok(Some(Bytes::from(v))),
390            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
391            Err(e) => Err(e.into()),
392        }
393    }
394
395    async fn get_range(
396        &self,
397        key: &str,
398        range: Range<u64>,
399    ) -> Result<Option<Bytes>, ObjectLogError> {
400        if range.start > range.end {
401            return Err(ObjectLogError::RangeOutOfBounds(format!(
402                "start {} > end {}",
403                range.start, range.end
404            )));
405        }
406        let path = self.path_for(key)?;
407        tokio::task::spawn_blocking(move || -> Result<Option<Bytes>, ObjectLogError> {
408            let mut f = match std::fs::File::open(&path) {
409                Ok(f) => f,
410                Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
411                Err(e) => return Err(e.into()),
412            };
413            let len = f.metadata()?.len();
414            if range.end > len {
415                return Err(ObjectLogError::RangeOutOfBounds(format!(
416                    "end {} > len {len}",
417                    range.end
418                )));
419            }
420            let count = (range.end - range.start) as usize;
421            let mut buf = vec![0u8; count];
422            f.seek(SeekFrom::Start(range.start))?;
423            f.read_exact(&mut buf)?;
424            Ok(Some(Bytes::from(buf)))
425        })
426        .await
427        .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?
428    }
429
430    async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
431        let root = self.root.clone();
432        let prefix = prefix.to_string();
433        let keys = tokio::task::spawn_blocking(move || -> std::io::Result<Vec<String>> {
434            let mut out = Vec::new();
435            collect_files(&root, &root, &mut out)?;
436            Ok(out)
437        })
438        .await
439        .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))??;
440        Ok(keys
441            .into_iter()
442            .filter(|k| k.starts_with(&prefix))
443            .collect())
444    }
445
446    async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
447        let path = self.path_for(key)?;
448        match tokio::fs::remove_file(&path).await {
449            Ok(()) => Ok(()),
450            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
451            Err(e) => Err(e.into()),
452        }
453    }
454
455    fn take_media_op_stats(&self) -> Option<MediaOpStats> {
456        Some(MediaOpStats {
457            media_ops: self.media_ops.swap(0, Ordering::Relaxed),
458            bytes: self.bytes_written.swap(0, Ordering::Relaxed),
459        })
460    }
461}
462
463fn collect_files(root: &Path, dir: &Path, out: &mut Vec<String>) -> std::io::Result<()> {
464    if !dir.exists() {
465        return Ok(());
466    }
467    for entry in std::fs::read_dir(dir)? {
468        let entry = entry?;
469        let path = entry.path();
470        if path.is_dir() {
471            collect_files(root, &path, out)?;
472        } else if !path.to_string_lossy().ends_with(TMP_SUFFIX)
473            && let Ok(rel) = path.strip_prefix(root)
474        {
475            out.push(rel.to_string_lossy().replace('\\', "/"));
476        }
477    }
478    Ok(())
479}
480
481#[cfg(test)]
482mod tests {
483    use super::*;
484
485    #[test]
486    fn diagnostic_store_tag_prefers_workload_shard_directory() {
487        assert_eq!(
488            diagnostic_store_tag(Path::new(
489                "/tmp/run/shard-12/log/fwlog/00000000000000000001"
490            )),
491            "shard-12"
492        );
493        assert_eq!(
494            diagnostic_store_tag(Path::new("/data/shard-0/projection.db")),
495            "shard-0"
496        );
497        assert_eq!(object_class("fwlog/1"), "segment");
498        assert_eq!(object_class("fwmeta/manifest/1"), "manifest");
499        assert_eq!(object_class("fwmeta/high-water/1"), "metadata");
500    }
501}