1use crate::ObjectLogError;
4use async_trait::async_trait;
5use bytes::{Bytes, BytesMut};
6use std::collections::BTreeMap;
7use std::io::{Read, Seek, SeekFrom, Write};
8use std::ops::Range;
9use std::os::unix::io::AsRawFd;
10use std::path::{Path, PathBuf};
11use std::sync::atomic::{AtomicU64, Ordering};
12use std::sync::{Arc, Mutex, OnceLock};
13use std::time::{Instant, SystemTime, UNIX_EPOCH};
14
15pub(crate) mod authority;
16pub use authority::{
17 BoundedMetadataStore, MetadataLimits, MetadataOwners, MetadataSnapshot, MetadataSwap,
18};
19
20#[derive(Clone, Debug, PartialEq, Eq)]
22pub enum CasOutcome {
23 Stored,
25 Conflict {
28 current: Option<Bytes>,
30 },
31}
32
33#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
35pub struct MediaOpStats {
36 pub media_ops: u64,
38 pub bytes: u64,
40}
41
42const TMP_SUFFIX: &str = ".olog-tmp";
45
46fn diagnostic_unix_us() -> u128 {
47 SystemTime::now()
48 .duration_since(UNIX_EPOCH)
49 .unwrap_or_default()
50 .as_micros()
51}
52
53fn object_class(key: &str) -> &'static str {
54 if key.starts_with("fwlog/") {
55 "segment"
56 } else if key.starts_with("fwmeta/manifest/") {
57 "manifest"
58 } else {
59 "metadata"
60 }
61}
62
63fn diagnostic_store_tag(path: &Path) -> String {
68 let mut fallback = None;
69 for component in path.components() {
70 let name = component.as_os_str().to_string_lossy();
71 if let Some(rest) = name.strip_prefix("shard-")
72 && !rest.is_empty()
73 && rest.bytes().all(|b| b.is_ascii_digit())
74 {
75 return name.into_owned();
76 }
77 if !matches!(
78 name.as_ref(),
79 "log" | "fwlog" | "fwmeta" | "manifest" | "projection.db" | "/" | "."
80 ) {
81 fallback = Some(name.into_owned());
82 }
83 }
84 fallback
85 .filter(|name| !name.is_empty())
86 .unwrap_or_else(|| "unknown".into())
87}
88
89#[async_trait]
101pub trait BlobStore: Send + Sync {
102 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError>;
106
107 async fn put_chunks(&self, key: &str, chunks: Vec<Bytes>) -> Result<(), ObjectLogError> {
113 match chunks.len() {
114 0 => self.put(key, Bytes::new()).await,
115 1 => self.put(key, chunks.into_iter().next().unwrap()).await,
116 _ => {
117 let total = chunks.iter().map(Bytes::len).sum();
118 let mut value = BytesMut::with_capacity(total);
119 for chunk in chunks {
120 value.extend_from_slice(&chunk);
121 }
122 self.put(key, value.freeze()).await
123 }
124 }
125 }
126
127 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError>;
129
130 async fn get_range(
135 &self,
136 key: &str,
137 range: Range<u64>,
138 ) -> Result<Option<Bytes>, ObjectLogError>;
139
140 async fn get_range_exact(
147 &self,
148 _key: &str,
149 _range: Range<u64>,
150 ) -> Result<Option<Bytes>, ObjectLogError> {
151 Err(ObjectLogError::InvalidConfig(
152 "strict bounded range reads are not implemented by this adapter".into(),
153 ))
154 }
155
156 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError>;
158
159 async fn delete(&self, key: &str) -> Result<(), ObjectLogError>;
161
162 async fn compare_and_swap(
169 &self,
170 key: &str,
171 expected: Option<Bytes>,
172 new_value: Bytes,
173 ) -> Result<CasOutcome, ObjectLogError> {
174 let current = self.get(key).await?;
175 let matches = match (&expected, ¤t) {
176 (None, None) => true,
177 (Some(expected), Some(current)) => expected == current,
178 _ => false,
179 };
180 if !matches {
181 return Ok(CasOutcome::Conflict { current });
182 }
183 self.put(key, new_value).await?;
184 Ok(CasOutcome::Stored)
185 }
186
187 fn take_media_op_stats(&self) -> Option<MediaOpStats> {
194 None
195 }
196}
197
198fn slice_range(bytes: &Bytes, range: Range<u64>) -> Result<Bytes, ObjectLogError> {
200 if range.start > range.end {
201 return Err(ObjectLogError::RangeOutOfBounds(format!(
202 "start {} > end {}",
203 range.start, range.end
204 )));
205 }
206 let len = bytes.len() as u64;
207 if range.end > len {
208 return Err(ObjectLogError::RangeOutOfBounds(format!(
209 "end {} > len {len}",
210 range.end
211 )));
212 }
213 Ok(bytes.slice(range.start as usize..range.end as usize))
214}
215
216#[derive(Clone, Default)]
219pub struct MemoryBlobStore {
220 objects: Arc<Mutex<BTreeMap<String, Bytes>>>,
221 bytes_written: Arc<AtomicU64>,
222}
223
224impl MemoryBlobStore {
225 pub fn new() -> Self {
227 Self::default()
228 }
229
230 pub fn object_count(&self) -> usize {
232 self.objects.lock().expect("poisoned").len()
233 }
234
235 pub fn total_bytes(&self) -> usize {
237 self.objects
238 .lock()
239 .expect("poisoned")
240 .values()
241 .map(|v| v.len())
242 .sum()
243 }
244}
245
246#[async_trait]
247impl BlobStore for MemoryBlobStore {
248 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
249 self.bytes_written
250 .fetch_add(value.len() as u64, Ordering::Relaxed);
251 self.objects
252 .lock()
253 .expect("poisoned")
254 .insert(key.to_string(), value);
255 Ok(())
256 }
257
258 async fn compare_and_swap(
259 &self,
260 key: &str,
261 expected: Option<Bytes>,
262 new_value: Bytes,
263 ) -> Result<CasOutcome, ObjectLogError> {
264 let mut objects = self.objects.lock().expect("poisoned");
265 let current = objects.get(key).cloned();
266 let matches = match (&expected, ¤t) {
267 (None, None) => true,
268 (Some(expected), Some(current)) => expected == current,
269 _ => false,
270 };
271 if !matches {
272 return Ok(CasOutcome::Conflict { current });
273 }
274 self.bytes_written
275 .fetch_add(new_value.len() as u64, Ordering::Relaxed);
276 objects.insert(key.to_string(), new_value);
277 Ok(CasOutcome::Stored)
278 }
279
280 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
281 Ok(self.objects.lock().expect("poisoned").get(key).cloned())
282 }
283
284 async fn get_range(
285 &self,
286 key: &str,
287 range: Range<u64>,
288 ) -> Result<Option<Bytes>, ObjectLogError> {
289 let object = self.objects.lock().expect("poisoned").get(key).cloned();
290 match object {
291 Some(bytes) => Ok(Some(slice_range(&bytes, range)?)),
292 None => Ok(None),
293 }
294 }
295
296 async fn get_range_exact(
297 &self,
298 key: &str,
299 range: Range<u64>,
300 ) -> Result<Option<Bytes>, ObjectLogError> {
301 self.get_range(key, range).await
302 }
303
304 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
305 Ok(self
306 .objects
307 .lock()
308 .expect("poisoned")
309 .keys()
310 .filter(|k| k.starts_with(prefix))
311 .cloned()
312 .collect())
313 }
314
315 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
316 self.objects.lock().expect("poisoned").remove(key);
317 Ok(())
318 }
319
320 fn take_media_op_stats(&self) -> Option<MediaOpStats> {
321 Some(MediaOpStats {
323 media_ops: 0,
324 bytes: self.bytes_written.swap(0, Ordering::Relaxed),
325 })
326 }
327}
328
329#[derive(Clone)]
340pub struct LocalBlobStore {
341 authority_identity: Arc<()>,
342 #[cfg(test)]
343 authority_hook: Arc<Mutex<Option<Arc<authority::LocalMetadataHook>>>>,
344 #[cfg(test)]
345 authority_lost_reply: Arc<Mutex<Option<String>>>,
346 #[cfg(test)]
347 authority_swap_calls: Arc<AtomicU64>,
348 root: Arc<PathBuf>,
349 media_ops: Arc<AtomicU64>,
350 bytes_written: Arc<AtomicU64>,
351}
352
353impl LocalBlobStore {
354 pub fn new(root: impl Into<PathBuf>) -> Self {
356 Self {
357 authority_identity: Arc::new(()),
358 #[cfg(test)]
359 authority_hook: Arc::new(Mutex::new(None)),
360 #[cfg(test)]
361 authority_lost_reply: Arc::new(Mutex::new(None)),
362 #[cfg(test)]
363 authority_swap_calls: Arc::new(AtomicU64::new(0)),
364 root: Arc::new(root.into()),
365 media_ops: Arc::new(AtomicU64::new(0)),
366 bytes_written: Arc::new(AtomicU64::new(0)),
367 }
368 }
369
370 #[cfg(any(test, feature = "native"))]
373 pub(crate) fn native_page_source_scratch(
374 &self,
375 key_len: usize,
376 ) -> Result<usize, ObjectLogError> {
377 self.root
378 .capacity()
379 .checked_add(key_len)
380 .and_then(|n| n.checked_add(TMP_SUFFIX.len() + 16))
381 .and_then(|n| n.checked_mul(8))
382 .and_then(|n| key_len.checked_mul(6).and_then(|keys| n.checked_add(keys)))
383 .and_then(|n| n.checked_add(4096))
384 .ok_or_else(|| {
385 ObjectLogError::InvalidConfig("native local path capacity overflow".into())
386 })
387 }
388
389 fn path_for(&self, key: &str) -> Result<PathBuf, ObjectLogError> {
390 if key.is_empty() || key.contains('\0') || key.split('/').any(|p| p == "..") {
391 return Err(ObjectLogError::InvalidObjectKey(key.to_string()));
392 }
393 Ok(self.root.join(key))
394 }
395
396 fn durable_publish_chunks(
399 path: PathBuf,
400 chunks: Vec<Bytes>,
401 key: &str,
402 ) -> std::io::Result<u64> {
403 static TRACE: OnceLock<bool> = OnceLock::new();
407 let trace =
408 *TRACE.get_or_init(|| std::env::var_os("OBJECT_LOG_LOCAL_PUBLISH_TRACE").is_some());
409 let started = trace.then(Instant::now);
410 let started_unix_us = trace.then(diagnostic_unix_us);
411 let mut phase = started;
412 let mut elapsed = || {
413 phase.map_or(0, |before| {
414 let now = Instant::now();
415 phase = Some(now);
416 now.duration_since(before).as_micros()
417 })
418 };
419 let parent = path.parent().expect("object path has a parent");
420 std::fs::create_dir_all(parent)?;
421 let mkdir_us = elapsed();
422 let mut tmp = path.clone().into_os_string();
423 tmp.push(TMP_SUFFIX);
424 let tmp = PathBuf::from(tmp);
425 let mut byte_len = 0u64;
426 let (create_us, write_us, data_sync_us);
427 {
428 let mut f = std::fs::File::create(&tmp)?;
429 create_us = elapsed();
430 for chunk in &chunks {
431 f.write_all(chunk)?;
432 byte_len += chunk.len() as u64;
433 }
434 write_us = elapsed();
435 f.sync_data()?;
437 data_sync_us = elapsed();
438 }
439 std::fs::rename(&tmp, &path)?;
440 let rename_us = elapsed(); let directory = std::fs::File::open(parent)?;
442 let dir_open_us = elapsed();
443 directory.sync_all()?;
444 let dir_sync_us = elapsed();
445 if let (Some(started), Some(started_unix_us)) = (started, started_unix_us) {
446 eprintln!(
447 "local_publish bytes={byte_len} mkdir_us={mkdir_us} create_us={create_us} write_us={write_us} data_sync_us={data_sync_us} rename_us={rename_us} dir_open_us={dir_open_us} dir_sync_us={dir_sync_us} total_us={} class={} store={} started_unix_us={started_unix_us}",
448 started.elapsed().as_micros(),
449 object_class(key),
450 diagnostic_store_tag(&path),
451 );
452 }
453 Ok(byte_len)
454 }
455
456 fn record_put(&self, byte_len: u64) {
457 self.media_ops.fetch_add(2, Ordering::Relaxed);
459 self.bytes_written.fetch_add(byte_len, Ordering::Relaxed);
460 }
461
462 async fn publish_async(
463 &self,
464 path: PathBuf,
465 chunks: Vec<Bytes>,
466 key: String,
467 ) -> Result<u64, ObjectLogError> {
468 let run = move || {
469 Self::durable_publish_chunks(path, chunks, &key)
470 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))
471 };
472 match tokio::runtime::Handle::try_current() {
473 Ok(h) if h.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => {
475 tokio::task::block_in_place(run)
476 }
477 Ok(_) => tokio::task::spawn_blocking(run)
479 .await
480 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?,
481 Err(_) => run(),
482 }
483 }
484}
485
486#[async_trait]
487impl BlobStore for LocalBlobStore {
488 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
489 let path = self.path_for(key)?;
490 let byte_len = self
491 .publish_async(path, vec![value], key.to_string())
492 .await?;
493 self.record_put(byte_len);
494 Ok(())
495 }
496
497 async fn put_chunks(&self, key: &str, chunks: Vec<Bytes>) -> Result<(), ObjectLogError> {
498 match chunks.len() {
499 0 => self.put(key, Bytes::new()).await,
500 1 => self.put(key, chunks.into_iter().next().unwrap()).await,
501 _ => {
502 let path = self.path_for(key)?;
503 let byte_len = self.publish_async(path, chunks, key.to_string()).await?;
504 self.record_put(byte_len);
505 Ok(())
506 }
507 }
508 }
509
510 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
511 let path = self.path_for(key)?;
512 match tokio::fs::read(&path).await {
513 Ok(v) => Ok(Some(Bytes::from(v))),
514 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
515 Err(e) => Err(e.into()),
516 }
517 }
518
519 async fn get_range(
520 &self,
521 key: &str,
522 range: Range<u64>,
523 ) -> Result<Option<Bytes>, ObjectLogError> {
524 if range.start > range.end {
525 return Err(ObjectLogError::RangeOutOfBounds(format!(
526 "start {} > end {}",
527 range.start, range.end
528 )));
529 }
530 let path = self.path_for(key)?;
531 tokio::task::spawn_blocking(move || -> Result<Option<Bytes>, ObjectLogError> {
532 let mut f = match std::fs::File::open(&path) {
533 Ok(f) => f,
534 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
535 Err(e) => return Err(e.into()),
536 };
537 let len = f.metadata()?.len();
538 if range.end > len {
539 return Err(ObjectLogError::RangeOutOfBounds(format!(
540 "end {} > len {len}",
541 range.end
542 )));
543 }
544 let count = usize::try_from(range.end - range.start).map_err(|_| {
545 ObjectLogError::RangeOutOfBounds("range exceeds address space".into())
546 })?;
547 let mut buf = vec![0u8; count];
548 f.seek(SeekFrom::Start(range.start))?;
549 f.read_exact(&mut buf)?;
550 Ok(Some(Bytes::from(buf)))
551 })
552 .await
553 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))?
554 }
555
556 async fn get_range_exact(
557 &self,
558 key: &str,
559 range: Range<u64>,
560 ) -> Result<Option<Bytes>, ObjectLogError> {
561 self.get_range(key, range).await
562 }
563
564 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
565 let root = self.root.clone();
566 let prefix = prefix.to_string();
567 let keys = tokio::task::spawn_blocking(move || -> std::io::Result<Vec<String>> {
568 let mut out = Vec::new();
569 collect_files(&root, &root, &mut out)?;
570 Ok(out)
571 })
572 .await
573 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))??;
574 Ok(keys
575 .into_iter()
576 .filter(|k| k.starts_with(&prefix))
577 .collect())
578 }
579
580 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
581 let path = self.path_for(key)?;
582 match tokio::fs::remove_file(&path).await {
583 Ok(()) => Ok(()),
584 Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
585 Err(e) => Err(e.into()),
586 }
587 }
588
589 async fn compare_and_swap(
590 &self,
591 key: &str,
592 expected: Option<Bytes>,
593 new_value: Bytes,
594 ) -> Result<CasOutcome, ObjectLogError> {
595 let path = self.path_for(key)?;
596 let key = key.to_string();
597 let byte_len = new_value.len() as u64;
598 let outcome = tokio::task::spawn_blocking(move || {
599 local_compare_and_swap(&path, &key, expected, new_value)
600 })
601 .await
602 .map_err(|e| ObjectLogError::StorageUnavailable(e.to_string()))??;
603 if matches!(outcome, CasOutcome::Stored) {
604 self.record_put(byte_len);
605 }
606 Ok(outcome)
607 }
608
609 fn take_media_op_stats(&self) -> Option<MediaOpStats> {
610 Some(MediaOpStats {
611 media_ops: self.media_ops.swap(0, Ordering::Relaxed),
612 bytes: self.bytes_written.swap(0, Ordering::Relaxed),
613 })
614 }
615}
616
617fn local_compare_and_swap(
618 path: &Path,
619 key: &str,
620 expected: Option<Bytes>,
621 new_value: Bytes,
622) -> Result<CasOutcome, ObjectLogError> {
623 if let Some(parent) = path.parent() {
624 std::fs::create_dir_all(parent)?;
625 }
626 let mut lock_name = path.as_os_str().to_os_string();
627 lock_name.push(".caslock");
628 let lock = std::fs::OpenOptions::new()
629 .create(true)
630 .truncate(false)
631 .read(true)
632 .write(true)
633 .open(lock_name)?;
634 flock_exclusive(&lock)?;
635 let current = match std::fs::read(path) {
636 Ok(bytes) => Some(Bytes::from(bytes)),
637 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
638 Err(error) => {
639 let _ = flock_unlock(&lock);
640 return Err(error.into());
641 }
642 };
643 let matches = match (&expected, ¤t) {
644 (None, None) => true,
645 (Some(expected), Some(current)) => expected == current,
646 _ => false,
647 };
648 if !matches {
649 let _ = flock_unlock(&lock);
650 return Ok(CasOutcome::Conflict { current });
651 }
652 let published =
653 LocalBlobStore::durable_publish_chunks(path.to_path_buf(), vec![new_value], key);
654 let _ = flock_unlock(&lock);
655 published?;
656 Ok(CasOutcome::Stored)
657}
658
659fn flock_exclusive(file: &std::fs::File) -> std::io::Result<()> {
660 let rc = unsafe { flock(file.as_raw_fd(), 2) };
661 if rc == 0 {
662 Ok(())
663 } else {
664 Err(std::io::Error::last_os_error())
665 }
666}
667
668fn flock_unlock(file: &std::fs::File) -> std::io::Result<()> {
669 let rc = unsafe { flock(file.as_raw_fd(), 8) };
670 if rc == 0 {
671 Ok(())
672 } else {
673 Err(std::io::Error::last_os_error())
674 }
675}
676
677unsafe extern "C" {
678 fn flock(fd: i32, operation: i32) -> i32;
679}
680
681fn collect_files(root: &Path, dir: &Path, out: &mut Vec<String>) -> std::io::Result<()> {
682 if !dir.exists() {
683 return Ok(());
684 }
685 for entry in std::fs::read_dir(dir)? {
686 let entry = entry?;
687 let path = entry.path();
688 if path.is_dir() {
689 collect_files(root, &path, out)?;
690 } else if !path.to_string_lossy().ends_with(TMP_SUFFIX)
691 && !path.to_string_lossy().ends_with(".caslock")
692 && let Ok(rel) = path.strip_prefix(root)
693 {
694 out.push(rel.to_string_lossy().replace('\\', "/"));
695 }
696 }
697 Ok(())
698}
699
700#[cfg(test)]
701mod tests {
702 use super::*;
703
704 #[test]
705 fn diagnostic_store_tag_prefers_workload_shard_directory() {
706 assert_eq!(
707 diagnostic_store_tag(Path::new(
708 "/tmp/run/shard-12/log/fwlog/00000000000000000001"
709 )),
710 "shard-12"
711 );
712 assert_eq!(
713 diagnostic_store_tag(Path::new("/data/shard-0/projection.db")),
714 "shard-0"
715 );
716 assert_eq!(object_class("fwlog/1"), "segment");
717 assert_eq!(object_class("fwmeta/manifest/1"), "manifest");
718 assert_eq!(object_class("fwmeta/high-water/1"), "metadata");
719 }
720}