use super::*;
use test_log::test;
#[cfg(zstd_any)]
fn no_dicts() -> crate::compression::ZstdDictionaries {
crate::compression::ZstdDictionaries::new()
}
#[test]
fn vlog_recovery_missing_blob_file_returns_unrecoverable() {
let dir = tempfile::tempdir().unwrap();
let result = recover_blob_files(
dir.path(),
&[(0, Checksum::from_raw(0), 0)],
0,
None,
&(Arc::new(crate::fs::StdFs) as Arc<dyn crate::fs::Fs>),
#[cfg(zstd_any)]
&no_dicts(),
);
assert!(matches!(result, Err(crate::Error::Unrecoverable)));
}
#[test]
fn vlog_recovery_nonexistent_folder_no_ids_returns_empty() {
let dir = tempfile::tempdir().unwrap();
let missing = dir.path().join("no_such_dir");
let (blob_files, orphans) = recover_blob_files(
&missing,
&[],
0,
None,
&(Arc::new(crate::fs::StdFs) as Arc<dyn crate::fs::Fs>),
#[cfg(zstd_any)]
&no_dicts(),
)
.unwrap();
assert!(blob_files.is_empty());
assert_eq!(orphans, Vec::<std::path::PathBuf>::new());
}
#[test]
fn vlog_recovery_nonexistent_folder_with_ids_returns_unrecoverable() {
let dir = tempfile::tempdir().unwrap();
let missing = dir.path().join("no_such_dir");
let result = recover_blob_files(
&missing,
&[(0, Checksum::from_raw(0), 0)],
0,
None,
&(Arc::new(crate::fs::StdFs) as Arc<dyn crate::fs::Fs>),
#[cfg(zstd_any)]
&no_dicts(),
);
assert!(matches!(result, Err(crate::Error::Unrecoverable)));
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn vlog_recovery_sweeps_a_crashed_salvage_copy() {
let dir = tempfile::tempdir().unwrap();
let leftover = dir.path().join("7.salvage-tmp");
std::fs::write(&leftover, b"partial salvage copy").unwrap();
let (blob_files, orphans) = recover_blob_files(
dir.path(),
&[],
0,
None,
&(Arc::new(crate::fs::StdFs) as Arc<dyn crate::fs::Fs>),
#[cfg(zstd_any)]
&no_dicts(),
)
.expect("a repair's own leftover must not make the tree unopenable");
assert!(blob_files.is_empty(), "the leftover is not a blob file");
assert_eq!(orphans, vec![leftover], "it is swept as an orphan");
}
#[test]
fn recover_blob_file_on_non_blob_file_errors() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("000042");
std::fs::write(&path, b"this is not a blob file").unwrap();
let fs: Arc<dyn crate::fs::Fs> = Arc::new(crate::fs::StdFs);
let result = recover_blob_file(
path.as_path(),
42,
Checksum::from_raw(0),
0,
&fs,
#[cfg(zstd_any)]
&no_dicts(),
);
assert!(result.is_err(), "recovering a non-blob file must fail");
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn successive_blob_views_punch_only_above_what_the_last_one_finished() -> crate::Result<()> {
use crate::fs::{FaultFs, Fs, MemFs};
let memfs = MemFs::new();
let root = std::path::absolute("/blobs")?;
memfs.create_dir_all(&root)?;
let path = root.join("0");
{
let fs_dyn: Arc<dyn Fs> = Arc::new(memfs.clone());
let mut w = super::blob_file::writer::Writer::new(path.clone(), 0, 0, &*fs_dyn)?;
for i in 0..3_000u32 {
w.write(
format!("k{i:06}").as_bytes(),
u64::from(i) + 1,
&[b'x'; 200],
)?;
}
w.finish()?;
}
let frame_end = |n: usize| -> crate::Result<u64> {
Ok(super::BlobFileScanner::new(&path, &memfs, 0)?
.nth(n)
.expect("frame")?
.frame_end)
};
let (f1, f2) = (frame_end(999)?, frame_end(1_999)?);
let fault = FaultFs::new(memfs.clone());
let injector = fault.injector();
let fs: Arc<dyn Fs> = Arc::new(fault);
let first = recover_blob_file(
&path,
0,
Checksum::from_raw(0),
0,
&fs,
#[cfg(zstd_any)]
&no_dicts(),
)?;
let second = first.reopen_restricted(f1)?;
first.mark_punch_on_drop(f1);
drop(first);
let third = second.reopen_restricted(f2)?;
second.mark_punch_on_drop(f2);
drop(second);
drop(third);
let punches = injector.punches_for(&path.to_string_lossy());
let floor = f1 - f1 % crate::table::PUNCH_ALIGN;
let [(first_start, first_len), (second_start, second_len)] = punches[..] else {
panic!("one punch per view: {punches:?}");
};
assert_eq!(
f1,
first_start + first_len,
"the first view punched up to its frontier"
);
assert_eq!(
(first_start.max(floor), f2),
(second_start, second_start + second_len),
"the second view punched from the aligned edge under the first one's frontier",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn blob_punch_on_drop_retains_the_reclaim_when_the_link_probe_cannot_answer() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
let memfs = MemFs::new();
let root = std::path::absolute("/blobs")?;
memfs.create_dir_all(&root)?;
let path = root.join("0");
{
let fs_dyn: Arc<dyn Fs> = Arc::new(memfs.clone());
let mut w = super::blob_file::writer::Writer::new(path.clone(), 0, 0, &*fs_dyn)?;
w.write(b"a", 1, &[b'x'; 64])?;
w.write(b"b", 2, &[b'y'; 64])?;
w.finish()?;
}
let first_frame_end = super::BlobFileScanner::new(&path, &memfs, 0)?
.next()
.expect("first frame")?
.frame_end;
let fault = FaultFs::new(memfs.clone());
let injector = fault.injector();
injector.arm(FaultRule::new(
FaultOp::HardLinkCount,
Fault::Error(ErrorKind::PermissionDenied),
));
let fs: Arc<dyn Fs> = Arc::new(fault);
let blob = recover_blob_file(
&path,
0,
Checksum::from_raw(0),
0,
&fs,
#[cfg(zstd_any)]
&no_dicts(),
)?;
let pause = crate::deletion_pause::DeletionPause::new_shared();
blob.install_deletion_pause(Arc::clone(&pause));
blob.mark_punch_on_drop(first_frame_end);
drop(blob);
assert!(
pause.has_pending_reclaims(),
"a blob reclaim whose link probe cannot answer must be retained for a \
retry, not discarded",
);
injector.clear();
pause.retry_pending_reclaims();
assert!(
!pause.has_pending_reclaims(),
"an exclusively-owned blob file's retained reclaim is finished by the retry",
);
let data_start = {
let mut file = memfs.open(&path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
reader
.toc()
.section(b"data")
.expect("blob file has a data section")
.pos()
};
let punched_len = usize::try_from(first_frame_end - data_start).expect("small fixture");
let file = memfs.open(&path, &crate::fs::FsOpenOptions::new().read(true))?;
let prefix = crate::file::read_exact(&*file, data_start, punched_len)?;
assert!(
prefix.iter().all(|&b| b == 0),
"the retried reclaim punches the consumed data prefix",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_pinned_blob_file_waits_for_its_pin() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
let memfs = MemFs::new();
let root = std::path::absolute("/blobs")?;
memfs.create_dir_all(&root)?;
let fs: Arc<dyn Fs> = Arc::new(memfs.clone());
let write = |path: &std::path::Path| -> crate::Result<()> {
let mut w = super::blob_file::writer::Writer::new(path, 0, 0, &*fs)?;
w.write(b"a", 1, &[b'x'; 64])?;
w.write(b"b", 2, &[b'y'; 64])?;
w.finish()?;
Ok(())
};
let recover = |path: &std::path::Path, id| {
recover_blob_file(
path,
id,
Checksum::from_raw(0),
0,
&fs,
#[cfg(zstd_any)]
&no_dicts(),
)
};
let pause = crate::deletion_pause::DeletionPause::new_shared();
let punched = root.join("0");
write(&punched)?;
let first_frame_end = super::BlobFileScanner::new(&punched, &memfs, 0)?
.next()
.expect("first frame")?
.frame_end;
let bytes = |path: &std::path::Path| -> crate::Result<Vec<u8>> {
let file = memfs.open(path, &crate::fs::FsOpenOptions::new().read(true))?;
let len = crate::fs::FsFile::metadata(&*file)?.len;
Ok(crate::file::read_exact(&*file, 0, usize::try_from(len).unwrap_or(0))?.to_vec())
};
let original = bytes(&punched)?;
let blob = recover(&punched, 0)?;
blob.install_deletion_pause(Arc::clone(&pause));
pause.pin_files(core::iter::once(punched.as_path()));
blob.mark_punch_on_drop(first_frame_end);
drop(blob);
assert_eq!(
bytes(&punched)?,
original,
"a pinned blob file keeps its bytes"
);
assert!(
pause.has_pending_reclaims(),
"the reclaim waits for the pin"
);
let removed = root.join("1");
write(&removed)?;
let blob = recover(&removed, 1)?;
blob.install_deletion_pause(Arc::clone(&pause));
pause.pin_files(core::iter::once(removed.as_path()));
blob.mark_as_deleted();
drop(blob);
assert!(memfs.exists(&removed)?, "a pinned blob file stays");
pause.unpin_files([punched.as_path(), removed.as_path()].into_iter());
assert!(!pause.has_pending_reclaims(), "the release ran the reclaim");
assert!(memfs.punched_bytes() > 0, "and punched the consumed prefix");
assert!(!memfs.exists(&removed)?, "and removed the deleted file");
Ok(())
}
#[test]
fn a_pinned_blob_file_outlives_an_open_checkpoint() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
let memfs = MemFs::new();
let root = std::path::absolute("/blobs")?;
memfs.create_dir_all(&root)?;
let fs: Arc<dyn Fs> = Arc::new(memfs.clone());
let path = root.join("0");
{
let mut w = super::blob_file::writer::Writer::new(&path, 0, 0, &*fs)?;
w.write(b"a", 1, &[b'x'; 64])?;
w.finish()?;
}
let blob = recover_blob_file(
&path,
0,
Checksum::from_raw(0),
0,
&fs,
#[cfg(zstd_any)]
&no_dicts(),
)?;
let pause = crate::deletion_pause::DeletionPause::new_shared();
blob.install_deletion_pause(Arc::clone(&pause));
pause.pin_files(core::iter::once(path.as_path()));
let checkpoint = pause.acquire();
blob.mark_as_deleted();
drop(blob);
drop(checkpoint);
assert!(memfs.exists(&path)?, "the pin still holds the file");
pause.unpin_files(core::iter::once(path.as_path()));
assert!(!memfs.exists(&path)?, "the pin's release removed it");
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
#[expect(clippy::indexing_slicing, reason = "test code")]
fn recovery_keeps_the_blob_copy_the_manifest_checksum_names() {
use crate::fs::{Fs, MemFs};
let memfs = MemFs::new();
let fs: Arc<dyn Fs> = Arc::new(memfs.clone());
let blobs = std::path::absolute("/blobs").expect("absolute");
memfs.create_dir_all(&blobs).expect("mkdir");
let mut writer = crate::vlog::BlobFileWriter::new(
crate::SequenceNumberCounter::default(),
&blobs,
0,
None,
Arc::clone(&fs),
)
.expect("writer");
for i in 0..20u32 {
writer
.write(
format!("k{i:04}").as_bytes(),
u64::from(i) + 1,
&[b'v'; 128],
)
.expect("write");
}
let written = writer.finish().expect("finish");
assert_eq!(written.len(), 1, "one blob file");
let authoritative = blobs.join("0");
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs
.open(&authoritative, &crate::fs::FsOpenOptions::new().read(true))
.expect("open"),
&mut bytes,
)
.expect("read");
let checksum = Checksum::from_raw(
crate::file::checksum_from_with_overrides(&*fs, &authoritative, 0, &[]).expect("digest"),
);
let mut rotted = bytes.clone();
let Some(byte) = rotted.get_mut(64) else {
panic!("the written blob reaches the flipped offset");
};
*byte ^= 0xFF;
let mut f = memfs
.open(
&blobs.join("00"),
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)
.expect("create twin");
std::io::Write::write_all(&mut f, &rotted).expect("write twin");
drop(f);
let (recovered, orphans) = recover_blob_files(
&blobs,
&[(0, checksum, 0)],
0,
None,
&fs,
#[cfg(zstd_any)]
&no_dicts(),
)
.expect("recover");
assert_eq!(recovered.len(), 1, "one copy per id survives");
assert_eq!(
&*recovered[0].0.path, &*authoritative,
"the copy whose digest matches the manifest is the one kept",
);
assert!(
orphans.iter().any(|p| p.ends_with("00")),
"the losing copy is reported as an orphan for the caller to sweep: {orphans:?}",
);
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_refused_digest_read_does_not_decide_the_blob_duplicate() {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
let memfs = MemFs::new();
let fault = FaultFs::new(memfs.clone());
let injector = fault.injector();
let fs: Arc<dyn Fs> = Arc::new(fault);
let blobs = std::path::absolute("/blobs").expect("absolute");
memfs.create_dir_all(&blobs).expect("mkdir");
let mut writer = crate::vlog::BlobFileWriter::new(
crate::SequenceNumberCounter::default(),
&blobs,
0,
None,
Arc::clone(&fs),
)
.expect("writer");
for i in 0..20u32 {
writer
.write(
format!("k{i:04}").as_bytes(),
u64::from(i) + 1,
&[b'v'; 128],
)
.expect("write");
}
assert_eq!(writer.finish().expect("finish").len(), 1, "one blob file");
let canonical = blobs.join("0");
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs
.open(&canonical, &crate::fs::FsOpenOptions::new().read(true))
.expect("open"),
&mut bytes,
)
.expect("read");
let checksum = Checksum::from_raw(
crate::file::checksum_from_with_overrides(&*fs, &canonical, 0, &[]).expect("digest"),
);
let authoritative = blobs.join("00");
memfs.rename(&canonical, &authoritative).expect("rename");
let mut rotted = bytes;
let Some(byte) = rotted.get_mut(64) else {
panic!("the written blob reaches the flipped offset");
};
*byte ^= 0xFF;
let mut f = memfs
.open(
&canonical,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)
.expect("create twin");
std::io::Write::write_all(&mut f, &rotted).expect("write twin");
drop(f);
injector.arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::PermissionDenied))
.on_path(authoritative.display().to_string())
.skip(1),
);
let result = recover_blob_files(
&blobs,
&[(0, checksum, 0)],
0,
None,
&fs,
#[cfg(zstd_any)]
&no_dicts(),
);
injector.clear();
assert!(
matches!(result, Err(crate::Error::Io(ref e)) if e.kind() == ErrorKind::PermissionDenied),
"a refused digest read must surface, not hand the id to the stale copy",
);
assert!(
memfs.exists(&authoritative).expect("stat"),
"the healthy copy must still be on disk for the retry",
);
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn recovery_refuses_when_no_blob_duplicate_matches_the_manifest() {
use crate::fs::{Fs, MemFs};
use std::io::Write;
let memfs = MemFs::new();
let fs: Arc<dyn Fs> = Arc::new(memfs.clone());
let blobs = std::path::absolute("/blobs_no_match").expect("absolute");
memfs.create_dir_all(&blobs).expect("mkdir");
let mut writer = crate::vlog::BlobFileWriter::new(
crate::SequenceNumberCounter::default(),
&blobs,
0,
None,
Arc::clone(&fs),
)
.expect("writer");
for i in 0..20u32 {
writer
.write(
format!("k{i:04}").as_bytes(),
u64::from(i) + 1,
&[b'v'; 128],
)
.expect("write");
}
assert_eq!(writer.finish().expect("finish").len(), 1, "one blob file");
let canonical = blobs.join("0");
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs
.open(&canonical, &crate::fs::FsOpenOptions::new().read(true))
.expect("open"),
&mut bytes,
)
.expect("read");
let checksum = Checksum::from_raw(
crate::file::checksum_from_with_overrides(&*fs, &canonical, 0, &[]).expect("digest"),
);
for (path, at) in [(canonical.clone(), 64usize), (blobs.join("00"), 96usize)] {
let mut damaged = bytes.clone();
let Some(byte) = damaged.get_mut(at) else {
panic!("the written blob reaches the flipped offset");
};
*byte ^= 0xFF;
memfs.remove_file(&path).ok();
let mut f = memfs
.open(
&path,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)
.expect("create copy");
f.write_all(&damaged).expect("write copy");
}
let result = recover_blob_files(
&blobs,
&[(0, checksum, 0)],
0,
None,
&fs,
#[cfg(zstd_any)]
&no_dicts(),
);
assert!(
matches!(result, Err(crate::Error::Unrecoverable)),
"with no copy matching the manifest, picking one by filename would open \
the wrong generation and delete the other",
);
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_reopen_reclaims_a_committed_blob_prefix() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, AnyTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::{Read as _, Seek as _, SeekFrom, Write as _};
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db_blob_reclaim")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(64),
))
};
{
let AnyTree::Blob(tree) = config().open()? else {
panic!("expected Blob tree");
};
for i in 0..64u64 {
tree.insert(format!("k{i:05}").as_bytes(), alloc::vec![b'v'; 256], i);
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join("blobs");
let blob = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("the flush spilled a blob file")
.path;
let data_start = {
let mut file = memfs.open(&blob, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
reader
.toc()
.section(b"data")
.expect("a blob file has a data section")
.pos()
};
let original = {
let mut bytes = Vec::new();
memfs
.open(&blob, &crate::fs::FsOpenOptions::new().read(true))?
.read_to_end(&mut bytes)?;
bytes
};
let frontier = super::BlobFileScanner::new(&blob, &*memfs, 0)?
.next()
.expect("the blob file holds at least one frame")?
.frame_end;
for e in memfs.read_dir(&root)? {
if crate::version::recovery::snapshot_id_of(&e.file_name).is_some()
|| crate::version::recovery::log_id_of(&e.file_name).is_some()
{
memfs.remove_file(&e.path)?;
}
}
memfs.punch_hole(&blob, data_start, frontier - data_start)?;
config().repair()?;
{
let mut file = memfs.open(&blob, &crate::fs::FsOpenOptions::new().write(true))?;
file.seek(SeekFrom::Start(data_start))?;
let end = usize::try_from(frontier).expect("test offsets fit usize");
let begin = usize::try_from(data_start).expect("test offsets fit usize");
file.write_all(original.get(begin..end).expect("prefix in range"))?;
file.flush()?;
}
let allocated_before = memfs
.allocated_size(&blob)?
.expect("MemFs reports allocation");
let AnyTree::Blob(tree) = config().open()? else {
panic!("expected Blob tree");
};
let allocated_after = memfs
.allocated_size(&blob)?
.expect("MemFs reports allocation");
assert!(
allocated_after < allocated_before,
"the reopen must re-derive the blob's punch intent \
(allocated {allocated_before} before, {allocated_after} after)",
);
assert!(
tree.get(b"k00063", crate::MAX_SEQNO)?.is_some(),
"the live suffix must still be served",
);
Ok(())
}