use super::{
commit_repair_tmp, compute_table_checksum, compute_table_checksum_with_overrides,
discard_unreferenced, highest_existing_version_id, repair_tmp_path, toc_may_hide_deletions,
verify_keep_decision,
};
use crate::fs::StdFs;
use test_log::test;
#[test]
fn toc_may_hide_deletions_propagates_a_transient_open_failure() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs};
use crate::io::ErrorKind;
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let source = dir.path().join("source");
let fault = FaultFs::new(StdFs);
let injector = fault.injector();
let fs: Arc<dyn Fs> = Arc::new(fault);
let mut writer = crate::table::Writer::new(source.clone(), 0, 0, Arc::clone(&fs))?;
writer.write(crate::InternalValue::from_components(
b"k".to_vec(),
b"v".to_vec(),
1,
crate::ValueType::Value,
))?;
assert!(writer.finish()?.is_some(), "source SST is non-empty");
injector.arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::Interrupted))
.on_path("source")
.once(),
);
let result = toc_may_hide_deletions(&fs, &source);
assert!(
matches!(result, Err(crate::Error::Io(_))),
"a transient open failure must propagate, not grade the catalogue as hiding a \
deletion section: {result:?}",
);
Ok(())
}
#[test]
fn toc_may_hide_deletions_fails_closed_on_a_persistent_open_failure() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs};
use crate::io::ErrorKind;
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let source = dir.path().join("source");
let fault = FaultFs::new(StdFs);
let injector = fault.injector();
let fs: Arc<dyn Fs> = Arc::new(fault);
let mut writer = crate::table::Writer::new(source.clone(), 0, 0, Arc::clone(&fs))?;
writer.write(crate::InternalValue::from_components(
b"k".to_vec(),
b"v".to_vec(),
1,
crate::ValueType::Value,
))?;
assert!(writer.finish()?.is_some(), "source SST is non-empty");
injector.arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::Other))
.on_path("source")
.once(),
);
let result = toc_may_hide_deletions(&fs, &source);
assert!(
matches!(result, Ok(true)),
"a persistent open failure must fail closed (drop), not propagate: {result:?}",
);
Ok(())
}
#[test]
fn checksum_with_overrides_skips_non_overlapping_overrides_across_chunks() -> crate::Result<()> {
use crate::fs::{Fs, FsOpenOptions};
use std::io::Write;
let dir = tempfile::tempdir()?;
let path = dir.path().join("multi-chunk.sst");
let len: usize = 3 * 256 * 1024 + 777;
let mut data: Vec<u8> = (0..len)
.map(|i| u8::try_from(i % 251).unwrap_or(0))
.collect();
let fs = StdFs;
{
let mut f = fs.open(
&path,
&FsOpenOptions::new().write(true).create(true).truncate(true),
)?;
f.write_all(&data)?;
}
let ov_off: usize = 100;
let ov_bytes = vec![0xABu8; 4096];
let overrides = vec![(ov_off as u64, ov_bytes.clone())];
if let Some(slot) = data.get_mut(ov_off..ov_off + ov_bytes.len()) {
slot.copy_from_slice(&ov_bytes);
}
let mut hasher = xxhash_rust::xxh3::Xxh3Default::new();
hasher.update(&data);
let expected = hasher.digest128();
let got = compute_table_checksum_with_overrides(&fs, &path, 0, &overrides)?;
assert_eq!(
got, expected,
"the spliced digest must match a manual splice and must not panic on \
chunks the override does not overlap",
);
Ok(())
}
#[test]
fn discard_unreferenced_syncs_the_directory() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, SyncMode};
use crate::io::ErrorKind;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let src = tables.join("junk-name");
std::fs::write(&src, b"orphan")?;
let fs = FaultFs::new(StdFs);
fs.injector().arm(
FaultRule::new(FaultOp::SyncDirectory, Fault::Error(ErrorKind::Other)).on_path("tables"),
);
assert!(
discard_unreferenced(&fs, &src, SyncMode::Full).is_err(),
"the directory fsync fault must surface",
);
Ok(())
}
#[test]
fn discard_unreferenced_treats_a_missing_file_as_done() -> crate::Result<()> {
use crate::fs::SyncMode;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
discard_unreferenced(&StdFs, &tables.join("7"), SyncMode::Normal)?;
Ok(())
}
#[test]
fn discard_unreferenced_removes_the_restriction_sidecar_too() -> crate::Result<()> {
use crate::fs::{Fs, SyncMode};
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("7");
std::fs::write(&sst, b"table")?;
let sidecar = crate::restrict_bound::sidecar_path(&sst);
std::fs::write(&sidecar, b"bound")?;
discard_unreferenced(&StdFs, &sst, SyncMode::Normal)?;
assert!(!StdFs.exists(&sst)?, "the table is removed");
assert!(
!StdFs.exists(&sidecar)?,
"its restriction bound must not outlive it",
);
Ok(())
}
#[test]
fn wal_replay_scope_derives_from_lost_coverage() {
let report = |lost: Vec<(
std::path::PathBuf,
crate::UserKey,
crate::UserKey,
Option<u64>,
)>| {
super::RepairReport {
recovered: 0,
salvaged: 0,
unreadable: 0,
unreadable_files: Vec::new(),
excluded_files: Vec::new(),
lost_coverage: lost,
unknowable_losses: Vec::new(),
blob_files_salvaged: Vec::new(),
method: "test",
warnings: Vec::new(),
}
};
let entry = |bound: Option<u64>| {
(
std::path::PathBuf::from("tables/0"),
crate::UserKey::from(b"a".as_slice()),
crate::UserKey::from(b"z".as_slice()),
bound,
)
};
assert_eq!(
report(Vec::new()).wal_replay_scope(),
super::WalReplayScope::TailOnly,
"no loss: the tail replay is sufficient",
);
assert_eq!(
report(vec![entry(Some(40)), entry(Some(70))]).wal_replay_scope(),
super::WalReplayScope::LostUpTo(70),
"bounded losses aggregate to the highest bound",
);
assert_eq!(
report(vec![entry(Some(40)), entry(None)]).wal_replay_scope(),
super::WalReplayScope::FullHistory,
"one unknown bound means no seqno scopes the damage",
);
let mut with_unknowable = report(vec![entry(Some(40))]);
with_unknowable
.unknowable_losses
.push(std::path::PathBuf::from("tables/9"));
assert_eq!(
with_unknowable.wal_replay_scope(),
super::WalReplayScope::FullHistory,
"an exclusion whose coverage never parsed cannot be scoped by any bound",
);
}
#[test]
fn commit_repair_tmp_replaces_the_source_and_carries_its_sidecar() -> crate::Result<()> {
use crate::fs::{Fs, SyncMode};
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("7");
std::fs::write(&sst, b"damaged")?;
std::fs::write(crate::restrict_bound::sidecar_path(&sst), b"stale-bound")?;
let tmp = repair_tmp_path(&sst);
std::fs::write(&tmp, b"replacement")?;
std::fs::write(crate::restrict_bound::sidecar_path(&tmp), b"fresh-bound")?;
commit_repair_tmp(&StdFs, &tmp, &sst, SyncMode::Normal, true)?;
assert_eq!(
std::fs::read(&sst)?,
b"replacement",
"the replacement takes the name the manifest gives it",
);
assert!(!StdFs.exists(&tmp)?, "nothing is left under the temp name");
assert_eq!(
std::fs::read(crate::restrict_bound::sidecar_path(&sst))?,
b"fresh-bound",
"the replacement's own bound replaces the source's",
);
Ok(())
}
#[test]
fn commit_repair_tmp_clears_a_stale_sidecar_when_the_replacement_has_none() -> crate::Result<()> {
use crate::fs::{Fs, SyncMode};
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("7");
std::fs::write(&sst, b"damaged")?;
let sidecar = crate::restrict_bound::sidecar_path(&sst);
std::fs::write(&sidecar, b"stale-bound")?;
let tmp = repair_tmp_path(&sst);
std::fs::write(&tmp, b"replacement")?;
commit_repair_tmp(&StdFs, &tmp, &sst, SyncMode::Normal, false)?;
assert_eq!(std::fs::read(&sst)?, b"replacement");
assert!(
!StdFs.exists(&sidecar)?,
"an unrestricted replacement must not inherit the source's bound",
);
Ok(())
}
#[test]
fn commit_repair_tmp_keeps_the_moved_sidecar_on_a_retried_swap() -> crate::Result<()> {
use crate::fs::SyncMode;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("7");
std::fs::write(&sst, b"damaged")?;
let tmp = repair_tmp_path(&sst);
std::fs::write(&tmp, b"replacement")?;
let dest_sidecar = crate::restrict_bound::sidecar_path(&sst);
std::fs::write(&dest_sidecar, b"fresh-bound")?;
commit_repair_tmp(&StdFs, &tmp, &sst, SyncMode::Normal, true)?;
assert_eq!(std::fs::read(&sst)?, b"replacement");
assert_eq!(
std::fs::read(&dest_sidecar)?,
b"fresh-bound",
"the replacement's already-moved sidecar must survive the retried swap",
);
Ok(())
}
#[test]
fn compute_table_checksum_matches_oneshot_xxh3() -> crate::Result<()> {
let dir = tempfile::tempdir()?;
let path = dir.path().join("000007");
let payload: Vec<u8> = (0..600_000u32).map(|i| (i % 251) as u8).collect();
std::fs::write(&path, &payload)?;
let got = compute_table_checksum(&StdFs, &path)?;
let expected = xxhash_rust::xxh3::xxh3_128(&payload);
assert_eq!(
got, expected,
"streamed digest must equal the one-shot xxh3-128 digest",
);
Ok(())
}
#[test]
fn highest_existing_version_id_picks_the_max_and_ignores_non_versions() -> crate::Result<()> {
let dir = tempfile::tempdir()?;
for name in ["v2", "v10", "v3", "current", "vNaN", "notaversion"] {
std::fs::write(dir.path().join(name), b"x")?;
}
assert_eq!(highest_existing_version_id(&StdFs, dir.path())?, Some(10));
Ok(())
}
#[test]
fn highest_existing_version_id_none_when_no_versions_present() -> crate::Result<()> {
let dir = tempfile::tempdir()?;
std::fs::write(dir.path().join("current"), b"x")?;
assert_eq!(highest_existing_version_id(&StdFs, dir.path())?, None);
Ok(())
}
#[test]
fn repair_rejects_an_exhausted_version_id_space() -> crate::Result<()> {
use crate::{Config, SequenceNumberCounter};
let dir = tempfile::tempdir()?;
std::fs::create_dir_all(dir.path().join("tables"))?;
std::fs::write(dir.path().join(format!("v{}", u64::MAX - 1)), b"x")?;
let result = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair();
assert!(
matches!(result, Err(crate::Error::Unrecoverable)),
"a rebuilt id of u64::MAX must fail the repair, not publish a tree \
whose next version edit overflows: {:?}",
result.map(|r| r.recovered),
);
Ok(())
}
#[test]
fn repair_reports_an_unknown_seqno_bound_for_a_lost_ingest_offset() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_bulk_ingested(Some(true));
for i in 0..4u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
b"v",
0,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(
report.unreadable, 1,
"the table is dropped: its sequence base is unrecoverable",
);
let [(path, first, last, seqno)] = report.lost_coverage.as_slice() else {
panic!(
"the dropped table's coverage must be reported, got {:?}",
report.lost_coverage,
);
};
assert_eq!(path, &sst, "the entry names the dropped file");
assert_eq!(&**first, b"k00000", "the key range IS knowable");
assert_eq!(&**last, b"k00003", "the key range IS knowable");
assert_eq!(
*seqno, None,
"the sequence bound is not: publishing the local maximum would scope \
the affected history far too low",
);
Ok(())
}
#[test]
fn repair_reports_the_key_coverage_an_excluded_table_lost() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
b"v",
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let offset = sole_data_block_offset(&recover_table(sst.clone(), &fs)?);
let flip = usize::try_from(offset).unwrap_or(0) + 16;
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(report.unreadable, 1, "the damaged table is excluded");
let [(path, first, last, seqno)] = report.lost_coverage.as_slice() else {
panic!(
"the excluded table's coverage must be reported, got {:?}",
report.lost_coverage,
);
};
assert_eq!(path, &sst, "the entry names the excluded file");
assert_eq!(&**first, b"k00000", "first key of the lost range");
assert_eq!(&**last, b"k00007", "last key of the lost range");
assert_eq!(
*seqno,
Some(8),
"the highest seqno the lost table held: at or below it, keys in that \
range may now serve a superseded value",
);
Ok(())
}
#[test]
fn plain_repair_drops_a_table_with_a_corrupt_data_block() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let offset = sole_data_block_offset(&recover_table(sst.clone(), &fs)?);
let flip = usize::try_from(offset).unwrap_or(0) + 16;
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(
report.recovered, 0,
"a table that errors on read must not be blessed into the manifest: {report:?}",
);
assert_eq!(
report.unreadable, 1,
"the damaged table is set aside and reported: {:?}",
report.unreadable_files,
);
let [(_, reason)] = report.unreadable_files.as_slice() else {
panic!(
"expected exactly one unreadable file, got {:?}",
report.unreadable_files,
);
};
assert!(
reason.contains("salvage"),
"the reason points the operator at the salvage-enabled repair, got: {reason}",
);
Ok(())
}
#[test]
fn repair_with_salvage_reports_a_sole_corrupt_block_as_unsalvageable() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let offset = sole_data_block_offset(&recover_table(sst.clone(), &fs)?);
let flip = usize::try_from(offset).unwrap_or(0) + 16;
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.salvaged, 0,
"the sole block is corrupt: nothing to salvage",
);
assert_eq!(report.recovered, 0, "no table joins the rebuilt manifest");
assert_eq!(
report.unreadable, 1,
"the unsalvageable SST is reported: {:?}",
report.unreadable_files,
);
let [(_, reason)] = report.unreadable_files.as_slice() else {
panic!(
"expected exactly one unreadable file, got {:?}",
report.unreadable_files,
);
};
assert!(
reason.contains("nothing salvageable"),
"the reason names the empty salvage, got: {reason}",
);
Ok(())
}
#[test]
fn repair_with_salvage_recovers_a_table_whose_whole_file_hash_faults() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs};
use crate::io::ErrorKind;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
{
let build_fs: Arc<dyn Fs> = Arc::new(StdFs);
let mut w = Writer::new(sst, 0, 0, Arc::clone(&build_fs))?.use_data_block_size(128);
for i in 0..64u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let fault = FaultFs::new(StdFs);
let injector = fault.injector();
injector.arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Other))
.on_path("tables")
.once(),
);
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
injector.clear();
assert_eq!(
report.unreadable, 0,
"a whole-file hash fault must not record the table unreadable: {:?}",
report.unreadable_files,
);
assert_eq!(
report.salvaged, 1,
"the table is recovered through block-salvage"
);
assert_eq!(
report.recovered, 1,
"the salvaged table joins the rebuilt manifest"
);
Ok(())
}
#[test]
fn repair_restricts_a_punched_sst_whose_whole_file_recovery_faults() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = build_punched_prefix_sst(&memfs, &fs, &tables)?;
let bound = b"k00050".to_vec();
crate::restrict_bound::write(&*fs, &sst, None, 0, &bound, crate::fs::SyncMode::Normal)?;
let fault = FaultFs::new(memfs.as_ref().clone());
let injector = fault.injector();
injector.arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Other))
.on_path("tables")
.once(),
);
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
injector.clear();
assert_eq!(
report.recovered, 1,
"the salvaged table joins the manifest: {report:?}",
);
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
for i in 0..50u32 {
let key = format!("k{i:05}").into_bytes();
assert!(
tree.get(&key, crate::MAX_SEQNO)?.is_none(),
"sub-bound key {} must not resurrect after recovery-failure salvage",
String::from_utf8_lossy(&key),
);
}
assert!(
tree.get(b"k00200", crate::MAX_SEQNO)?.is_some(),
"the live suffix must survive salvage",
);
Ok(())
}
#[test]
fn repair_with_salvage_drops_a_bulk_ingested_sst_that_fails_recovery() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs};
use crate::io::ErrorKind;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
{
let build_fs: Arc<dyn Fs> = Arc::new(StdFs);
let mut w = Writer::new(sst, 0, 0, Arc::clone(&build_fs))?
.use_bulk_ingested(Some(true))
.use_data_block_size(128);
for i in 0..64u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
0,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let fault = FaultFs::new(StdFs);
let injector = fault.injector();
injector.arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Other))
.on_path("tables")
.once(),
);
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
injector.clear();
assert_eq!(
report.recovered, 0,
"a bulk-ingested SST whose offset is lost must not be salvaged-and-kept: {report:?}",
);
assert_eq!(
report.unreadable, 1,
"the bulk-ingested SST is reported unreadable: {:?}",
report.unreadable_files,
);
let leftovers: Vec<String> = std::fs::read_dir(dir.path().join("tables"))
.into_iter()
.flatten()
.flatten()
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
assert!(
leftovers.is_empty(),
"the tables folder must be empty, found {leftovers:?}",
);
Ok(())
}
#[test]
fn repair_with_salvage_drops_a_legacy_seqno0_sst_that_fails_recovery() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs};
use crate::io::ErrorKind;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
{
let build_fs: Arc<dyn Fs> = Arc::new(StdFs);
let mut w = Writer::new(sst, 0, 0, Arc::clone(&build_fs))?
.use_bulk_ingested(None)
.use_data_block_size(128);
for i in 0..64u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
0,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let fault = FaultFs::new(StdFs);
let injector = fault.injector();
injector.arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Other))
.on_path("tables")
.once(),
);
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
injector.clear();
assert_eq!(
report.recovered, 0,
"a legacy seqno-0 SST of unknown provenance must not be salvaged-and-kept: {report:?}",
);
assert_eq!(
report.unreadable, 1,
"the legacy seqno-0 SST is reported unreadable: {:?}",
report.unreadable_files,
);
Ok(())
}
#[test]
fn repair_with_salvage_aborts_on_a_transient_salvage_failure() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs};
use crate::io::ErrorKind;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fault = FaultFs::new(StdFs);
let injector = fault.injector();
{
let build_fs: Arc<dyn Fs> = Arc::new(StdFs);
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&build_fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
let offset = sole_data_block_offset(&recover_table(sst.clone(), &build_fs)?);
let flip = usize::try_from(offset).unwrap_or(0) + 16;
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
}
injector.arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::Interrupted))
.on_path(super::REPAIR_TMP_SUFFIX)
.once(),
);
let result = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true);
injector.clear();
assert!(
result.is_err(),
"a transient salvage failure must abort the repair, not commit without the table: \
{result:?}",
);
assert!(
sst.exists(),
"the source stays at its path so a retry can salvage it",
);
assert!(
!super::repair_tmp_path(&sst).exists(),
"the half-written replacement must not be left where a retry could adopt it",
);
Ok(())
}
#[test]
fn repair_with_salvage_reports_a_range_tombstone_sst_as_unsalvageable() -> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let offset = sole_data_block_offset(&recover_table(sst.clone(), &fs)?);
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(usize::try_from(offset).unwrap_or(0) + 16) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.salvaged, 0,
"salvage refuses an SST with range tombstones",
);
assert_eq!(report.recovered, 0, "no table joins the rebuilt manifest");
let [(_, reason)] = report.unreadable_files.as_slice() else {
panic!(
"expected exactly one unreadable file, got {:?}",
report.unreadable_files,
);
};
assert!(
reason.contains("salvage failed") && reason.contains("range tombstones"),
"the reason names the failed salvage, got: {reason}",
);
Ok(())
}
#[test]
fn repair_aborts_on_a_transient_checksum_failure() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs};
use crate::io::ErrorKind;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fault = FaultFs::new(StdFs);
let injector = fault.injector();
{
let build_fs: Arc<dyn Fs> = Arc::new(StdFs);
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&build_fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
injector.arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Interrupted))
.on_path("tables")
.once(),
);
let result = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true);
injector.clear();
assert!(
result.is_err(),
"a transient checksum-hash failure must abort the repair, not record the table \
unreadable and commit without it: {result:?}",
);
assert!(
sst.exists(),
"the healthy SST must be left in place for a retry",
);
Ok(())
}
#[cfg(feature = "columnar")]
#[test]
fn repair_with_salvage_reports_a_corrupt_bitmap_and_block_sst_as_unsalvageable() -> crate::Result<()>
{
use crate::config::DeleteStrategy;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?
.use_columnar(true)
.use_zone_map(true)
.delete_strategy(DeleteStrategy::MergeOnRead);
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
for pos in [2u32, 6] {
w.delete_bitmap_mut().insert(pos);
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let block_offset = sole_data_block_offset(&recover_table(sst.clone(), &fs)?);
let bitmap = {
let mut f = std::fs::File::open(&sst)?;
let reader = match crate::sfa::Reader::from_reader(&mut f) {
Ok(r) => r,
Err(e) => panic!("reading the SFA trailer failed: {e:?}"),
};
let Some(entry) = reader.toc().iter().find(|e| e.name() == b"delete_bitmap") else {
panic!("the SST must carry a delete_bitmap section");
};
usize::try_from(entry.pos() + entry.len() / 2).unwrap_or(0)
};
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(usize::try_from(block_offset).unwrap_or(0) + 16) {
*b ^= 0xFF;
}
if let Some(b) = bytes.get_mut(bitmap) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.salvaged, 0,
"the bitmap is unreadable: automated salvage fails closed"
);
assert_eq!(report.recovered, 0, "no table joins the rebuilt manifest");
let [(_, reason)] = report.unreadable_files.as_slice() else {
panic!(
"expected exactly one unreadable file, got {:?}",
report.unreadable_files,
);
};
assert!(
reason.contains("salvage failed"),
"the reason names the failed salvage, got: {reason}",
);
Ok(())
}
#[test]
fn repair_with_salvage_rewrites_an_unrecognized_ecc_sst() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
let n = 200u32;
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..n {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
forge_unrecognized_ecc_descriptor(&sst)?;
assert!(
recover_table(sst.clone(), &fs)?.metadata.ecc_unrecognized,
"the forged descriptor must flag the table as unrecognized-ECC",
);
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.salvaged, 1,
"an unverifiable unrecognized-ECC table is rewritten, not accepted \
as-is: {report:?}",
);
assert!(
report.unreadable_files.is_empty(),
"every row is recoverable: {:?}",
report.unreadable_files,
);
let table = recover_table(sst, &fs)?;
assert!(
!table.metadata.ecc_unrecognized,
"the salvaged copy is re-stamped with a recognized descriptor",
);
assert_eq!(table.metadata.item_count, u64::from(n));
Ok(())
}
fn recover_table(
path: std::path::PathBuf,
fs: &std::sync::Arc<dyn crate::fs::Fs>,
) -> crate::Result<crate::table::Table> {
use std::sync::Arc;
let checksum = crate::Checksum::from_raw(super::compute_table_checksum(&**fs, &path)?);
let mut params = crate::table::RecoverParams::new(
path,
checksum,
0,
Arc::clone(fs),
crate::comparator::default_comparator(),
Arc::new(crate::cache::Cache::with_capacity_bytes(1 << 20)),
);
params.descriptor_table = Some(Arc::new(crate::descriptor_table::DescriptorTable::new(8)));
crate::table::Table::recover(params)
}
fn sole_data_block_offset(table: &crate::table::Table) -> u64 {
let offsets: alloc::vec::Vec<u64> = table
.data_block_handles()
.filter_map(Result::ok)
.map(|kh| *kh.as_ref().offset())
.collect();
let [only] = offsets.as_slice() else {
panic!("expected a single data block, got {offsets:?}");
};
*only
}
#[test]
fn repair_with_salvage_propagates_a_transient_verify_io_error() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule};
use crate::io::ErrorKind;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let offset = sole_data_block_offset(&recover_table(sst.clone(), &fs)?);
let fault = FaultFs::new(StdFs);
let injector = fault.injector();
injector.arm(
FaultRule::new(FaultOp::ReadAt, Fault::Error(ErrorKind::Interrupted))
.at_offset(offset)
.once(),
);
let result = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(Arc::new(fault))
.repair_with_salvage(true);
injector.clear();
assert!(
result.is_err(),
"a transient read error during block verification must abort the repair \
(not salvage a healthy block into missing data), got {result:?}",
);
assert!(
fs.metadata(&sst).is_ok(),
"the original file stays in place after the aborted repair",
);
Ok(())
}
#[test]
fn repair_preserves_a_pending_heal_attestation_sidecar() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let sidecar = tables.join("0.heal-attest");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst, 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
std::fs::write(&sidecar, b"pending-heal-marker")?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(report.recovered, 1, "the table is recovered: {report:?}");
assert!(
sidecar.exists(),
"the heal-attest sidecar stays next to its SST (not removed)",
);
Ok(())
}
#[test]
fn repair_drops_a_table_with_an_unrecoverable_ingest_offset() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst, 0, 0, Arc::clone(&fs))?.use_bulk_ingested(Some(true));
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
0,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(
report.recovered, 0,
"a table with an unrecoverable ingest offset must not join the manifest: {report:?}",
);
assert_eq!(
report.unreadable, 1,
"the ambiguous-offset table is reported unreadable: {:?}",
report.unreadable_files,
);
let [(_, reason)] = report.unreadable_files.as_slice() else {
panic!(
"expected exactly one unreadable file, got {:?}",
report.unreadable_files,
);
};
assert!(
reason.contains("sequence offset"),
"the reason names the unrecoverable ingest offset, got: {reason}",
);
Ok(())
}
#[test]
fn has_unrecoverable_ingest_offset_classifies_provenance() {
use super::has_unrecoverable_ingest_offset;
assert!(has_unrecoverable_ingest_offset(Some(true), 8, 0));
assert!(has_unrecoverable_ingest_offset(Some(true), 8, 5));
assert!(!has_unrecoverable_ingest_offset(Some(false), 8, 0));
assert!(!has_unrecoverable_ingest_offset(Some(false), 8, 5));
assert!(has_unrecoverable_ingest_offset(None, 8, 0));
assert!(!has_unrecoverable_ingest_offset(None, 8, 5));
assert!(!has_unrecoverable_ingest_offset(None, 0, 0));
}
fn recover_sst(
path: std::path::PathBuf,
fs: &std::sync::Arc<dyn crate::fs::Fs>,
) -> crate::Result<crate::Table> {
let checksum = crate::Checksum::from_raw(compute_table_checksum(&**fs, &path)?);
let mut params = crate::table::RecoverParams::new(
path,
checksum,
0,
std::sync::Arc::clone(fs),
crate::comparator::default_comparator(),
std::sync::Arc::new(crate::cache::Cache::with_capacity_bytes(1 << 20)),
);
params.descriptor_table = Some(std::sync::Arc::new(
crate::descriptor_table::DescriptorTable::new(8),
));
crate::Table::recover(params)
}
#[test]
fn repair_restricts_a_tight_space_punched_table() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::table::Writer;
use crate::{AbstractTree, Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..256u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let bound = b"k00130".to_vec();
{
let table = recover_sst(sst.clone(), &fs)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, &bound, crate::fs::SyncMode::Normal)?;
let punch_offset = table.punch_offset_for(&bound)?;
memfs.punch_hole(&sst, 0, punch_offset)?;
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair()?;
assert_eq!(
report.recovered, 1,
"the punched table is recovered restricted: {report:?}",
);
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
for i in 0..256u32 {
let key = format!("k{i:05}").into_bytes();
let served = tree.get(&key, crate::MAX_SEQNO)?.is_some();
assert_eq!(
served,
key.as_slice() >= bound.as_slice(),
"key {key:?} served={served}; expected served == (key >= {bound:?}) \
(a served key below the bound is resurrected; an unserved key at/above \
it is lost)",
);
}
Ok(())
}
#[test]
fn a_clean_manifest_restriction_survives_a_lost_sidecar() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::table::block_index::BlockIndex;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db_manifest_bound")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
};
write_multiblock_sst(&sst, &fs)?;
let bound;
{
let table = recover_sst(sst.clone(), &fs)?;
bound = 'pick: {
for i in 125..140u32 {
let cand = format!("k{i:05}").into_bytes();
let straddle_offset = table.punch_offset_for(&cand)?;
for handle in table.block_index.iter() {
let handle = handle?;
if handle.offset().0 == straddle_offset
&& handle.end_key().as_ref() > cand.as_slice()
{
break 'pick cand;
}
}
}
panic!("a strictly-mid-block bound exists in the fixture");
};
crate::restrict_bound::write(&*fs, &sst, None, 0, &bound, crate::fs::SyncMode::Normal)?;
let punch_offset = table.punch_offset_for(&bound)?;
memfs.punch_hole(&sst, 0, punch_offset)?;
}
let report = config().repair()?;
assert_eq!(report.recovered, 1, "fixture repair commits: {report:?}");
{
let tree = config().open()?;
tree.insert(b"zzz", b"v", 1000);
tree.flush_active_memtable(0)?;
}
memfs.remove_file(&crate::restrict_bound::sidecar_path(&sst))?;
let second = memfs
.read_dir(&tables)?
.into_iter()
.find(|e| !e.is_dir && e.file_name != "0" && e.file_name.parse::<u64>().is_ok())
.map(|e| e.path)
.ok_or_else(|| crate::Error::Unrecoverable)?;
memfs.remove_file(&second)?;
let report = config().repair()?;
assert_eq!(
report.recovered, 1,
"the restricted survivor is kept at the manifest's bound: {report:?}",
);
let tree = config().open()?;
for i in 0..256u32 {
let key = format!("k{i:05}").into_bytes();
let served = tree.get(&key, crate::MAX_SEQNO)?.is_some();
assert_eq!(
served,
key.as_slice() >= bound.as_slice(),
"key {key:?} served={served}; the manifest bound is exact — a \
served key below it is resurrected, an unserved key at/above it \
(the straddling block's live suffix) is lost to the geometry \
fallback",
);
}
Ok(())
}
#[test]
fn a_stale_sidecar_cannot_restrict_a_table_the_manifest_calls_unrestricted() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db_stale_sidecar")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
};
{
let tree = match config().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), b"v", u64::from(i) + 1);
}
tree.flush_active_memtable(0)?;
tree.insert(b"zzz", b"v", 100);
tree.flush_active_memtable(0)?;
}
let tables = root.join("tables");
let sst = tables.join("0");
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k0004", crate::fs::SyncMode::Normal)?;
memfs.remove_file(&tables.join("1"))?;
let report = config().repair()?;
assert_eq!(report.recovered, 1, "the healthy table is kept: {report:?}");
let tree = config().open()?;
for i in 0..8u32 {
let key = format!("k{i:04}").into_bytes();
assert!(
tree.get(&key, crate::MAX_SEQNO)?.is_some(),
"key {key:?} was hidden by a stale sidecar the manifest contradicts",
);
}
Ok(())
}
#[test]
fn repair_with_salvage_keeps_a_healthy_restricted_punched_table() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
let bound = b"k00130".to_vec();
{
let table = recover_sst(sst.clone(), &fs)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, &bound, crate::fs::SyncMode::Normal)?;
let punch = table.punch_offset_for(&bound)?;
memfs.punch_hole(&sst, 0, punch)?;
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"a healthy restricted SST must be kept restricted even with salvage on: {report:?}",
);
assert_eq!(
report.unreadable, 0,
"the healthy restricted SST must NOT be dropped: {:?}",
report.unreadable_files,
);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
for i in 0..256u32 {
let key = format!("k{i:05}").into_bytes();
let served = tree.get(&key, crate::MAX_SEQNO)?.is_some();
assert_eq!(
served,
key.as_slice() >= bound.as_slice(),
"key {key:?} served={served}; expected served == (key >= {bound:?})",
);
}
Ok(())
}
fn write_multiblock_sst(
path: &std::path::Path,
fs: &std::sync::Arc<dyn crate::fs::Fs>,
) -> crate::Result<()> {
use crate::{InternalValue, ValueType};
let mut w = crate::table::Writer::new(path.to_path_buf(), 0, 0, std::sync::Arc::clone(fs))?
.use_data_block_size(128);
for i in 0..256u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
Ok(())
}
fn build_punched_prefix_sst(
memfs: &std::sync::Arc<crate::fs::MemFs>,
fs: &std::sync::Arc<dyn crate::fs::Fs>,
tables: &std::path::Path,
) -> crate::Result<std::path::PathBuf> {
use crate::fs::Fs;
let sst = tables.join("0");
write_multiblock_sst(&sst, fs)?;
let committed = recover_sst(sst.clone(), fs)?.punch_offset_for(b"k00050")?;
memfs.punch_hole(&sst, 0, committed)?;
Ok(sst)
}
fn build_partially_punched_prefix_sst(
memfs: &std::sync::Arc<crate::fs::MemFs>,
fs: &std::sync::Arc<dyn crate::fs::Fs>,
tables: &std::path::Path,
) -> crate::Result<std::path::PathBuf> {
use crate::fs::Fs;
use crate::table::block_index::BlockIndex;
let sst = tables.join("0");
write_multiblock_sst(&sst, fs)?;
let table = recover_sst(sst.clone(), fs)?;
let committed = table.punch_offset_for(b"k00050")?;
let mut punched = 0u32;
for handle in table.block_index.iter() {
let handle = handle?;
let off = handle.offset().0;
if off > 0 && off < committed {
memfs.punch_hole(&sst, off, u64::from(handle.size()))?;
punched += 1;
}
}
assert!(
punched >= 2,
"precondition: the consumed prefix spans several blocks past the first",
);
Ok(sst)
}
#[test]
fn repair_sets_aside_a_partially_punched_sst_without_a_trustworthy_bound() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
build_partially_punched_prefix_sst(&memfs, &fs, &tables)?;
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 0,
"an irregularly punched SST with no exact bound must be set aside, not \
restricted to a guessed bound that resurrects consumed rows: {report:?}",
);
assert_eq!(report.unreadable, 1, "{:?}", report.unreadable_files);
assert!(
report
.unreadable_files
.first()
.is_some_and(|(_, reason)| reason.contains("punch failures")),
"the reason names the failed punches that made the bound unknowable: {:?}",
report.unreadable_files,
);
Ok(())
}
#[test]
fn repair_sets_aside_zeroed_blocks_the_backend_cannot_attribute() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
for allow_resurrection in [false, true] {
let memfs = Arc::new(MemFs::new());
memfs.set_hole_probe_supported(false);
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
build_punched_prefix_sst(&memfs, &fs, &tables)?;
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone())
.repair_with_resurrection(true, allow_resurrection)?;
if allow_resurrection {
assert_eq!(
report.recovered, 1,
"the resurrection repair keeps the readable region, restricted \
past the unattributable zeros: {report:?}",
);
} else {
assert_eq!(
report.recovered, 0,
"zeros the backend cannot attribute must never publish the \
table unrestricted (the punched prefix's sub-bound rows \
would reappear): {report:?}",
);
assert!(
report
.unreadable_files
.first()
.is_some_and(|(_, reason)| reason.contains("cannot attribute")),
"the reason names the unattributable zeros: {:?}",
report.unreadable_files,
);
}
}
Ok(())
}
#[test]
fn resurrection_restricts_a_partially_punched_sst_past_the_zeroed_blocks() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
build_partially_punched_prefix_sst(&memfs, &fs, &tables)?;
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_resurrection(false, true)?;
assert_eq!(report.recovered, 1, "{report:?}");
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
assert!(
tree.get(b"k00020", crate::MAX_SEQNO)?.is_none(),
"a key in a zeroed block must miss cleanly, not error",
);
assert!(
tree.get(b"k00000", crate::MAX_SEQNO)?.is_none(),
"the intact-but-superseded first block must not be served",
);
assert!(
tree.get(b"k00255", crate::MAX_SEQNO)?.is_some(),
"the live suffix must be served",
);
Ok(())
}
#[test]
fn repair_sets_aside_a_partially_punched_sidecarless_sst_that_fails_recovery() -> crate::Result<()>
{
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
build_partially_punched_prefix_sst(&memfs, &fs, &tables)?;
let fault = FaultFs::new(memfs.as_ref().clone());
fault.injector().arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Other))
.on_path("tables")
.once(),
);
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 0,
"a partially punched sidecar-less SST that fails recovery is set aside, \
not salvaged into an unrestricted output: {report:?}",
);
assert_eq!(report.unreadable, 1, "{:?}", report.unreadable_files);
assert!(
report
.unreadable_files
.first()
.is_some_and(|(_, reason)| reason.contains("punched extents found during salvage")),
"the reason names the punched extents the walk found: {:?}",
report.unreadable_files,
);
Ok(())
}
#[test]
fn a_recovery_failure_salvage_fails_closed_on_unattributable_zeros() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
memfs.set_hole_probe_supported(false);
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db_unattributable_recovery_fail")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
build_punched_prefix_sst(&memfs, &fs, &tables)?;
let fault = FaultFs::new(memfs.as_ref().clone());
fault.injector().arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Other))
.on_path("tables")
.once(),
);
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 0,
"an unattributable punched prefix must not salvage into an \
unrestricted output: {report:?}",
);
assert_eq!(report.unreadable, 1, "{:?}", report.unreadable_files);
assert!(
report
.unreadable_files
.first()
.is_some_and(|(_, reason)| reason.contains("punched extents found during salvage")),
"the post-walk guard is what caught it: {:?}",
report.unreadable_files,
);
Ok(())
}
#[test]
fn punch_failures_leave_a_classifiable_pattern() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let faultfs = FaultFs::new(memfs.as_ref().clone());
let injector = faultfs.injector();
let fs: Arc<dyn Fs> = Arc::new(faultfs);
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
let table = recover_sst(sst, &fs)?;
let committed = table.punch_offset_for(b"k00130")?;
table.mark_punch_on_drop(committed);
injector.arm(FaultRule::new(FaultOp::PunchHole, Fault::Error(ErrorKind::Other)).skip(2));
drop(table);
injector.clear();
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 0,
"a punch-failure pattern must be set aside, not restricted to a bound \
that resurrects the unpunched consumed blocks: {report:?}",
);
assert_eq!(report.unreadable, 1, "{:?}", report.unreadable_files);
assert!(
report
.unreadable_files
.first()
.is_some_and(|(_, reason)| reason.contains("punch failures")),
"{:?}",
report.unreadable_files,
);
Ok(())
}
#[test]
fn resurrection_keeps_an_irregularly_punched_table_the_default_repair_drops() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
build_partially_punched_prefix_sst(&memfs, &fs, &tables)?;
let kept = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_resurrection(true, true)?;
assert_eq!(kept.recovered, 1, "{kept:?}");
assert_eq!(kept.unreadable, 0, "{:?}", kept.unreadable_files);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
assert!(
tree.get(b"k00020", crate::MAX_SEQNO)?.is_none(),
"a key in a zeroed block still misses cleanly",
);
assert!(
tree.get(b"k00255", crate::MAX_SEQNO)?.is_some(),
"the readable region is served under resurrection",
);
drop(tree);
let memfs2 = Arc::new(MemFs::new());
let fs2: Arc<dyn Fs> = memfs2.clone();
let root2 = std::path::absolute("/db2")?;
let tables2 = root2.join("tables");
fs2.create_dir_all(&tables2)?;
build_partially_punched_prefix_sst(&memfs2, &fs2, &tables2)?;
let dropped = Config::new(
&root2,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs2.as_ref().clone())
.repair_with_salvage(true)?;
assert_eq!(dropped.recovered, 0, "{dropped:?}");
assert_eq!(dropped.unreadable, 1, "{:?}", dropped.unreadable_files);
assert!(
!fs2.exists(&tables2.join("0"))?,
"the dropped table's file is removed, not stashed",
);
Ok(())
}
#[test]
fn salvage_guard_finds_punched_blocks_deep_in_a_surrendered_extent() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
let table = recover_sst(sst.clone(), &fs)?;
let mut handles = Vec::new();
{
use crate::table::block_index::BlockIndex;
for handle in table.block_index.iter() {
let handle = handle?;
handles.push((handle.offset().0, handle.size()));
}
}
assert!(handles.len() >= 4, "fixture needs several data blocks");
let (Some(&(mid_off, mid_size)), Some(&(first_off, _))) =
(handles.get(handles.len() / 2), handles.first())
else {
panic!("fixture has >= 4 data blocks, so both lookups resolve");
};
drop(table);
memfs.punch_hole(&sst, mid_off, u64::from(mid_size))?;
let dropped = vec![crate::salvage::DroppedBlock {
offset: first_off,
section: b"data".to_vec(),
reason: crate::salvage::DropReason::HeaderCorrupted("surrendered tail".to_owned()),
key_range: None,
}];
assert!(
super::dropped_data_extent_is_zeroed(&*fs, &sst, &dropped)?,
"a punched block deeper inside the surrendered extent must be found",
);
let clean = tables.join("1");
write_multiblock_sst(&clean, &fs)?;
assert!(
!super::dropped_data_extent_is_zeroed(&*fs, &clean, &dropped)?,
"an unpunched source must never be reported as punched",
);
let destroyed = tables.join("2");
write_multiblock_sst(&destroyed, &fs)?;
{
use std::io::{Seek, SeekFrom, Write};
let mut file = fs.open(
&destroyed,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
file.seek(SeekFrom::Start(mid_off))?;
file.write_all(&vec![0u8; mid_size as usize])?;
}
assert!(
!super::dropped_data_extent_is_zeroed(&*fs, &destroyed, &dropped)?,
"zeros WRITTEN over a block are damage, not reclamation",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn salvage_guard_ignores_zero_filled_values_inside_a_surrendered_extent() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{InternalValue, ValueType};
use std::sync::Arc;
let fs: Arc<dyn Fs> = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
let mut w =
crate::table::Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..32u32 {
let mut value = vec![b'x'; 8];
value.extend_from_slice(&[0u8; 64]);
value.extend_from_slice(b"tail");
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
value,
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
{
let file = fs.open(&sst, &crate::fs::FsOpenOptions::new().read(true))?;
let len = crate::fs::FsFile::metadata(&*file)?.len;
let bytes = crate::file::read_exact(&*file, 0, usize::try_from(len).unwrap_or(0))?;
let min_run = crate::table::block::Header::MIN_LEN;
let has_run = bytes.windows(min_run).any(|w| w.iter().all(|&b| b == 0));
assert!(has_run, "fixture must embed a header-sized zero run");
}
let table = recover_sst(sst.clone(), &fs)?;
let first_off = {
use crate::table::block_index::BlockIndex;
let mut it = table.block_index.iter();
it.next().expect("at least one block")?.offset().0
};
drop(table);
let dropped = vec![crate::salvage::DroppedBlock {
offset: first_off,
section: b"data".to_vec(),
reason: crate::salvage::DropReason::HeaderCorrupted("surrendered tail".to_owned()),
key_range: None,
}];
assert!(
!super::dropped_data_extent_is_zeroed(&*fs, &sst, &dropped)?,
"a zero-filled value inside the surrendered extent is not punch evidence",
);
Ok(())
}
#[test]
fn repair_retry_after_a_failed_cleanup_does_not_duplicate_the_history() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
blob_tree_without_manifest(&memfs, &root)?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
punch_and_corrupt_blob(&memfs, &blobs.join("0"))?;
let config = |fs: Arc<dyn Fs>| {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(
FaultOp::RemoveFile,
Fault::Error(ErrorKind::PermissionDenied),
)
.on_path("blobs"),
);
assert!(
config(Arc::new(fault)).repair().is_err(),
"the post-commit cleanup failure must propagate",
);
let report = config(memfs.clone()).repair()?;
assert_eq!(
report.recovered, 1,
"one table, not the original beside its own rewrite: {report:?}",
);
let tree = match config(memfs).open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
assert_eq!(
tree.index.current_version().iter_tables().count(),
1,
"the rebuilt manifest holds one copy of the history",
);
Ok(())
}
#[test]
fn repair_orders_equal_seqno_tables_by_descending_id() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
for id in 0..6u64 {
use crate::{InternalValue, ValueType};
let path = tables.join(id.to_string());
let mut w = crate::table::Writer::new(path, id, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
format!("k{id:04}").into_bytes(),
b"v",
7,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs);
config.repair()?;
let tree = config.open()?;
let version = tree.current_version();
let ids: Vec<_> = version.iter_tables().map(crate::Table::id).collect();
assert_eq!(
ids,
vec![5, 4, 3, 2, 1, 0],
"equal-seqno tables must be ordered newest-first by id, not by scan order",
);
Ok(())
}
#[test]
fn repair_orders_l0_by_table_recency_not_by_maximum_seqno() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fs, MemFs};
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
for (id, other_seqno, value) in [(0u64, 100u64, "stale"), (1, 50, "fresh")] {
let mut w = crate::table::Writer::new(tables.join(id.to_string()), id, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
b"k",
value.as_bytes(),
10,
ValueType::Value,
))?;
w.write(InternalValue::from_components(
b"unrelated",
b"v",
other_seqno,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs);
config.repair()?;
let tree = config.open()?;
assert_eq!(
tree.get(b"k", crate::SeqNo::MAX)?.as_deref(),
Some(b"fresh".as_slice()),
"the later table's value must win: a maximum seqno drawn from an \
unrelated key says nothing about which table is newer",
);
Ok(())
}
struct SortedDirFs(crate::fs::MemFs, bool);
impl crate::fs::Fs for SortedDirFs {
fn open(
&self,
path: &std::path::Path,
options: &crate::fs::FsOpenOptions,
) -> crate::io::Result<Box<dyn crate::fs::FsFile>> {
self.0.open(path, options)
}
fn remove_file(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.0.remove_file(path)
}
fn rename(&self, from: &std::path::Path, to: &std::path::Path) -> crate::io::Result<()> {
self.0.rename(from, to)
}
fn create_dir_all(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.0.create_dir_all(path)
}
fn remove_dir_all(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.0.remove_dir_all(path)
}
fn sync_directory(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.0.sync_directory(path)
}
fn read_dir(&self, path: &std::path::Path) -> crate::io::Result<Vec<crate::fs::FsDirEntry>> {
let mut entries = self.0.read_dir(path)?;
entries.sort_by(|a, b| a.file_name.cmp(&b.file_name));
if self.1 {
entries.reverse();
}
Ok(entries)
}
fn metadata(&self, path: &std::path::Path) -> crate::io::Result<crate::fs::FsMetadata> {
self.0.metadata(path)
}
fn exists(&self, path: &std::path::Path) -> crate::io::Result<bool> {
self.0.exists(path)
}
fn capabilities(&self, path: &std::path::Path) -> crate::fs::FsCapabilities {
self.0.capabilities(path)
}
fn punch_hole(&self, path: &std::path::Path, offset: u64, len: u64) -> crate::io::Result<()> {
self.0.punch_hole(path, offset, len)
}
fn allocated_size(&self, path: &std::path::Path) -> crate::io::Result<Option<u64>> {
self.0.allocated_size(path)
}
fn extent_is_hole(
&self,
path: &std::path::Path,
offset: u64,
len: u64,
) -> crate::io::Result<Option<bool>> {
self.0.extent_is_hole(path, offset, len)
}
}
#[test]
fn repair_aborts_when_the_alias_probe_is_inconclusive() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let fault = crate::fs::FaultFs::new(MemFs::new());
fault.injector().arm(crate::fs::FaultRule::new(
crate::fs::FaultOp::SameFile,
crate::fs::Fault::Error(crate::io::ErrorKind::Interrupted),
));
let fs: Arc<dyn Fs> = Arc::new(fault);
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
for name in ["1", "01"] {
let mut w = crate::table::Writer::new(tables.join(name), 1, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
b"k".to_vec(),
b"v".to_vec(),
5,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs)
.repair();
assert!(
result.is_err(),
"an inconclusive alias probe must abort the repair instead of \
authorizing the duplicate's deletion: {:?}",
result.map(|r| r.recovered),
);
Ok(())
}
#[test]
fn std_fs_same_file_propagates_a_probe_failure() {
use crate::fs::Fs;
let missing_a = std::path::Path::new("/nonexistent-probe-a");
let missing_b = std::path::Path::new("/nonexistent-probe-b");
assert!(
StdFs.same_file(missing_a, missing_b).is_err(),
"a canonicalization failure must surface, never read as distinct",
);
}
#[test]
fn repair_prefers_the_canonical_spelling_of_a_duplicated_id() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
for reversed in [false, true] {
let fs: Arc<dyn Fs> = Arc::new(SortedDirFs(MemFs::new(), reversed));
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
for (name, key) in [("1", "canonical"), ("01", "alternate")] {
use crate::{InternalValue, ValueType};
let mut w = crate::table::Writer::new(tables.join(name), 1, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
key.as_bytes(),
b"v",
5,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(Arc::clone(&fs));
config.repair()?;
let tree = config.open()?;
assert!(
tree.get(b"canonical", crate::SeqNo::MAX)?.is_some(),
"the canonical `{{id}}` spelling must survive (reversed: {reversed})",
);
assert!(
tree.get(b"alternate", crate::SeqNo::MAX)?.is_none(),
"the alternate spelling must not displace it (reversed: {reversed})",
);
}
Ok(())
}
#[test]
fn repair_requires_a_hole_inside_the_zeroed_run() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let build = |memfs: &Arc<MemFs>| -> crate::Result<(std::path::PathBuf, u64)> {
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
let first_block_end = recover_sst(sst.clone(), &fs)?.punch_offset_for(b"k00050")?;
assert!(first_block_end > 0, "the fixture has a leading block");
{
use std::io::{Seek, SeekFrom, Write};
let mut file = fs.open(
&sst,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
file.seek(SeekFrom::Start(0))?;
file.write_all(&vec![0u8; usize::try_from(first_block_end).unwrap_or(0)])?;
}
Ok((sst, first_block_end))
};
let repair = |memfs: Arc<MemFs>| -> crate::Result<super::RepairReport> {
Config::new(
std::path::absolute("/db")?,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.repair()
};
let memfs = Arc::new(MemFs::new());
build(&memfs)?;
let report = repair(memfs)?;
assert_eq!(
report.recovered, 0,
"a zeroed run with no hole inside it is damage: {report:?}",
);
let memfs = Arc::new(MemFs::new());
let (sst, first_block_end) = build(&memfs)?;
memfs.punch_hole(&sst, first_block_end / 2, 8)?;
let report = repair(memfs)?;
assert_eq!(
report.recovered, 1,
"a hole contained in the zeroed run proves the punch: {report:?}",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_does_not_publish_a_rewrite_under_an_id_a_sidecar_names() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let kv = || Some(KvSeparationOptions::default().separation_threshold(16));
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(kv())
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let blob_path = memfs
.read_dir(&root.join(crate::file::BLOBS_FOLDER))?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
{
use std::io::{Seek, SeekFrom, Write};
let last = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.last()
.expect("a last frame")?;
let flip_at = last.frame_end - 8;
let mut file = fs_dyn.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*file, flip_at, 1)?;
file.seek(SeekFrom::Start(flip_at))?;
file.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
let tables = root.join("tables");
crate::restrict_bound::write(
&*fs_dyn,
&tables.join("1"),
None,
1,
b"k0004",
crate::fs::SyncMode::Normal,
)?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = crate::fs::FaultFs::new((*memfs).clone());
fault.injector().arm(
crate::fs::FaultRule::new(
crate::fs::FaultOp::RemoveFile,
crate::fs::Fault::Error(crate::io::ErrorKind::PermissionDenied),
)
.on_path("restrict-bound"),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(kv())
.repair();
assert!(
result.is_err(),
"an unremovable orphaned sidecar fails the repair; left in place the \
next open's sweep would hit the same refused unlink: {:?}",
result.map(|r| r.recovered),
);
let published: Vec<_> = memfs
.read_dir(&tables)?
.into_iter()
.filter(|e| e.file_name.parse::<crate::TableId>().is_ok())
.map(|e| e.file_name)
.collect();
for e in memfs.read_dir(&tables)? {
let Some(id) = e.file_name.strip_suffix(".restrict-bound") else {
continue;
};
assert!(
!published.iter().any(|name| name == id),
"table {id} is published while a stale sidecar names it: a later \
repair would restrict it at an unrelated bound",
);
}
Ok(())
}
#[test]
fn repair_does_not_read_a_zeroed_leading_block_as_a_reclaim() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
let first_block_end = recover_sst(sst.clone(), &fs)?.punch_offset_for(b"k00050")?;
assert!(first_block_end > 0, "the fixture has a leading block");
{
use std::io::{Seek, SeekFrom, Write};
let mut file = fs.open(
&sst,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
file.seek(SeekFrom::Start(0))?;
file.write_all(&vec![0u8; usize::try_from(first_block_end).unwrap_or(0)])?;
}
assert_eq!(
fs.allocated_size(&sst)?,
Some(
crate::fs::FsFile::metadata(
&*fs.open(&sst, &crate::fs::FsOpenOptions::new().read(true))?
)?
.len
),
"the fixture must be fully allocated: zeros written, not punched",
);
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs);
let report = config.repair()?;
assert_eq!(
report.recovered, 0,
"a table whose leading block was DESTROYED must not be published as a \
restricted view of its own suffix: {report:?}",
);
Ok(())
}
#[test]
fn verify_sst_file_bounds_the_skip_by_the_index_not_by_the_first_zeros() -> crate::Result<()> {
use crate::fs::{Fs, MemFs, SyncMode};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k00010", SyncMode::Normal)?;
let bound_offset = recover_sst(sst.clone(), &fs)?.punch_offset_for(b"k00010")?;
let far_above = recover_sst(sst.clone(), &fs)?.punch_offset_for(b"k00200")?;
assert!(
far_above > bound_offset,
"the destroyed block sits above the restriction's frontier",
);
memfs.punch_hole(&sst, bound_offset, far_above - bound_offset)?;
let report = crate::verify::verify_sst_file_with_fs(&fs, &sst);
assert!(
!report.is_ok(),
"destruction above the restriction frontier must be reported, not \
mistaken for the reclaimed prefix: errors {:?}, warnings {:?}",
report.errors,
report.warnings,
);
Ok(())
}
#[test]
fn verify_sst_file_rejects_a_sidecar_naming_another_table() -> crate::Result<()> {
use crate::fs::{Fs, MemFs, SyncMode};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
crate::restrict_bound::write(&*fs, &sst, None, 7, b"k00050", SyncMode::Normal)?;
let punch = recover_sst(sst.clone(), &fs)?.punch_offset_for(b"k00050")?;
assert!(punch > 0, "the fixture has a leading block to destroy");
memfs.punch_hole(&sst, 0, punch)?;
let report = crate::verify::verify_sst_file_with_fs(&fs, &sst);
assert!(
!report.is_ok(),
"a foreign sidecar must not let the zeroed prefix pass as reclaimed: \
errors {:?}, warnings {:?}",
report.errors,
report.warnings,
);
Ok(())
}
#[test]
fn verify_sst_file_skips_a_punched_prefix_within_a_read_budget() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs, SyncMode};
use crate::io::ErrorKind;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
{
use crate::{InternalValue, ValueType};
let mut w =
crate::table::Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(256);
for i in 0..4096u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
vec![b'v'; 32],
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k04000", SyncMode::Normal)?;
let punch = recover_sst(sst.clone(), &fs)?.punch_offset_for(b"k04000")?;
assert!(
punch > 100_000,
"the fixture has a sizable reclaimed prefix"
);
memfs.punch_hole(&sst, 0, punch)?;
let budget = FaultFs::new((*memfs).clone());
budget.injector().arm(
FaultRule::new(FaultOp::ReadAt, Fault::Error(ErrorKind::Other))
.on_path("tables")
.skip(512),
);
let budget_fs: Arc<dyn Fs> = Arc::new(budget);
let report = crate::verify::verify_sst_file_with_fs(&budget_fs, &sst);
assert!(
report.is_ok(),
"crossing a {punch}-byte reclaimed prefix must stay well inside a \
512-read budget: errors {:?}, warnings {:?}",
report.errors,
report.warnings,
);
Ok(())
}
#[test]
fn verify_sst_file_honors_a_restricted_punched_prefix() -> crate::Result<()> {
use crate::fs::{Fs, MemFs, SyncMode};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k00130", SyncMode::Normal)?;
let punch = recover_sst(sst.clone(), &fs)?.punch_offset_for(b"k00130")?;
memfs.punch_hole(&sst, 0, punch)?;
let report = crate::verify::verify_sst_file_with_fs(&fs, &sst);
assert!(
report.is_ok(),
"a healthy restricted punched SST must verify clean through the \
out-of-band walk: errors {:?}, warnings {:?}",
report.errors,
report.warnings,
);
assert!(
report.blocks_scanned > 0,
"the live suffix must actually be walked: {report:?}",
);
crate::restrict_bound::remove(&*fs, &sst, SyncMode::Normal);
let report = crate::verify::verify_sst_file_with_fs(&fs, &sst);
assert!(
!report.is_ok(),
"leading zeros without a restriction sidecar are destroyed data and \
must fail verification: {report:?}",
);
Ok(())
}
#[test]
fn a_zero_tailed_value_does_not_move_the_restricted_verify_frontier() -> crate::Result<()> {
use crate::fs::{Fs, MemFs, SyncMode};
use crate::{InternalValue, ValueType};
use std::sync::Arc;
let fs: Arc<dyn Fs> = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
{
let mut w =
crate::table::Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..64u32 {
let mut value = format!("v{i}").into_bytes();
value.extend(std::iter::repeat_n(0u8, 64));
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
value,
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let baseline = crate::verify::verify_sst_file_with_fs(&fs, &sst);
assert!(
baseline.is_ok(),
"the unpunched file must verify clean: {baseline:?}",
);
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k00000", SyncMode::Normal)?;
let restricted = crate::verify::verify_sst_file_with_fs(&fs, &sst);
assert!(
restricted.is_ok(),
"an unpunched file must verify clean under a restriction: {restricted:?}",
);
assert_eq!(
restricted.blocks_scanned, baseline.blocks_scanned,
"a zero-tailed value must not be mistaken for a punched extent: \
the frontier would advance past live blocks and the verifier would \
skip them (and any corruption inside them) while reporting OK",
);
Ok(())
}
#[test]
fn a_destroyed_suffix_block_does_not_move_the_restricted_verify_frontier() -> crate::Result<()> {
use crate::fs::{Fs, MemFs, SyncMode};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k00130", SyncMode::Normal)?;
let punch = recover_sst(sst.clone(), &fs)?.punch_offset_for(b"k00130")?;
memfs.punch_hole(&sst, 0, punch)?;
let destroyed = recover_sst(sst.clone(), &fs)?
.data_block_handles()
.filter_map(Result::ok)
.map(|keyed| *AsRef::<crate::table::BlockHandle>::as_ref(&keyed))
.filter(|h| h.offset().0 >= punch)
.nth(2)
.ok_or(crate::Error::Unrecoverable)?;
memfs.punch_hole(&sst, destroyed.offset().0, u64::from(destroyed.size()))?;
let report = crate::verify::verify_sst_file_with_fs(&fs, &sst);
assert!(
!report.is_ok(),
"a destroyed live block must be reported, not skipped as if it were \
another reclaimed extent: {report:?}",
);
Ok(())
}
#[test]
fn a_read_into_a_punched_extent_reports_it_as_excised() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
let table = recover_sst(sst.clone(), &fs)?;
let keyed = table
.data_block_handles()
.next()
.transpose()?
.ok_or(crate::Error::Unrecoverable)?;
let first: &crate::table::BlockHandle = keyed.as_ref();
memfs.punch_hole(&sst, first.offset().0, u64::from(first.size()))?;
let Err(err) = table.load_data_block(first) else {
panic!("a punched block cannot load");
};
assert!(
matches!(err, crate::Error::Excised { offset } if offset == first.offset().0),
"a punched extent must be reported as excised, not as damaged bytes: {err:?}",
);
Ok(())
}
#[test]
fn same_physical_file_requires_a_shared_namespace() -> crate::Result<()> {
use crate::fs::{Fs, MemFs, StdFs};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let real = dir.path().join("0");
std::fs::write(&real, b"real bytes")?;
let memfs: Arc<dyn Fs> = Arc::new(MemFs::new());
let stdfs: Arc<dyn Fs> = Arc::new(StdFs);
if let Some(parent) = real.parent() {
memfs.create_dir_all(parent)?;
}
{
use std::io::Write;
let mut f = memfs.open(
&real,
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?;
f.write_all(b"virtual bytes")?;
}
assert!(
!super::same_physical_file(&*memfs, &real, &*stdfs, &real)?,
"same path spelling in DIFFERENT namespaces is not an alias",
);
assert!(
super::same_physical_file(&*stdfs, &real, &*stdfs, &real)?,
"the same host path through one namespace IS an alias",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_skips_salvaging_an_unreferenced_invalid_blob() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..4u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let orphan = blobs.join("9");
{
let mut w = crate::vlog::blob_file::writer::Writer::new(orphan.clone(), 9, 0, &*fs_dyn)?;
w.write(b"o1", 1, &[b'x'; 300])?;
w.write(b"o2", 2, &[b'y'; 300])?;
w.write(b"o3", 3, &[b'z'; 300])?;
w.finish()?;
let last = crate::vlog::BlobFileScanner::new(&orphan, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?
.last()
.expect("a last frame")
.frame_end;
let mut f = memfs.open(
&orphan,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*f, last - 8, 1)?;
f.seek(SeekFrom::Start(last - 8))?;
f.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::Write, Fault::Error(ErrorKind::StorageFull))
.on_path(".salvage-tmp"),
);
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair()?;
assert_eq!(
report.recovered, 1,
"the healthy tree is recovered; the unreachable orphan is neither \
salvaged nor fatal: {report:?}",
);
Ok(())
}
#[test]
fn same_physical_file_fails_closed_without_a_backend_identity() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
struct NoIdentityFs(MemFs);
impl Fs for NoIdentityFs {
fn open(
&self,
path: &std::path::Path,
options: &crate::fs::FsOpenOptions,
) -> crate::io::Result<Box<dyn crate::fs::FsFile>> {
self.0.open(path, options)
}
fn remove_file(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.0.remove_file(path)
}
fn rename(&self, from: &std::path::Path, to: &std::path::Path) -> crate::io::Result<()> {
self.0.rename(from, to)
}
fn create_dir_all(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.0.create_dir_all(path)
}
fn remove_dir_all(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.0.remove_dir_all(path)
}
fn sync_directory(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.0.sync_directory(path)
}
fn read_dir(
&self,
path: &std::path::Path,
) -> crate::io::Result<Vec<crate::fs::FsDirEntry>> {
self.0.read_dir(path)
}
fn metadata(&self, path: &std::path::Path) -> crate::io::Result<crate::fs::FsMetadata> {
self.0.metadata(path)
}
fn exists(&self, path: &std::path::Path) -> crate::io::Result<bool> {
self.0.exists(path)
}
fn same_file(&self, a: &std::path::Path, b: &std::path::Path) -> crate::io::Result<bool> {
Ok(a.to_string_lossy()
.eq_ignore_ascii_case(&b.to_string_lossy()))
}
}
let fs_one = NoIdentityFs(MemFs::new());
let a = std::path::Path::new("/db/tables/AA");
let b = std::path::Path::new("/db/tables/aa");
assert!(
super::same_physical_file(&fs_one, a, &fs_one, b)?,
"one identity-less instance must still get to answer its own probe",
);
let fs_two = NoIdentityFs(MemFs::new());
assert!(
super::same_physical_file(&fs_one, a, &fs_two, b).is_err(),
"an unprovable identity must abort, not authorize a deletion",
);
Ok(())
}
#[cfg(unix)]
#[test]
fn same_physical_file_ignores_host_symlinks_for_a_virtual_backend() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::io::Write;
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let real_dir = dir.path().join("x");
std::fs::create_dir(&real_dir)?;
std::fs::write(real_dir.join("0"), b"host bytes")?;
let linked_dir = dir.path().join("y");
std::os::unix::fs::symlink(&real_dir, &linked_dir)?;
let memfs: Arc<dyn Fs> = Arc::new(MemFs::new());
let (real_path, linked_path) = (real_dir.join("0"), linked_dir.join("0"));
for (path, bytes) in [
(&real_path, b"first".as_slice()),
(&linked_path, b"second".as_slice()),
] {
if let Some(parent) = path.parent() {
memfs.create_dir_all(parent)?;
}
let mut file = memfs.open(
path,
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?;
file.write_all(bytes)?;
}
assert!(
!super::same_physical_file(&*memfs, &real_path, &*memfs, &linked_path)?,
"a host symlink must not alias two distinct virtual files",
);
Ok(())
}
#[test]
fn verify_sst_file_ignores_zero_runs_inside_live_values() -> crate::Result<()> {
use crate::fs::{Fs, MemFs, SyncMode};
use crate::table::Writer;
use crate::{InternalValue, ValueType};
use std::sync::Arc;
let fs: Arc<dyn Fs> = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..64u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
vec![0u8; 256],
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k00010", SyncMode::Normal)?;
let report = crate::verify::verify_sst_file_with_fs(&fs, &sst);
assert!(
report.is_ok(),
"zero-filled VALUES are legal payload and must not move the frontier: \
errors {:?}, warnings {:?}",
report.errors,
report.warnings,
);
assert!(
report.blocks_scanned > 0,
"every live block must still be walked: {report:?}",
);
Ok(())
}
#[test]
fn verify_sst_file_clears_partial_top_down_holes() -> crate::Result<()> {
use crate::fs::{Fs, MemFs, SyncMode};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k00130", SyncMode::Normal)?;
let table = recover_sst(sst.clone(), &fs)?;
let committed = table.punch_offset_for(b"k00130")?;
let mut consumed = Vec::new();
{
use crate::table::block_index::BlockIndex;
for handle in table.block_index.iter() {
let handle = handle?;
if handle.offset().0 < committed {
consumed.push((handle.offset().0, handle.size()));
}
}
}
assert!(consumed.len() >= 3, "fixture needs several consumed blocks");
drop(table);
for &(off, size) in consumed.iter().skip(1) {
memfs.punch_hole(&sst, off, u64::from(size))?;
}
let report = crate::verify::verify_sst_file_with_fs(&fs, &sst);
assert!(
report.is_ok(),
"a partially reclaimed sidecar-backed SST must verify clean: errors \
{:?}, warnings {:?}",
report.errors,
report.warnings,
);
assert!(
report.blocks_scanned > 0,
"the live suffix must still be walked: {report:?}",
);
Ok(())
}
#[test]
fn a_failed_repair_leaves_the_source_where_the_retry_finds_it() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
let offset = {
use crate::table::block_index::BlockIndex;
let table = recover_sst(sst.clone(), &fs)?;
let Some(handle) = table.block_index.iter().next().transpose()? else {
panic!("the fixture has data blocks");
};
handle.offset().0
};
{
use crate::fs::FsOpenOptions;
use std::io::{Read, Seek, SeekFrom, Write};
let mut bytes = Vec::new();
fs.open(&sst, &FsOpenOptions::new().read(true))?
.read_to_end(&mut bytes)?;
let flip = usize::try_from(offset).unwrap_or(0) + 16;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
let mut f = fs.open(&sst, &FsOpenOptions::new().write(true))?;
f.seek(SeekFrom::Start(0))?;
f.write_all(&bytes)?;
f.sync_all()?;
}
let fault = FaultFs::new(memfs.as_ref().clone());
fault.injector().arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::Interrupted))
.on_path(super::REPAIR_TMP_SUFFIX),
);
let failed = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true);
assert!(
failed.is_err(),
"the faulted replacement write must surface"
);
assert!(
fs.exists(&sst)?,
"the source must still be where the retry scans for it",
);
let retry = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_salvage(true)?;
assert_eq!(retry.recovered, 1, "{retry:?}");
assert_eq!(retry.salvaged, 1, "{retry:?}");
Ok(())
}
#[test]
fn resurrection_keeps_a_fully_punched_table_the_default_repair_drops() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let dropped_fs = Arc::new(MemFs::new());
let dropped_dyn: Arc<dyn Fs> = dropped_fs.clone();
let dropped_root = std::path::absolute("/db-dropped")?;
let dropped_tables = dropped_root.join("tables");
dropped_dyn.create_dir_all(&dropped_tables)?;
let dropped_sst = dropped_tables.join("0");
write_multiblock_sst(&dropped_sst, &dropped_dyn)?;
let punch = recover_sst(dropped_sst.clone(), &dropped_dyn)?.punch_offset_for(b"k00130")?;
dropped_fs.punch_hole(&dropped_sst, 0, punch)?;
let fault = FaultFs::new(dropped_fs.as_ref().clone());
fault.injector().arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Other))
.on_path("tables")
.once(),
);
let first = Config::new(
&dropped_root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
assert_eq!(first.recovered, 0, "{first:?}");
assert!(
!dropped_dyn.exists(&dropped_sst)?,
"the dropped table's file is removed, not stashed",
);
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
let punch = recover_sst(sst.clone(), &fs)?.punch_offset_for(b"k00130")?;
memfs.punch_hole(&sst, 0, punch)?;
let second = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_resurrection(true, true)?;
assert_eq!(second.recovered, 1, "{second:?}");
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
assert!(
tree.get(b"k00255", crate::MAX_SEQNO)?.is_some(),
"the readable region is served under resurrection",
);
Ok(())
}
#[test]
fn resurrection_keeps_a_punched_table_the_salvage_arm_would_drop() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let dropped_fs = Arc::new(MemFs::new());
let dropped_dyn: Arc<dyn Fs> = dropped_fs.clone();
let dropped_root = std::path::absolute("/db-dropped")?;
let dropped_tables = dropped_root.join("tables");
dropped_dyn.create_dir_all(&dropped_tables)?;
build_partially_punched_prefix_sst(&dropped_fs, &dropped_dyn, &dropped_tables)?;
let fault = FaultFs::new(dropped_fs.as_ref().clone());
fault.injector().arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Other))
.on_path("tables")
.once(),
);
let first = Config::new(
&dropped_root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
assert_eq!(first.recovered, 0, "{first:?}");
assert!(
!dropped_dyn.exists(&dropped_tables.join("0"))?,
"the dropped table's file is removed, not stashed",
);
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
build_partially_punched_prefix_sst(&memfs, &fs, &tables)?;
let second = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_resurrection(true, true)?;
assert_eq!(second.recovered, 1, "{second:?}");
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
assert!(
tree.get(b"k00255", crate::MAX_SEQNO)?.is_some(),
"the readable region is served under resurrection",
);
Ok(())
}
#[test]
fn writing_the_sidecar_does_not_mutate_the_sst() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::table::Writer;
use crate::{InternalValue, ValueType};
use std::sync::Arc;
let fs: Arc<dyn Fs> = Arc::new(MemFs::new());
let base = std::path::absolute("/d")?;
fs.create_dir_all(&base)?;
let sst = base.join("0");
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..64u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let before = compute_table_checksum(&*fs, &sst)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k00030", crate::fs::SyncMode::Normal)?;
let after = compute_table_checksum(&*fs, &sst)?;
assert_eq!(
before, after,
"publishing the sidecar must leave the SST byte-identical",
);
Ok(())
}
#[test]
fn repair_salvages_a_restricted_punched_sst_with_a_corrupt_suffix() -> crate::Result<()> {
use crate::fs::{Fs, FsOpenOptions, MemFs};
use crate::table::Writer;
use crate::{AbstractTree, Config, InternalValue, SequenceNumberCounter, ValueType};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..256u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let bound = b"k00130".to_vec();
let punch_offset = {
let table = recover_sst(sst.clone(), &fs)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, &bound, crate::fs::SyncMode::Normal)?;
let off = table.punch_offset_for(&bound)?;
memfs.punch_hole(&sst, 0, off)?;
off
};
{
let mut f = fs.open(&sst, &FsOpenOptions::new().write(true))?;
f.seek(SeekFrom::Start(punch_offset + 20))?;
f.write_all(&[0xFFu8])?;
f.sync_all()?;
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"the restricted SST's readable live suffix is salvaged, not stranded: {report:?}",
);
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
assert!(
crate::restrict_bound::exists(&*fs, &sst)?,
"the salvaged restricted SST re-writes its `.restrict-bound` sidecar",
);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
for i in 0..130u32 {
let key = format!("k{i:05}").into_bytes();
assert!(
tree.get(&key, crate::MAX_SEQNO)?.is_none(),
"sub-bound key {} must never be resurrected",
String::from_utf8_lossy(&key),
);
}
for i in [200u32, 255] {
let key = format!("k{i:05}").into_bytes();
assert!(
tree.get(&key, crate::MAX_SEQNO)?.is_some(),
"live suffix key {} must survive salvage",
String::from_utf8_lossy(&key),
);
}
Ok(())
}
#[cfg(feature = "std")]
fn build_restricted_corrupt_sst()
-> crate::Result<(std::sync::Arc<crate::fs::MemFs>, std::path::PathBuf)> {
use crate::fs::{Fs, FsOpenOptions, MemFs};
use crate::table::Writer;
use crate::{InternalValue, ValueType};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..256u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let bound = b"k00130".to_vec();
let punch_offset = {
let table = recover_sst(sst.clone(), &fs)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, &bound, crate::fs::SyncMode::Normal)?;
let off = table.punch_offset_for(&bound)?;
memfs.punch_hole(&sst, 0, off)?;
off
};
{
let mut f = fs.open(&sst, &FsOpenOptions::new().write(true))?;
f.seek(SeekFrom::Start(punch_offset + 20))?;
f.write_all(&[0xFFu8])?;
f.sync_all()?;
}
Ok((memfs, root))
}
#[cfg(feature = "std")]
fn assert_re_restriction_fault_leaves_the_source_alone(
fault_kind: crate::io::ErrorKind,
) -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs};
use crate::{Config, SequenceNumberCounter};
let (memfs, root) = build_restricted_corrupt_sst()?;
let sst = root.join("tables").join("0");
let fault = FaultFs::new(memfs.as_ref().clone());
fault.injector().arm(
FaultRule::new(FaultOp::Open, Fault::Error(fault_kind))
.on_path("restrict-bound")
.skip(1)
.once(),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true);
assert!(
matches!(&result, Err(crate::Error::Io(e)) if e.kind() == fault_kind),
"the re-restriction fault must propagate, got {result:?}",
);
assert!(
!memfs.exists(&super::repair_tmp_path(&sst))?,
"the fault must remove the unrestrictable replacement",
);
assert!(
memfs.exists(&sst)?,
"the source is untouched at its table path, ready for the retry",
);
Ok(())
}
#[test]
fn repair_keeps_the_source_when_re_restriction_faults_transiently() -> crate::Result<()> {
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, SequenceNumberCounter};
assert_re_restriction_fault_leaves_the_source_alone(ErrorKind::Interrupted)?;
let (memfs, root) = build_restricted_corrupt_sst()?;
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"the retry recovers the table: {report:?}"
);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
assert!(
tree.get(b"k00000", crate::MAX_SEQNO)?.is_none(),
"no sub-bound key resurrects after the retry",
);
Ok(())
}
#[test]
fn repair_keeps_the_source_when_re_restriction_faults_persistently() -> crate::Result<()> {
use crate::io::ErrorKind;
assert_re_restriction_fault_leaves_the_source_alone(ErrorKind::Other)
}
#[test]
fn repair_salvages_a_sidecarless_sst_whose_leading_bytes_were_destroyed() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..256u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
{
use std::io::{Seek, SeekFrom, Write};
let mut file = fs.open(
&sst,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
file.seek(SeekFrom::Start(0))?;
file.write_all(&[0u8; 128])?;
}
let fault = FaultFs::new(memfs.as_ref().clone());
fault.injector().arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Other))
.on_path("tables")
.once(),
);
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"destroyed leading bytes are not a lost punch bound: the readable blocks \
must still be salvaged rather than the whole table set aside: {report:?}",
);
Ok(())
}
#[test]
fn repair_sets_aside_a_punched_sidecarless_sst_that_fails_recovery() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..256u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let punch_offset = recover_sst(sst.clone(), &fs)?.punch_offset_for(b"k00130")?;
memfs.punch_hole(&sst, 0, punch_offset)?;
let fault = FaultFs::new(memfs.as_ref().clone());
fault.injector().arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Other))
.on_path("tables")
.once(),
);
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 0,
"a punched sidecar-less SST that fails recovery is set aside, not salvaged \
into an unrestricted output: {report:?}",
);
assert_eq!(report.unreadable, 1, "{:?}", report.unreadable_files);
assert!(
report
.unreadable_files
.first()
.is_some_and(|(_, reason)| reason.contains("no recoverable restriction bound")),
"the reason names the unrecoverable punched bound: {:?}",
report.unreadable_files,
);
Ok(())
}
#[test]
fn commit_repair_tmp_propagates_a_failed_swap() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, SyncMode};
use crate::io::ErrorKind;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
std::fs::write(&sst, b"damaged")?;
let tmp = repair_tmp_path(&sst);
std::fs::write(&tmp, b"replacement")?;
let fs = FaultFs::new(StdFs);
fs.injector().arm(FaultRule::new(
FaultOp::Rename,
Fault::Error(ErrorKind::Other),
));
assert!(
commit_repair_tmp(&fs, &tmp, &sst, SyncMode::Normal, false).is_err(),
"a failed swap must not be swallowed",
);
assert!(
fs.exists(&tmp)?,
"the replacement stays where the retry finds it",
);
Ok(())
}
fn assert_recovered_restricted_to(
memfs: &std::sync::Arc<crate::fs::MemFs>,
root: &std::path::Path,
bound: &[u8],
) -> crate::Result<()> {
use crate::{AbstractTree, Config, SequenceNumberCounter};
let report = Config::new(
root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"the SST is recovered restricted to its bound, not set aside: {report:?}",
);
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
let tree = Config::new(
root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
for i in 0..256u32 {
let key = format!("k{i:05}").into_bytes();
let served = tree.get(&key, crate::MAX_SEQNO)?.is_some();
assert_eq!(
served,
key.as_slice() >= bound,
"key {key:?} served={served}; expected served == (key >= {bound:?})",
);
}
Ok(())
}
#[test]
fn repair_restricts_a_punched_sst_whose_sidecar_bound_overshoots_the_punch() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = build_punched_prefix_sst(&memfs, &fs, &tables)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k00130", crate::fs::SyncMode::Normal)?;
assert_recovered_restricted_to(&memfs, &root, b"k00130")
}
#[test]
fn repair_honors_a_valid_sidecar_over_an_unpunched_sst() -> crate::Result<()> {
let (memfs, root) = build_unpunched_sidecar_sst(b"k00130")?;
assert_recovered_restricted_to(&memfs, &root, b"k00130")
}
fn build_unpunched_sidecar_sst(
bound: &[u8],
) -> crate::Result<(std::sync::Arc<crate::fs::MemFs>, std::path::PathBuf)> {
use crate::fs::{Fs, MemFs};
use crate::table::Writer;
use crate::{InternalValue, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..256u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
crate::restrict_bound::write(&*fs, &sst, None, 0, bound, crate::fs::SyncMode::Normal)?;
Ok((memfs, root))
}
#[test]
fn repair_with_resurrection_honors_a_committed_unpunched_sidecar() -> crate::Result<()> {
use crate::{AbstractTree, Config, SequenceNumberCounter};
let (memfs, root) = build_unpunched_sidecar_sst(b"k00130")?;
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_resurrection(true, true)?;
assert_eq!(
report.recovered, 1,
"the committed restriction is recovered restricted, not set aside: {report:?}",
);
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
assert!(
tree.get(b"k00000", crate::MAX_SEQNO)?.is_none(),
"a committed restriction must not be reopened whole by the resurrection flag",
);
assert!(
tree.get(b"k00129", crate::MAX_SEQNO)?.is_none(),
"the row just below the bound stays dropped",
);
assert!(
tree.get(b"k00130", crate::MAX_SEQNO)?.is_some(),
"the bound key is served",
);
assert!(
tree.get(b"k00255", crate::MAX_SEQNO)?.is_some(),
"the live suffix is served",
);
Ok(())
}
fn assert_punched_sst_recovered_restricted(
memfs: &std::sync::Arc<crate::fs::MemFs>,
root: &std::path::Path,
) -> crate::Result<()> {
use crate::{AbstractTree, Config, SequenceNumberCounter};
let report = Config::new(
root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"the punched SST is recovered restricted, not set aside: {report:?}",
);
assert_eq!(
report.unreadable, 0,
"nothing is set aside: {:?}",
report.unreadable_files,
);
let tree = Config::new(
root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
for i in [0u32, 10, 49] {
let key = format!("k{i:05}").into_bytes();
assert!(
tree.get(&key, crate::MAX_SEQNO)?.is_none(),
"key {key:?} is below the punch and must NOT be resurrected",
);
}
for i in [100u32, 200, 255] {
let key = format!("k{i:05}").into_bytes();
assert!(
tree.get(&key, crate::MAX_SEQNO)?.is_some(),
"live-suffix key {key:?} must be served",
);
}
Ok(())
}
#[test]
fn storage_stats_and_checkpoint_agree_on_a_restricted_tree() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = build_punched_prefix_sst(&memfs, &fs, &tables)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k00050", crate::fs::SyncMode::Normal)?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
};
config().repair()?;
let tree = config().open()?;
let stats = tree.storage_stats()?;
let cp_dir = root.join("checkpoint");
let info = tree.create_checkpoint(&cp_dir)?;
let len = memfs.metadata(&sst)?.len;
let punched = len - memfs.allocated_size(&sst)?.unwrap_or(len);
assert!(punched > 0, "the fixture's prefix is punched");
assert_eq!(
stats.used_bytes + punched,
info.total_bytes,
"both surfaces count the restricted table's live sidecar",
);
Ok(())
}
#[test]
fn repair_restricts_a_punched_sst_with_no_sidecar() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let _sst = build_punched_prefix_sst(&memfs, &fs, &tables)?;
assert_punched_sst_recovered_restricted(&memfs, &root)
}
#[test]
fn repair_reports_loss_from_a_geometry_derived_restriction() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let _sst = build_punched_prefix_sst(&memfs, &fs, &tables)?;
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"the punched SST is still recovered restricted: {report:?}",
);
assert!(
!report.lost_coverage.is_empty(),
"a geometry-derived bound can drop the straddling block's live \
suffix; the loss must be reported: {report:?}",
);
assert!(
!matches!(
report.wal_replay_scope(),
crate::repair::WalReplayScope::TailOnly
),
"a lossy restriction must widen the replay obligation: {report:?}",
);
Ok(())
}
#[test]
fn repair_restricts_a_punched_sst_with_a_corrupt_sidecar() -> crate::Result<()> {
use crate::fs::{Fs, FsOpenOptions, MemFs};
use std::io::{Read, Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = build_punched_prefix_sst(&memfs, &fs, &tables)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k00050", crate::fs::SyncMode::Normal)?;
let sidecar = crate::restrict_bound::sidecar_path(&sst);
{
let mut buf = Vec::new();
fs.open(&sidecar, &FsOpenOptions::new().read(true))?
.read_to_end(&mut buf)?;
if let Some(b) = buf.first_mut() {
*b ^= 0xFF;
}
let mut f = fs.open(&sidecar, &FsOpenOptions::new().write(true))?;
f.seek(SeekFrom::Start(0))?;
f.write_all(&buf)?;
f.sync_all()?;
}
assert_punched_sst_recovered_restricted(&memfs, &root)
}
#[test]
fn repair_restricts_a_punched_sst_with_a_mismatched_sidecar_id() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = build_punched_prefix_sst(&memfs, &fs, &tables)?;
crate::restrict_bound::write(
&*fs,
&sst,
None,
999,
b"k00050",
crate::fs::SyncMode::Normal,
)?;
assert_punched_sst_recovered_restricted(&memfs, &root)
}
#[test]
fn repair_with_resurrection_keeps_a_punched_sst_unrestricted() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let _sst = build_punched_prefix_sst(&memfs, &fs, &tables)?;
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_resurrection(true, true)?;
assert_eq!(
report.recovered, 1,
"resurrection recovers the punched SST unrestricted, not set aside: {report:?}",
);
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
assert!(
tree.get(b"k00255", crate::MAX_SEQNO)?.is_some(),
"the live suffix must be served under resurrection",
);
let resurrected = (50u32..100)
.filter_map(|i| {
let key = format!("k{i:05}").into_bytes();
tree.get(&key, crate::MAX_SEQNO).ok().flatten().map(|_| i)
})
.count();
assert!(
resurrected > 0,
"resurrection must serve readable keys the conservative derive would drop",
);
Ok(())
}
#[test]
fn repair_with_resurrection_but_no_salvage_does_not_expose_punched_blocks() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let _sst = build_punched_prefix_sst(&memfs, &fs, &tables)?;
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.repair_with_resurrection(false, true)?;
assert_eq!(
report.recovered, 1,
"the punched SST is recovered: {report:?}"
);
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
assert!(
tree.get(b"k00000", crate::MAX_SEQNO)?.is_none(),
"a key in the punched prefix must miss cleanly, not error on a zeroed block",
);
assert!(
tree.get(b"k00255", crate::MAX_SEQNO)?.is_some(),
"the live suffix must be served",
);
Ok(())
}
#[test]
fn repair_propagates_a_transient_restrict_bound_read() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let membase: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
membase.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &membase)?;
let bound = b"k00130".to_vec();
{
let table = recover_sst(sst.clone(), &membase)?;
crate::restrict_bound::write(
&*membase,
&sst,
None,
0,
&bound,
crate::fs::SyncMode::Normal,
)?;
let punch = table.punch_offset_for(&bound)?;
memfs.punch_hole(&sst, 0, punch)?;
}
let fault = FaultFs::new(memfs.as_ref().clone());
fault.injector().arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::Interrupted))
.on_path("restrict-bound"),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true);
assert!(
result.is_err(),
"a transient sidecar read on a punched SST must propagate, not silently \
open it unrestricted: {result:?}",
);
Ok(())
}
#[test]
fn repair_restricts_a_punched_sst_on_a_persistent_sidecar_read() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let membase: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
membase.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &membase)?;
let bound = b"k00130".to_vec();
{
let table = recover_sst(sst.clone(), &membase)?;
crate::restrict_bound::write(
&*membase,
&sst,
None,
0,
&bound,
crate::fs::SyncMode::Normal,
)?;
let punch = table.punch_offset_for(&bound)?;
memfs.punch_hole(&sst, 0, punch)?;
}
let fault = FaultFs::new(memfs.as_ref().clone());
fault.injector().arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::Other)).on_path("restrict-bound"),
);
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"the punched SST is recovered restricted, not set aside: {report:?}",
);
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
for i in [0u32, 50, 129] {
let key = format!("k{i:05}").into_bytes();
assert!(
tree.get(&key, crate::MAX_SEQNO)?.is_none(),
"key {key:?} is below the punch and must NOT be resurrected",
);
}
for i in [150u32, 200, 255] {
let key = format!("k{i:05}").into_bytes();
assert!(
tree.get(&key, crate::MAX_SEQNO)?.is_some(),
"live-suffix key {key:?} must be served",
);
}
Ok(())
}
#[test]
fn repair_honors_a_valid_sidecar_despite_a_persistent_dead_prefix_read() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = build_punched_prefix_sst(&memfs, &fs, &tables)?;
crate::restrict_bound::write(&*fs, &sst, None, 0, b"k00050", crate::fs::SyncMode::Normal)?;
let fault = FaultFs::new(memfs.as_ref().clone());
fault
.injector()
.arm(FaultRule::new(FaultOp::ReadAt, Fault::Error(ErrorKind::Other)).at_offset(0));
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(false)?;
assert_eq!(
report.recovered, 1,
"a valid sidecar must be honored despite an unreadable dead prefix, not \
dropped: {report:?}",
);
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(memfs.as_ref().clone())
.open()?;
for i in 0..256u32 {
let key = format!("k{i:05}").into_bytes();
let served = tree.get(&key, crate::MAX_SEQNO)?.is_some();
assert_eq!(
served,
key.as_slice() >= b"k00050".as_slice(),
"key {key:?} served={served}; expected served == (key >= k00050)",
);
}
Ok(())
}
#[test]
fn repair_leaves_a_foreign_healtmp_suffix_in_place() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let foreign = tables.join("5.healtmp-backup");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst, 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
std::fs::write(&foreign, b"not a real heal temp")?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(
report.recovered, 1,
"the real table is recovered: {report:?}"
);
assert!(
foreign.exists(),
"a name the engine does not own is the operator's file, not the \
repair's to remove",
);
Ok(())
}
#[test]
fn verify_keep_decision_grades_a_persistent_walk_io_error_as_corruption() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule};
use crate::io::ErrorKind;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let clean: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&clean))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let table = recover_table(sst.clone(), &clean)?;
let fault = FaultFs::new(StdFs);
fault
.injector()
.arm(FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::Other)).once());
let faulting: Arc<dyn crate::fs::Fs> = Arc::new(fault);
let config = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
);
let decision = verify_keep_decision(&config, &faulting, &sst, &table, false, true);
assert!(
decision.is_ok(),
"a persistent walk read error must be graded as corruption (a decision), not \
abort the whole repair, got {decision:?}",
);
Ok(())
}
#[cfg(feature = "page_ecc")]
#[test]
fn repair_with_salvage_keeps_a_parity_rotted_range_tombstone_sst() -> crate::Result<()> {
use crate::coding::Decode;
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::table::block::{EccParams, Header};
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?
.use_ecc(Some(EccParams::try_new(4, 2)?));
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let block_off = usize::try_from(sole_data_block_offset(&recover_table(sst.clone(), &fs)?))
.unwrap_or(usize::MAX);
let mut bytes = std::fs::read(&sst)?;
let Some(mut cursor) = bytes.get(block_off..) else {
panic!("data block within the file");
};
let header = Header::decode_from(&mut cursor)?;
let trailer_pos =
block_off + Header::header_len(header.block_type) + header.data_length as usize;
let Some(slot) = bytes.get_mut(trailer_pos) else {
panic!("parity trailer within the file");
};
*slot ^= 0xFF;
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert!(
report.unreadable_files.is_empty(),
"a readable table must not be dropped over parity-only rot: {:?}",
report.unreadable_files,
);
assert_eq!(
report.recovered, 1,
"the range-tombstone table joins the rebuilt manifest as-is: {report:?}",
);
assert_eq!(
report.salvaged, 0,
"salvage is never attempted when it cannot re-emit the range tombstones",
);
Ok(())
}
#[test]
fn repair_with_salvage_rejects_a_corrupt_unrecognized_ecc_tombstone_sst() -> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
forge_unrecognized_ecc_descriptor(&sst)?;
let block_off = usize::try_from(sole_data_block_offset(&recover_table(sst.clone(), &fs)?))
.unwrap_or(usize::MAX);
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(block_off + 40) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 0,
"a corrupt table whose sections the walk could not scan must not be \
kept over the range-tombstone escape hatch: {report:?}",
);
let [(_, reason)] = report.unreadable_files.as_slice() else {
panic!(
"expected exactly one unreadable file, got {:?}",
report.unreadable_files,
);
};
assert!(
reason.contains("range tombstones"),
"the reason names the refused range-tombstone salvage, got: {reason}",
);
Ok(())
}
#[test]
fn repair_with_salvage_drops_an_unrecognized_ecc_tombstone_sst() -> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
forge_unrecognized_ecc_descriptor(&sst)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 0,
"an unverifiable table never joins the rebuilt manifest: {report:?}",
);
assert_eq!(
report.salvaged, 0,
"salvage cannot re-emit range tombstones"
);
let [(_, reason)] = report.unreadable_files.as_slice() else {
panic!(
"expected exactly one unreadable file, got {:?}",
report.unreadable_files,
);
};
assert!(
reason.contains("excluded") && reason.contains("recompact"),
"the reason names the exclusion and the recovery path, got: {reason}",
);
assert!(
fs.metadata(&tables.join("0")).is_err(),
"the dropped table's file must not be left as an orphan for the next open",
);
Ok(())
}
#[test]
fn repair_with_salvage_keeps_a_healthy_rt_sst_with_a_forged_tail_meta() -> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("7");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 7, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
forge_tail_meta_table_id(&sst, Some(99), Some([0u8, 8, 2, 1]))?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"the healthy table joins the rebuilt manifest via the intact MID \
meta: {:?}",
report.unreadable_files,
);
assert_eq!(
report.unreadable, 0,
"a healthy table whose forged tail the id cross-check rejects is not \
dropped: {:?}",
report.unreadable_files,
);
Ok(())
}
#[test]
fn repair_with_salvage_keeps_a_healthy_rt_sst_with_a_forged_tail_descriptor_only()
-> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("7");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 7, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
forge_tail_meta_table_id(&sst, None, Some([0u8, 8, 2, 1]))?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"the healthy table joins the rebuilt manifest via the MID mirror's \
valid descriptor: {:?}",
report.unreadable_files,
);
assert_eq!(
report.unreadable, 0,
"a healthy table whose forged tail descriptor the MID fallback \
overrides is not dropped: {:?}",
report.unreadable_files,
);
Ok(())
}
#[test]
fn repair_fails_when_a_dropped_table_cannot_be_removed() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule};
use crate::io::ErrorKind;
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
forge_unrecognized_ecc_descriptor(&sst)?;
let fault = FaultFs::new(StdFs);
let injector = fault.injector();
injector.arm(
FaultRule::new(FaultOp::RemoveFile, Fault::Error(ErrorKind::Other))
.on_path("tables")
.once(),
);
let result = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(Arc::new(fault))
.repair_with_salvage(true);
injector.clear();
assert!(
result.is_err(),
"a removal the filesystem refuses must fail the repair — the next open \
would hit the same error sweeping it, got {result:?}",
);
assert!(
fs.metadata(&sst).is_ok(),
"the file the removal could not delete is still there for the retry",
);
Ok(())
}
#[test]
fn repair_with_salvage_rejects_an_unrecognized_ecc_tombstone_sst_with_a_corrupt_index()
-> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_partitioned_index();
for i in 0..200u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
forge_unrecognized_ecc_descriptor(&sst)?;
let (index_pos, index_len) = {
let mut f = std::fs::File::open(&sst)?;
let reader = match crate::sfa::Reader::from_reader(&mut f) {
Ok(r) => r,
Err(e) => panic!("reading the SFA trailer failed: {e:?}"),
};
let Some(entry) = reader.toc().iter().find(|e| e.name() == b"index") else {
panic!("a partitioned-index SST must carry an index section");
};
(entry.pos(), entry.len())
};
let flip = usize::try_from(index_pos + index_len / 2).unwrap_or(0);
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 0,
"a table whose data blocks the fallback scrub could not enumerate \
must not be kept: {report:?}",
);
assert_eq!(
report.unreadable_files.len(),
1,
"the unverifiable table is reported unreadable: {:?}",
report.unreadable_files,
);
Ok(())
}
#[cfg(feature = "page_ecc")]
#[test]
fn repair_with_salvage_rejects_a_corrupt_unrecognized_ecc_tombstone_sst_with_parity_errors()
-> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::table::block::EccParams;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?
.use_ecc(Some(EccParams::try_new(4, 2)?));
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
forge_unrecognized_ecc_descriptor(&sst)?;
let block_off = usize::try_from(sole_data_block_offset(&recover_table(sst.clone(), &fs)?))
.unwrap_or(usize::MAX);
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(block_off + 40) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 0,
"parity-only meta errors must not mask the unscanned data sections: {report:?}",
);
assert_eq!(
report.unreadable_files.len(),
1,
"the corrupt table is reported unreadable: {:?}",
report.unreadable_files,
);
Ok(())
}
#[test]
fn repair_with_salvage_rejects_an_unrecognized_ecc_tombstone_sst_with_a_corrupt_filter()
-> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..200u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
forge_unrecognized_ecc_descriptor(&sst)?;
let (filter_pos, filter_len) = {
let mut f = std::fs::File::open(&sst)?;
let reader = match crate::sfa::Reader::from_reader(&mut f) {
Ok(r) => r,
Err(e) => panic!("reading the SFA trailer failed: {e:?}"),
};
let Some(entry) = reader.toc().iter().find(|e| e.name() == b"filter") else {
panic!("the SST must carry a filter section");
};
(entry.pos(), entry.len())
};
let flip = usize::try_from(filter_pos + filter_len / 2).unwrap_or(0);
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 0,
"a table whose lazy filter is corrupt must not be kept over the \
range-tombstone escape hatch: {report:?}",
);
assert_eq!(
report.unreadable_files.len(),
1,
"the unverifiable table is reported unreadable: {:?}",
report.unreadable_files,
);
Ok(())
}
fn forge_tail_meta_table_id(
path: &std::path::Path,
forged_id: Option<u64>,
forge_descriptor: Option<[u8; 4]>,
) -> crate::Result<()> {
use crate::coding::{Decode, Encode};
use crate::table::block::Header;
let mut bytes = std::fs::read(path)?;
let (pos, section_len) = {
let mut f = std::fs::File::open(path)?;
let reader = match crate::sfa::Reader::from_reader(&mut f) {
Ok(r) => r,
Err(e) => panic!("reading the SFA trailer failed: {e:?}"),
};
let Some(entry) = reader.toc().iter().find(|e| e.name() == b"meta") else {
panic!("the SST must carry a meta section");
};
(entry.pos(), entry.len())
};
let block_off = usize::try_from(pos).unwrap_or(usize::MAX);
let Some(block) = bytes.get(block_off..) else {
panic!("meta block within the file");
};
let mut cursor = block;
let header = Header::decode_from(&mut cursor)?;
let header_len = Header::header_len(header.block_type);
let payload_range =
block_off + header_len..block_off + header_len + header.data_length as usize;
{
let Some(payload) = bytes.get_mut(payload_range.clone()) else {
panic!("meta payload within the file");
};
if let Some(forged_id) = forged_id {
let needle = b"table_id";
let Some(key_pos) = payload
.windows(needle.len())
.position(|w| w == needle.as_slice())
else {
panic!("table_id key present verbatim (restart interval 1)");
};
let val_at = key_pos + needle.len();
assert_eq!(
payload.get(val_at).copied(),
Some(8),
"table_id value length prefix",
);
let Some(value) = payload.get_mut(val_at + 1..val_at + 9) else {
panic!("table_id value within the payload");
};
value.copy_from_slice(&forged_id.to_le_bytes());
}
if let Some(descriptor) = forge_descriptor {
let needle = b"descriptor#page_ecc";
let Some(key_pos) = payload
.windows(needle.len())
.position(|w| w == needle.as_slice())
else {
panic!("descriptor key present verbatim (restart interval 1)");
};
let val_at = key_pos + needle.len();
assert_eq!(
payload.get(val_at).copied(),
Some(4),
"descriptor value length prefix",
);
let Some(value) = payload.get_mut(val_at + 1..val_at + 5) else {
panic!("descriptor value within the payload");
};
value.copy_from_slice(&descriptor);
}
}
#[cfg_attr(
not(feature = "page_ecc"),
expect(
clippy::redundant_clone,
reason = "the range is reused by the Page-ECC parity refresh below"
)
)]
let Some(payload) = bytes.get(payload_range.clone()) else {
panic!("meta payload within the file");
};
let new_checksum = crate::Checksum::from_raw(crate::hash::hash128(payload));
let new_header = Header {
checksum: new_checksum,
..header
};
let mut hdr_bytes = Vec::with_capacity(header_len);
new_header.encode_into(&mut hdr_bytes)?;
let Some(hdr_dst) = bytes.get_mut(block_off..block_off + header_len) else {
panic!("meta header within the file");
};
hdr_dst.copy_from_slice(&hdr_bytes);
#[cfg(feature = "page_ecc")]
{
let payload_end = payload_range.end;
let Ok(section_len) = usize::try_from(section_len) else {
panic!("meta section length fits usize");
};
let frame_end = block_off + section_len;
if frame_end > payload_end {
let Some(payload) = bytes.get(payload_range) else {
panic!("meta payload within the file");
};
let parity = crate::ecc::encode_parity(payload, 4, 2)?;
assert_eq!(
frame_end - payload_end,
parity.len(),
"the meta frame's trailer length matches the fixed RS(4,2) layout",
);
let Some(dst) = bytes.get_mut(payload_end..frame_end) else {
panic!("meta parity trailer within the file");
};
dst.copy_from_slice(&parity);
}
}
#[cfg(not(feature = "page_ecc"))]
let _ = section_len;
std::fs::write(path, &bytes)?;
Ok(())
}
#[test]
fn repair_with_salvage_preserves_the_file_name_id_over_a_forged_tail_meta_id() -> crate::Result<()>
{
use crate::table::Writer;
use crate::{AbstractTree, Config, InternalValue, MAX_SEQNO, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("7");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 7, 0, Arc::clone(&fs))?;
for i in 0..600u32 {
w.write(InternalValue::from_components(
format!("key{i:05}").into_bytes(),
format!("{i:08}").repeat(8).into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let offset = {
let checksum = crate::Checksum::from_raw(compute_table_checksum(&*fs, &sst)?);
let table = crate::table::Table::recover(crate::table::RecoverParams::new(
sst.clone(),
checksum,
7,
Arc::clone(&fs),
crate::comparator::default_comparator(),
Arc::new(crate::cache::Cache::with_capacity_bytes(1 << 20)),
))?;
let offsets: alloc::vec::Vec<u64> = table
.data_block_handles()
.filter_map(Result::ok)
.map(|kh| *kh.as_ref().offset())
.collect();
assert!(offsets.len() >= 2, "need several blocks, got {offsets:?}");
let Some(&first) = offsets.first() else {
panic!("a first data block exists");
};
first
};
let flip = usize::try_from(offset).unwrap_or(0) + 16;
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
forge_tail_meta_table_id(&sst, Some(99), None)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.salvaged, 1,
"the readable blocks are salvaged under the file-name id: {:?}",
report.unreadable_files,
);
assert_eq!(
report.unreadable, 0,
"a recoverable table is not dropped: {:?}",
report.unreadable_files,
);
let crate::AnyTree::Standard(tree) = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.open()?
else {
unreachable!("standard tree");
};
let got = tree.get(b"key00599", MAX_SEQNO)?;
assert!(
got.is_some(),
"a key outside the corrupt block survives the salvage",
);
Ok(())
}
fn forge_unrecognized_ecc_descriptor(path: &std::path::Path) -> crate::Result<()> {
use crate::coding::{Decode, Encode};
use crate::table::block::Header;
let mut bytes = std::fs::read(path)?;
let sections: Vec<(u64, bool)> = {
let mut f = std::fs::File::open(path)?;
let reader = match crate::sfa::Reader::from_reader(&mut f) {
Ok(r) => r,
Err(e) => panic!("reading the SFA trailer failed: {e:?}"),
};
[(b"meta".as_slice(), true), (b"meta_mid".as_slice(), false)]
.iter()
.map(|(name, is_tail)| {
let Some(entry) = reader.toc().iter().find(|e| e.name() == *name) else {
panic!(
"the SST must carry a {} section",
String::from_utf8_lossy(name)
);
};
(entry.pos(), *is_tail)
})
.collect()
};
for (pos, is_tail) in sections {
let block_off = usize::try_from(pos).unwrap_or(usize::MAX);
let Some(block) = bytes.get(block_off..) else {
panic!("meta block within the file");
};
let mut cursor = block;
let header = Header::decode_from(&mut cursor)?;
let header_len = Header::header_len(header.block_type);
let payload_range =
block_off + header_len..block_off + header_len + header.data_length as usize;
{
let Some(payload) = bytes.get_mut(payload_range.clone()) else {
panic!("meta payload within the file");
};
let needle = b"descriptor#page_ecc";
let Some(key_pos) = payload
.windows(needle.len())
.position(|w| w == needle.as_slice())
else {
panic!("descriptor key present verbatim (restart interval 1)");
};
let val_at = key_pos + needle.len();
assert_eq!(
payload.get(val_at).copied(),
Some(4),
"descriptor value length prefix",
);
let Some(value) = payload.get_mut(val_at + 1..val_at + 5) else {
panic!("descriptor value within the payload");
};
let forged = if is_tail || value.iter().any(|b| *b != 0) {
[0u8, 8, 2, 1]
} else {
[3u8, 4, 2, 0]
};
assert_ne!(value, forged, "descriptor not already forged");
value.copy_from_slice(&forged);
}
let new_checksum = crate::Checksum::from_raw(crate::hash::hash128(
bytes.get(payload_range).unwrap_or(&[]),
));
let new_header = Header {
checksum: new_checksum,
..header
};
let mut hdr_bytes = Vec::with_capacity(header_len);
new_header.encode_into(&mut hdr_bytes)?;
let Some(hdr_dst) = bytes.get_mut(block_off..block_off + header_len) else {
panic!("meta header within the file");
};
hdr_dst.copy_from_slice(&hdr_bytes);
}
std::fs::write(path, &bytes)?;
Ok(())
}
#[cfg(feature = "columnar")]
#[test]
fn repair_with_salvage_drops_a_corrupt_delete_bitmap_sst() -> crate::Result<()> {
use crate::config::DeleteStrategy;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
let n = 200u32;
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?
.use_columnar(true)
.use_zone_map(true)
.delete_strategy(DeleteStrategy::MergeOnRead);
for i in 0..n {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
for pos in [5u32, 50, 150] {
w.delete_bitmap_mut().insert(pos);
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let (pos, len) = {
let mut f = std::fs::File::open(&sst)?;
let reader = match crate::sfa::Reader::from_reader(&mut f) {
Ok(r) => r,
Err(e) => panic!("reading the SFA trailer failed: {e:?}"),
};
let Some(entry) = reader.toc().iter().find(|e| e.name() == b"delete_bitmap") else {
panic!("the SST must carry a delete_bitmap section");
};
(entry.pos(), entry.len())
};
let flip = usize::try_from(pos + len / 2).unwrap_or(0);
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.salvaged, 0,
"automated repair refuses to resurrect deleted rows: {:?}",
report.unreadable_files,
);
assert_eq!(report.recovered, 0, "no table joins the rebuilt manifest");
let [(_, reason)] = report.unreadable_files.as_slice() else {
panic!(
"expected exactly one unreadable file, got {:?}",
report.unreadable_files,
);
};
assert!(
reason.contains("salvage failed") && reason.contains("resurrect"),
"the reason names the refused delete resurrection, got: {reason}",
);
Ok(())
}
#[cfg(feature = "columnar")]
#[test]
fn repair_with_resurrection_recovers_a_corrupt_delete_bitmap_sst() -> crate::Result<()> {
use crate::config::DeleteStrategy;
use crate::table::Writer;
use crate::{AbstractTree, Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
let n = 200u32;
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?
.use_columnar(true)
.use_zone_map(true)
.delete_strategy(DeleteStrategy::MergeOnRead);
for i in 0..n {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
for pos in [5u32, 50, 150] {
w.delete_bitmap_mut().insert(pos);
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let (pos, len) = {
let mut f = std::fs::File::open(&sst)?;
let reader = crate::sfa::Reader::from_reader(&mut f)?;
let entry = reader
.toc()
.iter()
.find(|e| e.name() == b"delete_bitmap")
.ok_or(crate::Error::Unrecoverable)?;
(entry.pos(), entry.len())
};
let flip = usize::try_from(pos + len / 2).unwrap_or(0);
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_resurrection(true, true)?;
assert_eq!(
report.recovered, 1,
"resurrection recovers the table instead of excluding it: {report:?}",
);
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
let tree = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.open()?;
for i in [5u32, 50, 100, 150] {
let key = format!("k{i:05}").into_bytes();
assert!(
tree.get(&key, crate::MAX_SEQNO)?.is_some(),
"key {key:?} must be served under resurrection",
);
}
Ok(())
}
#[test]
fn repair_with_salvage_drops_a_toc_hidden_range_tombstone_sst() -> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
crate::test_forge::forge_section_omitted(&sst, b"range_tombstones")?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.salvaged, 0,
"salvage must not re-emit a TOC-hidden tombstone's covered keys as \
live: {:?}",
report.unreadable_files,
);
assert_eq!(report.recovered, 0, "no table joins the rebuilt manifest");
assert_eq!(
report.unreadable_files.len(),
1,
"the hidden-deletion table is reported unreadable: {:?}",
report.unreadable_files,
);
assert!(
report
.unreadable_files
.iter()
.any(|(_, reason)| reason.contains("may hide deletion metadata")),
"the refusal must come from the TOC concealment gate: {:?}",
report.unreadable_files,
);
assert!(
!tables.join("0").exists(),
"the dropped table's file must not stay in tables/ as an orphan the \
next open has to sweep: {:?}",
report.unreadable_files,
);
Ok(())
}
#[test]
fn repair_with_resurrection_still_excludes_a_toc_hidden_range_tombstone_sst() -> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::{AbstractTree, Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
crate::test_forge::forge_section_omitted(&sst, b"range_tombstones")?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_resurrection(true, true)?;
assert_eq!(
report.recovered, 0,
"salvage cannot re-emit a range-tombstone table, so flag-on still excludes it: {:?}",
report.unreadable_files,
);
assert_eq!(report.unreadable, 1, "{:?}", report.unreadable_files);
assert!(
report
.unreadable_files
.iter()
.any(|(_, reason)| reason.contains("range tombstones")),
"flag-on routes to salvage, which reports the range-tombstone table \
unsalvageable: {:?}",
report.unreadable_files,
);
let tree = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.open()?;
assert!(
tree.get(b"k00000", crate::MAX_SEQNO)?.is_none(),
"the whole excluded table is absent, not partially resurrected",
);
Ok(())
}
#[test]
fn repair_with_salvage_drops_a_range_tombstone_renamed_to_a_rebuildable_section()
-> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::table::block::BlockType;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
crate::test_forge::forge_duplicate_section_name(
&sst,
b"range_tombstones",
b"filter_tli",
BlockType::Index,
)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.salvaged, 0,
"a relabeled range_tombstones must not be salvaged into a live copy: {:?}",
report.unreadable_files,
);
assert_eq!(report.recovered, 0, "no table joins the rebuilt manifest");
assert!(
report
.unreadable_files
.iter()
.any(|(_, reason)| reason.contains("range tombstones")),
"the refusal must come from the range-tombstone gate: {:?}",
report.unreadable_files,
);
assert!(
!dir.path().join("tables").join("0").exists(),
"the relabeled table is dropped and its file removed: {:?}",
report.unreadable_files,
);
Ok(())
}
#[cfg(feature = "page_ecc")]
#[test]
fn verify_probe_distrusts_disagreeing_recognized_ecc_descriptors() -> crate::Result<()> {
use crate::table::Writer;
use crate::{InternalValue, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("7");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 7, 0, Arc::clone(&fs))?
.use_ecc(Some(crate::table::block::EccParams::RS_4_2));
for i in 0..64u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
forge_tail_meta_table_id(&sst, None, Some([0u8, 0, 0, 0]))?;
let report = crate::verify::verify_sst_file_with_fs(&fs, &sst);
assert!(
report
.errors
.iter()
.all(|e| matches!(e, crate::verify::BlockVerifyError::TocCorrupted { .. })),
"a forged recognized descriptor must not condemn the data blocks — \
the walk mis-sizes every parity trailer as a block header: {report:?}",
);
assert_eq!(
report.errors.len(),
1,
"exactly the mirror-divergence finding: {report:?}",
);
assert!(
report
.warnings
.iter()
.any(|w| matches!(w, crate::verify::BlockVerifyWarning::UnrecognizedEcc { .. })),
"disagreeing decodable descriptors must surface as an \
indeterminate-ECC warning: {report:?}",
);
Ok(())
}
#[cfg(feature = "page_ecc")]
#[test]
fn repair_grades_a_stale_kv_footer_corrupt_over_parity_only_degradation() -> crate::Result<()> {
use crate::range_tombstone::RangeTombstone;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, UserKey, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
use crate::runtime_config::{ChecksumAlgorithm, KvChecksumPolicy};
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?
.use_ecc(Some(crate::table::block::EccParams::RS_4_2))
.use_kv_checksums(KvChecksumPolicy::AllLevels, ChecksumAlgorithm::Xxh3_64);
for i in 0..8u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
w.write_range_tombstone(RangeTombstone::new(
UserKey::from(b"k00002".as_slice()),
UserKey::from(b"k00005".as_slice()),
2,
));
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
crate::test_forge::forge_stale_kv_footer(&sst)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 0,
"a stale-footer table must not be kept as parity-only degradation: {report:?}",
);
assert_eq!(
report.unreadable, 1,
"the corrupt table is reported and dropped: {report:?}",
);
Ok(())
}
#[test]
fn repair_routes_a_stale_kv_footer_through_salvage() -> crate::Result<()> {
use crate::runtime_config::KvChecksumPolicy;
use crate::{AbstractTree, Config, SequenceNumberCounter};
let dir = tempfile::tempdir()?;
let sst_path = {
let crate::AnyTree::Standard(tree) = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.data_block_compression_policy(crate::config::CompressionPolicy::all(
crate::CompressionType::None,
))
.open()?
else {
unreachable!("standard tree configured");
};
tree.update_runtime_config(|c| c.kv_checksums = KvChecksumPolicy::AllLevels)?;
for i in 0u64..500 {
tree.insert(format!("key-{i:06}"), format!("v{i:06}"), i);
}
tree.flush_active_memtable(500)?;
let binding = tree.version_history.read().latest_version();
let Some(table) = binding.version.iter_tables().next() else {
panic!("flush produced one table");
};
(*table.path).clone()
};
crate::test_forge::forge_stale_kv_footer(&sst_path)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.salvaged, 1,
"the stale-footer table must be routed through salvage: {report:?}",
);
let crate::AnyTree::Standard(tree) = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.open()?
else {
unreachable!("standard tree configured");
};
crate::verify::verify_kv_checksums(&tree)?;
Ok(())
}
#[test]
fn repair_routes_an_under_decoding_footerless_block_through_salvage() -> crate::Result<()> {
use crate::{AbstractTree, Config, SequenceNumberCounter};
let dir = tempfile::tempdir()?;
let sst_path = {
let crate::AnyTree::Standard(tree) = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.data_block_compression_policy(crate::config::CompressionPolicy::all(
crate::CompressionType::None,
))
.open()?
else {
unreachable!("standard tree configured");
};
for i in 0u64..500 {
tree.insert(format!("key-{i:06}"), format!("v{i:06}"), i);
}
tree.flush_active_memtable(500)?;
let binding = tree.version_history.read().latest_version();
let Some(table) = binding.version.iter_tables().next() else {
panic!("flush produced one table");
};
(*table.path).clone()
};
crate::test_forge::forge_inflated_item_count(&sst_path)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
report.salvaged, 1,
"the under-decoding footer-less table must be routed through salvage: {report:?}",
);
Ok(())
}
#[test]
fn is_corruption_routes_a_persistent_io_to_salvage() {
let persistent = crate::Error::Io(crate::io::Error::other("bad sector"));
assert!(
matches!(super::is_corruption(Err(persistent)), Ok(true)),
"a persistent I/O failure must grade as corruption, not abort the repair",
);
}
#[test]
fn is_corruption_aborts_the_repair_on_a_transient_io() {
let transient = crate::Error::Io(crate::io::Error::from_kind(
crate::io::ErrorKind::Interrupted,
));
assert!(
super::is_corruption(Err(transient)).is_err(),
"a transient I/O failure must propagate so the repair can retry",
);
}
fn write_three_frame_blob(
memfs: &std::sync::Arc<crate::fs::MemFs>,
path: &std::path::Path,
) -> crate::Result<()> {
let fs_dyn: std::sync::Arc<dyn crate::fs::Fs> = memfs.clone();
let mut w = crate::vlog::blob_file::writer::Writer::new(path, 0, 0, &*fs_dyn)?;
w.write(b"a", 1, &[b'x'; 300])?;
w.write(b"b", 2, &[b'y'; 300])?;
w.write(b"c", 3, &[b'z'; 300])?;
w.finish()?;
Ok(())
}
fn validate_frames(
config: &crate::Config,
path: &std::path::Path,
blob_id: crate::vlog::BlobFileId,
live_data_start: u64,
) -> crate::Result<Option<super::BlobLiveTotals>> {
let handle =
crate::vlog::recover_blob_file(path, blob_id, crate::Checksum::from_raw(0), 0, &config.fs)?;
super::validate_blob_frames(config, path, blob_id, live_data_start, &handle)
}
fn blob_validation_config(memfs: std::sync::Arc<crate::fs::MemFs>) -> crate::Config {
crate::Config::new(
std::path::PathBuf::from("/db"),
crate::SequenceNumberCounter::default(),
crate::SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
}
#[test]
fn blob_recovery_discards_a_file_whose_metadata_id_disagrees() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
memfs.create_dir_all(&blobs)?;
let fs_dyn: Arc<dyn Fs> = memfs.clone();
for id in 0u64..2 {
let path = blobs.join(id.to_string());
let mut w = crate::vlog::blob_file::writer::Writer::new(&path, id, 0, &*fs_dyn)?;
w.write(b"a", 1, &[b'x'; 200])?;
w.finish()?;
}
let (zero, one, tmp) = (blobs.join("0"), blobs.join("1"), blobs.join("swap-tmp"));
memfs.rename(&zero, &tmp)?;
memfs.rename(&one, &zero)?;
memfs.rename(&tmp, &one)?;
let config = blob_validation_config(Arc::clone(&memfs));
let mut published = super::PublishedBlobReplacements::new(&config);
let recovery =
super::recover_blob_files(&config, &mut published, &(0..10).collect(), None, None)?;
published.disarm();
assert!(
recovery.files.is_empty(),
"no swapped file may be published under its filename's id",
);
assert_eq!(
recovery.unreadable.len(),
2,
"both swapped files are set aside: {:?}",
recovery.unreadable,
);
assert!(
recovery
.unreadable
.iter()
.all(|(_, reason)| reason.contains("disagrees")),
"the reason names the id mismatch: {:?}",
recovery.unreadable,
);
assert_eq!(
recovery.discard.len(),
2,
"both are queued for removal once the manifest is durable: {:?}",
recovery.discard,
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn blob_frame_validation_rejects_reordered_frames() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
memfs.create_dir_all(&root)?;
let path = root.join("0");
write_three_frame_blob(&memfs, &path)?;
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(&path, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?;
let (a, b) = (
entries.first().expect("frame a"),
entries.get(1).expect("frame b"),
);
assert_eq!(
a.frame_end - a.offset,
b.frame_end - b.offset,
"equal-length frames swap cleanly",
);
let frame = |e: &crate::vlog::blob_file::scanner::ScanEntry| -> crate::Result<Vec<u8>> {
Ok(crate::file::read_exact(
&*memfs.open(&path, &crate::fs::FsOpenOptions::new().read(true))?,
e.offset,
usize::try_from(e.frame_end - e.offset).expect("frame fits usize"),
)?
.to_vec())
};
let (bytes_a, bytes_b) = (frame(a)?, frame(b)?);
{
let mut f = memfs.open(
&path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
f.seek(SeekFrom::Start(a.offset))?;
f.write_all(&bytes_b)?;
f.write_all(&bytes_a)?;
}
let config = blob_validation_config(memfs);
assert!(
validate_frames(&config, &path, 0, 0)?.is_none(),
"reordered (individually valid) frames must fail validation",
);
Ok(())
}
#[cfg(feature = "lz4")]
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn blob_frame_validation_rejects_a_restamped_compressed_payload() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
memfs.create_dir_all(&root)?;
let path = root.join("0");
let fs_dyn: Arc<dyn Fs> = memfs.clone();
{
let mut w = crate::vlog::blob_file::writer::Writer::new(&path, 0, 0, &*fs_dyn)?
.use_compression(crate::CompressionType::Lz4);
w.write(b"a", 1, b"compressible compressible compressible")?;
w.finish()?;
}
let entry = crate::vlog::BlobFileScanner::new(&path, &*fs_dyn, 0)?
.next()
.expect("one frame")?;
let header_len = u64::try_from(crate::vlog::blob_file::writer::BLOB_HEADER_LEN).expect("small");
let value_start = entry.offset + header_len + 1; let value_len = usize::try_from(entry.frame_end - value_start).expect("fits");
let crc_bytes = crate::file::read_exact(
&*memfs.open(&path, &crate::fs::FsOpenOptions::new().read(true))?,
entry.offset + 38, 4,
)?
.to_vec();
let garbage = vec![0xA5u8; value_len];
let restamped = {
let mut hasher = xxhash_rust::xxh3::Xxh3::default();
hasher.update(b"a");
hasher.update(&garbage);
hasher.update(&crc_bytes);
hasher.digest128()
};
{
let mut f = memfs.open(
&path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
f.seek(SeekFrom::Start(entry.offset + 4))?;
f.write_all(&restamped.to_le_bytes())?;
f.seek(SeekFrom::Start(value_start))?;
f.write_all(&garbage)?;
}
assert!(
crate::vlog::BlobFileScanner::new(&path, &*fs_dyn, 0)?
.next()
.expect("one frame")
.is_ok(),
"fixture: the restamped frame must still pass the raw scan",
);
let config = blob_validation_config(memfs);
assert!(
validate_frames(&config, &path, 0, 0)?.is_none(),
"a checksum-restamped, undecodable compressed payload must fail validation",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn blob_frame_validation_rejects_lying_metadata_counters() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
memfs.create_dir_all(&root)?;
let victim = root.join("0");
let donor = root.join("donor");
let fs_dyn: Arc<dyn Fs> = memfs.clone();
write_three_frame_blob(&memfs, &victim)?;
{
let mut w = crate::vlog::blob_file::writer::Writer::new(&donor, 0, 0, &*fs_dyn)?;
w.write(b"a", 1, &[b'x'; 300])?;
w.write(b"c", 3, &[b'z'; 300])?;
w.finish()?;
}
let section = |path: &std::path::Path| -> crate::Result<(u64, u64)> {
let mut f = memfs.open(path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut f)?;
let meta = reader.toc().section(b"meta").expect("meta section");
Ok((meta.pos(), meta.len()))
};
let (victim_pos, victim_len) = section(&victim)?;
let (donor_pos, donor_len) = section(&donor)?;
assert_eq!(
victim_len, donor_len,
"equal-width metadata transplants cleanly"
);
let donor_meta = crate::file::read_exact(
&*memfs.open(&donor, &crate::fs::FsOpenOptions::new().read(true))?,
donor_pos,
usize::try_from(donor_len).expect("fits"),
)?
.to_vec();
{
let mut f = memfs.open(
&victim,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
f.seek(SeekFrom::Start(victim_pos))?;
f.write_all(&donor_meta)?;
}
let config = blob_validation_config(memfs);
assert!(
validate_frames(&config, &victim, 0, 0)?.is_none(),
"metadata counters disagreeing with the scanned frames must fail validation",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn blob_frame_validation_rejects_understated_metadata_on_a_punched_file() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
memfs.create_dir_all(&root)?;
let victim = root.join("0");
let donor = root.join("donor");
let fs_dyn: Arc<dyn Fs> = memfs.clone();
write_three_frame_blob(&memfs, &victim)?;
{
let mut w = crate::vlog::blob_file::writer::Writer::new(&donor, 0, 0, &*fs_dyn)?;
w.write(b"a", 1, &[b'x'; 300])?;
w.finish()?;
}
let suffix_start = crate::vlog::BlobFileScanner::new(&victim, &*fs_dyn, 0)?
.next()
.expect("first frame")?
.frame_end;
let config = blob_validation_config(Arc::clone(&memfs));
assert!(
validate_frames(&config, &victim, 0, suffix_start)?.is_some(),
"fixture: the untampered punched file must pass validation",
);
let section = |path: &std::path::Path| -> crate::Result<(u64, u64)> {
let mut f = memfs.open(path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut f)?;
let meta = reader.toc().section(b"meta").expect("meta section");
Ok((meta.pos(), meta.len()))
};
let (victim_pos, victim_len) = section(&victim)?;
let (donor_pos, donor_len) = section(&donor)?;
assert_eq!(
victim_len, donor_len,
"equal-width metadata transplants cleanly"
);
let donor_meta = crate::file::read_exact(
&*memfs.open(&donor, &crate::fs::FsOpenOptions::new().read(true))?,
donor_pos,
usize::try_from(donor_len).expect("fits"),
)?
.to_vec();
{
let mut f = memfs.open(
&victim,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
f.seek(SeekFrom::Start(victim_pos))?;
f.write_all(&donor_meta)?;
}
assert!(
validate_frames(&config, &victim, 0, suffix_start)?.is_none(),
"metadata understating the live suffix must fail validation on a punched file",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn blob_frame_validation_rejects_a_range_not_containing_the_punched_suffix() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
memfs.create_dir_all(&root)?;
let victim = root.join("0");
let donor = root.join("donor");
let fs_dyn: Arc<dyn Fs> = memfs.clone();
write_three_frame_blob(&memfs, &victim)?;
{
let mut w = crate::vlog::blob_file::writer::Writer::new(&donor, 0, 0, &*fs_dyn)?;
w.write(b"a", 1, &[b'x'; 300])?;
w.write(b"b", 2, &[b'y'; 300])?;
w.finish()?;
}
let suffix_start = crate::vlog::BlobFileScanner::new(&victim, &*fs_dyn, 0)?
.next()
.expect("first frame")?
.frame_end;
let section = |path: &std::path::Path| -> crate::Result<(u64, u64)> {
let mut f = memfs.open(path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut f)?;
let meta = reader.toc().section(b"meta").expect("meta section");
Ok((meta.pos(), meta.len()))
};
let (victim_pos, victim_len) = section(&victim)?;
let (donor_pos, donor_len) = section(&donor)?;
assert_eq!(
victim_len, donor_len,
"equal-width metadata transplants cleanly"
);
let donor_meta = crate::file::read_exact(
&*memfs.open(&donor, &crate::fs::FsOpenOptions::new().read(true))?,
donor_pos,
usize::try_from(donor_len).expect("fits"),
)?
.to_vec();
{
let mut f = memfs.open(
&victim,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
f.seek(SeekFrom::Start(victim_pos))?;
f.write_all(&donor_meta)?;
}
let config = blob_validation_config(memfs);
assert!(
validate_frames(&config, &victim, 0, suffix_start)?.is_none(),
"a key range not containing the scanned suffix must fail validation \
even when the item and byte totals satisfy the lower bounds",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn blob_recovery_derives_the_frontier_of_a_punched_blob_file() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
memfs.create_dir_all(&blobs)?;
let punched_path = blobs.join("0");
let whole_path = blobs.join("1");
let fs_dyn: Arc<dyn Fs> = memfs.clone();
for (id, path) in [(0u64, &punched_path), (1u64, &whole_path)] {
let mut w = crate::vlog::blob_file::writer::Writer::new(path, id, 0, &*fs_dyn)?;
w.write(b"a", 1, &[b'x'; 300])?;
w.write(b"b", 2, &vec![0u8; 400])?; w.write(b"c", 3, &[b'y'; 300])?;
w.finish()?;
}
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(&punched_path, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?;
assert_eq!(entries.len(), 3, "three frames written");
let frontier = entries.first().expect("first frame").frame_end;
let data_start = {
let mut file = fs_dyn.open(&punched_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
reader
.toc()
.section(b"data")
.expect("blob file has a data section")
.pos()
};
memfs.punch_hole(&punched_path, data_start, frontier - data_start)?;
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs);
let mut published = super::PublishedBlobReplacements::new(&config);
let recovery =
super::recover_blob_files(&config, &mut published, &(0..10).collect(), None, None)?;
published.disarm();
let (files, unreadable) = (recovery.files, recovery.unreadable);
assert!(
unreadable.is_empty(),
"both blob files recover: {unreadable:?}"
);
assert_eq!(files.len(), 2, "both blob files recovered");
let punched = files.iter().find(|f| f.id() == 0).expect("punched file");
let whole = files.iter().find(|f| f.id() == 1).expect("whole file");
assert_eq!(
punched.live_data_start(),
frontier,
"repair must derive the punched file's frontier from its geometry"
);
let suffix_digest = crate::Checksum::from_raw(super::compute_table_checksum_from(
&*config.fs,
&punched_path,
frontier,
)?);
assert_eq!(
punched.checksum(),
suffix_digest,
"the recorded digest must cover the live suffix, not the zeroed prefix"
);
assert_eq!(
whole.live_data_start(),
0,
"an unpunched file keeps the whole-file frontier"
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_seeds_garbage_accounting_for_a_punched_blob_prefix() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fs, MemFs};
use crate::{Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let open_config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
{
let tree = match open_config().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(
format!("k{i:04}").as_bytes(),
alloc::vec![b'a' + u8::try_from(i).expect("small i"); 64],
u64::from(i),
);
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = blobs.join("0");
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?;
assert_eq!(entries.len(), 8, "eight separated values");
let prefix: Vec<_> = entries.iter().take(2).collect();
let frontier = prefix.last().expect("two frames").frame_end;
let prefix_bytes: u64 = prefix.iter().map(|e| u64::from(e.uncompressed_len)).sum();
let prefix_on_disk: u64 = prefix.iter().map(|e| e.value.len() as u64).sum();
let data_start = {
let mut f = fs_dyn.open(&blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut f)?;
reader
.toc()
.section(b"data")
.expect("blob file has a data section")
.pos()
};
memfs.punch_hole(&blob_path, data_start, frontier - data_start)?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
open_config().repair()?;
let tree = match open_config().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
let binding = tree.index.version_history.read().latest_version();
let entry = binding.version.gc_stats().get(&0).copied();
let entry = entry.expect(
"the rebuilt manifest must seed garbage accounting for the punched prefix, \
or the file can never be retired by blob GC",
);
assert_eq!(entry.bytes, prefix_bytes, "stale uncompressed bytes");
assert_eq!(entry.on_disk_bytes, prefix_on_disk, "stale on-disk bytes");
assert_eq!(entry.len, 2, "two punched-away records");
Ok(())
}
#[test]
fn repair_fails_when_its_own_salvage_temp_cannot_be_removed() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
tree.insert(b"k0", alloc::vec![b'x'; 64], 0);
tree.flush_active_memtable(0)?;
}
let blob_path = root.join(crate::file::BLOBS_FOLDER).join("0");
let (data_start, data_len) = {
let mut f = fs_dyn.open(&blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut f)?;
let data = reader
.toc()
.section(b"data")
.ok_or(crate::Error::InvalidHeader("BlobFile"))?;
(data.pos(), data.len())
};
{
#[expect(
clippy::cast_possible_truncation,
reason = "test fixture data section is tiny"
)]
let garbage = alloc::vec![0xA5u8; data_len as usize];
let mut f = memfs.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
f.seek(SeekFrom::Start(data_start))?;
f.write_all(&garbage)?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(
FaultOp::RemoveFile,
Fault::Error(ErrorKind::PermissionDenied),
)
.on_path("salvage-tmp"),
);
let fault_config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
));
let result = fault_config.repair();
assert!(
result.is_err(),
"repair must FAIL, not report success for a tree that cannot open: {result:?}",
);
let retry_config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
));
retry_config.repair()?;
assert!(
!memfs.exists(&root.join(crate::file::BLOBS_FOLDER).join("0.salvage-tmp"))?,
"the retry removes the temp",
);
retry_config.open()?;
Ok(())
}
#[test]
fn repair_fails_when_a_leftover_salvage_temp_cannot_be_removed() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
tree.insert(b"k0", alloc::vec![b'x'; 64], 0);
tree.flush_active_memtable(0)?;
}
let tmp_path = root.join(crate::file::BLOBS_FOLDER).join("0.salvage-tmp");
{
let mut f = memfs.open(
&tmp_path,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
f.write_all(b"partial salvage bytes")?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(
FaultOp::RemoveFile,
Fault::Error(ErrorKind::PermissionDenied),
)
.on_path("salvage-tmp"),
);
let fault_config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
));
let result = fault_config.repair();
assert!(
result.is_err(),
"repair must FAIL, not report success for a tree that cannot open: {result:?}",
);
let retry_config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
));
let report = retry_config.repair()?;
assert!(!memfs.exists(&tmp_path)?, "the retry removes the temp");
assert_eq!(
report.recovered, 1,
"the data survives untouched: {report:?}"
);
retry_config.open()?;
Ok(())
}
#[test]
fn repair_fails_when_a_fully_punched_blobs_drop_cannot_complete() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
tree.insert(b"k0", alloc::vec![b'x'; 64], 0);
tree.insert(b"k1", alloc::vec![b'y'; 64], 1);
tree.flush_active_memtable(0)?;
}
let blob_path = root.join(crate::file::BLOBS_FOLDER).join("0");
let (data_start, data_end) = {
let mut f = fs_dyn.open(&blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut f)?;
let data = reader
.toc()
.section(b"data")
.ok_or(crate::Error::InvalidHeader("BlobFile"))?;
(data.pos(), data.pos() + data.len())
};
memfs.punch_hole(&blob_path, data_start, data_end - data_start)?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(
FaultOp::RemoveFile,
Fault::Error(ErrorKind::PermissionDenied),
)
.on_path("blobs"),
);
let fault_config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
));
let result = fault_config.repair();
assert!(
result.is_err(),
"repair must FAIL, not report success for a tree that cannot open: {result:?}",
);
assert!(
memfs.exists(&blob_path)?,
"nothing is moved or discarded on the failure path",
);
let retry_config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
));
retry_config.repair()?;
assert!(
!memfs.exists(&blob_path)?,
"the retry completes the lagged drop",
);
retry_config.open()?;
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn blob_recovery_queues_the_drop_of_a_fully_punched_blob_file() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
memfs.create_dir_all(&blobs)?;
let consumed_path = blobs.join("0");
let live_path = blobs.join("1");
let fs_dyn: Arc<dyn Fs> = memfs.clone();
for (id, path) in [(0u64, &consumed_path), (1u64, &live_path)] {
let mut w = crate::vlog::blob_file::writer::Writer::new(path, id, 0, &*fs_dyn)?;
w.write(b"a", 1, &[b'x'; 300])?;
w.write(b"b", 2, &[b'y'; 300])?;
w.finish()?;
}
let (data_start, data_end) = {
let mut file = fs_dyn.open(&consumed_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
let data = reader
.toc()
.section(b"data")
.expect("blob file has a data section");
(data.pos(), data.pos() + data.len())
};
memfs.punch_hole(&consumed_path, data_start, data_end - data_start)?;
let config = blob_validation_config(Arc::clone(&memfs));
let mut published = super::PublishedBlobReplacements::new(&config);
let recovery =
super::recover_blob_files(&config, &mut published, &(0..10).collect(), None, None)?;
published.disarm();
assert!(
recovery.unreadable.is_empty(),
"a fully consumed file is a completed relocation, not damage: {:?}",
recovery.unreadable,
);
assert_eq!(
recovery.files.len(),
1,
"only the live blob file is published"
);
assert_eq!(
recovery.files.first().map(crate::vlog::BlobFile::id),
Some(1)
);
assert!(
memfs.exists(&consumed_path)?,
"the scan is read-only: the lagged drop belongs after the commit",
);
assert!(
recovery.discard.iter().any(|(p, _)| p == &consumed_path),
"the lagged drop is queued for after the commit: {:?}",
recovery.discard,
);
Ok(())
}
#[test]
fn repair_abort_leaves_a_fully_punched_blob_in_place() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
tree.insert(b"k0", alloc::vec![b'x'; 64], 0);
tree.flush_active_memtable(0)?;
tree.insert(b"m0", alloc::vec![b'y'; 64], 1);
tree.flush_active_memtable(0)?;
}
let blob_path = root.join(crate::file::BLOBS_FOLDER).join("0");
let (data_start, data_end) = {
let mut f = fs_dyn.open(&blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut f)?;
let data = reader
.toc()
.section(b"data")
.ok_or(crate::Error::InvalidHeader("BlobFile"))?;
(data.pos(), data.pos() + data.len())
};
memfs.punch_hole(&blob_path, data_start, data_end - data_start)?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::WouldBlock)).on_path(
root.join(crate::file::BLOBS_FOLDER)
.join("1")
.to_string_lossy(),
),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair();
assert!(
result.is_err(),
"the injected transient failure aborts the repair: {:?}",
result.map(|r| r.recovered),
);
assert!(
memfs.exists(&blob_path)?,
"a pre-commit abort must leave the fully punched blob in place",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_rewrites_a_restricted_table_that_references_a_reshaped_blob() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let kv = || Some(KvSeparationOptions::default().separation_threshold(16));
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(kv())
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let frontier = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.take(3)
.last()
.expect("three frames")?
.frame_end;
let data_start = {
let mut file = fs_dyn.open(&blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
reader
.toc()
.section(b"data")
.expect("blob file has a data section")
.pos()
};
memfs.punch_hole(&blob_path, data_start, frontier - data_start)?;
let source = root.join("tables").join("0");
crate::restrict_bound::write(
&*fs_dyn,
&source,
None,
0,
b"k0001",
crate::fs::SyncMode::Normal,
)?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_kv_separation(kv());
let report = config.repair()?;
assert_eq!(
report.recovered, 1,
"the restricted table is rewritten, not set aside: {report:?}",
);
let tree = match config.open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 3..8u32 {
let key = format!("k{i:04}");
assert!(
tree.get(key.as_bytes(), crate::SeqNo::MAX)?.is_some(),
"{key} is in the live suffix with an intact record and must survive",
);
}
assert!(
tree.get(b"k0000", crate::SeqNo::MAX)?.is_none(),
"the key below the restriction bound must stay hidden",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_swaps_a_routed_rewrite_through_its_own_backend() -> crate::Result<()> {
use crate::config::LevelRoute;
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let hotfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let hot = std::path::absolute("/hot")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.level_routes(vec![LevelRoute {
levels: 0..2,
path: hot.clone(),
fs: hotfs.clone(),
}])
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
{
let tree = match config().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let routed_source = hot.join("tables").join("0");
assert!(
hotfs.exists(&routed_source)?,
"the fixture's SST is on the routed tier",
);
let blob_path = memfs
.read_dir(&root.join(crate::file::BLOBS_FOLDER))?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
{
use std::io::{Seek, SeekFrom, Write};
let last = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.last()
.expect("a last frame")?;
let flip_at = last.frame_end - 8;
let mut file = fs_dyn.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*file, flip_at, 1)?;
file.seek(SeekFrom::Start(flip_at))?;
file.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
config().repair()?;
assert!(
hotfs.exists(&routed_source)?,
"the rewritten table takes its own name on the tier it lives on",
);
assert!(
!hotfs.exists(&super::repair_tmp_path(&routed_source))?,
"the swap completed on that tier, leaving no unpublished replacement",
);
config().open()?;
Ok(())
}
#[test]
fn repair_sweeps_orphaned_recovery_artifacts_before_success() -> crate::Result<()> {
use crate::fs::Fs;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(crate::fs::MemFs::new());
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.open()?
{
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
}
let tables = root.join("tables");
let disposable = [
tables.join("0.heal-attest.tmp"),
tables.join("7.healtmp-3"),
tables.join("0.restrict-bound.tmp"),
];
let orphan = tables.join("9.heal-attest");
let preserved = tables.join("0.heal-attest");
for path in disposable.iter().chain([&orphan, &preserved]) {
let mut f = memfs.open(
path,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
f.write_all(&[0xEE; 32])?;
}
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.repair()?;
for path in &disposable {
assert!(
!memfs.exists(path)?,
"a disposable crashed-heal artifact must not outlive a successful \
repair: {}",
path.display(),
);
}
assert!(
!memfs.exists(&orphan)?,
"an orphaned live sidecar (its table is gone) must not outlive a \
successful repair",
);
assert!(
memfs.exists(&preserved)?,
"a surviving table's live sidecar is preserved",
);
Ok(())
}
#[test]
fn a_post_commit_cleanup_failure_carries_the_report() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.open()?
{
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
}
{
let path = root.join("tables").join("0.heal-attest.tmp");
let mut f = memfs.open(
&path,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &[0xEE; 16])?;
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(
FaultOp::RemoveFile,
Fault::Error(ErrorKind::PermissionDenied),
)
.on_path("heal-attest.tmp"),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair();
match result {
Err(crate::Error::RepairedButUnopened { report, cause }) => {
assert_eq!(
report.recovered, 1,
"the committed repair's report rides in the error: {report:?}",
);
assert!(
matches!(*cause, crate::Error::Io(_)),
"the refused removal is the cause: {cause:?}",
);
}
other => panic!(
"a post-commit cleanup failure must carry the committed report, \
got {:?}",
other.map(|r| r.recovered),
),
}
Ok(())
}
#[test]
fn a_lost_routed_table_under_the_blobs_prefix_is_unknowable() -> crate::Result<()> {
use crate::config::LevelRoute;
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let cold = root.join(crate::file::BLOBS_FOLDER).join("cold");
let routed_tables = cold.join("tables");
memfs.create_dir_all(&root.join("tables"))?;
memfs.create_dir_all(&routed_tables)?;
let lost = routed_tables.join("7");
{
let mut f = memfs.open(
&lost,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
f.write_all(&[0xAB; 256])?;
}
let route_fs: Arc<dyn Fs> = memfs.clone();
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.level_routes(vec![LevelRoute {
levels: 0..2,
path: cold,
fs: route_fs,
}])
.repair()?;
assert!(
report.unknowable_losses.contains(&lost),
"an unreadable routed SST has unknowable coverage even under the \
blobs prefix: {report:?}",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_does_not_leave_an_unrestrictable_rewrite_in_the_scan() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let kv = || Some(KvSeparationOptions::default().separation_threshold(16));
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(kv())
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
{
use std::io::{Seek, SeekFrom, Write};
let last = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.last()
.expect("a last frame")?;
let flip_at = last.frame_end - 8; let mut file = fs_dyn.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*file, flip_at, 1)?;
file.seek(SeekFrom::Start(flip_at))?;
file.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
let source = root.join("tables").join("0");
crate::restrict_bound::write(
&*fs_dyn,
&source,
None,
0,
b"k0001",
crate::fs::SyncMode::Normal,
)?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::Rename, Fault::Error(ErrorKind::PermissionDenied))
.on_path("restrict-bound"),
);
fault.injector().arm(
FaultRule::new(
FaultOp::RemoveFile,
Fault::Error(ErrorKind::PermissionDenied),
)
.on_path("tables"),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(kv())
.repair();
assert!(
result.is_err(),
"the failed restriction propagates: {result:?}"
);
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_kv_separation(kv());
config.repair()?;
let tree = match config.open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
assert!(
tree.get(b"k0000", crate::SeqNo::MAX)?.is_none(),
"the row below the restriction bound must stay hidden: an unrestrictable \
rewrite left in tables/ resurrects it",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_keeps_the_rewrite_source_in_place_until_the_manifest_is_committed() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let kv = || Some(KvSeparationOptions::default().separation_threshold(16));
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(kv())
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let frontier = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.next()
.expect("a first frame")?
.frame_end;
let data_start = {
let mut file = fs_dyn.open(&blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
reader
.toc()
.section(b"data")
.expect("blob file has a data section")
.pos()
};
memfs.punch_hole(&blob_path, data_start, frontier - data_start)?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let source = root.join("tables").join("0");
assert!(memfs.exists(&source)?, "the fixture's SST is at tables/0");
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::Rename, Fault::Error(ErrorKind::PermissionDenied))
.on_path(crate::file::CURRENT_VERSION_FILE),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(kv())
.repair();
assert!(result.is_err(), "the commit fails: {result:?}");
assert!(
memfs.exists(&source)?,
"the source must still be in tables/ when the commit fails: a retry \
scans tables/, so a source moved out early is a table whose keys \
silently vanish from the rebuilt manifest",
);
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_kv_separation(kv());
config.repair()?;
let tree = match config.open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 1..8u32 {
let key = format!("k{i:04}");
assert!(
tree.get(key.as_bytes(), crate::SeqNo::MAX)?.is_some(),
"{key} survives the rewrite",
);
}
Ok(())
}
#[test]
fn repair_counts_only_blob_files_that_reach_the_manifest() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fs, MemFs};
use crate::{Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let kv = || Some(KvSeparationOptions::default().separation_threshold(16));
let progress = Arc::new(crate::RecoveryProgress::default());
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(kv())
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
tree.insert(b"k0", alloc::vec![b'x'; 64], 0);
tree.flush_active_memtable(0)?;
}
let orphan = root.join(crate::file::BLOBS_FOLDER).join("1");
let mut w = crate::vlog::blob_file::writer::Writer::new(&orphan, 1, 0, &*fs_dyn)?;
w.write(b"z", 9, &[b'z'; 300])?;
w.finish()?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_kv_separation(kv())
.with_recovery_progress(Arc::clone(&progress))
.repair()?;
let snap = progress.snapshot();
assert_eq!(
snap.blob_files_discovered, 2,
"both files are seen by the scan: {snap:?}",
);
assert_eq!(
snap.blob_files_recovered, 1,
"only the referenced file reaches the manifest, and the counter must \
say so rather than claim a recovery the reference filter undid: {snap:?}",
);
Ok(())
}
#[test]
fn repair_drops_a_blob_referenced_only_by_a_lineage_excluded_output() -> crate::Result<()> {
use crate::coding::Encode;
use crate::fs::{Fs, MemFs};
use crate::{Config, InternalValue, KvSeparationOptions, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
let tables = root.join("tables");
memfs.create_dir_all(&blobs)?;
memfs.create_dir_all(&tables)?;
for (id, key) in [(0u64, b"k".as_slice()), (1u64, b"m".as_slice())] {
let mut w =
crate::table::Writer::new(tables.join(id.to_string()), id, 0, Arc::clone(&fs_dyn))?;
w.write(InternalValue::from_components(
key.to_vec(),
b"a".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the input is non-empty");
}
let blob_path = blobs.join("7");
let (offset, on_disk_size) = {
let mut w = crate::vlog::blob_file::writer::Writer::new(&blob_path, 7, 0, &*fs_dyn)?;
let offset = w.offset();
let on_disk_size = w.write(b"k", 2, &[b'x'; 300])?;
w.finish()?;
(offset, on_disk_size)
};
let indirection = crate::blob_tree::handle::BlobIndirection {
vhandle: crate::vlog::ValueHandle {
blob_file_id: 7,
offset,
on_disk_size,
},
size: 300,
};
{
let mut w = crate::table::Writer::new(tables.join("2"), 2, 0, Arc::clone(&fs_dyn))?
.use_recency(Some(0))
.use_lineage(Some(vec![0, 1]));
w.link_blob_file(7, 1, 300, u64::from(on_disk_size));
w.write(InternalValue::from_components(
b"k".to_vec(),
indirection.encode_into_vec(),
2,
ValueType::Indirection,
))?;
assert!(w.finish()?.is_some(), "the output is non-empty");
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair()?;
assert_eq!(
report.recovered, 2,
"the derived output is excluded, its inputs are the history: {report:?}",
);
assert!(
!memfs.exists(&blob_path)?,
"a blob referenced only by the excluded output is unreachable in the \
rebuilt manifest and must be swept, not pinned forever",
);
Ok(())
}
#[test]
fn repair_fails_when_an_unreferenced_blob_cannot_be_removed() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let kv = || Some(KvSeparationOptions::default().separation_threshold(16));
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(kv())
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
tree.insert(b"k0", alloc::vec![b'x'; 64], 0);
tree.flush_active_memtable(0)?;
}
let orphan = root.join(crate::file::BLOBS_FOLDER).join("1");
let mut w = crate::vlog::blob_file::writer::Writer::new(&orphan, 1, 0, &*fs_dyn)?;
w.write(b"z", 9, &[b'z'; 300])?;
w.finish()?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(
FaultOp::RemoveFile,
Fault::Error(ErrorKind::PermissionDenied),
)
.on_path("blobs"),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(kv())
.repair();
assert!(
result.is_err(),
"repair must FAIL, not report success for a tree whose next open must \
sweep a file it cannot remove: {result:?}",
);
let retry = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(kv());
retry.repair()?;
assert!(
!memfs.exists(&orphan)?,
"the unreferenced blob is removed by the repair that omits it",
);
retry.open()?;
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn blob_recovery_keeps_a_file_whose_zeroed_tail_follows_live_frames() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
memfs.create_dir_all(&blobs)?;
let path = blobs.join("0");
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let mut w = crate::vlog::blob_file::writer::Writer::new(&path, 0, 0, &*fs_dyn)?;
w.write(b"a", 1, &[b'x'; 300])?;
w.write(b"b", 2, &[b'y'; 300])?;
w.write(b"c", 3, &[b'z'; 300])?;
w.finish()?;
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(&path, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?;
assert_eq!(entries.len(), 3, "three frames written");
let frontier = entries.first().expect("first frame").frame_end;
let tail_start = entries.get(1).expect("second frame").frame_end;
let (data_start, data_end) = {
let mut file = fs_dyn.open(&path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
let data = reader
.toc()
.section(b"data")
.expect("blob file has a data section");
(data.pos(), data.pos() + data.len())
};
memfs.punch_hole(&path, data_start, frontier - data_start)?;
memfs.punch_hole(&path, tail_start, data_end - tail_start)?;
let config = blob_validation_config(Arc::clone(&memfs));
let mut replacements_guard = super::PublishedBlobReplacements::new(&config);
let recovery = super::recover_blob_files(
&config,
&mut replacements_guard,
&(0..10).collect(),
None,
None,
)?;
replacements_guard.disarm();
assert!(
memfs.exists(&path)?,
"a zeroed tail below live frames is damage, not a completed relocation: \
the file must not be dropped",
);
assert_eq!(
recovery.files.len(),
1,
"the live frame is published: {:?}",
recovery.unreadable,
);
let published = recovery.files.first().expect("a published blob file");
let keys: Vec<_> = crate::vlog::BlobFileScanner::new(
blobs.join(published.id().to_string()),
&*fs_dyn,
published.id(),
)?
.map(|entry| entry.map(|entry| entry.key))
.collect::<crate::Result<Vec<_>>>()?;
assert_eq!(
keys,
vec![crate::UserKey::from(b"b".as_slice())],
"only the intact frame survives: the relocated one was punched away, \
the destroyed one is unreadable",
);
Ok(())
}
#[test]
fn a_crashed_blob_salvage_temp_is_swept_not_fatal() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..4u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let temp = root.join(crate::file::BLOBS_FOLDER).join("0.salvage-tmp");
{
let mut f = memfs.open(
&temp,
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?;
f.write_all(b"partial salvage output")?;
}
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..4u32 {
assert!(
tree.get(format!("k{i:04}").as_bytes(), crate::MAX_SEQNO)?
.is_some(),
"record k{i:04} must still read with a stranded salvage temp present",
);
}
drop(tree);
assert!(
!memfs.exists(&temp)?,
"the plain open must sweep the stranded salvage temp",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_excludes_a_compaction_output_whose_inputs_survived() -> crate::Result<()> {
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
struct SumMerge;
impl crate::MergeOperator for SumMerge {
fn merge(
&self,
_key: &[u8],
base_value: Option<&[u8]>,
operands: &[&[u8]],
) -> crate::Result<crate::UserValue> {
let mut sum: i64 = base_value.map_or(0, |b| {
i64::from_le_bytes(b.try_into().expect("8-byte counter"))
});
for op in operands {
sum += i64::from_le_bytes((*op).try_into().expect("8-byte operand"));
}
Ok(sum.to_le_bytes().to_vec().into())
}
}
let dir = tempfile::tempdir()?;
let config = || {
Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_merge_operator(Some(Arc::new(SumMerge)))
};
let one = 1i64.to_le_bytes();
let inputs: Vec<(std::path::PathBuf, Vec<u8>)>;
{
let tree = config().open()?;
tree.merge(b"counter", one, 1);
tree.flush_active_memtable(0)?;
tree.merge(b"counter", one, 2);
tree.flush_active_memtable(0)?;
let tables = dir.path().join("tables");
inputs = std::fs::read_dir(&tables)?
.map(|e| {
let path = e.expect("dir entry").path();
let bytes = std::fs::read(&path).expect("read input");
(path, bytes)
})
.collect();
assert_eq!(inputs.len(), 2, "two flushed inputs");
tree.major_compact(u64::MAX, 3)?;
let version = tree.current_version();
let output = version.iter_tables().next().expect("one compacted table");
assert_eq!(
output.metadata.lineage.as_deref(),
Some(&[0, 1][..]),
"a compaction output persists the sorted ids of its inputs",
);
}
for (path, bytes) in &inputs {
std::fs::write(path, bytes)?;
}
for entry in std::fs::read_dir(dir.path())? {
let entry = entry?;
let name = entry.file_name();
let name = name.to_string_lossy();
let is_version = name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || name == "current" {
std::fs::remove_file(entry.path())?;
}
}
let report = config().repair()?;
assert_eq!(
report.recovered, 2,
"the derived output is excluded; only the surviving inputs are \
published: {report:?}",
);
assert_eq!(report.unreadable, 0, "{report:?}");
assert!(
report
.excluded_files
.iter()
.any(|(_, reason)| reason.contains("derived output")),
"the exclusion is reported in its own field: {report:?}",
);
let tree = config().open()?;
let value = tree
.get(b"counter", crate::SeqNo::MAX)?
.expect("the counter survives");
assert_eq!(
i64::from_le_bytes(value.as_ref().try_into().expect("8-byte counter")),
2,
"publishing both histories would double-apply the merge operands",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_keep_only_filtered_compaction_output_keeps_its_lineage() -> crate::Result<()> {
use crate::compaction::filter::{
CompactionFilter, Context as FilterContext, Factory, ItemAccessor, Verdict,
};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
struct KeepAll;
impl CompactionFilter for KeepAll {
fn filter_item(
&mut self,
_item: ItemAccessor<'_>,
_ctx: &FilterContext,
) -> crate::Result<Verdict> {
Ok(Verdict::Keep)
}
}
struct KeepAllFactory;
impl Factory for KeepAllFactory {
fn name(&self) -> &'static str {
"KeepAll"
}
fn make_filter(&self, _ctx: &FilterContext) -> Box<dyn CompactionFilter> {
Box::new(KeepAll)
}
}
let dir = tempfile::tempdir()?;
let tree = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_compaction_filter_factory(Some(Arc::new(KeepAllFactory)))
.open()?;
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
tree.insert(b"b", b"v", 2);
tree.flush_active_memtable(0)?;
tree.major_compact(u64::MAX, 3)?;
let version = tree.current_version();
let output = version.iter_tables().next().expect("one compacted table");
assert_eq!(
output.metadata.lineage.as_deref(),
Some(&[0, 1][..]),
"a Keep-only filter transforms nothing; the output stays derivable",
);
assert!(
!output.metadata.lineage_transformed,
"a Keep-only window carries no transformed marker",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_transforming_compaction_output_keeps_its_lineage_marked() -> crate::Result<()> {
use crate::compaction::filter::{
CompactionFilter, Context as FilterContext, Factory, ItemAccessor, Verdict,
};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
struct RemoveB;
impl CompactionFilter for RemoveB {
fn filter_item(
&mut self,
item: ItemAccessor<'_>,
_ctx: &FilterContext,
) -> crate::Result<Verdict> {
Ok(if item.key() == b"b" {
Verdict::Remove
} else {
Verdict::Keep
})
}
}
struct RemoveBFactory;
impl Factory for RemoveBFactory {
fn name(&self) -> &'static str {
"RemoveB"
}
fn make_filter(&self, _ctx: &FilterContext) -> Box<dyn CompactionFilter> {
Box::new(RemoveB)
}
}
let dir = tempfile::tempdir()?;
let tree = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_compaction_filter_factory(Some(Arc::new(RemoveBFactory)))
.open()?;
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
tree.insert(b"b", b"v", 2);
tree.flush_active_memtable(0)?;
tree.major_compact(u64::MAX, 3)?;
let version = tree.current_version();
let output = version.iter_tables().next().expect("one compacted table");
assert_eq!(
output.metadata.lineage.as_deref(),
Some(&[0, 1][..]),
"a transformed output keeps its lineage — it supersedes the inputs it covers",
);
assert!(
output.metadata.lineage_transformed,
"the transformed marker keeps the dedup from trading it back for its inputs",
);
assert!(
output.metadata.lineage_last,
"a serial run's final output closes the first-to-last completeness proof",
);
Ok(())
}
#[test]
fn a_committed_empty_compaction_republishes_inputs_on_manifest_loss() -> crate::Result<()> {
use crate::compaction::filter::{
CompactionFilter, Context as FilterContext, Factory, ItemAccessor, Verdict,
};
use crate::fs::Fs;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
struct DestroyAll;
impl CompactionFilter for DestroyAll {
fn filter_item(
&mut self,
_item: ItemAccessor<'_>,
_ctx: &FilterContext,
) -> crate::Result<Verdict> {
Ok(Verdict::Destroy)
}
}
struct DestroyAllFactory;
impl Factory for DestroyAllFactory {
fn name(&self) -> &'static str {
"DestroyAll"
}
fn make_filter(&self, _ctx: &FilterContext) -> Box<dyn CompactionFilter> {
Box::new(DestroyAll)
}
}
let memfs = Arc::new(crate::fs::MemFs::new());
let root = std::path::absolute("/db")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_compaction_filter_factory(Some(Arc::new(DestroyAllFactory)))
};
{
let tree = match config().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
tree.insert(b"b", b"v", 2);
tree.flush_active_memtable(0)?;
let mut inputs = Vec::new();
for id in 0u64..2 {
let path = root.join("tables").join(id.to_string());
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs.open(&path, &crate::fs::FsOpenOptions::new().read(true))?,
&mut bytes,
)?;
inputs.push((path, bytes));
}
tree.major_compact(u64::MAX, 3)?;
assert_eq!(
tree.table_count(),
0,
"the filter destroyed everything; the committed compaction has \
ZERO outputs",
);
drop(tree);
for (path, bytes) in inputs {
let mut f = memfs.open(
&path,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &bytes)?;
}
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let report = config().repair()?;
assert_eq!(
report.recovered, 2,
"with no output and no lineage the inputs are the only history: {report:?}",
);
assert!(
report
.warnings
.iter()
.any(|w| w.contains("unlogged version edits")),
"the standing warning covers the lost empty compaction: {report:?}",
);
let tree = match config().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
assert!(
tree.get(b"a", crate::MAX_SEQNO)?.is_some(),
"the pre-compaction history is republished whole (transient \
re-exposure, not corruption)",
);
tree.major_compact(u64::MAX, 4)?;
assert!(
tree.get(b"a", crate::MAX_SEQNO)?.is_none(),
"the re-run filter restores the committed outcome",
);
Ok(())
}
#[test]
fn a_valid_routed_duplicate_is_excluded_not_unreadable() -> crate::Result<()> {
use crate::config::LevelRoute;
use crate::fs::{Fs, MemFs};
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let cold = std::path::absolute("/cold")?;
let tables = root.join("tables");
let cold_tables = cold.join("tables");
memfs.create_dir_all(&tables)?;
memfs.create_dir_all(&cold_tables)?;
{
let mut w = crate::table::Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs_dyn))?;
w.write(InternalValue::from_components(
b"a".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the table is non-empty");
}
{
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs.open(
&tables.join("0"),
&crate::fs::FsOpenOptions::new().read(true),
)?,
&mut bytes,
)?;
let mut f = memfs.open(
&cold_tables.join("0"),
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &bytes)?;
}
let route_fs: Arc<dyn Fs> = memfs.clone();
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.level_routes(vec![LevelRoute {
levels: 0..2,
path: cold,
fs: route_fs,
}])
.repair()?;
assert_eq!(report.recovered, 1, "one copy is retained: {report:?}");
assert_eq!(
report.unreadable, 0,
"a valid duplicate is not an unreadable file: {report:?}",
);
assert!(
report
.excluded_files
.iter()
.any(|(_, reason)| reason.contains("duplicate")),
"the duplicate is reported as a healthy exclusion: {report:?}",
);
Ok(())
}
#[test]
fn an_ambiguous_repair_temp_does_not_shut_out_a_routed_manifest_copy() -> crate::Result<()> {
use crate::config::LevelRoute;
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let cold = std::path::absolute("/cold")?;
let tables = root.join("tables");
let cold_tables = cold.join("tables");
memfs.create_dir_all(&tables)?;
memfs.create_dir_all(&cold_tables)?;
{
let mut w = crate::table::Writer::new(cold_tables.join("0"), 0, 0, Arc::clone(&fs_dyn))?;
w.write(InternalValue::from_components(
b"a".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the table is non-empty");
}
let config = || {
let route_fs: Arc<dyn Fs> = memfs.clone();
let shared: Arc<dyn Fs> = memfs.clone();
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(shared)
.level_routes(vec![LevelRoute {
levels: 0..2,
path: cold.clone(),
fs: route_fs,
}])
};
assert_eq!(
config().repair()?.recovered,
1,
"the routed copy is published"
);
{
let mut w =
crate::table::Writer::new(tables.join("0.repair-tmp"), 0, 0, Arc::clone(&fs_dyn))?;
w.write(InternalValue::from_components(
b"a".to_vec(),
b"other".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the temp is non-empty");
let mut f = memfs.open(
&tables.join("0"),
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, b"not an sst")?;
}
let tree = match config().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
assert_eq!(
tree.get(b"a", crate::MAX_SEQNO)?.as_deref(),
Some(&b"v"[..]),
"the open settles the ambiguous temp against the routed copy the manifest names",
);
assert!(
!memfs.exists(&tables.join("0.repair-tmp"))?,
"a temp proven not to be the manifest's copy is an abandoned build and is \
removed by the open that proved it, not left for someone else",
);
Ok(())
}
#[test]
fn a_routed_duplicate_with_a_rotted_data_block_is_unreadable() -> crate::Result<()> {
use crate::config::LevelRoute;
use crate::fs::{Fs, MemFs};
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let cold = std::path::absolute("/cold")?;
let tables = root.join("tables");
let cold_tables = cold.join("tables");
memfs.create_dir_all(&tables)?;
memfs.create_dir_all(&cold_tables)?;
{
let mut w = crate::table::Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs_dyn))?
.use_data_block_size(128);
for i in 0..200u32 {
w.write(InternalValue::from_components(
format!("key{i:06}").into_bytes(),
vec![0xABu8; 64],
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the table is non-empty");
}
{
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs.open(
&tables.join("0"),
&crate::fs::FsOpenOptions::new().read(true),
)?,
&mut bytes,
)?;
let Some(byte) = bytes.get_mut(64) else {
panic!("the written file reaches the flipped offset");
};
*byte ^= 0xFF;
let mut f = memfs.open(
&cold_tables.join("0"),
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &bytes)?;
}
let route_fs: Arc<dyn Fs> = memfs.clone();
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.level_routes(vec![LevelRoute {
levels: 0..2,
path: cold,
fs: route_fs,
}])
.repair()?;
assert_eq!(
report.recovered, 1,
"the intact copy is retained: {report:?}"
);
assert_eq!(
report.unreadable, 1,
"a duplicate that opens but whose data rotted is still the corruption \
signal: {report:?}",
);
assert!(
report.unknowable_losses.is_empty() && report.lost_coverage.is_empty(),
"a redundant damaged copy costs no coverage: {report:?}",
);
assert_eq!(
report.wal_replay_scope(),
crate::repair::WalReplayScope::TailOnly,
"nothing was lost, so the usual tail replay is enough: {report:?}",
);
Ok(())
}
#[test]
fn a_damaged_routed_duplicate_is_unreadable_not_a_healthy_exclusion() -> crate::Result<()> {
use crate::config::LevelRoute;
use crate::fs::{Fs, MemFs};
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let cold = std::path::absolute("/cold")?;
let tables = root.join("tables");
let cold_tables = cold.join("tables");
memfs.create_dir_all(&tables)?;
memfs.create_dir_all(&cold_tables)?;
{
let mut w = crate::table::Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs_dyn))?;
w.write(InternalValue::from_components(
b"a".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the table is non-empty");
}
{
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs.open(
&tables.join("0"),
&crate::fs::FsOpenOptions::new().read(true),
)?,
&mut bytes,
)?;
bytes.truncate(bytes.len() / 2);
let mut f = memfs.open(
&cold_tables.join("0"),
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &bytes)?;
}
let route_fs: Arc<dyn Fs> = memfs.clone();
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.level_routes(vec![LevelRoute {
levels: 0..2,
path: cold,
fs: route_fs,
}])
.repair()?;
assert_eq!(
report.recovered, 1,
"the intact copy is retained: {report:?}"
);
assert_eq!(
report.unreadable, 1,
"the damaged duplicate must raise the corruption signal: {report:?}",
);
assert!(
!report
.excluded_files
.iter()
.any(|(p, _)| p.ends_with("0") && p.starts_with("/cold")),
"a file that never opened cannot be reported as a healthy exclusion: {report:?}",
);
Ok(())
}
#[test]
fn a_damaged_duplicate_in_an_earlier_folder_does_not_shut_out_the_manifest_copy()
-> crate::Result<()> {
use crate::config::LevelRoute;
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let cold = std::path::absolute("/cold")?;
let tables = root.join("tables");
let cold_tables = cold.join("tables");
memfs.create_dir_all(&tables)?;
memfs.create_dir_all(&cold_tables)?;
{
let mut w = crate::table::Writer::new(cold_tables.join("0"), 0, 0, Arc::clone(&fs_dyn))?;
w.write(InternalValue::from_components(
b"a".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the table is non-empty");
}
let config = || {
let route_fs: Arc<dyn Fs> = memfs.clone();
let shared: Arc<dyn Fs> = memfs.clone();
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(shared)
.level_routes(vec![LevelRoute {
levels: 0..2,
path: cold.clone(),
fs: route_fs,
}])
};
let report = config().repair()?;
assert_eq!(
report.recovered, 1,
"the routed copy is published: {report:?}"
);
{
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs.open(
&cold_tables.join("0"),
&crate::fs::FsOpenOptions::new().read(true),
)?,
&mut bytes,
)?;
bytes.truncate(bytes.len() / 2);
let mut f = memfs.open(
&tables.join("0"),
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &bytes)?;
}
let tree = match config().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
assert_eq!(
tree.get(b"a", crate::MAX_SEQNO)?.as_deref(),
Some(&b"v"[..]),
"the open falls through to the routed folder holding the manifest's copy",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_bottommost_tombstone_elision_marks_the_output_transformed() -> crate::Result<()> {
use crate::{AbstractTree, Config, SequenceNumberCounter};
let dir = tempfile::tempdir()?;
let tree = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.open()?;
tree.insert(b"a", b"v", 1);
tree.insert(b"z", b"v", 2);
tree.flush_active_memtable(0)?;
tree.remove(b"a", 3);
tree.flush_active_memtable(0)?;
tree.major_compact(u64::MAX, 4)?;
let version = tree.current_version();
let output = version.iter_tables().next().expect("one compacted table");
assert_eq!(
output.metadata.lineage.as_deref(),
Some(&[0, 1][..]),
"the output keeps its lineage",
);
assert!(
output.metadata.lineage_transformed,
"eliding the tombstone (and draining the covered value) is a \
visibility transform: the marker must be set",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_weak_annihilation_marks_the_output_transformed() -> crate::Result<()> {
use crate::{AbstractTree, Config, SequenceNumberCounter};
let dir = tempfile::tempdir()?;
let tree = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.open()?;
tree.insert(b"k", b"v", 1);
tree.insert(b"z", b"v", 2);
tree.flush_active_memtable(0)?;
tree.remove_weak(b"k", 3);
tree.flush_active_memtable(0)?;
tree.major_compact(u64::MAX, 4)?;
let version = tree.current_version();
let output = version.iter_tables().next().expect("one compacted table");
assert!(
output.metadata.lineage_transformed,
"the weak annihilation removed the consumed value and its weak \
delete: the marker must be set",
);
Ok(())
}
#[test]
fn a_committed_transforming_compaction_survives_manifest_loss() -> crate::Result<()> {
use crate::compaction::filter::{
CompactionFilter, Context as FilterContext, Factory, ItemAccessor, Verdict,
};
use crate::fs::Fs;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
struct DestroyB;
impl CompactionFilter for DestroyB {
fn filter_item(
&mut self,
item: ItemAccessor<'_>,
_ctx: &FilterContext,
) -> crate::Result<Verdict> {
Ok(if item.key() == b"b" {
Verdict::Destroy
} else {
Verdict::Keep
})
}
}
struct DestroyBFactory;
impl Factory for DestroyBFactory {
fn name(&self) -> &'static str {
"DestroyB"
}
fn make_filter(&self, _ctx: &FilterContext) -> Box<dyn CompactionFilter> {
Box::new(DestroyB)
}
}
let memfs = Arc::new(crate::fs::MemFs::new());
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_compaction_filter_factory(Some(Arc::new(DestroyBFactory)))
.open()?
{
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
tree.insert(b"b", b"v", 2);
tree.flush_active_memtable(0)?;
let mut inputs = Vec::new();
for id in 0u64..2 {
let path = root.join("tables").join(id.to_string());
let mut bytes = Vec::new();
memfs
.open(&path, &crate::fs::FsOpenOptions::new().read(true))?
.read_to_end(&mut bytes)?;
inputs.push((path, bytes));
}
tree.major_compact(u64::MAX, 3)?;
assert!(
tree.get(b"b", crate::MAX_SEQNO)?.is_none(),
"the filter removed b at compaction",
);
drop(tree);
for (path, bytes) in inputs {
let mut f = memfs.open(
&path,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
f.write_all(&bytes)?;
}
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.repair()?;
assert_eq!(
report.recovered, 1,
"the transformed output supersedes both lingering inputs: {report:?}",
);
let reopened = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.open()?;
assert!(
reopened.get(b"a", crate::MAX_SEQNO)?.is_some(),
"the kept record must survive the rebuild",
);
assert!(
reopened.get(b"b", crate::MAX_SEQNO)?.is_none(),
"a record the committed filter removed must not resurrect from a \
lingering input",
);
Ok(())
}
#[test]
fn repair_keeps_an_output_whose_inputs_survived_only_lossily() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs))?;
for i in 0..200u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the input is non-empty");
}
forge_unrecognized_ecc_descriptor(&tables.join("0"))?;
{
let mut w = Writer::new(tables.join("1"), 1, 0, Arc::clone(&fs))?
.use_recency(Some(0))
.use_lineage(Some(vec![0]));
w.write(InternalValue::from_components(
b"k00000".to_vec(),
b"folded".to_vec(),
2,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the output is non-empty");
}
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(
(report.recovered, report.salvaged),
(2, 1),
"the healthy output must be KEPT — the lossily recovered input does \
not prove its contents survive elsewhere: {report:?}",
);
Ok(())
}
#[test]
fn repair_supersedes_an_input_a_partial_output_fully_covers() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
b"k".to_vec(),
b"a".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the input is non-empty");
}
{
let mut w = Writer::new(tables.join("2"), 2, 0, Arc::clone(&fs))?
.use_recency(Some(1))
.use_lineage(Some(vec![0, 1]));
w.write(InternalValue::from_components(
b"k".to_vec(),
b"b".to_vec(),
2,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the output is non-empty");
}
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(
report.recovered, 1,
"the covered input is superseded by the kept output: {report:?}",
);
assert!(
report.lost_coverage.is_empty(),
"a superseded input lost nothing — its content lives in the output: \
{report:?}",
);
Ok(())
}
#[test]
fn repair_supersedes_an_input_covered_by_an_unbroken_sibling_union() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs))?;
for key in [b"a".as_slice(), b"z".as_slice()] {
w.write(InternalValue::from_components(
key.to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the input is non-empty");
}
for (id, keys, prev) in [
(2u64, [b"a".as_slice(), b"m".as_slice()], None),
(3u64, [b"n".as_slice(), b"z".as_slice()], Some(2)),
] {
let mut w = Writer::new(tables.join(id.to_string()), id, 0, Arc::clone(&fs))?
.use_recency(Some(1))
.use_lineage(Some(vec![0, 1]))
.use_lineage_prev(prev);
for key in keys {
w.write(InternalValue::from_components(
key.to_vec(),
b"w".to_vec(),
2,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the output is non-empty");
}
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(
report.recovered, 2,
"the unbroken sibling union covers the input, which is superseded: \
{report:?}",
);
assert!(
report.lost_coverage.is_empty(),
"a superseded input lost nothing: {report:?}",
);
Ok(())
}
#[test]
fn repair_keeps_an_input_when_the_sibling_chain_is_broken() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs))?;
for key in [b"a".as_slice(), b"z".as_slice()] {
w.write(InternalValue::from_components(
key.to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the input is non-empty");
}
for (id, keys, prev) in [
(2u64, [b"a".as_slice(), b"h".as_slice()], None),
(4u64, [b"n".as_slice(), b"z".as_slice()], Some(3)),
] {
let mut w = Writer::new(tables.join(id.to_string()), id, 0, Arc::clone(&fs))?
.use_recency(Some(1))
.use_lineage(Some(vec![0, 1]))
.use_lineage_prev(prev);
for key in keys {
w.write(InternalValue::from_components(
key.to_vec(),
b"w".to_vec(),
2,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the output is non-empty");
}
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(
report.recovered, 3,
"a broken chain proves nothing; the input holds the gap's only \
complete copy and is kept: {report:?}",
);
assert!(
!report.lost_coverage.is_empty(),
"the residual overlaps must be reported: {report:?}",
);
Ok(())
}
#[test]
fn a_transformed_output_without_the_run_end_fails_on_reaching_inputs() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs))?;
for key in [b"a".as_slice(), b"z".as_slice()] {
w.write(InternalValue::from_components(
key.to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the input is non-empty");
}
{
let mut w = Writer::new(tables.join("2"), 2, 0, Arc::clone(&fs))?
.use_recency(Some(1))
.use_lineage(Some(vec![0, 1]))
.use_lineage_transformed(true);
for key in [b"a".as_slice(), b"m".as_slice()] {
w.write(InternalValue::from_components(
key.to_vec(),
b"w".to_vec(),
2,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the output is non-empty");
}
let result = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair();
assert!(
matches!(result, Err(crate::Error::Unrecoverable)),
"a residual overlap against a TRANSFORMED output resurrects \
filter-removed keys no replay can re-delete; the repair must fail \
closed even without a merge operator: {:?}",
result.map(|r| r.recovered),
);
Ok(())
}
#[test]
fn a_manifest_referenced_table_missing_from_disk_is_unknowable() -> crate::Result<()> {
use crate::fs::Fs;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(crate::fs::MemFs::new());
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.open()?
{
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
tree.insert(b"b", b"v", 2);
tree.flush_active_memtable(0)?;
}
memfs.remove_file(&root.join("tables").join("1"))?;
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.repair()?;
assert_eq!(
report.recovered, 1,
"the surviving table is rebuilt: {report:?}"
);
assert!(
report.unknowable_losses.iter().any(|p| p.ends_with("1")),
"the manifest-referenced missing table is an unscopable loss: {report:?}",
);
assert!(
matches!(
report.wal_replay_scope(),
crate::repair::WalReplayScope::FullHistory
),
"a vanished table widens the replay obligation: {report:?}",
);
Ok(())
}
#[cfg(feature = "encryption")]
#[test]
fn a_repair_under_the_wrong_encryption_key_fails_closed() -> crate::Result<()> {
use crate::fs::Fs;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(crate::fs::MemFs::new());
let root = std::path::absolute("/db_enc")?;
let good: Arc<dyn crate::encryption::EncryptionProvider> =
Arc::new(crate::encryption::Aes256GcmProvider::new(&[0x42; 32]));
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_encryption(Some(Arc::clone(&good)))
.open()?
{
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
tree.insert(b"b", b"v", 2);
tree.flush_active_memtable(0)?;
}
memfs.remove_file(&root.join("current"))?;
let wrong: Arc<dyn crate::encryption::EncryptionProvider> =
Arc::new(crate::encryption::Aes256GcmProvider::new(&[0x13; 32]));
let Err(err) = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_encryption(Some(Arc::clone(&wrong)))
.repair() else {
panic!("a wrong-key repair must fail, not commit an empty manifest");
};
assert!(
matches!(&err, crate::Error::Decrypt(_)),
"the failure names the decrypt cause: {err:?}",
);
assert!(
memfs.exists(&root.join("tables").join("0"))?
&& memfs.exists(&root.join("tables").join("1"))?,
"no ciphertext file may be deleted by a wrong-key repair",
);
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_encryption(Some(good))
.repair()?;
assert_eq!(
report.recovered, 2,
"the right key recovers every table: {report:?}"
);
Ok(())
}
#[test]
fn an_inconclusive_current_probe_never_claims_a_committed_repair() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
for skip in 0..8u64 {
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.open()?
{
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::Rename, Fault::Error(ErrorKind::Other))
.on_path("current")
.times(1),
);
fault.injector().arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::PermissionDenied))
.on_path("current")
.skip(skip)
.times(1),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair();
if !memfs.exists(&root.join("current"))?
&& let Err(e) = &result
{
assert!(
!matches!(e, crate::Error::RepairedButUnopened { .. }),
"CURRENT never switched, so no report may claim a committed \
repair (skip {skip}): {e:?}",
);
}
}
Ok(())
}
#[test]
fn repair_progress_credits_a_removed_temp_companion() -> crate::Result<()> {
use crate::fs::Fs;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(crate::fs::MemFs::new());
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.open()?
{
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
}
let tables = root.join("tables");
for name in ["9.repair-tmp", "9.repair-tmp.restrict-bound"] {
let mut f = memfs.open(
&tables.join(name),
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &[0xEE; 64])?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let progress = Arc::new(crate::RecoveryProgress::default());
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_recovery_progress(Arc::clone(&progress))
.repair()?;
let snap = progress.snapshot();
assert!(
snap.bytes_processed >= snap.bytes_total,
"a successful repair reaches 100%: the removed companion's bytes \
are credited at removal time: {snap:?}",
);
Ok(())
}
#[test]
fn a_switched_current_sync_failure_carries_the_report() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
for skip in 0..64u64 {
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.open()?
{
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::SyncDirectory, Fault::Error(ErrorKind::Other))
.skip(skip)
.times(1),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair();
let switched = memfs.exists(&root.join("current"))?;
match result {
Ok(_) => {
}
Err(e) if !switched => {
assert!(
!matches!(e, crate::Error::RepairedButUnopened { .. }),
"no report may claim a repair the pointer disproves",
);
}
Err(crate::Error::RepairedButUnopened { report, .. }) => {
assert_eq!(
report.recovered, 1,
"the switched manifest's report rides in the error: {report:?}",
);
return Ok(());
}
Err(other) => panic!(
"a failure after the CURRENT switch must carry the report, \
got {other:?}",
),
}
}
panic!("no skip count landed the fault in the post-switch window");
}
#[test]
fn a_clean_manifest_record_preserves_a_bulk_ingested_offset() -> crate::Result<()> {
use crate::fs::Fs;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(crate::fs::MemFs::new());
let root = std::path::absolute("/db_ingest")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
};
{
let tree = match config().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
let mut ingestion = crate::tree::ingest::Ingestion::new(&tree)?;
ingestion.write(b"ing".as_slice().into(), b"V".as_slice().into())?;
ingestion.finish()?;
}
memfs.remove_file(&root.join("tables").join("0"))?;
let report = config().repair()?;
assert_eq!(
report.recovered, 1,
"the healthy bulk-ingested table is kept, offset from the clean \
manifest record: {report:?}",
);
assert!(
!report
.unreadable_files
.iter()
.any(|(_, why)| why.contains("bulk-ingest")),
"no healthy ingested table may be dropped over a recoverable \
offset: {report:?}",
);
let tree = config().open()?;
assert!(
tree.get(b"ing", crate::MAX_SEQNO)?.is_some(),
"the ingested row survives the repair",
);
assert!(
tree.get(b"ing", 1)?.is_none(),
"the ingested row keeps its real sequence position, not offset 0",
);
Ok(())
}
#[cfg(zstd_any)]
#[test]
fn a_repair_without_the_dictionary_never_discards_the_blob_file() -> crate::Result<()> {
use crate::compression::ZstdDictionary;
use crate::fs::{Fs, MemFs};
use crate::{
AbstractTree, CompressionType, Config, KvSeparationOptions, SequenceNumberCounter,
};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db_no_dict")?;
let samples: alloc::vec::Vec<u8> = (0..4000u32).map(|i| (i % 251) as u8).collect();
let dict = Arc::new(ZstdDictionary::new(&samples));
let with_dict = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.zstd_dictionary(Some(Arc::clone(&dict)))
.with_kv_separation(Some(
KvSeparationOptions::default()
.separation_threshold(16)
.compression(CompressionType::ZstdDict {
level: 3,
dict_id: dict.id(),
})
.dict(Arc::clone(&dict)),
))
};
{
let tree = match with_dict().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
tree.insert(b"zzz", vec![b'w'; 64], 100);
tree.flush_active_memtable(0)?;
}
memfs.remove_file(&root.join("tables").join("1"))?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_count = memfs.read_dir(&blobs)?.len();
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair_with_salvage(true);
assert!(
result.is_err(),
"a repair without the dictionary must fail closed, not report success \
over a file it could not read: {result:?}",
);
assert_eq!(
memfs.read_dir(&blobs)?.len(),
blob_count,
"no blob file may be removed by a repair that lacked the dictionary",
);
let report = with_dict().repair_with_salvage(true)?;
assert!(
report.recovered >= 1,
"the store repairs with it: {report:?}"
);
let tree = with_dict().open()?;
assert!(
tree.get(b"k0000", crate::MAX_SEQNO)?.is_some(),
"the values survive the refused repair",
);
Ok(())
}
#[test]
fn an_abandoned_temp_is_swept_before_its_source_is_salvaged() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let fs: Arc<dyn Fs> = Arc::new(MemFs::new());
let root = std::path::absolute("/db_temp_before_source")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
{
let table = recover_sst(sst.clone(), &fs)?;
let offset = table
.data_block_handles()
.filter_map(Result::ok)
.map(|kh| *kh.as_ref().offset())
.next()
.unwrap_or(0);
drop(table);
let mut f = fs.open(&sst, &crate::fs::FsOpenOptions::new().write(true))?;
std::io::Seek::seek(&mut f, std::io::SeekFrom::Start(offset + 16))?;
std::io::Write::write_all(&mut f, &[0xFFu8])?;
crate::fs::FsFile::sync_all(&*f)?;
}
{
let mut f = fs.open(
&tables.join("0.repair-tmp"),
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &[0xEE; 64])?;
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone())
.repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"the source must salvage: the abandoned temp holding its destination \
name is swept first, not after the source is condemned: {report:?}",
);
Ok(())
}
#[test]
fn a_clean_manifest_tree_type_refuses_a_contradicting_repair() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db_tree_type")?;
let standard = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
};
{
let tree = match standard().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
tree.insert(b"b", b"v", 2);
tree.flush_active_memtable(0)?;
}
memfs.remove_file(&root.join("tables").join("1"))?;
let result = standard()
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair();
assert!(
matches!(
&result,
Err(crate::Error::TreeTypeMismatch {
requested: crate::TreeType::Blob,
actual: crate::TreeType::Standard,
})
),
"the committed type must refuse the contradicting configuration: {result:?}",
);
let report = standard().repair()?;
assert_eq!(
report.recovered, 1,
"the refused repair must leave the store repairable: {report:?}",
);
let tree = standard().open()?;
assert!(
tree.get(b"a", crate::MAX_SEQNO)?.is_some(),
"the surviving table's rows must still read",
);
Ok(())
}
#[test]
fn an_unreferenced_blob_is_removed_at_its_recovered_path() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db_blob_spelling")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
{
let tree = match config().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
tree.insert(b"k", vec![b'v'; 64], 1);
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let original = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.map(|e| e.path)
.ok_or(crate::Error::Unrecoverable)?;
let respelled = blobs.join("00");
memfs.rename(&original, &respelled)?;
for entry in memfs.read_dir(&root.join("tables"))? {
memfs.remove_file(&entry.path)?;
}
for entry in memfs.read_dir(&root)? {
let is_version = entry
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || entry.file_name == "current" {
memfs.remove_file(&entry.path)?;
}
}
let report = config().repair()?;
assert!(
!memfs.exists(&respelled)?,
"the unreferenced blob must be removed at the path it was recovered \
from, not at a name rebuilt from its id: {report:?}",
);
Ok(())
}
#[test]
fn a_duplicate_blob_with_a_rotted_frame_is_unreadable() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db_blob_dup_rot")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
{
let tree = match config().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..40u32 {
tree.insert(format!("k{i:04}"), vec![b'v'; 128], u64::from(i) + 1);
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let original = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.map(|e| e.path)
.ok_or(crate::Error::Unrecoverable)?;
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs.open(&original, &crate::fs::FsOpenOptions::new().read(true))?,
&mut bytes,
)?;
let Some(byte) = bytes.get_mut(64) else {
panic!("the written blob reaches the flipped offset");
};
*byte ^= 0xFF;
let mut f = memfs.open(
&blobs.join("00"),
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &bytes)?;
drop(f);
let report = config().repair()?;
assert_eq!(
report.unreadable, 1,
"a duplicate blob whose frames rotted is the corruption signal, not a \
healthy exclusion: {report:?}",
);
Ok(())
}
#[test]
fn a_displaced_salvage_drops_its_queued_swap() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{InternalValue, ValueType};
use std::sync::Arc;
let fs: Arc<dyn Fs> = Arc::new(MemFs::new());
let dir = std::path::absolute("/displaced")?;
fs.create_dir_all(&dir)?;
let build = |name: &str| -> crate::Result<crate::table::Table> {
let path = dir.join(name);
let mut w = crate::table::Writer::new(path.clone(), 0, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
b"k".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the table is non-empty");
recover_sst(path, &fs)
};
let salvaged = build("salvaged_source")?;
let intact = build("intact")?;
let salvaged_source = (*salvaged.path).clone();
let mut map = crate::HashMap::default();
let mut unreadable = Vec::new();
let mut discard = Vec::new();
let mut swaps = Vec::new();
let mut redundant = crate::HashSet::default();
super::keep_salvaged_replacement(
&mut map,
&mut unreadable,
&mut redundant,
&mut discard,
&mut swaps,
0,
salvaged,
&fs,
&salvaged_source,
dir.join("salvaged_source.repair-tmp"),
)?;
assert_eq!(swaps.len(), 1, "the salvage queued its swap");
super::record_best(
&mut map,
&mut unreadable,
&mut redundant,
&mut discard,
&mut swaps,
0,
intact,
super::Fidelity::Complete,
&fs,
&dir.join("intact"),
false,
)?;
assert!(
swaps.is_empty(),
"the displaced salvage's swap must go with it — the manifest now names \
the intact copy, so publishing the replacement onto that name is wrong \
(still queued: {})",
swaps.len(),
);
let tmp = dir.join("salvaged_source.repair-tmp");
assert!(
discard.iter().any(|(_, path, _)| path == &tmp),
"the unpublished replacement must be queued for removal: {:?}",
discard.iter().map(|(_, p, _)| p).collect::<Vec<_>>(),
);
Ok(())
}
#[test]
fn the_most_complete_lossy_duplicate_is_kept() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{InternalValue, ValueType};
use std::sync::Arc;
let fs: Arc<dyn Fs> = Arc::new(MemFs::new());
let dir = std::path::absolute("/dup")?;
fs.create_dir_all(&dir)?;
let build = |name: &str, rows: u32| -> crate::Result<crate::table::Table> {
let path = dir.join(name);
let mut w = crate::table::Writer::new(path.clone(), 0, 0, Arc::clone(&fs))?;
for i in 0..rows {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
b"v".to_vec(),
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the table is non-empty");
recover_sst(path, &fs)
};
let thin = build("thin", 2)?;
let full = build("full", 40)?;
let (thin_path, full_path) = ((*thin.path).clone(), (*full.path).clone());
let mut map = crate::HashMap::default();
let displaced = super::keep_best_candidate(
&mut map,
0,
super::TableCandidate {
table: thin,
fidelity: super::Fidelity::Salvaged,
fs: Arc::clone(&fs),
path: thin_path.clone(),
matches_manifest: false,
},
);
assert!(
displaced.is_none(),
"nothing to displace on the first sighting"
);
let Some(displaced) = super::keep_best_candidate(
&mut map,
0,
super::TableCandidate {
table: full,
fidelity: super::Fidelity::Salvaged,
fs: Arc::clone(&fs),
path: full_path.clone(),
matches_manifest: false,
},
) else {
panic!("the second sighting displaces one of the two");
};
assert_eq!(
displaced.path, thin_path,
"the THIN salvage must be the one displaced — keeping it would discard \
the rows the fuller salvage recovered",
);
assert_eq!(
map.get(&0).map(|c| c.path.clone()),
Some(full_path),
"the fuller salvage is what the rebuilt manifest keeps",
);
Ok(())
}
#[test]
fn a_restricted_original_proves_itself_on_its_live_suffix() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db_restricted_suffix")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
let bound: crate::UserKey = b"k00130".to_vec().into();
let suffix_digest = {
let table = recover_sst(sst.clone(), &fs)?;
let punch_offset = table.punch_offset_for(bound.as_ref())?;
drop(table);
memfs.punch_hole(&sst, 0, punch_offset)?;
crate::Checksum::from_raw(crate::repair::compute_table_checksum_from(
&*fs,
&sst,
punch_offset,
)?)
};
let tmp = tables.join("0.repair-tmp");
{
let mut f = fs.open(
&tmp,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &[0xCD; 128])?;
}
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone());
let published =
crate::repair::repair_tmp_is_published(&config, &fs, &tmp, 0, suffix_digest, Some(&bound))?;
assert!(
!published,
"the healthy restricted original matches the manifest on its live \
suffix, which proves the temp a disposable abandoned build",
);
Ok(())
}
#[cfg(feature = "std")]
#[test]
fn a_readable_temp_with_a_mismatching_digest_is_not_condemned_alone() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let fs: Arc<dyn Fs> = Arc::new(MemFs::new());
let root = std::path::absolute("/db_mismatching_temp")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
write_multiblock_sst(&tables.join("0"), &fs)?;
let tmp = tables.join("0.repair-tmp");
write_multiblock_sst(&tmp, &fs)?;
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone());
let published = crate::repair::repair_tmp_is_published(
&config,
&fs,
&tmp,
0,
crate::Checksum::from_raw(0xDEAD_BEEF),
None,
);
assert!(
published.is_err(),
"a mismatching digest alone must not condemn the temp while the \
original proves nothing: {published:?}",
);
let original_digest = crate::Checksum::from_raw(crate::repair::compute_table_checksum(
&*fs,
&tables.join("0"),
)?);
let published =
crate::repair::repair_tmp_is_published(&config, &fs, &tmp, 0, original_digest, None)?;
assert!(
!published,
"an original matching the manifest proves the temp abandoned",
);
Ok(())
}
#[cfg(feature = "std")]
#[test]
fn a_structurally_broken_restricted_temp_is_not_condemned_by_an_unproven_original()
-> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let fs: Arc<dyn Fs> = Arc::new(MemFs::new());
let root = std::path::absolute("/db_restricted_temp")?;
let tables = root.join("tables");
fs.create_dir_all(&tables)?;
let sst = tables.join("0");
write_multiblock_sst(&sst, &fs)?;
let tmp = tables.join("0.repair-tmp");
{
let mut f = fs.open(
&tmp,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &[0xAB; 512])?;
}
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone());
let bound: crate::UserKey = b"k00130".to_vec().into();
let published = crate::repair::repair_tmp_is_published(
&config,
&fs,
&tmp,
0,
crate::Checksum::from_raw(0xDEAD_BEEF),
Some(&bound),
);
assert!(
published.is_err(),
"with neither copy proven authoritative, the temp's failure must \
propagate instead of condemning the manifest's own replacement: \
{published:?}",
);
Ok(())
}
#[test]
fn an_unreadable_repair_temp_yields_to_a_proven_original() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let fault = Arc::new(FaultFs::new(crate::fs::MemFs::new()));
let fs: Arc<dyn Fs> = fault.clone();
let root = std::path::absolute("/db_tmp_rot")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone())
};
{
let tree = match config().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
}
let tmp = root.join("tables").join("0.repair-tmp");
{
let mut f = fs.open(
&tmp,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &[0xEE; 64])?;
}
for op in [FaultOp::Read, FaultOp::ReadAt] {
fault
.injector()
.arm(FaultRule::new(op, Fault::Error(ErrorKind::InvalidData)).on_path("0.repair-tmp"));
}
let tree = config().open()?;
assert!(
tree.get(b"a", crate::MAX_SEQNO)?.is_some(),
"the healthy manifest reopens past the unreadable disposable temp",
);
assert!(
!fs.exists(&tmp)?,
"the unreadable temp is swept once the original proves authoritative",
);
Ok(())
}
#[test]
fn an_abandoned_repair_temp_sweep_takes_its_restriction_sidecar() -> crate::Result<()> {
use crate::fs::Fs;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(crate::fs::MemFs::new());
let root = std::path::absolute("/db")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
};
{
let tree = match config().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"a", b"v", 1);
tree.flush_active_memtable(0)?;
}
let tables = root.join("tables");
let tmp = tables.join("9.repair-tmp");
let sidecar = tables.join("9.repair-tmp.restrict-bound");
for path in [&tmp, &sidecar] {
let mut f = memfs.open(
path,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
std::io::Write::write_all(&mut f, &[0xEE; 16])?;
}
let tree = config().open()?;
assert!(
tree.get(b"a", crate::MAX_SEQNO)?.is_some(),
"the healthy manifest reopens",
);
assert!(!memfs.exists(&tmp)?, "the abandoned replacement is swept",);
assert!(
!memfs.exists(&sidecar)?,
"its restriction sidecar goes with it — left behind it classifies \
as Foreign and fails the open",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_corrupt_reference_section_does_not_abort_the_blob_repair() -> crate::Result<()> {
use crate::coding::Encode;
use crate::fs::{Fs, MemFs};
use crate::{Config, InternalValue, KvSeparationOptions, SequenceNumberCounter, ValueType};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
let tables = root.join("tables");
memfs.create_dir_all(&blobs)?;
memfs.create_dir_all(&tables)?;
{
let mut w = crate::table::Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs_dyn))?;
w.write(InternalValue::from_components(
b"a".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "table 0 is non-empty");
}
let blob_path = blobs.join("7");
let (offset, on_disk_size) = {
let mut w = crate::vlog::blob_file::writer::Writer::new(&blob_path, 7, 0, &*fs_dyn)?;
let offset = w.offset();
let on_disk = w.write(b"m", 2, &[b'x'; 300])?;
w.finish()?;
(offset, on_disk)
};
let checksum = {
let mut w = crate::table::Writer::new(tables.join("1"), 1, 0, Arc::clone(&fs_dyn))?;
w.link_blob_file(7, 1, 300, u64::from(on_disk_size));
let ind = crate::blob_tree::handle::BlobIndirection {
vhandle: crate::vlog::ValueHandle {
blob_file_id: 7,
offset,
on_disk_size,
},
size: 300,
};
w.write(InternalValue::from_components(
b"m".to_vec(),
ind.encode_into_vec(),
2,
ValueType::Indirection,
))?;
w.finish()?.expect("table written").1
};
{
let table = crate::table::Table::recover(crate::table::RecoverParams::new(
tables.join("1"),
checksum,
1,
Arc::clone(&fs_dyn),
crate::comparator::default_comparator(),
Arc::new(crate::Cache::with_capacity_bytes(1_000_000)),
))?;
let handle = table
.regions
.linked_blob_files
.expect("table 1 has a reference section");
drop(table);
let mut f = memfs.open(
&tables.join("1"),
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
f.seek(SeekFrom::Start(*handle.offset()))?;
f.write_all(&u32::MAX.to_le_bytes())?;
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair()?;
assert_eq!(
report.recovered, 1,
"the corrupt-section table is set aside; its healthy sibling is \
recovered: {report:?}",
);
assert!(
report
.unreadable_files
.iter()
.any(|(path, _)| path.ends_with("1")),
"the corrupt-section table is reported: {report:?}",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_corrupt_reference_section_is_never_published_as_standard() -> crate::Result<()> {
use crate::coding::Encode;
use crate::fs::{Fs, MemFs};
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
let tables = root.join("tables");
memfs.create_dir_all(&blobs)?;
memfs.create_dir_all(&tables)?;
{
let mut w = crate::table::Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs_dyn))?;
w.write(InternalValue::from_components(
b"a".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "table 0 is non-empty");
}
let blob_path = blobs.join("7");
let (offset, on_disk_size) = {
let mut w = crate::vlog::blob_file::writer::Writer::new(&blob_path, 7, 0, &*fs_dyn)?;
let offset = w.offset();
let on_disk = w.write(b"m", 2, &[b'x'; 300])?;
w.finish()?;
(offset, on_disk)
};
let checksum = {
let mut w = crate::table::Writer::new(tables.join("1"), 1, 0, Arc::clone(&fs_dyn))?;
w.link_blob_file(7, 1, 300, u64::from(on_disk_size));
let ind = crate::blob_tree::handle::BlobIndirection {
vhandle: crate::vlog::ValueHandle {
blob_file_id: 7,
offset,
on_disk_size,
},
size: 300,
};
w.write(InternalValue::from_components(
b"m".to_vec(),
ind.encode_into_vec(),
2,
ValueType::Indirection,
))?;
w.finish()?.expect("table written").1
};
{
let table = crate::table::Table::recover(crate::table::RecoverParams::new(
tables.join("1"),
checksum,
1,
Arc::clone(&fs_dyn),
crate::comparator::default_comparator(),
Arc::new(crate::Cache::with_capacity_bytes(1_000_000)),
))?;
let handle = table
.regions
.linked_blob_files
.expect("table 1 has a reference section");
drop(table);
let mut f = memfs.open(
&tables.join("1"),
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
f.seek(SeekFrom::Start(*handle.offset()))?;
f.write_all(&u32::MAX.to_le_bytes())?;
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.repair()?;
assert_eq!(
report.recovered, 1,
"only the provably-standard sibling is published: {report:?}",
);
assert!(
report
.unreadable_files
.iter()
.any(|(path, _)| path.ends_with("1")),
"the unprovable table is set aside and reported: {report:?}",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn blob_salvage_never_allocates_a_missing_referenced_id() -> crate::Result<()> {
use crate::coding::Encode;
use crate::fs::{Fs, MemFs};
use crate::{Config, InternalValue, KvSeparationOptions, SequenceNumberCounter, ValueType};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
let tables = root.join("tables");
memfs.create_dir_all(&blobs)?;
memfs.create_dir_all(&tables)?;
let blob0 = blobs.join("0");
let (offset0, on_disk0) = {
let mut w = crate::vlog::blob_file::writer::Writer::new(&blob0, 0, 0, &*fs_dyn)?;
let offset = w.offset();
let on_disk = w.write(b"a", 1, &[b'x'; 300])?;
w.write(b"b", 2, &[b'y'; 300])?;
w.write(b"c", 3, &[b'z'; 300])?;
w.finish()?;
(offset, on_disk)
};
{
let last = crate::vlog::BlobFileScanner::new(&blob0, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?
.last()
.expect("a last frame")
.frame_end;
let mut f = memfs.open(
&blob0,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*f, last - 8, 1)?;
f.seek(SeekFrom::Start(last - 8))?;
f.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
{
let mut w = crate::table::Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs_dyn))?;
w.link_blob_file(0, 1, 300, u64::from(on_disk0));
w.link_blob_file(1, 1, 300, 100);
let ind0 = crate::blob_tree::handle::BlobIndirection {
vhandle: crate::vlog::ValueHandle {
blob_file_id: 0,
offset: offset0,
on_disk_size: on_disk0,
},
size: 300,
};
let ind1 = crate::blob_tree::handle::BlobIndirection {
vhandle: crate::vlog::ValueHandle {
blob_file_id: 1,
offset: 0,
on_disk_size: 100,
},
size: 300,
};
w.write(InternalValue::from_components(
b"a".to_vec(),
ind0.encode_into_vec(),
1,
ValueType::Indirection,
))?;
w.write(InternalValue::from_components(
b"m".to_vec(),
ind1.encode_into_vec(),
2,
ValueType::Indirection,
))?;
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair()?;
assert_eq!(
report.recovered, 0,
"blob 1 is referenced but gone; the salvage replacement must not be \
allocated onto that id, so the table's dependency stays missing and \
the table is excluded: {report:?}",
);
Ok(())
}
#[test]
fn repair_rejects_blob_backed_ssts_without_kv_separation() -> crate::Result<()> {
use crate::fs::Fs;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(crate::fs::MemFs::new());
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
tree.insert(b"k", vec![b'v'; 64], 1);
tree.flush_active_memtable(0)?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.repair();
assert!(
matches!(
result,
Err(crate::Error::TreeTypeMismatch {
requested: crate::TreeType::Standard,
actual: crate::TreeType::Blob,
})
),
"blob-backed SSTs must not be rebuilt into a Standard manifest: {:?}",
result.map(|r| r.recovered),
);
Ok(())
}
#[test]
fn repair_supersedes_ancestry_through_an_excluded_intermediate() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
b"k".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "A is non-empty");
}
{
let mut w = Writer::new(tables.join("1"), 1, 0, Arc::clone(&fs))?
.use_recency(Some(0))
.use_lineage(Some(vec![0]))
.use_lineage_last(true);
w.write(InternalValue::from_components(
b"k".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "B is non-empty");
}
{
let mut w = Writer::new(tables.join("2"), 2, 0, Arc::clone(&fs))?
.use_recency(Some(1))
.use_lineage(Some(vec![1]))
.use_lineage_transformed(true)
.use_lineage_last(true);
w.write(InternalValue::from_components(
b"m".to_vec(),
b"w".to_vec(),
2,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "C is non-empty");
}
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(
report.recovered, 1,
"C's complete run covers B, and superseding B transitively \
supersedes A — publishing A would resurrect the filtered k: {report:?}",
);
Ok(())
}
#[test]
fn repair_rejects_a_transformed_output_with_a_live_history_carrier() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
b"k".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "A is non-empty");
}
{
let mut w = Writer::new(tables.join("1"), 1, 0, Arc::clone(&fs))?
.use_recency(Some(0))
.use_lineage(Some(vec![0]))
.use_lineage_last(true);
w.write(InternalValue::from_components(
b"k".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "B is non-empty");
}
{
let mut w = Writer::new(tables.join("2"), 2, 0, Arc::clone(&fs))?
.use_recency(Some(1))
.use_lineage(Some(vec![1]))
.use_lineage_transformed(true);
w.write(InternalValue::from_components(
b"m".to_vec(),
b"w".to_vec(),
2,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "C is non-empty");
}
let result = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair();
assert!(
matches!(result, Err(crate::Error::Unrecoverable)),
"a retained transformed output whose input history keeps a live \
unsuperseded carrier must fail the repair: {:?}",
result.map(|r| r.recovered),
);
Ok(())
}
#[test]
fn repair_rejects_a_residual_overlap_under_a_merge_operator() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
struct SumMerge;
impl crate::MergeOperator for SumMerge {
fn merge(
&self,
_key: &[u8],
_base_value: Option<&[u8]>,
_operands: &[&[u8]],
) -> crate::Result<crate::UserValue> {
Ok(b"sum".to_vec().into())
}
}
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs))?;
for key in [b"a".as_slice(), b"z".as_slice()] {
w.write(InternalValue::from_components(
key.to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the input is non-empty");
}
{
let mut w = Writer::new(tables.join("2"), 2, 0, Arc::clone(&fs))?
.use_recency(Some(1))
.use_lineage(Some(vec![0, 1]));
for key in [b"a".as_slice(), b"m".as_slice()] {
w.write(InternalValue::from_components(
key.to_vec(),
b"w".to_vec(),
2,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the output is non-empty");
}
let result = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_merge_operator(Some(Arc::new(SumMerge)))
.repair();
assert!(
matches!(result, Err(crate::Error::Unrecoverable)),
"a residual overlap under a merge operator must fail the repair, not \
publish a double-applying pair: {:?}",
result.map(|r| r.recovered),
);
Ok(())
}
#[test]
fn repair_reports_the_overlap_of_a_partially_covered_input() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs))?;
for key in [b"a".as_slice(), b"z".as_slice()] {
w.write(InternalValue::from_components(
key.to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the input is non-empty");
}
{
let mut w = Writer::new(tables.join("2"), 2, 0, Arc::clone(&fs))?
.use_recency(Some(1))
.use_lineage(Some(vec![0, 1]));
for key in [b"a".as_slice(), b"m".as_slice()] {
w.write(InternalValue::from_components(
key.to_vec(),
b"v".to_vec(),
2,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the output is non-empty");
}
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(
report.recovered, 2,
"an input reaching beyond the output holds live records and is kept: \
{report:?}",
);
assert!(
!report.lost_coverage.is_empty(),
"the residual overlap must be reported — reads there may double-apply \
merge operands: {report:?}",
);
Ok(())
}
#[test]
fn open_or_repair_carries_the_report_when_the_follow_up_open_fails() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, MemFs};
use crate::io::ErrorKind;
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::Interrupted))
.on_path(root.join("v0").to_string_lossy())
.skip(1),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.open_or_repair(crate::repair::RepairPolicy::default());
match result {
Err(err @ crate::Error::RepairedButUnopened { .. }) => {
assert!(
matches!(
core::error::Error::source(&err).and_then(|s| s.downcast_ref::<crate::Error>()),
Some(crate::Error::Io(_)),
),
"source() must expose the follow-up open's failure",
);
let crate::Error::RepairedButUnopened { report, cause } = err else {
unreachable!("matched above");
};
assert_eq!(
report.recovered, 1,
"the committed repair's report rides with the error: {report:?}",
);
assert!(
matches!(&*cause, crate::Error::Io(e) if e.kind() == ErrorKind::Interrupted),
"the open's own failure is preserved as the cause: {cause:?}",
);
}
other => panic!(
"a failed post-repair open must carry the completed report, not \
drop it: {:?}",
other.map(|(_, r)| r),
),
}
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn an_unreadable_current_probe_preserves_published_replacements() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
let fixture = || -> crate::Result<Arc<MemFs>> {
let memfs = Arc::new(MemFs::new());
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let last = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?
.last()
.expect("a last frame")
.frame_end;
{
let mut file = memfs.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*file, last - 8, 1)?;
file.seek(SeekFrom::Start(last - 8))?;
file.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
Ok(memfs)
};
for skip in 0u64..10 {
let memfs = fixture()?;
let fault = FaultFs::new((*memfs).clone());
fault
.injector()
.arm(FaultRule::new(FaultOp::SyncDirectory, Fault::Error(ErrorKind::Other)).skip(skip));
fault.injector().arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::PermissionDenied))
.on_path(crate::file::CURRENT_VERSION_FILE)
.skip(1),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair();
if memfs.exists(&root.join("current"))? {
let tree = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?;
for i in 0..7u32 {
assert!(
tree.get(format!("k{i:04}").as_bytes(), crate::SeqNo::MAX)?
.is_some(),
"skip={skip}, repair={:?}: a committed manifest must \
resolve its values — a referenced blob file was deleted",
result.as_ref().map(|r| r.recovered),
);
}
}
}
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_failed_manifest_write_unwinds_published_blob_replacements() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
let memfs = Arc::new(MemFs::new());
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let last = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?
.last()
.expect("a last frame")
.frame_end;
{
let mut file = memfs.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*file, last - 8, 1)?;
file.seek(SeekFrom::Start(last - 8))?;
file.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::Write, Fault::Error(ErrorKind::StorageFull))
.on_path(root.join("v0").to_string_lossy()),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair();
assert!(
result.is_err(),
"the manifest write fault must fail the repair: {:?}",
result.map(|r| r.recovered),
);
assert!(
!memfs.exists(&root.join("current"))?,
"self-check: the fault fired before the pointer switch",
);
let leftover: Vec<String> = memfs
.read_dir(&blobs)?
.into_iter()
.filter(|e| !e.is_dir)
.map(|e| e.file_name)
.collect();
assert_eq!(
leftover,
vec![
blob_path
.file_name()
.expect("blob file name")
.to_string_lossy()
.to_string()
],
"an uncommitted fresh-id replacement must be unwound, or every \
failed attempt stacks another copy",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_never_half_publishes_a_salvaged_blob_under_transient_faults() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
let read_all = |fs: &Arc<dyn Fs>, path: &std::path::Path| -> crate::Result<Vec<u8>> {
let file = fs.open(path, &crate::fs::FsOpenOptions::new().read(true))?;
let len = crate::fs::FsFile::metadata(&*file)?.len;
Ok(crate::file::read_exact(&*file, 0, usize::try_from(len).unwrap_or(0))?.to_vec())
};
let fixture = || -> crate::Result<(Arc<MemFs>, std::path::PathBuf, Vec<u8>)> {
let memfs = Arc::new(MemFs::new());
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let last = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?
.last()
.expect("a last frame")
.frame_end;
{
let mut file = memfs.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*file, last - 8, 1)?;
file.seek(SeekFrom::Start(last - 8))?;
file.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let corrupted = read_all(&fs_dyn, &blob_path)?;
Ok((memfs, blob_path, corrupted))
};
for skip in [0u64, 4, 8, 16, 24, 32, 48, 64, 96, 128] {
let (memfs, blob_path, corrupted) = fixture()?;
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::ReadAt, Fault::Error(ErrorKind::Interrupted))
.on_path("blobs")
.skip(skip),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair();
assert!(
!memfs.exists(&blobs.join("0.salvage-tmp"))?,
"skip={skip}: a salvage temp must never survive the repair",
);
if result.is_err() {
assert_eq!(
read_all(&fs_dyn, &blob_path)?,
corrupted,
"skip={skip}: an aborted repair must leave the original blob at \
the canonical path, never an unverified replacement",
);
} else {
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..7u32 {
assert!(
tree.get(format!("k{i:04}").as_bytes(), crate::MAX_SEQNO)?
.is_some(),
"skip={skip}: record k{i:04} must survive a completed repair",
);
}
}
}
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_leaves_every_table_in_place_when_the_rewrite_fails_transiently() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let frontier = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.next()
.expect("a first frame")?
.frame_end;
let data_start = {
let mut file = fs_dyn.open(&blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
reader
.toc()
.section(b"data")
.expect("blob file has a data section")
.pos()
};
memfs.punch_hole(&blob_path, data_start, frontier - data_start)?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::ReadAt, Fault::Error(ErrorKind::Interrupted)).on_path("tables"),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair();
assert!(
result.is_err(),
"a transient rewrite failure must propagate for a retry: {result:?}",
);
assert!(
memfs.exists(&root.join("tables").join("0"))?,
"the source SST must still be in tables/ when the retryable error \
propagates, or the retry rebuilds a manifest without its keys",
);
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
));
config.repair()?;
let tree = match config.open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 1..8u32 {
let key = format!("k{i:04}");
assert!(
tree.get(key.as_bytes(), crate::SeqNo::MAX)?.is_some(),
"{key} is recovered by the retry",
);
}
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_salvages_a_frame_corrupt_blob_and_remaps_handles() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let value = |i: u32| alloc::vec![b'a' + u8::try_from(i).expect("small i"); 64];
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), value(i), u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?;
assert_eq!(entries.len(), 8, "eight separated values");
let last = entries.last().expect("last frame");
let flip_at = last.frame_end - 8; {
let mut file = memfs.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*file, flip_at, 1)?;
file.seek(SeekFrom::Start(flip_at))?;
file.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair()?;
assert_eq!(
report.recovered, 1,
"the referencing table survives, rewritten through the remap: {report:?}",
);
assert_eq!(
report.unreadable, 0,
"an installed salvaged blob is not an unreadable file: {report:?}",
);
assert_eq!(
report.blob_files_salvaged.len(),
1,
"the salvage outcome is reported in its own field: {report:?}",
);
assert!(
report
.blob_files_salvaged
.first()
.is_some_and(|(_, note)| note.contains("records salvaged")),
"the note describes what was recovered: {report:?}",
);
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..7u32 {
assert_eq!(
tree.get(format!("k{i:04}").as_bytes(), crate::MAX_SEQNO)?
.as_deref(),
Some(value(i).as_slice()),
"intact record k{i:04} must survive the blob salvage + handle remap",
);
}
assert_eq!(
tree.get(b"k0007", crate::MAX_SEQNO)?,
None,
"the corrupt record's key reads as absent, never as an error",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_failed_repair_leaves_the_originals_intact_and_the_retry_succeeds() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let value = |i: u32| alloc::vec![b'a' + u8::try_from(i).expect("small i"); 64];
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), value(i), u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = blobs.join("0");
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?;
assert_eq!(entries.len(), 8, "eight separated values");
let frontier = entries.get(1).expect("second frame").frame_end;
let data_start = {
let mut file = fs_dyn.open(&blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
reader
.toc()
.section(b"data")
.expect("blob file has a data section")
.pos()
};
memfs.punch_hole(&blob_path, data_start, frontier - data_start)?;
let last = entries.last().expect("last frame");
let flip_at = last.frame_end - 8; {
let mut file = memfs.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*file, flip_at, 1)?;
file.seek(SeekFrom::Start(flip_at))?;
file.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let config = |fs: Arc<dyn Fs>| {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::ReadAt, Fault::Error(ErrorKind::Interrupted)).on_path("tables"),
);
let result = config(Arc::new(fault)).repair();
assert!(
result.is_err(),
"the transient table-stage failure must propagate: {result:?}",
);
assert!(
memfs.exists(&blob_path)?,
"a failed repair must leave the damaged original in place — the \
tables it has not rewritten yet still reference it",
);
let report = config(memfs.clone()).repair()?;
assert_eq!(
report.recovered, 1,
"the referencing table survives the retry, rewritten: {report:?}",
);
let tree = match config(memfs).open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 2..7u32 {
assert_eq!(
tree.get(format!("k{i:04}").as_bytes(), crate::MAX_SEQNO)?
.as_deref(),
Some(value(i).as_slice()),
"surviving record k{i:04} must read through the retry-finished remap",
);
}
for lost in [0u32, 1, 7] {
assert_eq!(
tree.get(format!("k{lost:04}").as_bytes(), crate::MAX_SEQNO)?,
None,
"punched/corrupt record k{lost:04} reads as absent, never an error",
);
}
Ok(())
}
#[expect(clippy::expect_used, reason = "test code")]
fn punch_and_corrupt_blob(
memfs: &crate::fs::MemFs,
blob_path: &std::path::Path,
) -> crate::Result<()> {
use crate::fs::Fs;
use std::io::{Seek, SeekFrom, Write};
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(blob_path, memfs, 0)?
.collect::<crate::Result<Vec<_>>>()?;
assert_eq!(entries.len(), 8, "eight separated values");
let frontier = entries.get(1).expect("second frame").frame_end;
let data_start = {
let mut file = memfs.open(blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
reader
.toc()
.section(b"data")
.expect("blob file has a data section")
.pos()
};
memfs.punch_hole(blob_path, data_start, frontier - data_start)?;
let last = entries.last().expect("last frame");
let flip_at = last.frame_end - 8; let mut file = memfs.open(
blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*file, flip_at, 1)?;
file.seek(SeekFrom::Start(flip_at))?;
file.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_retry_handles_damage_that_appeared_after_the_failed_attempt() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let value = |prefix: u8, i: u32| alloc::vec![prefix + u8::try_from(i).expect("small i"); 64];
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("a{i:04}").as_bytes(), value(b'a', i), u64::from(i));
}
tree.flush_active_memtable(0)?;
for i in 0..8u32 {
tree.insert(
format!("b{i:04}").as_bytes(),
value(b'b', i),
u64::from(8 + i),
);
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
punch_and_corrupt_blob(&memfs, &blobs.join("0"))?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let config = |fs: Arc<dyn Fs>| {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::ReadAt, Fault::Error(ErrorKind::Interrupted)).on_path("tables"),
);
let result = config(Arc::new(fault)).repair();
assert!(
result.is_err(),
"the table-stage failure must propagate: {result:?}",
);
punch_and_corrupt_blob(&memfs, &blobs.join("1"))?;
let report = config(memfs.clone()).repair()?;
assert_eq!(
report.recovered, 2,
"both referencing tables survive the retry: {report:?}",
);
let tree = match config(memfs).open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for (prefix, tag) in [(b'a', "a"), (b'b', "b")] {
for i in 2..7u32 {
assert_eq!(
tree.get(format!("{tag}{i:04}").as_bytes(), crate::MAX_SEQNO)?
.as_deref(),
Some(value(prefix, i).as_slice()),
"surviving record {tag}{i:04} must read after the retry \
(a stamped table passed through an applied remap again would \
drop its live records)",
);
}
for lost in [0u32, 1, 7] {
assert_eq!(
tree.get(format!("{tag}{lost:04}").as_bytes(), crate::MAX_SEQNO)?,
None,
"punched/corrupt record {tag}{lost:04} reads as absent, never an error",
);
}
}
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_zeroed_blob_prefix_without_a_hole_is_not_a_frontier() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
blob_tree_without_manifest(&memfs, &root)?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let frontier =
crate::vlog::BlobFileScanner::new(&blob_path, &*(memfs.clone() as Arc<dyn Fs>), 0)?
.nth(1)
.expect("a second frame")?
.frame_end;
let data_start = {
let mut file = memfs.open(&blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
reader
.toc()
.section(b"data")
.expect("blob file has a data section")
.pos()
};
{
let mut file = memfs.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
file.seek(SeekFrom::Start(data_start))?;
file.write_all(&vec![
0u8;
usize::try_from(frontier - data_start).unwrap_or(0)
])?;
file.sync_all()?;
}
let fs: Arc<dyn Fs> = memfs;
let derived = super::derive_blob_frontier(&fs, &blob_path, 0)?;
assert!(
matches!(derived, super::BlobFrontier::Whole),
"allocated zeros are corruption the validation scan must surface, not a \
frontier that silently drops the handles below it, got {derived:?}",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_wholly_zeroed_blob_section_without_holes_is_not_fully_consumed() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
blob_tree_without_manifest(&memfs, &root)?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let (data_start, data_len) = {
let mut file = memfs.open(&blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
let data = reader
.toc()
.section(b"data")
.expect("blob file has a data section");
(data.pos(), data.len())
};
{
let mut file = memfs.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
file.seek(SeekFrom::Start(data_start))?;
file.write_all(&vec![0u8; usize::try_from(data_len).unwrap_or(0)])?;
file.sync_all()?;
}
let fs: Arc<dyn Fs> = memfs;
let derived = super::derive_blob_frontier(&fs, &blob_path, 0)?;
assert!(
matches!(derived, super::BlobFrontier::Whole),
"a fully allocated zeroed section is destroyed data, not a relocation \
whose file removal lagged a crash, got {derived:?}",
);
Ok(())
}
#[expect(clippy::expect_used, reason = "test code")]
fn blob_tree_without_manifest(
memfs: &std::sync::Arc<crate::fs::MemFs>,
root: &std::path::Path,
) -> crate::Result<()> {
use crate::fs::Fs;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
{
let tree = match Config::new(
root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(
format!("k{i:04}").as_bytes(),
alloc::vec![b'a' + u8::try_from(i).expect("small i"); 64],
u64::from(i),
);
}
tree.flush_active_memtable(0)?;
}
for e in memfs.read_dir(root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
Ok(())
}
#[cfg(feature = "lz4")]
#[test]
fn repair_leaves_an_unreferenced_blob_out_of_the_manifest() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
blob_tree_without_manifest(&memfs, &root)?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
punch_and_corrupt_blob(&memfs, &blobs.join("0"))?;
let config = |fs: Arc<dyn Fs>| {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
{
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let mut w = crate::vlog::blob_file::writer::Writer::new(blobs.join("1"), 1, 0, &*fs_dyn)?;
w.write(b"k0000", 1, &[b'x'; 200])?;
w.finish()?;
let table = root.join("tables").join("0");
let bytes = {
let file = memfs.open(&table, &crate::fs::FsOpenOptions::new().read(true))?;
let len = crate::fs::FsFile::metadata(&*file)?.len;
#[expect(clippy::expect_used, reason = "test code")]
let len = usize::try_from(len).expect("small file");
crate::file::read_exact(&*file, 0, len)?.to_vec()
};
let mut tmp = memfs.open(
&super::repair_tmp_path(&table),
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
tmp.write_all(&bytes)?;
}
assert!(
memfs.exists(&blobs.join("1"))?,
"the abandoned replacement is what this test is about",
);
assert!(
memfs.exists(&super::repair_tmp_path(&root.join("tables").join("0")))?,
"the unpublished replacement is what leaves the blob unreferenced",
);
config(memfs.clone()).repair()?;
let tree = config(memfs.clone()).open()?;
assert_eq!(
tree.blob_file_count(),
1,
"only the blob the surviving table references belongs in the manifest; \
admitting the abandoned copy would pin it forever, since repair cannot \
rebuild the fragmentation stats GC needs to retire it",
);
assert!(
!memfs.exists(&blobs.join("1"))?,
"the unreferenced copy is removed by the repair that omits it",
);
Ok(())
}
#[cfg(feature = "lz4")]
#[test]
fn repair_fails_when_a_superseded_blob_original_cannot_be_removed() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, MemFs};
use crate::io::ErrorKind;
use crate::{Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
blob_tree_without_manifest(&memfs, &root)?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
punch_and_corrupt_blob(&memfs, &blobs.join("0"))?;
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(
FaultOp::RemoveFile,
Fault::Error(ErrorKind::PermissionDenied),
)
.on_path("blobs"),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair();
assert!(
result.is_err(),
"a superseded original that cannot be removed must fail the repair, \
not be reported as a success the next open cannot honour: {result:?}",
);
Ok(())
}
#[test]
fn a_salvaged_blob_takes_a_fresh_id_and_the_original_is_removed() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
blob_tree_without_manifest(&memfs, &root)?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
punch_and_corrupt_blob(&memfs, &blobs.join("0"))?;
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair()?;
assert_eq!(
report.blob_files_salvaged.len(),
1,
"the damaged blob was salvaged: {report:?}",
);
assert!(
memfs.exists(&blobs.join("1"))?,
"the replacement took the next free id rather than the original's",
);
assert!(
!memfs.exists(&blobs.join("0"))?,
"the superseded original is removed once the manifest is committed",
);
Ok(())
}
#[cfg(feature = "std")]
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_blob_rewrite_preserves_a_zero_ingest_offset() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db_zero_ingest")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
{
let tree = match config().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
let mut ingestion = crate::blob_tree::ingest::BlobIngestion::new(&tree)?;
for i in 0..8u32 {
ingestion.write(format!("k{i:04}").as_bytes().into(), vec![b'v'; 64].into())?;
}
ingestion.finish()?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?;
assert!(entries.len() >= 3, "several frames written");
let second = entries.get(1).expect("second frame");
{
let offset = second.frame_end - 4;
let mut file = fs_dyn.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let mut byte = [0u8; 1];
let n = crate::fs::FsFile::read_at(&*file, &mut byte, offset)?;
assert_eq!(n, 1, "one byte read back for the flip");
byte[0] ^= 0xFF;
std::io::Seek::seek(&mut file, std::io::SeekFrom::Start(offset))?;
std::io::Write::write_all(&mut file, &byte)?;
}
let report = config().repair_with_salvage(true)?;
assert_eq!(
report.recovered, 1,
"the ingested SST survives the blob rewrite with its ZERO offset \
carried over, never rejected as offset-less: {report:?}",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_clean_manifest_frontier_survives_a_backend_that_cannot_probe_holes() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::version::{BlobFileList, Level, Run, Version};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db_blob_frontier_authority")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
{
let tree = match config().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
tree.insert(b"zzz", vec![b'w'; 64], 100);
tree.flush_active_memtable(0)?;
}
let (tables, blob_files) = {
let tree = match config().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
let version = tree.index.current_version();
let blob = version
.blob_files
.iter()
.next()
.cloned()
.expect("the flush spilled a blob file");
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(blob.path(), &*fs_dyn, blob.id())?
.collect::<crate::Result<Vec<_>>>()?;
assert!(entries.len() >= 2, "several frames written");
let frontier = entries.first().expect("first frame").frame_end;
let data_start = {
let mut file = fs_dyn.open(blob.path(), &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
reader
.toc()
.section(b"data")
.expect("blob file has a data section")
.pos()
};
memfs.punch_hole(blob.path(), data_start, frontier - data_start)?;
let restricted = blob.reopen_restricted(frontier)?;
let tables: Vec<_> = version.iter_tables().cloned().collect();
let mut map = crate::HashMap::default();
map.insert(restricted.id(), restricted);
(tables, BlobFileList::new(map))
};
let run = Arc::new(Run::new(tables).expect("non-empty run"));
let version = Version::from_levels(
2,
crate::TreeType::Blob,
alloc::vec![Level::from_runs(alloc::vec![run])],
blob_files,
crate::blob_tree::FragmentationMap::default(),
);
crate::version::persist_version(
&root,
&version,
crate::comparator::default_comparator().name(),
&*fs_dyn,
Arc::new(crate::runtime_config::RuntimeConfig::default()),
None,
crate::fs::SyncMode::Normal,
)?;
memfs.set_hole_probe_supported(false);
memfs.remove_file(&root.join("tables").join("1"))?;
let report = config().repair()?;
assert!(
report
.unreadable_files
.iter()
.all(|(p, _)| !p.to_string_lossy().contains("blobs")),
"the committed frontier keeps the punched blob file recoverable \
without any hole probe: {report:?}",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_rewrites_tables_with_handles_below_a_blob_frontier() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), vec![b'v'; 64], u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(&blob_path, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?;
assert!(entries.len() >= 2, "several frames written");
let frontier = entries.first().expect("first frame").frame_end;
let data_start = {
let mut file = fs_dyn.open(&blob_path, &crate::fs::FsOpenOptions::new().read(true))?;
let reader = crate::sfa::Reader::from_reader(&mut file)?;
reader
.toc()
.section(b"data")
.expect("blob file has a data section")
.pos()
};
memfs.punch_hole(&blob_path, data_start, frontier - data_start)?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(Arc::clone(&memfs) as Arc<dyn Fs>)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair()?;
assert_eq!(
report.recovered, 1,
"the table survives, rewritten with its stale handles dropped: {report:?}",
);
assert_eq!(
report.salvaged, 1,
"the rewrite runs through the salvage pipeline and counts as such: {report:?}",
);
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
assert_eq!(
tree.get(b"k0000", crate::MAX_SEQNO)?,
None,
"the sub-frontier record's key reads as absent, never as an error",
);
for i in 1..8u32 {
assert!(
tree.get(format!("k{i:04}").as_bytes(), crate::MAX_SEQNO)?
.is_some(),
"record k{i:04} above the frontier must survive the rewrite",
);
}
Ok(())
}
#[test]
fn blob_recovery_discards_a_persistently_unreadable_blob_file() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
memfs.create_dir_all(&blobs)?;
{
let mut f = memfs.open(
&blobs.join("3"),
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?;
f.write_all(b"not a blob file")?;
}
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone());
let mut published = super::PublishedBlobReplacements::new(&config);
let recovery =
super::recover_blob_files(&config, &mut published, &(0..10).collect(), None, None)?;
published.disarm();
let (files, unreadable, discard) = (recovery.files, recovery.unreadable, recovery.discard);
assert!(files.is_empty(), "nothing recoverable");
assert_eq!(unreadable.len(), 1, "the bad blob is reported");
assert_eq!(
discard.iter().map(|(p, _)| p).collect::<Vec<_>>(),
vec![&blobs.join("3")],
"the unreadable blob is queued for removal after the commit",
);
assert!(
memfs.exists(&blobs.join("3"))?,
"the scan removes nothing itself",
);
Ok(())
}
#[test]
fn blob_recovery_discards_a_duplicate_blob_id() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
memfs.create_dir_all(&blobs)?;
let fs_dyn: Arc<dyn Fs> = memfs.clone();
for (name, val) in [("1", b"canonical".as_slice()), ("01", b"stale-dup")] {
let mut w = crate::vlog::blob_file::writer::Writer::new(blobs.join(name), 1, 0, &*fs_dyn)?;
w.write(b"k", 1, val)?;
w.finish()?;
}
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone());
let mut published = super::PublishedBlobReplacements::new(&config);
let recovery =
super::recover_blob_files(&config, &mut published, &(0..10).collect(), None, None)?;
published.disarm();
let (files, unreadable, excluded, discard) = (
recovery.files,
recovery.unreadable,
recovery.excluded,
recovery.discard,
);
assert_eq!(files.len(), 1, "one blob file per id");
assert_eq!(unreadable.len(), 0, "{unreadable:?}");
assert_eq!(
excluded.len(),
1,
"the displaced duplicate is reported as an exclusion: {excluded:?}"
);
assert!(
memfs.exists(&blobs.join("1"))?,
"the canonical spelling is the kept file"
);
assert_eq!(
discard.iter().map(|(p, _)| p).collect::<Vec<_>>(),
vec![&blobs.join("01")],
"the duplicate is queued for removal once the manifest is durable",
);
assert!(
memfs.exists(&blobs.join("01"))?,
"the scan itself removes nothing: a crash here must leave the directory \
exactly as the retry expects to find it",
);
let kept = crate::Checksum::from_raw(super::compute_table_checksum(
&*config.fs,
&blobs.join("1"),
)?);
assert_eq!(
files.first().map(crate::vlog::BlobFile::checksum),
Some(kept),
"the manifest checksum must describe the kept canonical file"
);
Ok(())
}
#[test]
fn blob_recovery_propagates_a_transient_checksum_failure() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
memfs.create_dir_all(&blobs)?;
{
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let mut w = crate::vlog::blob_file::writer::Writer::new(blobs.join("0"), 0, 0, &*fs_dyn)?;
w.write(b"k", 1, b"blob bytes")?;
w.finish()?;
}
let fault = FaultFs::new(memfs.as_ref().clone());
fault
.injector()
.arm(FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::WouldBlock)).on_path("blobs"));
let config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault);
let result = super::recover_blob_files(
&config,
&mut super::PublishedBlobReplacements::new(&config),
&(0..10).collect(),
None,
None,
);
assert!(
result.is_err(),
"a transient blob checksum failure must propagate for a retry, not be \
recorded as unreadable: {:?}",
result.map(|r| (r.files.len(), r.unreadable)),
);
Ok(())
}
#[test]
fn open_discards_an_uncommitted_repair_replacement() -> crate::Result<()> {
use crate::fs::{Fs, FsOpenOptions, MemFs};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
};
{
let crate::AnyTree::Standard(tree) = config().open()? else {
panic!("expected a standard tree");
};
tree.insert(b"k", b"v", 1);
tree.flush_active_memtable(0)?;
}
let sst = root.join("tables").join("0");
let tmp = super::repair_tmp_path(&sst);
{
let mut file = memfs.open(&tmp, &FsOpenOptions::new().write(true).create_new(true))?;
file.write_all(b"half-built garbage")?;
}
let crate::AnyTree::Standard(tree) = config().open()? else {
panic!("expected a standard tree");
};
assert_eq!(
tree.get(b"k", u64::MAX)?.as_deref(),
Some(b"v".as_ref()),
"the source the manifest describes must survive the leftover temp",
);
assert!(
!memfs.exists(&tmp)?,
"the unpublished replacement is garbage and is swept",
);
Ok(())
}
#[test]
fn repair_discards_an_uncommitted_repair_replacement() -> crate::Result<()> {
use crate::fs::{Fs, FsOpenOptions, MemFs};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
};
{
let crate::AnyTree::Standard(tree) = config().open()? else {
panic!("expected a standard tree");
};
tree.insert(b"k", b"v", 1);
tree.flush_active_memtable(0)?;
}
let sst = root.join("tables").join("0");
let tmp = super::repair_tmp_path(&sst);
{
let mut file = memfs.open(&tmp, &FsOpenOptions::new().write(true).create_new(true))?;
file.write_all(b"half-built garbage")?;
}
let report = config().repair()?;
assert_eq!(
report.unreadable, 0,
"nothing is damaged: the source is intact and the temp is not a table",
);
let crate::AnyTree::Standard(tree) = config().open()? else {
panic!("expected a standard tree");
};
assert_eq!(
tree.get(b"k", u64::MAX)?.as_deref(),
Some(b"v".as_ref()),
"the intact source must survive the repair re-run",
);
assert!(
!memfs.exists(&tmp)?,
"the unpublished replacement is garbage and is swept",
);
Ok(())
}
fn standard_tree_without_manifest(
memfs: &std::sync::Arc<crate::fs::MemFs>,
root: &std::path::Path,
) -> crate::Result<()> {
use crate::fs::Fs;
use crate::{AbstractTree, Config, SequenceNumberCounter};
{
let crate::AnyTree::Standard(tree) = Config::new(
root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.open()?
else {
panic!("expected a standard tree");
};
tree.insert(b"k", b"v", 1);
tree.flush_active_memtable(0)?;
}
for e in memfs.read_dir(root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
Ok(())
}
#[test]
fn lossy_salvage_contributes_the_source_coverage() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..64u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let offsets: alloc::vec::Vec<u64> = recover_table(sst.clone(), &fs)?
.data_block_handles()
.filter_map(Result::ok)
.map(|kh| *kh.as_ref().offset())
.collect();
let (Some(corrupt), true) = (offsets.last().copied(), offsets.len() > 2) else {
panic!("need several blocks, got {offsets:?}");
};
let flip = usize::try_from(corrupt).unwrap_or(0) + 16;
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair_with_salvage(true)?;
assert_eq!(report.salvaged, 1, "the table survives as a lossy copy");
assert_eq!(report.recovered, 1, "the lossy copy joins the manifest");
assert_eq!(report.unreadable, 0, "{:?}", report.unreadable_files);
let [(path, lo, hi, bound)] = report.lost_coverage.as_slice() else {
panic!(
"the kept lossy copy must contribute a lost-coverage entry: {:?}",
report.lost_coverage,
);
};
assert_eq!(path, &sst, "the entry names the damaged source");
assert!(
lo.as_ref() <= b"k00000".as_slice() && hi.as_ref() >= b"k00063".as_slice(),
"the entry scopes the SOURCE's key range, not the shrunken copy's: [{lo:?}, {hi:?}]",
);
assert_eq!(
*bound,
Some(64),
"the ceiling is the SOURCE's highest seqno, not the copy's",
);
assert_eq!(
report.wal_replay_scope(),
super::WalReplayScope::LostUpTo(64),
"the WAL must be told how deep the loss reaches",
);
Ok(())
}
#[test]
fn lossy_salvage_without_source_metadata_is_unknowable() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs};
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
{
let build_fs: Arc<dyn Fs> = Arc::new(StdFs);
let mut w = Writer::new(sst, 0, 0, Arc::clone(&build_fs))?.use_data_block_size(128);
for i in 0..64u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let fault = FaultFs::new(StdFs);
let injector = fault.injector();
injector.arm(
FaultRule::new(FaultOp::Read, Fault::Error(crate::io::ErrorKind::Other))
.on_path("tables")
.once(),
);
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true)?;
injector.clear();
assert_eq!(report.salvaged, 1, "the table survives as a salvaged copy");
assert_eq!(
report.unknowable_losses.len(),
1,
"a lossy copy without source metadata cannot be scoped: {report:?}",
);
assert_eq!(
report.wal_replay_scope(),
super::WalReplayScope::FullHistory,
"no bound derived from survivors can scope what was lost",
);
Ok(())
}
#[test]
fn unparseable_exclusion_forces_full_history_replay() -> crate::Result<()> {
use crate::fs::{Fs, FsOpenOptions, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
{
let mut file = memfs.open(
&root.join("tables").join("7"),
&FsOpenOptions::new().write(true).create_new(true),
)?;
file.write_all(b"not a table at all")?;
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.repair()?;
assert_eq!(report.recovered, 1, "the healthy table is kept");
assert_eq!(report.unreadable, 1, "{:?}", report.unreadable_files);
assert_eq!(
report.unknowable_losses.len(),
1,
"the unparseable exclusion is recorded as unscopable: {report:?}",
);
assert_eq!(
report.wal_replay_scope(),
super::WalReplayScope::FullHistory,
"no bound can prove a retained record unaffected",
);
Ok(())
}
#[test]
fn a_foreign_name_is_neither_read_nor_reported_by_repair() -> crate::Result<()> {
use crate::fs::{Fs, FsOpenOptions, MemFs};
use crate::{Config, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
{
let mut file = memfs.open(
&root.join("tables").join("notes.txt"),
&FsOpenOptions::new().write(true).create_new(true),
)?;
file.write_all(b"operator scribbles, not a table")?;
}
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.repair()?;
assert_eq!(report.recovered, 1, "the healthy table is kept");
assert_eq!(
report.unreadable, 0,
"a name the engine does not own is not an unreadable ENGINE file: {:?}",
report.unreadable_files,
);
assert!(
memfs.exists(&root.join("tables").join("notes.txt"))?,
"the operator's file survives the repair untouched",
);
assert!(
report.unknowable_losses.is_empty(),
"a foreign name held no table data — it is not a loss: {report:?}",
);
assert_eq!(
report.wal_replay_scope(),
super::WalReplayScope::TailOnly,
"nothing was lost, so the WAL replays its tail as usual",
);
Ok(())
}
#[test]
fn repair_propagates_a_transient_manifest_probe_failure() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
{
let crate::AnyTree::Standard(tree) = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.open()?
else {
panic!("expected a standard tree");
};
tree.insert(b"k", b"v", 1);
tree.flush_active_memtable(0)?;
}
let manifest_names: Vec<String> = memfs
.read_dir(&root)?
.into_iter()
.filter(|e| e.file_name.starts_with('v') || e.file_name == "current")
.map(|e| e.file_name)
.collect();
let fault = FaultFs::new((*memfs).clone());
fault
.injector()
.arm(FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::WouldBlock)).on_path("current"));
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair();
assert!(
matches!(result, Err(crate::Error::Io(ref e)) if e.kind().is_transient()),
"a transient probe fault must surface for a retry, not trigger a \
manifest-loss rebuild: {:?}",
result.map(|r| r.recovered),
);
let after: Vec<String> = memfs
.read_dir(&root)?
.into_iter()
.filter(|e| e.file_name.starts_with('v') || e.file_name == "current")
.map(|e| e.file_name)
.collect();
assert_eq!(
manifest_names, after,
"a rebuild would have written a fresh manifest generation",
);
Ok(())
}
#[test]
fn repair_rejects_an_exhausted_table_id_space() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join(u64::MAX.to_string());
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst, u64::MAX, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
b"k".to_vec(),
b"v".to_vec(),
1,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let result = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair();
assert!(
matches!(result, Err(crate::Error::Unrecoverable)),
"an exhausted table id space must fail the repair, not commit a \
manifest the next open cannot allocate past: {:?}",
result.map(|r| r.recovered),
);
Ok(())
}
#[test]
fn repair_rejects_an_exhausted_blob_id_space() -> crate::Result<()> {
use crate::coding::Encode;
use crate::fs::{Fs, MemFs};
use crate::{Config, InternalValue, KvSeparationOptions, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db")?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
let tables = root.join("tables");
memfs.create_dir_all(&blobs)?;
memfs.create_dir_all(&tables)?;
let blob_path = blobs.join(u64::MAX.to_string());
let (offset, on_disk_size) = {
let mut w = crate::vlog::blob_file::writer::Writer::new(&blob_path, u64::MAX, 0, &*fs_dyn)?;
let offset = w.offset();
let on_disk_size = w.write(b"k", 1, &[b'x'; 300])?;
w.finish()?;
(offset, on_disk_size)
};
let indirection = crate::blob_tree::handle::BlobIndirection {
vhandle: crate::vlog::ValueHandle {
blob_file_id: u64::MAX,
offset,
on_disk_size,
},
size: 300,
};
{
let mut w = crate::table::Writer::new(tables.join("0"), 0, 0, Arc::clone(&fs_dyn))?;
w.link_blob_file(u64::MAX, 1, 300, u64::from(on_disk_size));
w.write(InternalValue::from_components(
b"k".to_vec(),
indirection.encode_into_vec(),
1,
ValueType::Indirection,
))?;
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair();
assert!(
matches!(result, Err(crate::Error::Unrecoverable)),
"an exhausted blob id space must fail the repair, not commit a \
manifest the next open cannot allocate past: {:?}",
result.map(|r| r.recovered),
);
Ok(())
}
#[test]
fn repair_propagates_a_permission_denied_open() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, MemFs};
use crate::io::ErrorKind;
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
let fault = FaultFs::new((*memfs).clone());
let injector = fault.injector();
injector.arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::PermissionDenied))
.on_path(root.join("tables").join("0").to_string_lossy()),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair();
assert!(
matches!(result, Err(crate::Error::Io(ref e)) if e.kind() == ErrorKind::PermissionDenied),
"an access failure must abort the repair, never grade the file: {:?}",
result.map(|r| (r.recovered, r.unreadable)),
);
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.repair()?;
assert_eq!(report.recovered, 1, "the intact table survives the mistake");
Ok(())
}
#[test]
fn repair_propagates_a_full_destination_during_salvage() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule};
use crate::io::ErrorKind;
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let sst = tables.join("0");
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(sst.clone(), 0, 0, Arc::clone(&fs))?.use_data_block_size(128);
for i in 0..64u32 {
w.write(InternalValue::from_components(
format!("k{i:05}").into_bytes(),
format!("v{i}").into_bytes(),
1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the SST is non-empty");
}
let offsets: alloc::vec::Vec<u64> = recover_table(sst.clone(), &fs)?
.data_block_handles()
.filter_map(Result::ok)
.map(|kh| *kh.as_ref().offset())
.collect();
let Some(corrupt) = offsets.get(1).copied() else {
panic!("need several blocks, got {offsets:?}");
};
let flip = usize::try_from(corrupt).unwrap_or(0) + 16;
let mut bytes = std::fs::read(&sst)?;
if let Some(b) = bytes.get_mut(flip) {
*b ^= 0xFF;
}
std::fs::write(&sst, &bytes)?;
let fault = FaultFs::new(StdFs);
fault.injector().arm(
FaultRule::new(FaultOp::Write, Fault::Error(ErrorKind::StorageFull)).on_path("repair-tmp"),
);
let result = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair_with_salvage(true);
assert!(
matches!(result, Err(crate::Error::Io(ref e)) if e.kind() == ErrorKind::StorageFull),
"a full destination must abort the repair, never grade the source: {:?}",
result.map(|r| (r.recovered, r.unreadable)),
);
assert!(
std::fs::metadata(&sst).is_ok(),
"the damaged-but-salvageable source stays for the retry",
);
Ok(())
}
#[test]
fn repair_propagates_a_network_timeout() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, MemFs};
use crate::io::ErrorKind;
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::Open, Fault::Error(ErrorKind::TimedOut))
.on_path(root.join("tables").join("0").to_string_lossy()),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair();
assert!(
matches!(result, Err(crate::Error::Io(ref e)) if e.kind() == ErrorKind::TimedOut),
"a transport timeout must abort the repair, never grade the file: {:?}",
result.map(|r| (r.recovered, r.unreadable)),
);
Ok(())
}
#[test]
fn repair_publishes_phase_and_byte_progress() -> crate::Result<()> {
use crate::fs::MemFs;
use crate::{Config, RecoveryPhase, RecoveryProgress, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
let progress = Arc::new(RecoveryProgress::default());
assert_eq!(progress.snapshot().phase, RecoveryPhase::Idle);
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_recovery_progress(Arc::clone(&progress))
.repair()?;
let snap = progress.snapshot();
assert_eq!(
snap.phase,
RecoveryPhase::Done,
"a successful run ends Done"
);
assert!(
snap.bytes_total > 0,
"the upfront listing published a total"
);
assert_eq!(
snap.bytes_processed, snap.bytes_total,
"a completed run took up exactly what it announced",
);
assert_eq!(snap.tables_recovered, 1, "the flushed SST was recovered");
Ok(())
}
#[test]
fn repair_progress_reaches_the_total_past_rejected_blob_entries() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, RecoveryPhase, RecoveryProgress, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
blob_tree_without_manifest(&memfs, &root)?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
for junk in ["not-a-blob-id", "7.salvage-tmp"] {
let mut f = memfs.open(
&blobs.join(junk),
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
f.write_all(b"leftover bytes")?;
}
let progress = Arc::new(RecoveryProgress::default());
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.with_kv_separation(Some(
crate::KvSeparationOptions::default().separation_threshold(16),
))
.with_recovery_progress(Arc::clone(&progress))
.repair()?;
let snap = progress.snapshot();
assert_eq!(
snap.phase,
RecoveryPhase::Done,
"a successful run ends Done"
);
assert!(
snap.bytes_total > 0,
"the upfront listing published a total"
);
assert_eq!(
snap.bytes_processed, snap.bytes_total,
"rejected blob entries count as processed when their arms dispose of \
them — {} of {} leaves a percentage UI below 100%",
snap.bytes_processed, snap.bytes_total,
);
Ok(())
}
enum HookOn {
Open,
RenameTo,
}
struct HookFs {
inner: crate::fs::MemFs,
on: HookOn,
needle: String,
fired: std::sync::atomic::AtomicBool,
hook: Box<dyn Fn() + Send + Sync>,
}
impl HookFs {
fn fire_once(&self) {
if !self.fired.swap(true, std::sync::atomic::Ordering::SeqCst) {
(self.hook)();
}
}
}
impl crate::fs::Fs for HookFs {
fn open(
&self,
path: &std::path::Path,
options: &crate::fs::FsOpenOptions,
) -> crate::io::Result<Box<dyn crate::fs::FsFile>> {
if matches!(self.on, HookOn::Open) && path.to_string_lossy().contains(&self.needle) {
self.fire_once();
}
self.inner.open(path, options)
}
fn remove_file(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.inner.remove_file(path)
}
fn rename(&self, from: &std::path::Path, to: &std::path::Path) -> crate::io::Result<()> {
if matches!(self.on, HookOn::RenameTo) && to.to_string_lossy().contains(&self.needle) {
self.fire_once();
}
self.inner.rename(from, to)
}
fn create_dir_all(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.inner.create_dir_all(path)
}
fn remove_dir_all(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.inner.remove_dir_all(path)
}
fn sync_directory(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.inner.sync_directory(path)
}
fn read_dir(&self, path: &std::path::Path) -> crate::io::Result<Vec<crate::fs::FsDirEntry>> {
self.inner.read_dir(path)
}
fn metadata(&self, path: &std::path::Path) -> crate::io::Result<crate::fs::FsMetadata> {
self.inner.metadata(path)
}
fn exists(&self, path: &std::path::Path) -> crate::io::Result<bool> {
self.inner.exists(path)
}
fn capabilities(&self, path: &std::path::Path) -> crate::fs::FsCapabilities {
self.inner.capabilities(path)
}
fn create_dir(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.inner.create_dir(path)
}
fn sync_directory_with(
&self,
path: &std::path::Path,
mode: crate::fs::SyncMode,
) -> crate::io::Result<()> {
self.inner.sync_directory_with(path, mode)
}
fn same_file(&self, a: &std::path::Path, b: &std::path::Path) -> crate::io::Result<bool> {
self.inner.same_file(a, b)
}
fn hard_link(&self, src: &std::path::Path, dst: &std::path::Path) -> crate::io::Result<()> {
self.inner.hard_link(src, dst)
}
fn backend_id(&self) -> Option<u64> {
self.inner.backend_id()
}
fn volume_id(&self, path: &std::path::Path) -> Option<u64> {
self.inner.volume_id(path)
}
fn punch_hole(&self, path: &std::path::Path, offset: u64, len: u64) -> crate::io::Result<()> {
self.inner.punch_hole(path, offset, len)
}
fn truncate_file(&self, path: &std::path::Path) -> crate::io::Result<()> {
self.inner.truncate_file(path)
}
fn hard_link_count(&self, path: &std::path::Path) -> crate::io::Result<u64> {
self.inner.hard_link_count(path)
}
fn available_space(&self, path: &std::path::Path) -> crate::io::Result<u64> {
self.inner.available_space(path)
}
fn allocated_size(&self, path: &std::path::Path) -> crate::io::Result<Option<u64>> {
self.inner.allocated_size(path)
}
fn extent_is_hole(
&self,
path: &std::path::Path,
offset: u64,
len: u64,
) -> crate::io::Result<Option<bool>> {
self.inner.extent_is_hole(path, offset, len)
}
fn extent_contains_hole(
&self,
path: &std::path::Path,
offset: u64,
len: u64,
) -> crate::io::Result<Option<bool>> {
self.inner.extent_contains_hole(path, offset, len)
}
}
#[test]
fn repair_cancel_during_the_final_file_still_aborts_before_commit() -> crate::Result<()> {
use crate::fs::MemFs;
use crate::{Config, RecoveryProgress, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
let progress = Arc::new(RecoveryProgress::default());
let hook_progress = Arc::clone(&progress);
let fs = HookFs {
inner: (*memfs).clone(),
on: HookOn::Open,
needle: "tables".into(),
fired: std::sync::atomic::AtomicBool::new(false),
hook: Box::new(move || hook_progress.request_cancel()),
};
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fs)
.with_recovery_progress(Arc::clone(&progress))
.repair();
assert!(
matches!(result, Err(crate::Error::Cancelled)),
"a cancel during the final file must abort before the commit: {:?}",
result.map(|r| r.recovered),
);
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.repair()?;
assert_eq!(report.recovered, 1);
Ok(())
}
#[test]
fn repair_cancel_removes_published_blob_replacements() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{Config, KvSeparationOptions, RecoveryProgress, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
blob_tree_without_manifest(&memfs, &root)?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
punch_and_corrupt_blob(&memfs, &blobs.join("0"))?;
let progress = Arc::new(RecoveryProgress::default());
let hook_progress = Arc::clone(&progress);
let replacement = blobs.join("1");
let fs = HookFs {
inner: (*memfs).clone(),
on: HookOn::RenameTo,
needle: replacement.to_string_lossy().into_owned(),
fired: std::sync::atomic::AtomicBool::new(false),
hook: Box::new(move || hook_progress.request_cancel()),
};
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.with_recovery_progress(Arc::clone(&progress))
.repair();
assert!(
matches!(result, Err(crate::Error::Cancelled)),
"the cancel lands before the commit and must abort: {:?}",
result.map(|r| r.recovered),
);
assert!(
!memfs.exists(&replacement)?,
"the published replacement must be removed by the abort — an orphan \
here makes the retry build a second copy beside it",
);
assert!(
memfs.exists(&blobs.join("0"))?,
"the damaged original stays for the retry to re-salvage",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn repair_error_removes_published_blob_replacements() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(
format!("k{i:04}").as_bytes(),
alloc::vec![b'a' + u8::try_from(i).expect("small i"); 64],
u64::from(i),
);
}
tree.flush_active_memtable(0)?;
for i in 0..8u32 {
tree.insert(
format!("m{i:04}").as_bytes(),
alloc::vec![b'a' + u8::try_from(i).expect("small i"); 64],
u64::from(i) + 8,
);
}
tree.flush_active_memtable(0)?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
punch_and_corrupt_blob(&memfs, &blobs.join("0"))?;
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::WouldBlock))
.on_path(blobs.join("1").to_string_lossy()),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
.repair();
assert!(
matches!(result, Err(crate::Error::Io(ref e)) if e.kind().is_transient()),
"the transient failure propagates for a retry: {:?}",
result.map(|r| r.recovered),
);
assert!(
!memfs.exists(&blobs.join("2"))?,
"the published replacement must be removed by the abort — an orphan \
here makes the retry build a second copy beside it",
);
assert!(
memfs.exists(&blobs.join("0"))? && memfs.exists(&blobs.join("1"))?,
"the originals stay untouched for the retry",
);
Ok(())
}
#[test]
fn repair_cancel_aborts_before_commit_and_a_retry_succeeds() -> crate::Result<()> {
use crate::fs::MemFs;
use crate::{Config, RecoveryProgress, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
let config = |memfs: &Arc<MemFs>| {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
};
let progress = Arc::new(RecoveryProgress::default());
progress.request_cancel();
let result = config(&memfs)
.with_recovery_progress(Arc::clone(&progress))
.repair();
assert!(
matches!(result, Err(crate::Error::Cancelled)),
"a requested cancel must surface as the dedicated error: {result:?}",
);
let report = config(&memfs).repair()?;
assert_eq!(
report.recovered, 1,
"the retry rebuilds from untouched bytes"
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn open_or_repair_repairs_a_structural_failure_only() -> crate::Result<()> {
use crate::fs::MemFs;
use crate::{AbstractTree, Config, RepairPolicy, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
};
let (tree, repaired) = config().open_or_repair(RepairPolicy::default().salvage(true))?;
let report = repaired.expect("a manifest-loss open only succeeds after a repair");
assert_eq!(report.recovered, 1);
assert_eq!(
report.wal_replay_scope(),
super::WalReplayScope::TailOnly,
"nothing was lost, so the WAL replays its tail as usual",
);
assert_eq!(
tree.get(b"k", u64::MAX)?.as_deref(),
Some(b"v".as_ref()),
"the repaired tree serves the flushed write",
);
drop(tree);
let (tree, repaired) = config().open_or_repair(RepairPolicy::default())?;
assert!(repaired.is_none(), "a healthy open must not repair");
assert_eq!(tree.get(b"k", u64::MAX)?.as_deref(), Some(b"v".as_ref()));
Ok(())
}
#[test]
fn open_or_repair_propagates_transient_io_without_repairing() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, RepairPolicy, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.repair()?;
let manifest_names: Vec<String> = memfs
.read_dir(&root)?
.into_iter()
.filter(|e| e.file_name.starts_with('v'))
.map(|e| e.file_name)
.collect();
let fault = FaultFs::new((*memfs).clone());
fault
.injector()
.arm(FaultRule::new(FaultOp::Read, Fault::Error(ErrorKind::WouldBlock)).on_path("current"));
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.open_or_repair(RepairPolicy::default().salvage(true));
assert!(
matches!(result, Err(crate::Error::Io(ref e)) if e.kind().is_transient()),
"a transient hiccup must propagate for a retry: {:?}",
result.map(|_| "opened"),
);
let after: Vec<String> = memfs
.read_dir(&root)?
.into_iter()
.filter(|e| e.file_name.starts_with('v'))
.map(|e| e.file_name)
.collect();
assert_eq!(
manifest_names, after,
"repairing a transient failure would have rewritten the manifest",
);
Ok(())
}
#[test]
fn open_or_repair_propagates_an_unsupported_format_version() -> crate::Result<()> {
use crate::fs::{Fs, FsOpenOptions, MemFs};
use crate::{Config, RepairPolicy, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.repair()?;
{
let mut file = memfs.open(
&root.join("version"),
&FsOpenOptions::new().write(true).create_new(true),
)?;
file.write_all(b"1")?;
}
let manifest_names: Vec<String> = memfs
.read_dir(&root)?
.into_iter()
.filter(|e| e.file_name.starts_with('v') && e.file_name != "version")
.map(|e| e.file_name)
.collect();
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.open_or_repair(RepairPolicy::default().salvage(true));
assert!(
matches!(result, Err(crate::Error::InvalidVersion(_))),
"an unsupported format needs a converter, never a destructive rebuild: {:?}",
result.map(|_| "opened"),
);
let after: Vec<String> = memfs
.read_dir(&root)?
.into_iter()
.filter(|e| e.file_name.starts_with('v') && e.file_name != "version")
.map(|e| e.file_name)
.collect();
assert_eq!(
manifest_names, after,
"repairing an unsupported format would have rewritten the manifest",
);
Ok(())
}
#[test]
fn repair_orders_l0_by_recency_key_not_table_id() -> crate::Result<()> {
use crate::table::Writer;
use crate::{AbstractTree, Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
{
let mut w = Writer::new(tables.join("2"), 2, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
b"k",
b"new",
10,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the flush table is non-empty");
}
{
let mut w = Writer::new(tables.join("5"), 5, 0, Arc::clone(&fs))?.use_recency(Some(1));
w.write(InternalValue::from_components(
b"k",
b"old",
10,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the output table is non-empty");
}
let config = || {
Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
};
config().repair()?;
let crate::AnyTree::Standard(tree) = config().open()? else {
panic!("expected a standard tree");
};
assert_eq!(
tree.get(b"k", u64::MAX)?.as_deref(),
Some(b"new".as_ref()),
"the flush's newer write must shadow the compaction output's superseded \
one, exactly as the live tree ordered them",
);
Ok(())
}
#[test]
fn repair_reports_ambiguous_order_of_legacy_overlapping_tables() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
for (id, payload) in [(0u64, b"old".as_slice()), (1u64, b"new".as_slice())] {
let mut w = Writer::new(tables.join(id.to_string()), id, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
b"k5".to_vec(),
payload.to_vec(),
7,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the table is non-empty");
}
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(report.recovered, 2, "both tables are still recovered");
assert!(
!report.lost_coverage.is_empty(),
"the ambiguous overlap must be reported: {report:?}",
);
assert!(
!matches!(
report.wal_replay_scope(),
crate::repair::WalReplayScope::TailOnly
),
"an unorderable tied history must widen the replay obligation: {report:?}",
);
Ok(())
}
#[test]
fn repair_rejects_ambiguous_legacy_overlap_under_a_merge_operator() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
struct SumMerge;
impl crate::MergeOperator for SumMerge {
fn merge(
&self,
_key: &[u8],
_base_value: Option<&[u8]>,
_operands: &[&[u8]],
) -> crate::Result<crate::UserValue> {
Ok(b"sum".to_vec().into())
}
}
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
for id in [0u64, 1] {
let mut w = Writer::new(tables.join(id.to_string()), id, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
b"k5".to_vec(),
b"v".to_vec(),
7,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the table is non-empty");
}
let result = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_merge_operator(Some(Arc::new(SumMerge)))
.repair();
assert!(
matches!(result, Err(crate::Error::Unrecoverable)),
"an unorderable legacy overlap under a merge operator must fail the \
repair, not publish a double-applying pair: {:?}",
result.map(|r| r.recovered),
);
Ok(())
}
#[test]
fn repair_stays_tail_only_for_disjoint_seqno_legacy_tables() -> crate::Result<()> {
use crate::table::Writer;
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let dir = tempfile::tempdir()?;
let tables = dir.path().join("tables");
std::fs::create_dir_all(&tables)?;
let fs: Arc<dyn crate::fs::Fs> = Arc::new(StdFs);
for (id, seqno) in [(0u64, 3u64), (1u64, 8u64)] {
let mut w = Writer::new(tables.join(id.to_string()), id, 0, Arc::clone(&fs))?;
w.write(InternalValue::from_components(
b"k5".to_vec(),
b"v".to_vec(),
seqno,
ValueType::Value,
))?;
assert!(w.finish()?.is_some(), "the table is non-empty");
}
let report = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.repair()?;
assert_eq!(report.recovered, 2, "both tables are recovered");
assert!(
matches!(
report.wal_replay_scope(),
crate::repair::WalReplayScope::TailOnly
),
"disjoint seqno ranges leave nothing tied — no report: {report:?}",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn flush_persists_an_own_id_recency_key() -> crate::Result<()> {
use crate::{AbstractTree, Config, SequenceNumberCounter};
let dir = tempfile::tempdir()?;
let crate::AnyTree::Standard(tree) = Config::new(
dir.path(),
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.open()?
else {
panic!("expected a standard tree");
};
tree.insert(b"k", b"v", 1);
tree.flush_active_memtable(0)?;
let version = tree.current_version();
let table = version.iter_tables().next().expect("one flushed table");
assert_eq!(
table.metadata.recency,
Some(table.id()),
"a flush stamps its own id as its recency",
);
Ok(())
}
#[cfg(feature = "lz4")]
#[test]
fn open_finishes_a_committed_repair_swap() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
blob_tree_without_manifest(&memfs, &root)?;
let blobs = root.join(crate::file::BLOBS_FOLDER);
punch_and_corrupt_blob(&memfs, &blobs.join("0"))?;
let config = |fs: Arc<dyn Fs>| {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(fs)
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
let swap_dest = root.join("tables").join("0");
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::Rename, Fault::Error(ErrorKind::PermissionDenied))
.on_path(swap_dest.to_string_lossy().into_owned()),
);
assert!(
config(Arc::new(fault)).repair().is_err(),
"a swap the filesystem refuses must fail the repair after its commit",
);
let tmp = super::repair_tmp_path(&root.join("tables").join("0"));
assert!(
memfs.exists(&tmp)?,
"the committed replacement is still at its temp name",
);
let tree = config(memfs.clone()).open()?;
assert!(
!memfs.exists(&tmp)?,
"the pending swap is finished by the open",
);
assert!(
tree.get(b"k0004", u64::MAX)?.is_some(),
"the salvaged content the manifest describes is served",
);
Ok(())
}
#[test]
fn a_restricted_routed_duplicate_is_not_read_as_corrupt() -> crate::Result<()> {
use crate::config::LevelRoute;
use crate::fs::{Fs, MemFs};
use crate::{Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db_restricted_dup")?;
let cold = std::path::absolute("/cold_restricted_dup")?;
let tables = root.join("tables");
let cold_tables = cold.join("tables");
memfs.create_dir_all(&tables)?;
memfs.create_dir_all(&cold_tables)?;
let bound = b"key000100".to_vec();
for dir in [&tables, &cold_tables] {
let path = dir.join("0");
{
let mut w = crate::table::Writer::new(path.clone(), 0, 0, Arc::clone(&fs_dyn))?
.use_data_block_size(128);
for i in 0..200u32 {
w.write(InternalValue::from_components(
format!("key{i:06}").into_bytes(),
vec![0xABu8; 64],
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the table is non-empty");
}
let probe_config = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
);
let table = crate::table::Table::recover(super::repair_recover_params(
&probe_config,
path.clone(),
crate::Checksum::from_raw(super::compute_table_checksum(&*fs_dyn, &path)?),
0,
Arc::clone(&fs_dyn),
None,
))?;
let offset = table.punch_offset_for(&bound)?;
drop(table);
memfs.punch_hole(&path, 0, offset)?;
crate::restrict_bound::write(
&*fs_dyn,
&path,
None,
0,
&bound,
crate::fs::SyncMode::Normal,
)?;
}
let route_fs: Arc<dyn Fs> = memfs.clone();
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.level_routes(vec![LevelRoute {
levels: 0..2,
path: cold,
fs: route_fs,
}])
.repair()?;
assert_eq!(report.recovered, 1, "one copy is retained: {report:?}");
assert_eq!(
report.unreadable, 0,
"the duplicate's punched prefix is its restriction, not damage: {report:?}",
);
assert!(
report
.excluded_files
.iter()
.any(|(_, reason)| reason.contains("duplicate")),
"it is a healthy exclusion: {report:?}",
);
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn an_intact_blob_duplicate_wins_over_a_salvage_of_the_canonical_copy() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db_blob_intact_dup")?;
let value = |i: u32| alloc::vec![b'a' + u8::try_from(i).expect("small i"); 64];
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
{
let tree = match config().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), value(i), u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let canonical = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let intact = blobs.join("00");
{
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs.open(&canonical, &crate::fs::FsOpenOptions::new().read(true))?,
&mut bytes,
)?;
let mut f = memfs.open(
&intact,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
f.write_all(&bytes)?;
}
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(&canonical, &*fs_dyn, 0)?
.collect::<crate::Result<Vec<_>>>()?;
assert_eq!(entries.len(), 8, "eight separated values");
let flip_at = entries.last().expect("last frame").frame_end - 8;
{
let mut file = memfs.open(
&canonical,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*file, flip_at, 1)?;
file.seek(SeekFrom::Start(flip_at))?;
file.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let report = config().repair()?;
assert_eq!(
report.blob_files_salvaged.len(),
0,
"an intact copy of the id exists, so nothing needs salvaging: {report:?}",
);
let tree = match config().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
assert_eq!(
tree.get(format!("k{i:04}").as_bytes(), crate::MAX_SEQNO)?
.as_deref(),
Some(value(i).as_slice()),
"key {i} must survive: the intact duplicate held every frame",
);
}
Ok(())
}
#[test]
#[expect(clippy::expect_used, reason = "test code")]
fn a_salvaged_blob_is_reported_even_when_its_original_cannot_be_removed() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::{Seek, SeekFrom, Write};
use std::sync::Arc;
let memfs = MemFs::new();
let fault = FaultFs::new(memfs.clone());
let injector = fault.injector();
let fs: Arc<dyn Fs> = Arc::new(fault);
let root = std::path::absolute("/db_salvage_report")?;
let value = |i: u32| alloc::vec![b'a' + u8::try_from(i).expect("small i"); 64];
let config = || {
Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(Arc::clone(&fs))
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
{
let tree = match config().open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
for i in 0..8u32 {
tree.insert(format!("k{i:04}").as_bytes(), value(i), u64::from(i));
}
tree.flush_active_memtable(0)?;
}
let blobs = root.join(crate::file::BLOBS_FOLDER);
let blob_path = memfs
.read_dir(&blobs)?
.into_iter()
.find(|e| !e.is_dir)
.expect("one blob file")
.path;
let entries: Vec<_> = crate::vlog::BlobFileScanner::new(&blob_path, &memfs, 0)?
.collect::<crate::Result<Vec<_>>>()?;
let flip_at = entries.last().expect("last frame").frame_end - 8;
{
let mut file = memfs.open(
&blob_path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let byte = crate::file::read_exact(&*file, flip_at, 1)?;
file.seek(SeekFrom::Start(flip_at))?;
file.write_all(&[byte.first().expect("one byte") ^ 0xFF])?;
}
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
injector.arm(
FaultRule::new(
FaultOp::RemoveFile,
Fault::Error(ErrorKind::PermissionDenied),
)
.on_path(blob_path.display().to_string()),
);
let outcome = config().repair();
injector.clear();
let Err(crate::Error::RepairedButUnopened { report, .. }) = outcome else {
panic!("a post-commit cleanup failure must carry the finished report, got {outcome:?}");
};
assert_eq!(
report.blob_files_salvaged.len(),
1,
"the durable replacement must be reported whatever the cleanup did: {report:?}",
);
assert!(
report
.blob_files_salvaged
.first()
.is_some_and(
|(_, note)| note.contains("records salvaged") && note.contains("NOT removed")
),
"the note says what was recovered AND that the original is still there: {report:?}",
);
Ok(())
}
#[test]
fn the_manifest_checksum_picks_the_blob_copy_not_directory_order() -> crate::Result<()> {
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, KvSeparationOptions, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let stale_root = std::path::absolute("/db_blob_pick_stale")?;
let live_root = std::path::absolute("/db_blob_pick_live")?;
let config = |root: &std::path::Path| {
Config::new(
root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(Some(
KvSeparationOptions::default().separation_threshold(16),
))
};
let stale_value = alloc::vec![b's'; 64];
let live_value = alloc::vec![b'L'; 64];
for (root, value) in [(&stale_root, &stale_value), (&live_root, &live_value)] {
let tree = match config(root).open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
tree.insert(b"k", value.clone(), 1);
tree.flush_active_memtable(0)?;
}
let blobs = live_root.join(crate::file::BLOBS_FOLDER);
let canonical = blobs.join("0");
memfs.rename(&canonical, &blobs.join("00"))?;
{
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs.open(
&stale_root.join(crate::file::BLOBS_FOLDER).join("0"),
&crate::fs::FsOpenOptions::new().read(true),
)?,
&mut bytes,
)?;
let mut f = memfs.open(
&canonical,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
f.write_all(&bytes)?;
}
config(&live_root).repair()?;
let tree = match config(&live_root).open()? {
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
assert_eq!(
tree.get(b"k", crate::MAX_SEQNO)?.as_deref(),
Some(live_value.as_slice()),
"the copy the manifest named must win over the canonical spelling",
);
Ok(())
}
#[test]
fn the_manifest_checksum_picks_the_table_copy_not_scan_order() -> crate::Result<()> {
use crate::config::LevelRoute;
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, SequenceNumberCounter};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let stale_root = std::path::absolute("/db_table_pick_stale")?;
let live_root = std::path::absolute("/db_table_pick_live")?;
let cold = std::path::absolute("/cold_table_pick")?;
for (root, value) in [(&stale_root, "stale"), (&live_root, "live")] {
let tree = match Config::new(
root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.open()?
{
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
tree.insert(b"k", value.as_bytes(), 1);
tree.flush_active_memtable(0)?;
}
let cold_tables = cold.join("tables");
memfs.create_dir_all(&cold_tables)?;
let primary = live_root.join("tables").join("0");
memfs.rename(&primary, &cold_tables.join("0"))?;
{
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs.open(
&stale_root.join("tables").join("0"),
&crate::fs::FsOpenOptions::new().read(true),
)?,
&mut bytes,
)?;
let mut f = memfs.open(
&primary,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
f.write_all(&bytes)?;
}
let config = || {
let route_fs: Arc<dyn Fs> = memfs.clone();
Config::new(
&live_root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.level_routes(vec![LevelRoute {
levels: 0..2,
path: cold.clone(),
fs: route_fs,
}])
};
let report = config().repair()?;
assert!(
report.lost_coverage.is_empty() && report.unknowable_losses.is_empty(),
"a superseded duplicate is not lost coverage: {report:?}",
);
let tree = match config().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
assert_eq!(
tree.get(b"k", crate::MAX_SEQNO)?.as_deref(),
Some(&b"live"[..]),
"the copy the manifest named must win over the one scanned first",
);
Ok(())
}
#[test]
fn the_manifest_digest_picks_a_restricted_table_copy_too() -> crate::Result<()> {
use crate::config::LevelRoute;
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, InternalValue, SequenceNumberCounter, ValueType};
use std::io::Write;
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let live_root = std::path::absolute("/db_restricted_pick")?;
let cold = std::path::absolute("/cold_restricted_pick")?;
let tables = live_root.join("tables");
let cold_tables = cold.join("tables");
memfs.create_dir_all(&tables)?;
memfs.create_dir_all(&cold_tables)?;
let bound: crate::UserKey = b"key000100".to_vec().into();
let write_copy = |path: &std::path::Path, marker: u8| -> crate::Result<u64> {
let mut w = crate::table::Writer::new(path.to_path_buf(), 0, 0, Arc::clone(&fs_dyn))?
.use_data_block_size(128);
for i in 0..200u32 {
w.write(InternalValue::from_components(
format!("key{i:06}").into_bytes(),
alloc::vec![marker; 64],
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the table is non-empty");
let probe = Config::new(
&live_root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
);
let table = crate::table::Table::recover(super::repair_recover_params(
&probe,
path.to_path_buf(),
crate::Checksum::from_raw(super::compute_table_checksum(&*fs_dyn, path)?),
0,
Arc::clone(&fs_dyn),
None,
))?;
let offset = table.punch_offset_for(&bound)?;
drop(table);
memfs.punch_hole(path, 0, offset)?;
crate::restrict_bound::write(&*fs_dyn, path, None, 0, &bound, crate::fs::SyncMode::Normal)?;
Ok(offset)
};
let live_offset = write_copy(&cold_tables.join("0"), b'L')?;
write_copy(&tables.join("0"), b'S')?;
let stale_bytes = {
let path = tables.join("0");
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut memfs.open(&path, &crate::fs::FsOpenOptions::new().read(true))?,
&mut bytes,
)?;
memfs.remove_file(&path)?;
memfs.remove_file(&crate::restrict_bound::sidecar_path(&path))?;
bytes
};
let config = || {
let route_fs: Arc<dyn Fs> = memfs.clone();
Config::new(
&live_root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.level_routes(vec![LevelRoute {
levels: 0..2,
path: cold.clone(),
fs: route_fs,
}])
};
let first = config().repair()?;
assert_eq!(first.recovered, 1, "the live copy is published: {first:?}");
{
let path = tables.join("0");
let mut f = memfs.open(
&path,
&crate::fs::FsOpenOptions::new().write(true).create_new(true),
)?;
f.write_all(&stale_bytes)?;
crate::restrict_bound::write(
&*fs_dyn,
&path,
None,
0,
&bound,
crate::fs::SyncMode::Normal,
)?;
}
let _ = live_offset;
config().repair()?;
let tree = match config().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
assert_eq!(
tree.get(b"key000150", crate::MAX_SEQNO)?.as_deref(),
Some(&[b'L'; 64][..]),
"the restricted copy the manifest digested must win over scan order",
);
Ok(())
}
#[test]
fn a_restricted_table_with_a_routed_duplicate_still_opens() -> crate::Result<()> {
use crate::config::LevelRoute;
use crate::fs::{Fs, MemFs};
use crate::{AbstractTree, Config, InternalValue, SequenceNumberCounter, ValueType};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let live_root = std::path::absolute("/db_restricted_dup_open")?;
let cold = std::path::absolute("/cold_restricted_dup_open")?;
let tables = live_root.join("tables");
let cold_tables = cold.join("tables");
memfs.create_dir_all(&tables)?;
memfs.create_dir_all(&cold_tables)?;
let bound: crate::UserKey = b"key000100".to_vec().into();
let authoritative = cold_tables.join("0");
{
let mut w = crate::table::Writer::new(authoritative.clone(), 0, 0, Arc::clone(&fs_dyn))?
.use_data_block_size(128);
for i in 0..200u32 {
w.write(InternalValue::from_components(
format!("key{i:06}").into_bytes(),
alloc::vec![b'L'; 64],
u64::from(i) + 1,
ValueType::Value,
))?;
}
assert!(w.finish()?.is_some(), "the table is non-empty");
let probe = Config::new(
&live_root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
);
let table = crate::table::Table::recover(super::repair_recover_params(
&probe,
authoritative.clone(),
crate::Checksum::from_raw(super::compute_table_checksum(&*fs_dyn, &authoritative)?),
0,
Arc::clone(&fs_dyn),
None,
))?;
let offset = table.punch_offset_for(&bound)?;
drop(table);
memfs.punch_hole(&authoritative, 0, offset)?;
crate::restrict_bound::write(
&*fs_dyn,
&authoritative,
None,
0,
&bound,
crate::fs::SyncMode::Normal,
)?;
}
let config = || {
let route_fs: Arc<dyn Fs> = memfs.clone();
Config::new(
&live_root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.level_routes(vec![LevelRoute {
levels: 0..2,
path: cold.clone(),
fs: route_fs,
}])
};
config().repair()?;
let displaced = tables.join("0");
memfs.hard_link(&authoritative, &displaced)?;
crate::restrict_bound::write(
&*fs_dyn,
&displaced,
None,
0,
&bound,
crate::fs::SyncMode::Normal,
)?;
let tree = match config().open()? {
crate::AnyTree::Standard(t) => t,
crate::AnyTree::Blob(_) => panic!("expected Standard tree"),
};
assert_eq!(
tree.get(b"key000150", crate::MAX_SEQNO)?.as_deref(),
Some(&[b'L'; 64][..]),
"the restricted table must still be served with a duplicate present",
);
Ok(())
}
#[test]
fn repair_syncs_the_blobs_directory_after_removing_an_unreferenced_file() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, Fs, MemFs};
use crate::io::ErrorKind;
use crate::{Config, KvSeparationOptions, SequenceNumberCounter};
use std::sync::Arc;
let memfs = Arc::new(MemFs::new());
let fs_dyn: Arc<dyn Fs> = memfs.clone();
let root = std::path::absolute("/db_blob_dir_sync")?;
let kv = || Some(KvSeparationOptions::default().separation_threshold(16));
{
let tree = match Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs.clone())
.with_kv_separation(kv())
.open()?
{
crate::AnyTree::Blob(t) => t,
crate::AnyTree::Standard(_) => panic!("expected blob tree"),
};
tree.insert(b"k0", alloc::vec![b'x'; 64], 0);
tree.flush_active_memtable(0)?;
}
let orphan = root.join(crate::file::BLOBS_FOLDER).join("1");
let mut w = crate::vlog::blob_file::writer::Writer::new(&orphan, 1, 0, &*fs_dyn)?;
w.write(b"z", 9, &[b'z'; 300])?;
w.finish()?;
for e in memfs.read_dir(&root)? {
let is_version = e
.file_name
.strip_prefix('v')
.is_some_and(|rest| rest.parse::<u64>().is_ok());
if is_version || e.file_name == "current" {
memfs.remove_file(&e.path)?;
}
}
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(
FaultOp::SyncDirectory,
Fault::Error(ErrorKind::PermissionDenied),
)
.on_path(crate::file::BLOBS_FOLDER),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.with_kv_separation(kv())
.repair();
assert!(
result.is_err(),
"a repair that cannot make the removal durable must not report success: {result:?}",
);
Ok(())
}
#[test]
fn repair_propagates_a_transport_failure() -> crate::Result<()> {
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule, MemFs};
use crate::io::ErrorKind;
use crate::{Config, SequenceNumberCounter};
use std::sync::Arc;
for kind in [
ErrorKind::ConnectionReset,
ErrorKind::ConnectionAborted,
ErrorKind::ConnectionRefused,
ErrorKind::NotConnected,
ErrorKind::BrokenPipe,
] {
let memfs = Arc::new(MemFs::new());
let root = std::path::absolute("/db")?;
standard_tree_without_manifest(&memfs, &root)?;
let fault = FaultFs::new((*memfs).clone());
fault.injector().arm(
FaultRule::new(FaultOp::Open, Fault::Error(kind))
.on_path(root.join("tables").join("0").to_string_lossy()),
);
let result = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_fs(fault)
.repair();
assert!(
matches!(result, Err(crate::Error::Io(ref e)) if e.kind() == kind),
"a {kind:?} transport failure must abort the repair, never grade the \
file: {:?}",
result.map(|r| (r.recovered, r.unreadable)),
);
let report = Config::new(
&root,
SequenceNumberCounter::default(),
SequenceNumberCounter::default(),
)
.with_shared_fs(memfs)
.repair()?;
assert_eq!(report.recovered, 1, "the intact table survives the outage");
}
Ok(())
}