use super::*;
use crate::coding::Encode;
use crate::fs::MemFs;
use crate::io::{LittleEndian, WriteBytesExt};
use crate::version::edit::{AddedBlobFile, ChangedLevel, TableDesc, VersionEdit};
use std::io::Write;
fn recovery_with(version_id: u64, table_ids: Vec<Vec<Vec<RecoveredTable>>>) -> Recovery {
Recovery {
tree_type: TreeType::Standard,
snapshot_id: version_id,
curr_version_id: version_id,
table_ids,
blob_file_ids: Vec::new(),
gc_stats: crate::blob_tree::FragmentationMap::default(),
restrictions: crate::HashMap::default(),
blob_restrictions: crate::HashMap::default(),
retention_floor: 0,
dicts: Vec::new(),
repair_report: None,
}
}
#[test]
fn apply_edit_overwrites_the_retention_floor_only_when_carried() {
let mut rec = recovery_with(1, vec![]);
rec.apply_edit(&VersionEdit {
new_version_id: 2,
retention_floor: Some(41),
..Default::default()
})
.expect("apply");
assert_eq!(rec.retention_floor, 41);
rec.apply_edit(&VersionEdit {
new_version_id: 3,
retention_floor: None,
..Default::default()
})
.expect("apply");
assert_eq!(rec.retention_floor, 41, "an edit without a floor keeps it");
rec.apply_edit(&VersionEdit {
new_version_id: 4,
retention_floor: Some(99),
..Default::default()
})
.expect("apply");
assert_eq!(rec.retention_floor, 99);
}
#[test]
fn parse_retention_floor_section_is_strict() {
let mut bytes = Vec::new();
bytes.write_u64::<LittleEndian>(77).expect("write");
assert_eq!(parse_retention_floor_section(&bytes).expect("parse"), 77);
assert!(parse_retention_floor_section(&bytes[..7]).is_err(), "short");
bytes.push(0);
assert!(
parse_retention_floor_section(&bytes).is_err(),
"trailing byte"
);
}
#[test]
fn parse_dicts_section_reads_every_id() {
let mut bytes = Vec::new();
bytes.write_u32::<LittleEndian>(2).expect("write");
bytes.write_u32::<LittleEndian>(7).expect("write");
bytes.write_u32::<LittleEndian>(9).expect("write");
assert_eq!(parse_dicts_section(&bytes).expect("parse"), vec![7, 9]);
}
#[test]
fn parse_dicts_section_rejects_a_count_the_section_cannot_hold() {
let mut bytes = Vec::new();
bytes.write_u32::<LittleEndian>(u32::MAX).expect("write");
bytes.write_u32::<LittleEndian>(7).expect("write");
assert!(matches!(
parse_dicts_section(&bytes),
Err(crate::Error::InvalidHeader("dicts section")),
));
}
#[test]
fn parse_dicts_section_rejects_a_count_below_the_ids_present() {
let mut bytes = Vec::new();
bytes.write_u32::<LittleEndian>(1).expect("write");
bytes.write_u32::<LittleEndian>(7).expect("write");
bytes.write_u32::<LittleEndian>(9).expect("write");
assert!(parse_dicts_section(&bytes).is_err());
}
fn rtable(id: u64, seqno: u64) -> RecoveredTable {
RecoveredTable {
id,
checksum: Checksum::from_raw(u128::from(id) * 31),
global_seqno: seqno,
recency: id + 1_000,
}
}
fn tdesc(id: u64, seqno: u64) -> TableDesc {
TableDesc {
id,
checksum: u128::from(id) * 31,
global_seqno: seqno,
recency: id + 1_000,
}
}
#[test]
fn apply_replaces_a_changed_levels_run_layout_wholesale() {
let mut rec = recovery_with(1, vec![vec![vec![rtable(1, 10)]]]);
let edit = VersionEdit {
new_version_id: 2,
changed_levels: vec![ChangedLevel {
level: 0,
runs: vec![vec![tdesc(1, 10)], vec![tdesc(2, 11)]],
}],
..Default::default()
};
rec.apply_edit(&edit).expect("apply");
assert_eq!(rec.curr_version_id, 2);
assert_eq!(
rec.table_ids,
vec![vec![vec![rtable(1, 10)], vec![rtable(2, 11)]]],
"changed level's run grouping must be reconstructed exactly",
);
}
#[test]
fn apply_leaves_unmentioned_levels_untouched() {
let mut rec = recovery_with(1, vec![vec![vec![rtable(1, 10)]], vec![vec![rtable(9, 5)]]]);
let edit = VersionEdit {
new_version_id: 2,
changed_levels: vec![ChangedLevel {
level: 0,
runs: vec![vec![tdesc(3, 12)]],
}],
..Default::default()
};
rec.apply_edit(&edit).expect("apply");
assert_eq!(rec.table_ids[0], vec![vec![rtable(3, 12)]]);
assert_eq!(
rec.table_ids[1],
vec![vec![rtable(9, 5)]],
"a level the edit does not mention is left as-is",
);
}
#[test]
fn apply_empties_a_drained_level() {
let mut rec = recovery_with(1, vec![vec![vec![rtable(1, 10)]]]);
let edit = VersionEdit {
new_version_id: 2,
changed_levels: vec![ChangedLevel {
level: 0,
runs: vec![],
}],
..Default::default()
};
rec.apply_edit(&edit).expect("apply");
assert!(
rec.table_ids[0].is_empty(),
"a compaction that drains a level leaves zero runs",
);
}
#[test]
fn apply_grows_levels_for_a_higher_index() {
let mut rec = recovery_with(1, vec![vec![vec![rtable(1, 10)]]]);
let edit = VersionEdit {
new_version_id: 2,
changed_levels: vec![ChangedLevel {
level: 2,
runs: vec![vec![tdesc(5, 20)]],
}],
..Default::default()
};
rec.apply_edit(&edit).expect("apply");
assert_eq!(rec.table_ids.len(), 3, "levels grew to fit index 2");
assert!(rec.table_ids[1].is_empty(), "the gap level is empty");
assert_eq!(rec.table_ids[2], vec![vec![rtable(5, 20)]]);
}
#[test]
fn apply_edit_advances_restrictions_per_slice() {
let mut rec = recovery_with(1, vec![vec![vec![rtable(1, 10)]]]);
assert!(rec.restrictions.is_empty(), "starts unrestricted");
rec.apply_edit(&VersionEdit {
new_version_id: 2,
restrictions: vec![(1, crate::UserKey::from(&b"ccc"[..]))],
..Default::default()
})
.expect("apply");
assert_eq!(
rec.restrictions.get(&1),
Some(&crate::UserKey::from(&b"ccc"[..])),
);
rec.apply_edit(&VersionEdit {
new_version_id: 3,
restrictions: vec![(1, crate::UserKey::from(&b"mmm"[..]))],
..Default::default()
})
.expect("apply");
assert_eq!(
rec.restrictions.get(&1),
Some(&crate::UserKey::from(&b"mmm"[..])),
"a later slice's higher bound overwrites the earlier one",
);
}
#[test]
fn apply_edit_advances_blob_restrictions_per_slice() {
let mut rec = recovery_with(1, vec![vec![vec![rtable(1, 10)]]]);
assert!(rec.blob_restrictions.is_empty(), "starts unreclaimed");
rec.apply_edit(&VersionEdit {
new_version_id: 2,
blob_restrictions: vec![(9, 4_096)],
..Default::default()
})
.expect("apply");
assert_eq!(rec.blob_restrictions.get(&9), Some(&4_096));
rec.apply_edit(&VersionEdit {
new_version_id: 3,
blob_restrictions: vec![(9, 65_536)],
..Default::default()
})
.expect("apply");
assert_eq!(
rec.blob_restrictions.get(&9),
Some(&65_536),
"a later slice's higher frontier overwrites the earlier one",
);
}
#[test]
fn apply_edit_clears_a_removed_blob_files_frontier() {
let mut rec = recovery_with(1, vec![vec![vec![rtable(1, 10)]]]);
rec.apply_edit(&VersionEdit {
new_version_id: 2,
added_blob_files: vec![AddedBlobFile { id: 9, checksum: 1 }],
blob_restrictions: vec![(9, 4_096)],
..Default::default()
})
.expect("apply");
assert_eq!(rec.blob_restrictions.get(&9), Some(&4_096));
rec.apply_edit(&VersionEdit {
new_version_id: 3,
removed_blob_file_ids: vec![9],
..Default::default()
})
.expect("apply");
assert!(
rec.blob_restrictions.is_empty(),
"a removed blob file's frontier must not outlive it: {:?}",
rec.blob_restrictions,
);
rec.apply_edit(&VersionEdit {
new_version_id: 4,
added_blob_files: vec![AddedBlobFile { id: 9, checksum: 2 }],
..Default::default()
})
.expect("apply");
assert_eq!(
rec.blob_restrictions.get(&9),
None,
"a reused id's whole replacement file must not inherit a stale frontier",
);
}
#[test]
fn apply_edit_drops_restrictions_absent_from_a_later_edit() {
let mut rec = recovery_with(1, vec![vec![vec![rtable(1, 10)]]]);
rec.apply_edit(&VersionEdit {
new_version_id: 2,
restrictions: vec![(1, crate::UserKey::from(&b"ccc"[..]))],
..Default::default()
})
.expect("apply");
assert!(rec.restrictions.contains_key(&1));
rec.apply_edit(&VersionEdit {
new_version_id: 3,
..Default::default()
})
.expect("apply");
assert!(
rec.restrictions.is_empty(),
"a lifted restriction must not survive replay: {:?}",
rec.restrictions,
);
}
#[test]
fn parse_restrictions_section_roundtrips_entries() {
let mut bytes = Vec::new();
bytes
.write_u32::<LittleEndian>(2)
.expect("encode test bytes");
bytes
.write_u64::<LittleEndian>(7)
.expect("encode test bytes");
bytes
.write_u32::<LittleEndian>(3)
.expect("encode test bytes");
bytes.write_all(b"mmm").expect("encode test bytes");
bytes
.write_u64::<LittleEndian>(42)
.expect("encode test bytes");
bytes
.write_u32::<LittleEndian>(4)
.expect("encode test bytes");
bytes.write_all(b"zzzz").expect("encode test bytes");
let map = parse_restrictions_section(&bytes).expect("parse");
assert_eq!(map.len(), 2);
assert_eq!(map.get(&7), Some(&crate::UserKey::from(&b"mmm"[..])));
assert_eq!(map.get(&42), Some(&crate::UserKey::from(&b"zzzz"[..])));
}
#[test]
fn parse_restrictions_section_rejects_a_truncated_key() {
let mut bytes = Vec::new();
bytes
.write_u32::<LittleEndian>(1)
.expect("encode test bytes");
bytes
.write_u64::<LittleEndian>(1)
.expect("encode test bytes");
bytes
.write_u32::<LittleEndian>(8)
.expect("encode test bytes");
bytes.write_all(b"xy").expect("encode test bytes");
assert!(
parse_restrictions_section(&bytes).is_err(),
"a key shorter than its length prefix must not silently un-clamp",
);
}
#[test]
fn parse_restrictions_section_rejects_a_duplicate_table_id() {
let mut bytes = Vec::new();
bytes
.write_u32::<LittleEndian>(2)
.expect("encode test bytes");
bytes
.write_u64::<LittleEndian>(5)
.expect("encode test bytes");
bytes
.write_u32::<LittleEndian>(3)
.expect("encode test bytes");
bytes.write_all(b"mmm").expect("encode test bytes");
bytes
.write_u64::<LittleEndian>(5)
.expect("encode test bytes");
bytes
.write_u32::<LittleEndian>(3)
.expect("encode test bytes");
bytes.write_all(b"ccc").expect("encode test bytes");
assert!(
parse_restrictions_section(&bytes).is_err(),
"a duplicate table id must not silently un-clamp an advanced bound",
);
}
#[test]
fn apply_adds_updates_and_removes_blob_files() {
let mut rec = recovery_with(1, vec![]);
rec.blob_file_ids = vec![(100, Checksum::from_raw(1)), (200, Checksum::from_raw(2))];
let edit = VersionEdit {
new_version_id: 2,
added_blob_files: vec![
AddedBlobFile {
id: 300,
checksum: 9,
},
AddedBlobFile {
id: 100,
checksum: 7,
},
],
removed_blob_file_ids: vec![200],
..Default::default()
};
rec.apply_edit(&edit).expect("apply");
assert!(
!rec.blob_file_ids.iter().any(|(id, _)| *id == 200),
"removed blob is gone",
);
assert_eq!(
rec.blob_file_ids
.iter()
.find(|(id, _)| *id == 100)
.map(|(_, c)| *c),
Some(Checksum::from_raw(7)),
"existing blob's checksum updated in place",
);
assert!(
rec.blob_file_ids
.iter()
.any(|(id, c)| *id == 300 && *c == Checksum::from_raw(9)),
"new blob appended",
);
}
#[test]
fn apply_overwrites_gc_stats_when_present() {
let mut rec = recovery_with(1, vec![]);
let mut gc = crate::blob_tree::FragmentationMap::default();
gc.insert(42, crate::blob_tree::FragmentationEntry::new(2, 50, 60));
let mut bytes = Vec::new();
gc.encode_into(&mut bytes).expect("encode gc");
let edit = VersionEdit {
new_version_id: 2,
gc_stats: Some(bytes),
..Default::default()
};
rec.apply_edit(&edit).expect("apply");
assert_eq!(rec.gc_stats, gc, "GC stats overwritten from the edit");
}
fn install_fixture(folder: &Path, version_id: u64, fs: &dyn Fs) -> crate::Result<()> {
crate::version::edit_log::create_log(fs, folder, version_id, crate::fs::SyncMode::Full)
}
type FixtureWriter = crate::manifest_blocks::writer::ManifestArchiveWriter;
fn open_fixture_writer(folder: &Path, id: u64, fs: &dyn Fs) -> crate::Result<FixtureWriter> {
let path = folder.join(format!("v{id}"));
let mut writer = FixtureWriter::create(
&path,
fs,
alloc::sync::Arc::new(crate::runtime_config::RuntimeConfig::default()),
None,
crate::fs::SyncMode::Normal,
)?;
writer.start("version_id")?;
writer.write_u64::<LittleEndian>(id)?;
Ok(writer)
}
fn write_header(w: &mut FixtureWriter) -> crate::Result<()> {
write_header_at(w, crate::FormatVersion::V6.into())
}
fn write_header_at(w: &mut FixtureWriter, format_version: u8) -> crate::Result<()> {
w.start("format_version")?;
w.write_u8(format_version)?;
w.start("tree_type")?;
w.write_u8(0)?;
Ok(())
}
fn start_tables(w: &mut FixtureWriter, levels: u8) -> crate::Result<()> {
w.start("tables")?;
w.write_u8(levels)?;
Ok(())
}
fn write_empty_tables(w: &mut FixtureWriter) -> crate::Result<()> {
start_tables(w, 1)
}
fn table_payload(level: u8, run: u32, id: u64, checksum_type: u8) -> crate::Result<Vec<u8>> {
let mut payload = Vec::new();
payload.write_u8(level)?;
payload.write_u32::<LittleEndian>(run)?;
payload.write_u64::<LittleEndian>(id)?;
payload.write_u8(checksum_type)?;
payload.write_u128::<LittleEndian>(0)?;
payload.write_u64::<LittleEndian>(0)?;
payload.write_u64::<LittleEndian>(id)?;
Ok(payload)
}
fn write_torn_table_record(w: &mut FixtureWriter, keep: usize) -> crate::Result<()> {
let mut framed = Vec::new();
write_good_table_record(&mut framed, 0, 0, 999)?;
let Some(head) = framed.get(..keep) else {
panic!("a torn record keeps fewer bytes than a whole one");
};
w.write_all(head)?;
Ok(())
}
fn write_empty_blob_files(w: &mut FixtureWriter) -> crate::Result<()> {
w.start("blob_files")?;
w.write_u32::<LittleEndian>(0)?;
Ok(())
}
fn write_empty_blob_gc_stats(w: &mut FixtureWriter) -> crate::Result<()> {
w.start("blob_gc_stats")?;
w.write_u32::<LittleEndian>(0)?;
Ok(())
}
fn write_tables(
folder: &Path,
id: u64,
levels: u8,
records: &[(u8, u32, u64)],
fs: &dyn Fs,
) -> crate::Result<()> {
let mut w = open_fixture_writer(folder, id, fs)?;
write_header(&mut w)?;
start_tables(&mut w, levels)?;
for &(level, run, table) in records {
write_good_table_record(&mut w, level, run, table)?;
}
write_empty_blob_files(&mut w)?;
write_empty_blob_gc_stats(&mut w)?;
w.finish()?;
Ok(())
}
fn recover_tables(levels: u8, records: &[(u8, u32, u64)]) -> crate::Result<Recovery> {
let fs = MemFs::new();
let folder = Path::new("/tables");
fs.create_dir_all(folder)?;
write_tables(folder, 1, levels, records, &fs)?;
install_fixture(folder, 1, &fs)?;
recover(folder, &fs, ManifestRecoveryMode::AbsoluteConsistency, None)
}
#[test]
fn recover_groups_table_records_by_the_place_they_name() -> crate::Result<()> {
let recovery = recover_tables(3, &[(0, 0, 1), (0, 0, 2), (0, 1, 3), (2, 0, 4)])?;
assert_eq!(
recovery.table_ids,
vec![
vec![vec![rtable_zero(1), rtable_zero(2)], vec![rtable_zero(3)]],
vec![],
vec![vec![rtable_zero(4)]],
],
);
Ok(())
}
fn rtable_zero(id: u64) -> RecoveredTable {
RecoveredTable {
id,
checksum: Checksum::from_raw(0),
global_seqno: 0,
recency: id,
}
}
#[test]
fn recover_rejects_a_table_record_out_of_place() {
let out_of_place = |records: &[(u8, u32, u64)]| {
matches!(
recover_tables(2, records),
Err(crate::Error::InvalidHeader("tables record out of place"))
)
};
assert!(out_of_place(&[(0, 1, 1)]), "a first run numbered 1");
assert!(
out_of_place(&[(0, 0, 1), (0, 2, 2)]),
"a skipped run ordinal"
);
assert!(
out_of_place(&[(1, 0, 1), (0, 0, 2)]),
"a level going backwards"
);
assert!(
out_of_place(&[(0, 1, 1), (0, 0, 2)]),
"a run going backwards"
);
assert!(out_of_place(&[(1, 1, 1)]), "a later level opening at run 1");
assert!(out_of_place(&[(2, 0, 1)]), "a level beyond the level count");
}
#[test]
fn place_table_accepts_run_ordinals_past_u16() -> crate::Result<()> {
let mut levels = vec![Vec::new()];
let mut previous = Some(TablePlace {
level: 0,
run: 69_999,
});
levels[0] = vec![vec![rtable_zero(1)]; 70_000];
place_table(
&mut levels,
&mut previous,
TablePlace {
level: 0,
run: 70_000,
},
rtable_zero(2),
)?;
assert_eq!(levels[0].len(), 70_001);
assert_eq!(levels[0][70_000], vec![rtable_zero(2)]);
Ok(())
}
fn write_corrupt_blob_count(folder: &Path, id: u64, fs: &dyn Fs) -> crate::Result<()> {
let mut w = open_fixture_writer(folder, id, fs)?;
write_header(&mut w)?;
write_empty_tables(&mut w)?;
w.start("blob_files")?;
w.write_u32::<LittleEndian>(u32::MAX)?;
w.start("blob_gc_stats")?;
w.write_u32::<LittleEndian>(0)?;
w.finish()?;
Ok(())
}
#[test]
fn recover_rejects_corrupt_blob_file_count() -> crate::Result<()> {
let fs = MemFs::new();
let folder = Path::new("/corrupt/blobs");
fs.create_dir_all(folder)?;
write_corrupt_blob_count(folder, 1, &fs)?;
install_fixture(folder, 1, &fs)?;
let Err(err) = recover(folder, &fs, ManifestRecoveryMode::AbsoluteConsistency, None) else {
panic!("corrupt blob_file_count should fail");
};
assert!(
matches!(err, crate::Error::InvalidHeader("blob_files section")),
"expected the blob_files section refusal, got: {err:?}"
);
Ok(())
}
fn assert_refused_in_every_mode(
folder: &Path,
fs: &dyn Fs,
refused: impl Fn(&crate::Error) -> bool,
) {
for mode in [
ManifestRecoveryMode::AbsoluteConsistency,
ManifestRecoveryMode::TolerateCorruptedTailRecords,
] {
match recover(folder, fs, mode, None) {
Err(e) if refused(&e) => {}
Err(e) => panic!("{mode:?}: wrong refusal: {e:?}"),
Ok(_) => panic!("{mode:?}: a defective snapshot section must not recover"),
}
}
}
fn write_truncated_tables_tail(
folder: &Path,
id: u64,
complete: u64,
fs: &dyn Fs,
) -> crate::Result<()> {
let mut w = open_fixture_writer(folder, id, fs)?;
write_header(&mut w)?;
start_tables(&mut w, 1)?;
for entry_id in 0..complete {
write_good_table_record(&mut w, 0, 0, entry_id)?;
}
write_torn_table_record(&mut w, 20)?;
write_empty_blob_files(&mut w)?;
write_empty_blob_gc_stats(&mut w)?;
w.finish()?;
Ok(())
}
#[test]
fn recover_rejects_a_truncated_tables_section_in_every_mode() -> crate::Result<()> {
let fs = MemFs::new();
let folder = Path::new("/truncated/tables");
fs.create_dir_all(folder)?;
write_truncated_tables_tail(folder, 1, 1, &fs)?;
install_fixture(folder, 1, &fs)?;
assert_refused_in_every_mode(
folder,
&fs,
|e| matches!(e, crate::Error::Io(io) if io.kind() == crate::io::ErrorKind::UnexpectedEof),
);
Ok(())
}
#[test]
fn recover_tolerate_tail_does_not_swallow_invalid_tag() -> crate::Result<()> {
let fs = MemFs::new();
let folder = Path::new("/tolerate/bad_tag");
fs.create_dir_all(folder)?;
let mut w = open_fixture_writer(folder, 1, &fs)?;
write_header(&mut w)?;
start_tables(&mut w, 1)?;
let corrupt = table_payload(0, 0, 0, 0xFF)?;
crate::version::framing::write_framed_record(&mut w, &mut Vec::new(), |payload| {
payload.extend_from_slice(&corrupt);
Ok(())
})?;
write_empty_blob_files(&mut w)?;
w.start("blob_gc_stats")?;
w.write_u32::<LittleEndian>(0)?;
w.finish()?;
install_fixture(folder, 1, &fs)?;
let result = recover(
folder,
&fs,
ManifestRecoveryMode::TolerateCorruptedTailRecords,
None,
);
let err = result.expect_err("InvalidTag must still abort under TolerateCorruptedTailRecords");
assert!(
matches!(err, crate::Error::InvalidTag(("ChecksumType", 0xFF))),
"expected InvalidTag, got: {err:?}",
);
Ok(())
}
fn write_truncated_at_second_run(folder: &Path, id: u64, fs: &dyn Fs) -> crate::Result<()> {
let mut w = open_fixture_writer(folder, id, fs)?;
write_header(&mut w)?;
start_tables(&mut w, 1)?;
write_good_table_record(&mut w, 0, 0, 42)?;
write_torn_table_record(&mut w, 2)?;
write_empty_blob_files(&mut w)?;
w.start("blob_gc_stats")?;
w.write_u32::<LittleEndian>(0)?;
w.finish()?;
Ok(())
}
#[test]
fn recover_rejects_a_tables_section_cut_inside_a_frame_header() -> crate::Result<()> {
let fs = MemFs::new();
let folder = Path::new("/truncated/header");
fs.create_dir_all(folder)?;
write_truncated_at_second_run(folder, 1, &fs)?;
install_fixture(folder, 1, &fs)?;
assert_refused_in_every_mode(
folder,
&fs,
|e| matches!(e, crate::Error::Io(io) if io.kind() == crate::io::ErrorKind::UnexpectedEof),
);
Ok(())
}
fn write_truncated_blob_tail(
folder: &Path,
id: u64,
declared: u32,
actual: u32,
fs: &dyn Fs,
) -> crate::Result<()> {
assert!(
actual < declared,
"actual must be < declared for truncation"
);
let mut w = open_fixture_writer(folder, id, fs)?;
write_header(&mut w)?;
write_empty_tables(&mut w)?;
w.start("blob_files")?;
w.write_u32::<LittleEndian>(declared)?;
for entry_id in 0..actual {
crate::version::framing::write_framed_record(&mut w, &mut Vec::new(), |payload| {
payload.write_u64::<LittleEndian>(u64::from(entry_id))?;
payload.write_u8(0)?; payload.write_u128::<LittleEndian>(0)?; Ok(())
})?;
}
w.start("blob_gc_stats")?;
w.write_u32::<LittleEndian>(0)?;
w.finish()?;
Ok(())
}
#[test]
fn recover_rejects_a_truncated_blob_files_section_in_every_mode() -> crate::Result<()> {
let fs = MemFs::new();
let folder = Path::new("/truncated/blobs");
fs.create_dir_all(folder)?;
write_truncated_blob_tail(folder, 1, 5, 1, &fs)?;
install_fixture(folder, 1, &fs)?;
assert_refused_in_every_mode(folder, &fs, |e| {
matches!(e, crate::Error::InvalidHeader("blob_files section"))
});
Ok(())
}
fn write_truncated_blob_gc_stats(folder: &Path, id: u64, fs: &dyn Fs) -> crate::Result<()> {
let mut w = open_fixture_writer(folder, id, fs)?;
write_header(&mut w)?;
write_empty_tables(&mut w)?;
w.start("blob_files")?;
w.write_u32::<LittleEndian>(0)?;
w.start("blob_gc_stats")?;
w.finish()?;
Ok(())
}
#[test]
fn recover_rejects_a_truncated_blob_gc_stats_section_in_every_mode() -> crate::Result<()> {
let fs = MemFs::new();
let folder = Path::new("/truncated/gc_stats");
fs.create_dir_all(folder)?;
write_truncated_blob_gc_stats(folder, 1, &fs)?;
install_fixture(folder, 1, &fs)?;
assert_refused_in_every_mode(
folder,
&fs,
|e| matches!(e, crate::Error::Io(io) if io.kind() == crate::io::ErrorKind::UnexpectedEof),
);
Ok(())
}
fn write_good_table_record<W: std::io::Write>(
w: &mut W,
level: u8,
run: u32,
id: u64,
) -> crate::Result<()> {
let payload = table_payload(level, run, id, 0)?;
crate::version::framing::write_framed_record(w, &mut Vec::new(), |out| {
out.extend_from_slice(&payload);
Ok(())
})
}
fn write_bad_table_record<W: std::io::Write>(
w: &mut W,
level: u8,
run: u32,
id: u64,
) -> crate::Result<()> {
let payload = table_payload(level, run, id, 0)?;
let len = u32::try_from(payload.len()).expect("a table record fits a u32 length");
w.write_u32::<LittleEndian>(len)?;
w.write_u64::<LittleEndian>(0xDEAD_BEEF_DEAD_BEEF)?;
w.write_all(&payload)?;
Ok(())
}
fn write_manifest_with_mid_record_corruption(
folder: &Path,
id: u64,
fs: &dyn Fs,
) -> crate::Result<()> {
let mut w = open_fixture_writer(folder, id, fs)?;
write_header(&mut w)?;
start_tables(&mut w, 2)?;
write_good_table_record(&mut w, 0, 0, 100)?;
write_bad_table_record(&mut w, 0, 0, 101)?;
write_good_table_record(&mut w, 0, 0, 102)?;
write_good_table_record(&mut w, 1, 0, 200)?;
write_good_table_record(&mut w, 1, 0, 201)?;
write_empty_blob_files(&mut w)?;
w.start("blob_gc_stats")?;
w.write_u32::<LittleEndian>(0)?;
w.finish()?;
Ok(())
}
#[test]
fn recover_rejects_a_corrupt_table_record_in_every_mode() -> crate::Result<()> {
let fs = MemFs::new();
let folder = Path::new("/corrupt/table_record");
fs.create_dir_all(folder)?;
write_manifest_with_mid_record_corruption(folder, 1, &fs)?;
install_fixture(folder, 1, &fs)?;
assert_refused_in_every_mode(folder, &fs, |e| {
matches!(
e,
crate::Error::ManifestFrameChecksumMismatch {
section: "tables",
..
}
)
});
Ok(())
}
fn write_manifest_with_corrupt_blob_record(
folder: &Path,
id: u64,
fs: &dyn Fs,
) -> crate::Result<()> {
let mut w = open_fixture_writer(folder, id, fs)?;
write_header(&mut w)?;
write_empty_tables(&mut w)?;
w.start("blob_files")?;
w.write_u32::<LittleEndian>(3)?;
crate::version::framing::write_framed_record(&mut w, &mut Vec::new(), |payload| {
payload.write_u64::<LittleEndian>(10)?;
payload.write_u8(0)?;
payload.write_u128::<LittleEndian>(0)?;
Ok(())
})?;
let mut payload: Vec<u8> = Vec::new();
payload.write_u64::<LittleEndian>(11)?;
payload.write_u8(0)?;
payload.write_u128::<LittleEndian>(0)?;
#[expect(
clippy::cast_possible_truncation,
reason = "payload is 25 bytes — fits in u32"
)]
let len = payload.len() as u32;
w.write_u32::<LittleEndian>(len)?;
w.write_u64::<LittleEndian>(0xDEAD_BEEF_DEAD_BEEF)?;
w.write_all(&payload)?;
crate::version::framing::write_framed_record(&mut w, &mut Vec::new(), |payload| {
payload.write_u64::<LittleEndian>(12)?;
payload.write_u8(0)?;
payload.write_u128::<LittleEndian>(0)?;
Ok(())
})?;
w.start("blob_gc_stats")?;
w.write_u32::<LittleEndian>(0)?;
w.finish()?;
Ok(())
}
#[test]
fn recover_rejects_a_corrupt_blob_file_record_in_every_mode() -> crate::Result<()> {
let fs = MemFs::new();
let folder = Path::new("/corrupt/blob_record");
fs.create_dir_all(folder)?;
write_manifest_with_corrupt_blob_record(folder, 1, &fs)?;
install_fixture(folder, 1, &fs)?;
assert_refused_in_every_mode(folder, &fs, |e| {
matches!(
e,
crate::Error::ManifestFrameChecksumMismatch {
section: "blob_files",
..
}
)
});
Ok(())
}
fn write_manifest_labelled(folder: &Path, fs: &dyn Fs, format_version: u8) -> crate::Result<()> {
let mut w = open_fixture_writer(folder, 1, fs)?;
write_header_at(&mut w, format_version)?;
write_empty_tables(&mut w)?;
write_empty_blob_files(&mut w)?;
write_empty_blob_gc_stats(&mut w)?;
w.finish()?;
install_fixture(folder, 1, fs)
}
#[test]
fn recover_refuses_a_manifest_of_another_format_version() -> crate::Result<()> {
let fs = MemFs::new();
let folder = Path::new("/format/v5");
fs.create_dir_all(folder)?;
write_manifest_labelled(folder, &fs, 5)?;
let err = recover(folder, &fs, ManifestRecoveryMode::AbsoluteConsistency, None)
.expect_err("a V5 manifest must be refused");
assert!(
matches!(err, crate::Error::InvalidVersion(5)),
"expected InvalidVersion(5), got {err:?}",
);
Ok(())
}
#[test]
fn a_store_with_a_version_pointer_is_refused_as_another_format() -> crate::Result<()> {
use std::io::Write;
let dir = tempfile::tempdir()?;
let fs = crate::fs::StdFs;
let folder = dir.path();
fs.create_dir_all(&folder.join(crate::file::TABLES_FOLDER))?;
write_manifest_labelled(folder, &fs, 5)?;
fs.open(
&folder.join("current"),
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?
.write_all(b"1")?;
fs.open(
&folder.join("v2"),
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?
.write_all(b"torn")?;
let config = || {
crate::Config::new(
folder,
crate::SequenceNumberCounter::default(),
crate::SequenceNumberCounter::default(),
)
};
match config().open() {
Err(crate::Error::InvalidVersion(5)) => {}
Err(e) => panic!("open: expected InvalidVersion(5), got {e:?}"),
Ok(_) => panic!("open: a store with a version pointer opened"),
}
match config().repair() {
Err(crate::Error::InvalidVersion(5)) => {}
Err(e) => panic!("repair: expected InvalidVersion(5), got {e:?}"),
Ok(_) => panic!("repair: a store with a version pointer was rebuilt"),
}
assert_eq!(
std::fs::read(folder.join("v2"))?,
b"torn",
"the store is left as it was"
);
Ok(())
}
#[test]
fn a_snapshot_under_another_number_is_refused() -> crate::Result<()> {
use std::io::{Read, Write};
let fs = std::sync::Arc::new(MemFs::new());
let config = crate::Config::new(
"/renumbered",
crate::SequenceNumberCounter::default(),
crate::SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone());
let folder = config.path.clone();
let folder = folder.as_path();
drop(config.open()?);
let id = installed_snapshot(folder, &*fs)?.ok_or(crate::Error::Unrecoverable)?;
let mut bytes = Vec::new();
fs.open(
&folder.join(format!("v{id}")),
&crate::fs::FsOpenOptions::new().read(true),
)?
.read_to_end(&mut bytes)?;
fs.open(
&folder.join(format!("v{}", id + 1)),
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?
.write_all(&bytes)?;
let err = recover(
folder,
&*fs,
ManifestRecoveryMode::AbsoluteConsistency,
None,
)
.expect_err("a snapshot under another number must be refused");
assert!(
matches!(
err,
crate::Error::ManifestFooterInvalid(
"manifest snapshot names another version than its file"
)
),
"got {err:?}",
);
Ok(())
}
#[cfg(zstd_any)]
#[test]
fn an_open_refuses_a_renumbered_snapshot_before_it_touches_the_dictionaries() -> crate::Result<()> {
use std::io::{Read, Write};
let fs = std::sync::Arc::new(MemFs::new());
let config = crate::Config::new(
"/renumbered-dicts",
crate::SequenceNumberCounter::default(),
crate::SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone());
let folder = config.path.clone();
drop(config.open()?);
let id = installed_snapshot(&folder, &*fs)?.ok_or(crate::Error::Unrecoverable)?;
let mut bytes = Vec::new();
fs.open(
&folder.join(format!("v{id}")),
&crate::fs::FsOpenOptions::new().read(true),
)?
.read_to_end(&mut bytes)?;
fs.open(
&folder.join(format!("v{}", id + 1)),
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?
.write_all(&bytes)?;
let dicts = folder.join(crate::file::DICTS_FOLDER);
fs.create_dir_all(&dicts)?;
let registration = dicts.join("7.tmp");
fs.open(
®istration,
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?;
let err = crate::Config::new(
&folder,
crate::SequenceNumberCounter::default(),
crate::SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone())
.open()
.err()
.ok_or(crate::Error::Unrecoverable)?;
assert!(
matches!(
err,
crate::Error::ManifestFooterInvalid(
"manifest snapshot names another version than its file"
)
),
"got {err:?}",
);
assert!(
fs.exists(®istration)?,
"the dictionary folder is left as it was"
);
Ok(())
}
fn a_tree_at_version(fs: &std::sync::Arc<MemFs>, id: u64) -> crate::Result<crate::path::PathBuf> {
use crate::AbstractTree;
let config = top_id_config(fs);
let folder = config.path.clone();
let version = {
let tree = config.open()?;
tree.insert("a", "1", 0);
tree.flush_active_memtable(0)?;
tree.current_version()
};
crate::version::persist_version(
&folder,
&version.with_id_and_dicts(id, version.dicts().to_vec()),
crate::comparator::default_comparator().name(),
&**fs,
alloc::sync::Arc::new(crate::runtime_config::RuntimeConfig::default()),
None,
crate::fs::SyncMode::Normal,
)?;
Ok(folder)
}
fn top_id_config(fs: &std::sync::Arc<MemFs>) -> crate::Config {
crate::Config::new(
"/top-id",
crate::SequenceNumberCounter::default(),
crate::SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone())
}
fn snapshots(fs: &MemFs, folder: &crate::path::Path) -> crate::Result<Vec<String>> {
let mut names: Vec<String> = fs
.read_dir(folder)?
.into_iter()
.map(|entry| entry.file_name)
.filter(|name| name.starts_with('v'))
.collect();
names.sort();
Ok(names)
}
#[test]
fn an_open_at_the_top_of_the_id_range_reinstalls_the_tree_as_version_0() -> crate::Result<()> {
use crate::AbstractTree;
let fs = std::sync::Arc::new(MemFs::new());
let folder = a_tree_at_version(&fs, u64::MAX)?;
{
let tree = top_id_config(&fs).open()?;
assert!(tree.get("a", 1)?.is_some(), "the data is kept");
assert_eq!(snapshots(&fs, &folder)?, vec!["v0"], "nothing above it");
tree.insert("b", "2", 1);
tree.flush_active_memtable(0)?;
}
let tree = top_id_config(&fs).open()?;
assert!(tree.get("a", 3)?.is_some());
assert!(tree.get("b", 3)?.is_some(), "the install after it is kept");
assert_eq!(installed_snapshot(&folder, &*fs)?, Some(2));
Ok(())
}
#[test]
fn a_damaged_identity_section_does_not_make_a_report_owed() -> crate::Result<()> {
let fs = std::sync::Arc::new(MemFs::new());
let folder = a_tree_at_version(&fs, 7)?;
let entry_at = {
let archive = open_snapshot(&folder, 7, &*fs, None)?;
assert!(archive.section("repair_report").is_none(), "self-check");
let entry = archive
.section("version_id")
.ok_or(crate::Error::Unrecoverable)?;
entry.block_offset + u64::from(entry.block_size) - 1
};
let path = folder.join("v7");
let mut file = fs.open(
&path,
&crate::fs::FsOpenOptions::new().read(true).write(true),
)?;
let mut byte = [0u8; 1];
file.seek(std::io::SeekFrom::Start(entry_at))?;
std::io::Read::read_exact(&mut file, &mut byte)?;
file.seek(std::io::SeekFrom::Start(entry_at))?;
file.write_all(&[byte[0] ^ 0xFF])?;
drop(file);
assert_eq!(owed_repair_report(&folder, &*fs, None)?, None);
Ok(())
}
#[test]
fn a_snapshot_naming_another_version_cannot_say_what_is_owed() -> crate::Result<()> {
let fs = std::sync::Arc::new(MemFs::new());
let folder = a_tree_at_version(&fs, 7)?;
let len = usize::try_from(fs.metadata(&folder.join("v7"))?.len)
.map_err(|_| crate::Error::Unrecoverable)?;
let bytes = crate::file::read_exact(
&*fs.open(
&folder.join("v7"),
&crate::fs::FsOpenOptions::new().read(true),
)?,
0,
len,
)?;
fs.open(
&folder.join("v8"),
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?
.write_all(&bytes)?;
assert_eq!(
owed_repair_report(&folder, &*fs, None)?,
Some(OwedReport::Unreadable(8))
);
Ok(())
}
#[test]
fn a_renumbering_keeps_foreign_files_in_the_tree_root() -> crate::Result<()> {
let fs = std::sync::Arc::new(MemFs::new());
let folder = a_tree_at_version(&fs, crate::version::RENUMBER_FROM)?;
let foreign = ["vendor-notes.txt", "v1.bak", "edits-old", "v007"];
for name in foreign {
fs.open(
&folder.join(name),
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?
.write_all(b"not the engine's")?;
}
let temporary = folder.join(format!("v{}.tmp", crate::version::RENUMBER_FROM - 1));
fs.open(
&temporary,
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?
.write_all(b"torn")?;
drop(top_id_config(&fs).open()?);
for name in foreign {
assert!(fs.exists(&folder.join(name))?, "{name} is kept");
}
assert!(!fs.exists(&temporary)?, "the temporary snapshot goes");
Ok(())
}
#[test]
fn a_renumbering_cut_short_keeps_the_edits_of_the_snapshot_it_read() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule};
let top = crate::version::RENUMBER_FROM;
for _ in 0..32 {
let fs = std::sync::Arc::new(MemFs::new());
a_tree_at_version(&fs, top - 1)?;
{
let tree = top_id_config(&fs).open()?;
tree.insert("b", "2", 1);
tree.flush_active_memtable(0)?;
}
let fault = FaultFs::new(fs.as_ref().clone());
fault.injector().arm(
FaultRule::new(
FaultOp::RemoveFile,
Fault::Error(crate::io::ErrorKind::Other),
)
.on_path(format!("v{top}")),
);
let failed = crate::Config::new(
"/top-id",
crate::SequenceNumberCounter::default(),
crate::SequenceNumberCounter::default(),
)
.with_fs(fault)
.open();
assert!(failed.is_err(), "the snapshot read cannot be removed");
let tree = top_id_config(&fs).open()?;
assert!(
tree.get("b", 3)?.is_some(),
"the edit in the log of the snapshot read is kept"
);
}
Ok(())
}
#[test]
fn a_renumbering_cut_short_is_done_again_by_the_next_open() -> crate::Result<()> {
use crate::AbstractTree;
use crate::fs::{Fault, FaultFs, FaultOp, FaultRule};
let fs = std::sync::Arc::new(MemFs::new());
let folder = a_tree_at_version(&fs, crate::version::RENUMBER_FROM)?;
let fault = FaultFs::new(fs.as_ref().clone());
fault.injector().arm(
FaultRule::new(
FaultOp::RemoveFile,
Fault::Error(crate::io::ErrorKind::Other),
)
.on_path(format!("v{}", crate::version::RENUMBER_FROM)),
);
let failed = crate::Config::new(
"/top-id",
crate::SequenceNumberCounter::default(),
crate::SequenceNumberCounter::default(),
)
.with_fs(fault)
.open();
assert!(failed.is_err(), "the snapshot read cannot be removed");
let tree = top_id_config(&fs).open()?;
assert!(tree.get("a", 1)?.is_some(), "the data is kept");
assert_eq!(snapshots(&fs, &folder)?, vec!["v0"]);
Ok(())
}
#[test]
fn a_snapshot_whose_version_id_is_not_one_id_wide_is_refused() -> crate::Result<()> {
let fs = MemFs::new();
let folder = Path::new("/narrow-id");
fs.create_dir_all(folder)?;
let mut w = FixtureWriter::create(
&folder.join("v1"),
&fs,
alloc::sync::Arc::new(crate::runtime_config::RuntimeConfig::default()),
None,
crate::fs::SyncMode::Normal,
)?;
w.start("version_id")?;
w.write_u32::<LittleEndian>(1)?;
write_header(&mut w)?;
write_empty_tables(&mut w)?;
write_empty_blob_files(&mut w)?;
write_empty_blob_gc_stats(&mut w)?;
w.finish()?;
install_fixture(folder, 1, &fs)?;
let err = recover(folder, &fs, ManifestRecoveryMode::AbsoluteConsistency, None)
.expect_err("a narrow version id must be refused");
assert!(
matches!(
err,
crate::Error::ManifestFooterInvalid("manifest version_id section is not 8 bytes")
),
"got {err:?}",
);
Ok(())
}
#[test]
fn only_canonical_ids_name_a_snapshot_or_a_log() {
assert_eq!(snapshot_id_of("v0"), Some(0));
assert_eq!(snapshot_id_of("v10"), Some(10));
assert_eq!(snapshot_id_of("v010"), None);
assert_eq!(snapshot_id_of("v00"), None);
assert_eq!(log_id_of("edits-0"), Some(0));
assert_eq!(log_id_of("edits-07"), None);
}
#[test]
fn a_stray_zero_padded_snapshot_name_does_not_hide_the_installed_one() -> crate::Result<()> {
use crate::AbstractTree;
let fs = std::sync::Arc::new(MemFs::new());
let config = crate::Config::new(
"/zero-padded",
crate::SequenceNumberCounter::default(),
crate::SequenceNumberCounter::default(),
)
.with_shared_fs(fs.clone());
let folder = config.path.clone();
{
let tree = config.open()?;
tree.insert("key", "value", 0);
tree.flush_active_memtable(0)?;
}
fs.open(
&folder.join("v0100"),
&crate::fs::FsOpenOptions::new().write(true).create(true),
)?;
let tree = crate::Config::new(
&folder,
crate::SequenceNumberCounter::default(),
crate::SequenceNumberCounter::default(),
)
.with_shared_fs(fs)
.open()?;
assert!(tree.get("key", crate::MAX_SEQNO)?.is_some());
Ok(())
}
#[test]
fn repair_refuses_a_store_of_another_format_version() -> crate::Result<()> {
let dir = tempfile::tempdir()?;
let fs = crate::fs::StdFs;
fs.create_dir_all(&dir.path().join(crate::file::TABLES_FOLDER))?;
write_manifest_labelled(dir.path(), &fs, 5)?;
let err = crate::Config::new(
dir.path(),
crate::SequenceNumberCounter::default(),
crate::SequenceNumberCounter::default(),
)
.repair()
.expect_err("repair must not rebuild a V5 store");
assert!(
matches!(err, crate::Error::InvalidVersion(5)),
"expected InvalidVersion(5), got {err:?}",
);
Ok(())
}