use std::sync::Arc;
use super::config::SegConfig;
use super::core::SegCore;
use super::manifest::load;
use super::seal::{sdat_name, sidx_name};
use super::testkit::{CoreShape, TestKey, copy_collection, fill_core as fill, key_of, value};
use crate::error::DbError;
fn wide(shard_count: usize) -> SegConfig {
SegConfig {
seal_max_keys: 1_000_000,
max_frozen_generations: 8,
shard_count,
..SegConfig::test()
}
}
fn assert_holds(core: &SegCore<TestKey>, keys: std::ops::Range<u64>) {
assert!(core.verify_all().unwrap().is_clean());
for i in keys {
assert_eq!(core.get(&key_of(i)).unwrap(), Some(value(i)), "key {i}");
}
}
fn restart(
core: &SegCore<TestKey>,
config: SegConfig,
shape: CoreShape,
) -> (tempfile::TempDir, Arc<SegCore<TestKey>>) {
core.durable_flush().unwrap();
let dir = tempfile::tempdir().unwrap();
copy_collection(&core.dir, dir.path());
let reopened = shape.open_core(dir.path(), config);
(dir, reopened)
}
fn a_freeze_without_its_seal_survives_a_restart(shape: CoreShape) {
let dir = tempfile::tempdir().unwrap();
let core = shape.open_core(dir.path(), wide(2));
fill(&core, 0..100);
core.seal().unwrap();
fill(&core, 100..150);
let (_copy, reopened) = restart(&core, wide(2), shape);
let view = reopened.view();
assert_eq!(view.frozen.len(), 1);
assert!(view.frozen[0].is_frozen());
assert_eq!(view.frozen[0].keys(), 100);
assert_eq!(view.active.keys(), 50);
assert_holds(&reopened, 0..150);
assert!(reopened.seal_oldest().unwrap());
assert_eq!(reopened.view().sealed.len(), 1);
assert_holds(&reopened, 0..150);
fill(&reopened, 150..160);
reopened.seal().unwrap();
let manifest = load(&reopened.dir).unwrap().unwrap();
manifest.check().unwrap();
assert_eq!(manifest.segments[0].max_gsn, 100);
assert_eq!(manifest.generations[0].min_gsn, 101);
assert_eq!(manifest.generations[0].max_gsn, 160);
}
fn a_crashed_seal_is_started_over(shape: CoreShape) {
let dir = tempfile::tempdir().unwrap();
let core = shape.open_core(dir.path(), wide(2));
fill(&core, 0..100);
core.seal().unwrap();
core.durable_flush().unwrap();
let copy = tempfile::tempdir().unwrap();
copy_collection(dir.path(), copy.path());
let manifest = load(copy.path()).unwrap().unwrap();
let (reserved, burned_id) = manifest.reserve_segment_id().unwrap();
super::manifest::publish(&*super::fs::real(), copy.path(), &reserved).unwrap();
let seg = copy.path().join("seg");
std::fs::write(seg.join(sdat_name(burned_id, 1)), b"half a segment").unwrap();
std::fs::write(
seg.join(format!("{}.tmp", sidx_name(burned_id, 1))),
b"torn",
)
.unwrap();
let reopened = shape.open_core(copy.path(), wide(2));
assert_eq!(
std::fs::read_dir(&seg).unwrap().count(),
0,
"the leftovers are gone"
);
assert_eq!(reopened.view().frozen.len(), 1);
assert_eq!(
load(copy.path()).unwrap().unwrap().next_segment_id,
burned_id + 1
);
assert!(reopened.seal_oldest().unwrap());
assert_eq!(reopened.view().sealed.len(), 1);
assert_eq!(reopened.view().sealed[0].id(), burned_id + 1);
assert_holds(&reopened, 0..100);
assert!(reopened.verify_all().unwrap().is_clean());
}
fn a_frozen_generation_that_lost_its_whole_tail_is_retired(shape: CoreShape) {
let dir = tempfile::tempdir().unwrap();
let core = shape.open_core(dir.path(), wide(2));
fill(&core, 0..100);
core.seal().unwrap();
assert!(core.seal_oldest().unwrap());
fill(&core, 100..110);
core.seal().unwrap();
let copy = tempfile::tempdir().unwrap();
copy_collection(dir.path(), copy.path());
let reopened = shape.open_core(copy.path(), wide(2));
assert_eq!(reopened.view().frozen.len(), 1);
assert_eq!(reopened.view().frozen[0].keys(), 0, "the tail was lost");
assert_eq!(reopened.view().sealed.len(), 1);
assert!(reopened.seal_oldest().unwrap());
assert!(reopened.view().frozen.is_empty());
assert_eq!(reopened.view().sealed.len(), 1);
assert!(!copy.path().join("gen/0000000000000002").exists());
assert_holds(&reopened, 0..100);
assert_eq!(reopened.get(&key_of(105)).unwrap(), None);
fill(&reopened, 200..210);
reopened.seal().unwrap();
assert!(reopened.seal_oldest().unwrap());
assert_eq!(reopened.view().sealed.len(), 2);
assert_holds(&reopened, 200..210);
load(copy.path()).unwrap().unwrap().check().unwrap();
assert!(reopened.verify_all().unwrap().is_clean());
}
fn a_frozen_journal_past_its_recorded_last_gsn_is_refused(shape: CoreShape) {
let dir = tempfile::tempdir().unwrap();
shape.build_collection(dir.path());
let mut manifest = load(dir.path()).unwrap().unwrap();
manifest.generations[0].max_gsn = 300;
manifest.check().unwrap();
super::manifest::publish(&*super::fs::real(), dir.path(), &manifest).unwrap();
let err = shape
.try_open_core(
dir.path(),
SegConfig::test(),
super::fs::real(),
&super::testkit::TestClock::new(1),
)
.map(drop)
.unwrap_err();
match err {
DbError::GenerationMissing { gen_id, reason, .. } => {
assert_eq!(gen_id, 1);
assert!(reason.contains("GSN 350"), "{reason}");
}
other => panic!("{other:?}"),
}
}
fn a_generation_that_outlived_its_seal_is_removed(shape: CoreShape) {
let dir = tempfile::tempdir().unwrap();
let core = shape.open_core(dir.path(), wide(2));
fill(&core, 0..100);
core.seal().unwrap();
let held = core.view();
core.seal_oldest().unwrap();
assert!(dir.path().join("gen/0000000000000001").exists());
let (copy, reopened) = restart(&core, wide(2), shape);
drop(held);
assert!(!copy.path().join("gen/0000000000000001").exists());
assert!(reopened.view().frozen.is_empty());
assert_eq!(reopened.view().sealed.len(), 1);
assert_holds(&reopened, 0..100);
}
fn files_of_a_dropped_segment_are_removed(shape: CoreShape) {
let dir = tempfile::tempdir().unwrap();
let core = shape.open_core(dir.path(), wide(2));
fill(&core, 0..50);
core.seal().unwrap();
core.seal_oldest().unwrap();
fill(&core, 50..100);
core.seal().unwrap();
core.seal_oldest().unwrap();
let held = core.view();
assert_eq!(core.drop_segments_before(u64::MAX).unwrap(), 2);
assert!(dir.path().join("seg").join(sdat_name(1, 1)).exists());
let (copy, reopened) = restart(&core, wide(2), shape);
drop(held);
assert_eq!(
std::fs::read_dir(copy.path().join("seg")).unwrap().count(),
0
);
assert!(reopened.view().sealed.is_empty());
assert_eq!(reopened.get(&key_of(10)).unwrap(), None);
}
fn the_shard_count_changes_across_restarts(shape: CoreShape) {
let dir = tempfile::tempdir().unwrap();
let core = shape.open_core(dir.path(), wide(4));
fill(&core, 0..100);
core.seal().unwrap();
core.seal_oldest().unwrap();
fill(&core, 100..200);
core.seal().unwrap();
fill(&core, 200..300);
core.clean_shutdown().unwrap();
drop(core);
let core = shape.open_core(dir.path(), wide(16));
assert_eq!(core.view().active.shard_count(), 4);
assert_eq!(core.view().frozen[0].shard_count(), 4);
assert_holds(&core, 0..300);
fill(&core, 300..400);
core.seal().unwrap();
assert_eq!(core.view().active.shard_count(), 16);
fill(&core, 400..500);
core.clean_shutdown().unwrap();
drop(core);
let core = shape.open_core(dir.path(), wide(2));
let counts: Vec<usize> = std::iter::once(&core.view().active)
.chain(core.view().frozen.iter())
.map(|g| g.shard_count())
.collect();
assert_eq!(counts, [16, 4, 4]);
assert_holds(&core, 0..500);
assert!(matches!(
core.insert(&key_of(250), b"x"),
Err(DbError::KeyExists)
));
core.insert(&key_of(900), b"new").unwrap();
core.delete(&key_of(50)).unwrap();
assert_eq!(core.get(&key_of(50)).unwrap(), None);
assert_eq!(core.count_exact().unwrap(), 500);
while core.seal_oldest().unwrap() {}
core.seal().unwrap();
core.seal_oldest().unwrap();
assert_eq!(core.view().active.shard_count(), 2);
assert_holds(&core, 0..50);
assert_holds(&core, 51..500);
assert_eq!(core.get(&key_of(50)).unwrap(), None);
assert_eq!(core.get(&key_of(900)).unwrap(), Some(b"new".to_vec()));
assert_eq!(core.count_exact().unwrap(), 500);
let mut scanned = std::collections::BTreeMap::new();
core.scan(&mut |row| {
let (key, value) = row.unwrap();
assert!(scanned.insert(key, value).is_none(), "a key came twice");
std::ops::ControlFlow::Continue(())
});
let mut expected: std::collections::BTreeMap<TestKey, Vec<u8>> = (0..500)
.filter(|&i| i != 50)
.map(|i| (key_of(i), value(i)))
.collect();
expected.insert(key_of(900), b"new".to_vec());
assert_eq!(scanned, expected);
assert!(core.verify_all().unwrap().is_clean());
}
fn a_restart_changes_no_answer(shape: CoreShape) {
let dir = tempfile::tempdir().unwrap();
let core = shape.open_core(dir.path(), wide(2));
fill(&core, 0..200);
for i in (0..200).step_by(7) {
core.delete(&key_of(i)).unwrap();
}
core.seal().unwrap();
core.seal_oldest().unwrap();
for i in (0..200).step_by(5) {
core.put(&key_of(i), b"again").unwrap();
}
core.seal().unwrap();
for i in (0..200).step_by(11) {
core.delete(&key_of(i)).unwrap();
}
let before: Vec<Option<Vec<u8>>> = (0..220).map(|i| core.get(&key_of(i)).unwrap()).collect();
let (_copy, reopened) = restart(&core, wide(2), shape);
let after: Vec<Option<Vec<u8>>> = (0..220)
.map(|i| reopened.get(&key_of(i)).unwrap())
.collect();
assert_eq!(before, after);
assert_eq!(core.count_exact().unwrap(), reopened.count_exact().unwrap());
}
fn recovered_saturated_queue_refuses_every_mutation_across_reopens(shape: CoreShape) {
for by_bytes in [false, true] {
let dir = tempfile::tempdir().unwrap();
let config = SegConfig {
seal_max_keys: if by_bytes { 1_000_000 } else { 2 },
seal_max_bytes: if by_bytes {
2 * crate::entry::entry_size(32, 2)
} else {
u64::MAX
},
max_frozen_generations: 1,
seal_wait_timeout: std::time::Duration::ZERO,
..SegConfig::test()
};
let core = shape.open_core(dir.path(), config.clone());
fill(&core, 0..4);
assert_eq!(core.view().frozen.len(), 1);
assert!(core.view().active.is_full());
core.clean_shutdown().unwrap();
drop(core);
let mut baseline = None;
for _ in 0..3 {
let core = shape.open_core(dir.path(), config.clone());
let active = core.view().active.clone();
let before = (active.keys(), active.bytes(), active.next_gsn());
assert_eq!(*baseline.get_or_insert(before), before);
let locations = active.snapshot_range(0, u64::MAX, core.seed);
for result in [
core.put(&key_of(9), b"new"),
core.put(&key_of(2), b"overwrite"),
core.insert(&key_of(10), b"new"),
core.delete(&key_of(2)),
] {
assert!(
matches!(result, Err(DbError::SealBackpressure { .. })),
"{result:?}"
);
}
assert_eq!((active.keys(), active.bytes(), active.next_gsn()), before);
assert_eq!(active.snapshot_range(0, u64::MAX, core.seed), locations);
assert_eq!(core.get(&key_of(9)).unwrap(), None);
assert_eq!(core.get(&key_of(10)).unwrap(), None);
assert_holds(&core, 0..4);
core.clean_shutdown().unwrap();
}
}
}
fn recovered_lowered_limits_freeze_before_write_or_tick(shape: CoreShape) {
for by_bytes in [false, true] {
for by_tick in [false, true] {
let dir = tempfile::tempdir().unwrap();
let core = shape.open_core(dir.path(), wide(2));
fill(&core, 0..3);
core.clean_shutdown().unwrap();
let bytes = core.view().active.bytes();
drop(core);
let config = SegConfig {
seal_max_keys: if by_bytes { 1_000_000 } else { 3 },
seal_max_bytes: if by_bytes { bytes } else { u64::MAX },
..wide(2)
};
let core = shape.open_core(dir.path(), config);
let recovered = core.view().active.clone();
assert!(
recovered.is_full(),
"keys/bytes limits must apply before exposure"
);
let before = (recovered.keys(), recovered.bytes(), recovered.next_gsn());
if by_tick {
core.tick().unwrap();
assert_ne!(
core.view().active.id(),
recovered.id(),
"tick must freeze before any write"
);
}
core.put(&key_of(9), b"new").unwrap();
assert_ne!(core.view().active.id(), recovered.id());
assert!(recovered.is_frozen());
assert_eq!(
(recovered.keys(), recovered.bytes(), recovered.next_gsn()),
before
);
assert_eq!(core.get(&key_of(9)).unwrap(), Some(b"new".to_vec()));
assert_holds(&core, 0..3);
}
}
}
fn unrepresentable_configured_seal_wait_is_rejected(shape: CoreShape) {
let bad = SegConfig {
seal_wait_timeout: std::time::Duration::MAX,
..SegConfig::test()
};
assert!(matches!(bad.validate(), Err(DbError::Config(_))));
let dir = tempfile::tempdir().unwrap();
assert!(matches!(
shape.try_open_core(
dir.path(),
bad,
super::fs::real(),
&super::testkit::TestClock::new(1_000_000)
),
Err(DbError::Config(_))
));
assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);
}
fn unrepresentable_public_seal_wait_returns_error_without_mutation(shape: CoreShape) {
let dir = tempfile::tempdir().unwrap();
let core = shape.open_core(dir.path(), wide(2));
fill(&core, 0..3);
let active = core.view().active.clone();
let before = (active.keys(), active.bytes(), active.next_gsn());
assert!(matches!(
core.wait_sealed(std::time::Duration::MAX),
Err(DbError::Client(_))
));
assert_eq!((active.keys(), active.bytes(), active.next_gsn()), before);
assert_holds(&core, 0..3);
assert!(core.wait_sealed(std::time::Duration::ZERO).unwrap());
core.seal().unwrap();
assert!(!core.wait_sealed(std::time::Duration::ZERO).unwrap());
}
mod by_hash {
#[test]
fn a_freeze_without_its_seal_survives_a_restart() {
super::a_freeze_without_its_seal_survives_a_restart(super::CoreShape::ByHash);
}
#[test]
fn a_crashed_seal_is_started_over() {
super::a_crashed_seal_is_started_over(super::CoreShape::ByHash);
}
#[test]
fn a_frozen_generation_that_lost_its_whole_tail_is_retired() {
super::a_frozen_generation_that_lost_its_whole_tail_is_retired(super::CoreShape::ByHash);
}
#[test]
fn a_frozen_journal_past_its_recorded_last_gsn_is_refused() {
super::a_frozen_journal_past_its_recorded_last_gsn_is_refused(super::CoreShape::ByHash);
}
#[test]
fn a_generation_that_outlived_its_seal_is_removed() {
super::a_generation_that_outlived_its_seal_is_removed(super::CoreShape::ByHash);
}
#[test]
fn files_of_a_dropped_segment_are_removed() {
super::files_of_a_dropped_segment_are_removed(super::CoreShape::ByHash);
}
#[test]
fn the_shard_count_changes_across_restarts() {
super::the_shard_count_changes_across_restarts(super::CoreShape::ByHash);
}
#[test]
fn a_restart_changes_no_answer() {
super::a_restart_changes_no_answer(super::CoreShape::ByHash);
}
#[test]
fn recovered_saturated_queue_refuses_every_mutation_across_reopens() {
super::recovered_saturated_queue_refuses_every_mutation_across_reopens(
super::CoreShape::ByHash,
);
}
#[test]
fn recovered_lowered_limits_freeze_before_write_or_tick() {
super::recovered_lowered_limits_freeze_before_write_or_tick(super::CoreShape::ByHash);
}
#[test]
fn unrepresentable_configured_seal_wait_is_rejected() {
super::unrepresentable_configured_seal_wait_is_rejected(super::CoreShape::ByHash);
}
#[test]
fn unrepresentable_public_seal_wait_returns_error_without_mutation() {
super::unrepresentable_public_seal_wait_returns_error_without_mutation(
super::CoreShape::ByHash,
);
}
}
mod by_key {
#[test]
fn a_freeze_without_its_seal_survives_a_restart() {
super::a_freeze_without_its_seal_survives_a_restart(super::CoreShape::ByKey);
}
#[test]
fn a_crashed_seal_is_started_over() {
super::a_crashed_seal_is_started_over(super::CoreShape::ByKey);
}
#[test]
fn a_frozen_generation_that_lost_its_whole_tail_is_retired() {
super::a_frozen_generation_that_lost_its_whole_tail_is_retired(super::CoreShape::ByKey);
}
#[test]
fn a_frozen_journal_past_its_recorded_last_gsn_is_refused() {
super::a_frozen_journal_past_its_recorded_last_gsn_is_refused(super::CoreShape::ByKey);
}
#[test]
fn a_generation_that_outlived_its_seal_is_removed() {
super::a_generation_that_outlived_its_seal_is_removed(super::CoreShape::ByKey);
}
#[test]
fn files_of_a_dropped_segment_are_removed() {
super::files_of_a_dropped_segment_are_removed(super::CoreShape::ByKey);
}
#[test]
fn the_shard_count_changes_across_restarts() {
super::the_shard_count_changes_across_restarts(super::CoreShape::ByKey);
}
#[test]
fn a_restart_changes_no_answer() {
super::a_restart_changes_no_answer(super::CoreShape::ByKey);
}
#[test]
fn recovered_saturated_queue_refuses_every_mutation_across_reopens() {
super::recovered_saturated_queue_refuses_every_mutation_across_reopens(
super::CoreShape::ByKey,
);
}
#[test]
fn recovered_lowered_limits_freeze_before_write_or_tick() {
super::recovered_lowered_limits_freeze_before_write_or_tick(super::CoreShape::ByKey);
}
#[test]
fn unrepresentable_configured_seal_wait_is_rejected() {
super::unrepresentable_configured_seal_wait_is_rejected(super::CoreShape::ByKey);
}
#[test]
fn unrepresentable_public_seal_wait_returns_error_without_mutation() {
super::unrepresentable_public_seal_wait_returns_error_without_mutation(
super::CoreShape::ByKey,
);
}
}