use std::cmp::Reverse;
use std::collections::{BinaryHeap, VecDeque};
use std::hash::Hash;
use std::ops::Bound;
use std::sync::atomic::Ordering;
use std::sync::{Arc, Weak};
use std::time::{Duration, Instant};
use super::core::{SegCore, View};
use super::generation::Generation;
use super::hash::{key_hash, shard_range};
use super::manifest::{self, PublishError, SegRecord};
use super::seal::{SegTimes, SegmentWriter, sdat_name, sidx_name};
use super::segment::Segment;
use super::sidecar::{OrderKind, SegHeader};
use crate::error::{DbError, DbResult};
use crate::key::Key;
use crate::map_index::Dir;
use crate::sync;
const IDLE_PAUSE: Duration = Duration::from_secs(1);
const SKELETON_PASS_BUDGET: Duration = Duration::from_millis(250);
const MIN_BACKOFF: Duration = Duration::from_secs(1);
const MAX_BACKOFF: Duration = Duration::from_secs(60);
const WAIT_SLICE: Duration = Duration::from_millis(20);
const SHUTDOWN_CHECK_EVERY: usize = 1024;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SealStatus {
pub frozen: usize,
pub in_flight: Option<u64>,
pub last_error: Option<String>,
pub retry_at_ms: Option<u64>,
pub last_sealed_at_ms: Option<u64>,
pub fenced: Option<String>,
}
impl<K: Key + Hash + Eq + Send + Sync + 'static> SegCore<K> {
#[cfg(test)]
pub(crate) fn seal_oldest(&self) -> DbResult<bool> {
self.seal_oldest_with_retry(None)
}
fn seal_oldest_with_retry(&self, retry_delay: Option<Duration>) -> DbResult<bool> {
let _attempt = sync::lock(&self.seal_attempt);
let started = Instant::now();
let result = self.seal_oldest_inner();
{
let mut state = sync::lock(&self.seal);
state.in_flight = None;
match &result {
Ok(true) => {
state.last_error = None;
state.retry_at_ms = None;
state.last_sealed_at_ms = Some(self.now_ms());
metrics::histogram!("armdb.seg.seal_seconds", "collection" => self.meta.name)
.record(started.elapsed().as_secs_f64());
}
Ok(false) => {}
Err(_) if self.shut_down.load(Ordering::Acquire) => {}
Err(e) => {
state.last_error = Some(e.to_string());
state.failures += 1;
if !matches!(e, DbError::CollectionFenced { .. })
&& let Some(delay) = retry_delay
{
state.retry_at_ms =
Some(self.now_ms().saturating_add(
u64::try_from(delay.as_millis()).unwrap_or(u64::MAX),
));
}
metrics::counter!("armdb.seg.seal_failures", "collection" => self.meta.name)
.increment(1);
}
}
}
self.seal_cv.notify_all();
result
}
fn check_seal_shutdown(&self) -> DbResult<()> {
if self.shut_down.load(Ordering::Acquire) {
return Err(DbError::Internal("the seal was aborted by a shutdown"));
}
Ok(())
}
fn seal_oldest_inner(&self) -> DbResult<bool> {
let (generation, segment_id, nothing_below) = {
let mut manifest = sync::lock(&self.manifest);
self.check_writable()?;
self.check_seal_shutdown()?;
let Some(rec) = manifest.oldest_frozen().copied() else {
return Ok(false);
};
let view = self.view();
let generation = view
.frozen
.last()
.cloned()
.filter(|g| g.id() == rec.gen_id)
.ok_or(DbError::Internal(
"the view and the manifest disagree on the oldest frozen generation",
))?;
let (reserved, segment_id) = manifest.reserve_segment_id()?;
self.publish_next(&mut manifest, reserved)?;
(generation, segment_id, manifest.segments.is_empty())
};
sync::lock(&self.seal).in_flight = Some(generation.id());
generation.durable_flush()?;
let header = SegHeader {
segment_id,
version: 1,
order: self.order,
key_len: self.key_len(),
seed: self.seed,
block_size: self.config.block_size,
};
let record = self.write_segment(&generation, header, nothing_below)?;
let segment = match &record {
Some(rec) => match Segment::open(
self.fs.clone(),
&self.seg_root,
rec,
self.key_len(),
self.seed,
&self.shared,
) {
Ok(segment) => Some(Arc::new(segment)),
Err(e) => {
self.discard_segment_files(rec);
return Err(e);
}
},
None => None,
};
let mut manifest = sync::lock(&self.manifest);
let discard = |segment: &Option<Arc<Segment>>| {
if let Some(segment) = segment {
segment.mark_doomed();
}
};
if let Err(e) = self
.check_writable()
.and_then(|()| self.check_seal_shutdown())
{
discard(&segment);
return Err(e);
}
let transition = if record.is_none() && generation.keys() == 0 {
manifest.with_empty_sealed(generation.id())
} else {
manifest.with_sealed(generation.id(), record)
};
let next = match transition {
Ok(next) => next,
Err(e) => {
discard(&segment);
return Err(e);
}
};
match manifest::publish(&*self.fs, &self.dir, &next) {
Ok(()) => {
*manifest = next;
super::failpoint!(SEAL_PUBLISHED);
}
Err(PublishError::NotPublished(e)) => {
discard(&segment);
return Err(e);
}
Err(PublishError::Indeterminate(e)) => {
return Err(self.fence(format!(
"the outcome of the seal of generation {:#x} is unknown: {e}",
generation.id()
)));
}
}
let view = self.view();
let mut frozen = view.frozen.clone();
frozen.pop();
let mut sealed = Vec::with_capacity(view.sealed.len() + 1);
sealed.extend(segment);
sealed.extend(view.sealed.iter().cloned());
let costs: Vec<_> = sealed.iter().map(|s| s.cost()).collect();
let hot = super::residency::assign(&costs, self.config.resident_metadata_bytes);
for segment in sealed.iter_mut().skip(hot) {
if segment.is_hot() {
*segment = Arc::new(segment.cooled());
}
}
self.set_view(View {
active: view.active.clone(),
frozen,
sealed,
});
drop(manifest);
generation.doom();
Ok(true)
}
fn discard_segment_files(&self, rec: &SegRecord) {
for name in [
sdat_name(rec.segment_id, rec.version),
sidx_name(rec.segment_id, rec.version),
] {
if let Err(e) = self.fs.remove_file(&self.seg_root.join(&name)) {
tracing::warn!(collection = self.meta.name, file = %name, error = %e, "cannot remove an unnamed segment file");
}
}
if let Err(e) = self.fs.sync_dir(&self.seg_root) {
tracing::warn!(collection = self.meta.name, error = %e, "cannot sync seg/ after removing an unnamed segment");
}
}
fn write_segment(
&self,
generation: &Arc<Generation<K>>,
header: SegHeader,
nothing_below: bool,
) -> DbResult<Option<SegRecord>> {
let mut writer = if self.order == OrderKind::ByKey {
SegmentWriter::create_key(
self.fs.clone(),
&self.seg_root,
header,
&self.prefix_filters,
generation.keys(),
)?
} else {
SegmentWriter::create(self.fs.clone(), &self.seg_root, header, generation.keys())?
};
let shards = generation.shard_count();
if self.order == OrderKind::ByKey {
const BATCH: usize = 256;
let mut batches: Vec<VecDeque<_>> = Vec::with_capacity(shards);
for shard in 0..shards {
self.check_seal_shutdown()?;
batches.push(
generation
.ordered_batch(shard, Bound::Unbounded, Bound::Unbounded, Dir::Asc, BATCH)
.into(),
);
}
let mut heap = BinaryHeap::new();
for (shard, batch) in batches.iter().enumerate() {
if let Some((key, _)) = batch.front() {
heap.push(Reverse((key.as_bytes().to_vec(), shard)));
}
}
let mut consumed = 0usize;
while let Some(Reverse((_, shard))) = heap.pop() {
if consumed.is_multiple_of(SHUTDOWN_CHECK_EVERY) {
self.check_seal_shutdown()?;
}
consumed += 1;
let (key, loc) = batches[shard].pop_front().expect("heap head has an entry");
let h = key_hash(key.as_bytes(), self.seed);
if !(loc.tombstone && nothing_below) {
writer.push(h, &generation.read_entry(h, &loc.loc)?)?;
}
if batches[shard].is_empty() {
batches[shard] = generation
.ordered_batch(
shard,
Bound::Excluded(&key),
Bound::Unbounded,
Dir::Asc,
BATCH,
)
.into();
}
if let Some((key, _)) = batches[shard].front() {
heap.push(Reverse((key.as_bytes().to_vec(), shard)));
}
}
} else {
for shard in 0..shards {
self.check_seal_shutdown()?;
let (lo, hi) = shard_range(shard, shards);
for (n, (h, _key, loc)) in generation
.snapshot_range(lo, hi, self.seed)
.into_iter()
.enumerate()
{
if n % SHUTDOWN_CHECK_EVERY == SHUTDOWN_CHECK_EVERY - 1 {
self.check_seal_shutdown()?;
}
if loc.tombstone && nothing_below {
continue;
}
let entry = generation.read_entry(h, &loc.loc)?;
writer.push(h, &entry)?;
}
}
}
if writer.entries() == 0 {
return Ok(None);
}
self.check_seal_shutdown()?;
let times = SegTimes {
opened_at_ms: generation.opened_at_ms(),
frozen_at_ms: generation.frozen_at_ms(),
sealed_at_ms: self.now_ms().max(generation.frozen_at_ms()),
};
let files = writer.finish(times, self.meta.value_version, self.meta.typ_hash)?;
metrics::counter!("armdb.seg.seal_bytes", "collection" => self.meta.name)
.increment(files.sdat_len + files.sidx_len);
Ok(Some(files.into_record()))
}
pub(crate) fn wait_sealed(&self, timeout: Duration) -> DbResult<bool> {
let deadline = Instant::now()
.checked_add(timeout)
.ok_or(DbError::Client("seal timeout exceeds the clock range"))?;
loop {
self.check_writable()?;
let state = sync::lock(&self.seal);
if self.view().frozen.is_empty() && state.in_flight.is_none() {
return Ok(true);
}
let now = Instant::now();
if now >= deadline || self.shut_down.load(Ordering::Acquire) {
return Ok(false);
}
drop(sync::cv_wait_timeout(
&self.seal_cv,
&self.seal,
state,
WAIT_SLICE.min(deadline - now),
));
}
}
pub(crate) fn seal_status(&self) -> SealStatus {
let state = sync::lock(&self.seal).clone();
SealStatus {
frozen: self.view().frozen.len(),
in_flight: state.in_flight,
last_error: state.last_error,
retry_at_ms: state.retry_at_ms,
last_sealed_at_ms: state.last_sealed_at_ms,
fenced: self.fence_reason(),
}
}
}
impl<K: Key + Hash + Eq + Send + Sync + 'static> SegCore<K> {
fn freeze_if_due(&self, now_ms: u64, aged: bool) -> DbResult<()> {
let view = self.view();
let active = &view.active;
if active.keys() == 0 {
return Ok(());
}
let max_age_ms = u64::try_from(self.config.seal_max_age.as_millis()).unwrap_or(u64::MAX);
let old = aged && now_ms.saturating_sub(active.opened_at_ms()) >= max_age_ms;
if !active.is_full() && !old {
return Ok(());
}
let mut manifest = sync::lock(&self.manifest);
if !Arc::ptr_eq(&self.view().active, active)
|| manifest.frozen_count() >= self.config.max_frozen_generations
{
return Ok(());
}
self.freeze_locked(&mut manifest)
}
#[cfg(test)]
pub(crate) fn tick(&self) -> DbResult<bool> {
let (result, more) = self.round();
result.map(|()| more)
}
fn round(&self) -> (DbResult<()>, bool) {
if self.shut_down.load(Ordering::Acquire) || self.fence_reason().is_some() {
return (Ok(()), false);
}
let now_ms = self.now_ms();
let interval_ms =
u64::try_from(self.config.maintenance_interval.as_millis()).unwrap_or(u64::MAX);
let last = self.last_tick_ms.load(Ordering::Acquire);
let due = last == 0 || now_ms < last || now_ms - last >= interval_ms;
if due {
self.last_tick_ms.store(now_ms.max(1), Ordering::Release);
}
let flushed = self.flush_frozen();
let frozen = self.freeze_if_due(now_ms, due);
let retained = if due { self.maintain(now_ms) } else { Ok(()) };
let (skeleton, more) = self.build_pending_skeletons(SKELETON_PASS_BUDGET);
self.publish_gauges();
(flushed.and(frozen).and(retained).and(skeleton), more)
}
pub(crate) fn build_pending_skeletons(&self, budget: Duration) -> (DbResult<()>, bool) {
let started = Instant::now();
let mut result = Ok(());
let mut built = 0usize;
loop {
if self.shut_down.load(Ordering::Acquire) || self.fence_reason().is_some() {
return (result, false);
}
let now = super::skeleton::now_ms();
let Some(segment) = self
.view()
.sealed
.iter()
.find(|s| s.files.skeleton.is_pending(now))
.cloned()
else {
return (result, false);
};
if built > 0 && started.elapsed() >= budget {
return (result, true);
}
if let Err(e) = segment.skeleton_background()
&& result.is_ok()
{
result = Err(e);
}
built += 1;
}
}
fn flush_frozen(&self) -> DbResult<()> {
let view = self.view();
let mut first = None;
for generation in view.frozen.iter().rev() {
if generation.dirty_shards() == 0 {
continue;
}
if let Err(e) = generation.durable_flush()
&& first.is_none()
{
first = Some(e);
}
}
first.map_or(Ok(()), Err)
}
fn drop_oldest_if(&self, due: impl Fn(&SegRecord) -> bool) -> DbResult<bool> {
let mut manifest = sync::lock(&self.manifest);
self.check_writable()?;
let Some(oldest) = manifest.segments.first() else {
return Ok(false);
};
if !due(oldest) {
return Ok(false);
}
let (segment_id, frozen_at_ms) = (oldest.segment_id, oldest.frozen_at_ms);
let view = self.view();
let mut sealed = view.sealed.clone();
let dropped = sealed
.pop()
.filter(|s| s.id() == segment_id)
.ok_or(DbError::Internal(
"the view and the manifest disagree on the oldest segment",
))?;
let next = manifest.without_oldest_segment()?;
self.publish_next(&mut manifest, next)?;
super::failpoint!(DROP_PUBLISHED);
dropped.mark_doomed();
self.set_view(View {
active: view.active.clone(),
frozen: view.frozen.clone(),
sealed,
});
drop(manifest);
tracing::warn!(
collection = self.meta.name,
segment_id,
frozen_at_ms,
"segment dropped"
);
metrics::counter!("armdb.seg.retention_drops", "collection" => self.meta.name).increment(1);
Ok(true)
}
pub(crate) fn drop_segments_before(&self, ts_ms: u64) -> DbResult<usize> {
let mut dropped = 0;
while self.drop_oldest_if(|rec| rec.frozen_at_ms < ts_ms)? {
dropped += 1;
}
Ok(dropped)
}
fn maintain(&self, now_ms: u64) -> DbResult<()> {
let Some(retention) = self.config.retention else {
return Ok(());
};
let retention_ms = u64::try_from(retention.as_millis()).unwrap_or(u64::MAX);
self.drop_oldest_if(|rec| now_ms.saturating_sub(rec.frozen_at_ms) >= retention_ms)?;
Ok(())
}
pub(crate) fn start_worker(self: &Arc<Self>) -> DbResult<()> {
#[cfg(any(test, feature = "test-faults"))]
if super::failpoints::worker_paused() {
return Ok(());
}
let mut worker = sync::lock(&self.worker);
if worker.is_some() {
return Ok(());
}
let weak = Arc::downgrade(self);
let signal = self.signal.clone();
let handle = std::thread::Builder::new()
.name(format!("armdb-seg-{}", self.meta.name))
.spawn(move || worker_loop(weak, signal))?;
*worker = Some(handle);
Ok(())
}
pub(crate) fn worker_running(&self) -> bool {
sync::lock(&self.worker)
.as_ref()
.is_some_and(|handle| !handle.is_finished())
}
pub(crate) fn stop_worker(&self) {
{
let _manifest = sync::lock(&self.manifest);
self.shut_down.store(true, Ordering::Release);
}
self.signal.stop();
let handle = sync::lock(&self.worker).take();
if let Some(handle) = handle
&& handle.thread().id() != std::thread::current().id()
{
let _ = handle.join();
}
}
}
fn round_pause(busy: bool, retry_not_before: Option<Instant>, now: Instant) -> Duration {
let pause = if busy { Duration::ZERO } else { IDLE_PAUSE };
retry_not_before.map_or(pause, |at| pause.min(at.saturating_duration_since(now)))
}
fn worker_loop<K: Key + Hash + Eq + Send + Sync + 'static>(
weak: Weak<SegCore<K>>,
signal: Arc<super::core::WorkerSignal>,
) {
let mut backoff = MIN_BACKOFF;
let mut retry_not_before: Option<Instant> = None;
loop {
let pause = {
let Some(core) = weak.upgrade() else { return };
let mut sealed_one = false;
let backing_off = retry_not_before.is_some_and(|at| Instant::now() < at);
if !backing_off && core.fence_reason().is_none() {
match core.seal_oldest_with_retry(Some(backoff)) {
Ok(sealed) => {
backoff = MIN_BACKOFF;
retry_not_before = None;
sealed_one = sealed;
}
Err(_) if core.shut_down.load(Ordering::Acquire) => {}
Err(DbError::CollectionFenced { .. }) => {}
Err(e) => {
tracing::warn!(collection = core.meta.name, error = %e, retry_in = ?backoff, "seal failed");
retry_not_before = Instant::now().checked_add(backoff);
backoff = (backoff * 2).min(MAX_BACKOFF);
}
}
}
let (result, skeletons_remain) = core.round();
if let Err(e) = result {
tracing::warn!(collection = core.meta.name, error = %e, "maintenance failed");
}
round_pause(
sealed_one || skeletons_remain,
retry_not_before,
Instant::now(),
)
};
if signal.wait(pause) {
return;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::DbError;
use crate::seg::config::SegConfig;
use crate::seg::fs::{FsOpKind, TraceFs};
use crate::seg::manifest::{MANIFEST_NAME, load};
use crate::seg::testkit::{
TestClock, fill_core as fill, key_of, open_core, open_core_with_worker, try_open_core,
value,
};
use std::time::Duration;
#[test]
fn the_worker_builds_pending_skeletons_newest_first() {
let dir = tempfile::tempdir().unwrap();
crate::seg::testkit::build_legacy_collection(dir.path(), 3);
let mut config = roomy();
config.resident_metadata_bytes = Some(0);
let core = open_core(dir.path(), config);
assert_eq!(core.stats().skeletons_pending, 3);
let view = core.view();
assert!(!core.tick().unwrap(), "no skeleton work remains");
assert_eq!(core.stats().skeletons_pending, 0);
assert!(Arc::ptr_eq(&view, &core.view()), "no view publication");
assert_eq!(*sync::lock(&core.shared.built), vec![3, 2, 1]);
assert_eq!(core.stats().skeleton_builds_foreground, 0);
assert_eq!(core.stats().hot_segments, 0);
let threaded_dir = tempfile::tempdir().unwrap();
crate::seg::testkit::build_legacy_collection(threaded_dir.path(), 3);
let mut config = roomy();
config.resident_metadata_bytes = Some(0);
let threaded = open_core(threaded_dir.path(), config);
assert_eq!(threaded.stats().skeletons_pending, 3);
threaded.start_worker().unwrap();
let deadline = Instant::now() + Duration::from_secs(10);
while threaded.stats().skeletons_pending != 0 && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(10));
}
threaded.stop_worker();
assert_eq!(threaded.stats().skeletons_pending, 0);
assert_eq!(*sync::lock(&threaded.shared.built), vec![3, 2, 1]);
}
#[test]
fn a_skeleton_pass_stops_at_its_budget_and_reports_remaining_work() {
let dir = tempfile::tempdir().unwrap();
crate::seg::testkit::build_legacy_collection(dir.path(), 3);
let mut config = roomy();
config.resident_metadata_bytes = Some(0);
let core = open_core(dir.path(), config);
for (built, remains) in [(vec![3], true), (vec![3, 2], true), (vec![3, 2, 1], false)] {
let (result, more) = core.build_pending_skeletons(Duration::ZERO);
result.unwrap();
assert_eq!(more, remains);
assert_eq!(*sync::lock(&core.shared.built), built);
}
let (result, more) = core.build_pending_skeletons(Duration::ZERO);
result.unwrap();
assert!(!more);
assert_eq!(core.stats().skeleton_builds_foreground, 0);
}
#[test]
fn a_skeleton_pass_skips_failed_builds_and_returns_the_first_error() {
let dir = tempfile::tempdir().unwrap();
crate::seg::testkit::build_legacy_collection(dir.path(), 4);
let mut config = roomy();
config.resident_metadata_bytes = Some(0);
let core = open_core(dir.path(), config);
for segment in [&core.view().sealed[1], &core.view().sealed[2]] {
std::fs::OpenOptions::new()
.write(true)
.open(dir.path().join("seg").join(sidx_name(
segment.record().segment_id,
segment.record().version,
)))
.unwrap()
.set_len(0)
.unwrap();
}
let (result, more) = core.build_pending_skeletons(Duration::from_secs(60));
assert!(result.is_err());
assert!(!more, "failed builds wait for their backoff");
assert_eq!(*sync::lock(&core.shared.built), vec![4, 1]);
assert_eq!(core.stats().skeleton_build_failures, 2);
}
#[test]
fn the_worker_does_not_sleep_while_work_remains() {
let now = Instant::now();
assert_eq!(round_pause(true, None, now), Duration::ZERO);
assert_eq!(round_pause(false, None, now), IDLE_PAUSE);
let soon = now + Duration::from_millis(300);
assert_eq!(
round_pause(false, Some(soon), now),
Duration::from_millis(300)
);
assert_eq!(round_pause(true, Some(soon), now), Duration::ZERO);
assert_eq!(
round_pause(false, Some(now), now + Duration::from_secs(5)),
Duration::ZERO
);
}
#[test]
fn a_failing_skeleton_does_not_block_others() {
let dir = tempfile::tempdir().unwrap();
crate::seg::testkit::build_legacy_collection(dir.path(), 3);
let mut config = roomy();
config.resident_metadata_bytes = Some(0);
let core = open_core(dir.path(), config);
let middle = core.view().sealed[1].clone();
std::fs::OpenOptions::new()
.write(true)
.open(dir.path().join("seg").join(sidx_name(
middle.record().segment_id,
middle.record().version,
)))
.unwrap()
.set_len(0)
.unwrap();
assert!(core.tick().is_err());
assert_eq!(core.stats().skeletons_pending, 1, "Failed is not ready");
assert_eq!(core.stats().skeleton_build_failures, 1);
assert!(!core.tick().unwrap(), "backoff hides the failed build");
assert_eq!(core.stats().skeleton_build_failures, 1);
assert_eq!(*sync::lock(&core.shared.built), vec![3, 1]);
for segment in [&core.view().sealed[0], &core.view().sealed[2]] {
let id = segment.record().segment_id;
let key = key_of(id);
assert_eq!(
segment
.lookup(key_hash(&key, crate::seg::testkit::SEED), &key, &core.cache)
.unwrap(),
crate::seg::segment::Lookup::Live(crate::seg::testkit::value_of(id))
);
}
assert_eq!(
core.get(&key_of(3)).unwrap(),
Some(crate::seg::testkit::value_of(3))
);
assert!(matches!(core.get(&key_of(2)), Err(DbError::Io(_))));
assert!(
matches!(core.get(&key_of(1)), Err(DbError::Io(_))),
"newer failed layer is never absence"
);
assert_eq!(
core.stats().skeleton_build_failures,
1,
"backoff avoids repeated builds"
);
}
#[test]
fn shutdown_stops_background_skeleton_work_but_readers_can_build() {
let dir = tempfile::tempdir().unwrap();
crate::seg::testkit::build_legacy_collection(dir.path(), 1);
let mut config = roomy();
config.resident_metadata_bytes = Some(0);
let core = open_core(dir.path(), config);
core.stop_worker();
core.tick().unwrap();
assert_eq!(core.stats().skeletons_pending, 1);
assert_eq!(
core.get(&key_of(1)).unwrap(),
Some(crate::seg::testkit::value_of(1))
);
assert_eq!(core.stats().skeletons_pending, 0);
assert_eq!(core.stats().skeleton_builds_foreground, 1);
}
struct ShutdownFs {
inner: Arc<dyn crate::seg::fs::SegFs>,
core: std::sync::Mutex<Option<Weak<SegCore<crate::seg::testkit::TestKey>>>>,
point: &'static str,
gates: Option<(Arc<std::sync::Barrier>, Arc<std::sync::Barrier>)>,
writes: std::sync::atomic::AtomicUsize,
sdat_named: std::sync::atomic::AtomicBool,
}
impl ShutdownFs {
fn stop_at(&self, point: &str, path: &std::path::Path) {
if self.point == point && path.to_string_lossy().ends_with(".sdat.tmp") {
let weak = self.core.lock().unwrap().take();
if let Some(core) = weak.and_then(|core| core.upgrade()) {
if let Some((entered, release)) = &self.gates {
entered.wait();
release.wait();
} else {
core.stop_worker();
}
}
}
}
}
impl crate::seg::fs::SegFs for ShutdownFs {
fn create(&self, path: &std::path::Path) -> std::io::Result<std::fs::File> {
self.inner.create(path)
}
fn create_new(&self, path: &std::path::Path) -> std::io::Result<std::fs::File> {
self.inner.create_new(path)
}
fn write_all(
&self,
out: &mut dyn std::io::Write,
path: &std::path::Path,
bytes: &[u8],
) -> std::io::Result<()> {
if path.to_string_lossy().ends_with(".sdat.tmp")
&& self.writes.fetch_add(1, Ordering::Relaxed) == 1
{
self.stop_at("last_shard", path);
}
self.inner.write_all(out, path, bytes)
}
fn sync_file(&self, file: &std::fs::File, path: &std::path::Path) -> std::io::Result<()> {
self.stop_at("finish", path);
self.inner.sync_file(file, path)
}
fn rename(&self, from: &std::path::Path, to: &std::path::Path) -> std::io::Result<()> {
if to.to_string_lossy().ends_with(".sdat") {
self.sdat_named.store(true, Ordering::Relaxed);
}
self.inner.rename(from, to)
}
fn sync_dir(&self, dir: &std::path::Path) -> std::io::Result<()> {
self.inner.sync_dir(dir)
}
fn create_dir_all(&self, dir: &std::path::Path) -> std::io::Result<()> {
self.inner.create_dir_all(dir)
}
fn remove_file(&self, path: &std::path::Path) -> std::io::Result<()> {
self.inner.remove_file(path)
}
fn remove_dir_all(&self, dir: &std::path::Path) -> std::io::Result<()> {
self.inner.remove_dir_all(dir)
}
}
fn roomy() -> SegConfig {
SegConfig {
seal_max_keys: 1_000_000,
max_frozen_generations: 64,
..SegConfig::test()
}
}
#[test]
fn by_key_seal_merges_bounded_shard_batches_and_reopens() {
use crate::seg::testkit::{TestClock, open_raw_tree_core, raw_ordered_key};
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1000);
let core = open_raw_tree_core(dir.path(), roomy(), clock.clone(), &[4, 8]);
for n in (0..3000).rev() {
core.put(&raw_ordered_key(n), b"tree").unwrap();
}
core.seal().unwrap();
core.seal_oldest().unwrap();
assert!(core.verify_all().unwrap().is_clean());
let view = core.view();
assert_eq!(view.sealed[0].record().order, OrderKind::ByKey);
let mut keys = Vec::new();
for b in 0..view.sealed[0].block_count().unwrap() {
let payload = view.sealed[0].block_payload(b, None).unwrap();
for entry in crate::seg::block::EntryIter::new(&payload, 32) {
keys.push(entry.unwrap().key.to_vec());
}
}
assert_eq!(keys.len(), 3000);
assert!(keys.windows(2).all(|w| w[0] < w[1]));
drop(view);
drop(core);
let core = open_raw_tree_core(dir.path(), roomy(), clock, &[16]);
for n in 0..3000 {
assert_eq!(
core.get(&raw_ordered_key(n)).unwrap(),
Some(b"tree".to_vec())
);
}
}
#[test]
fn sealing_turns_a_generation_into_a_segment() {
let dir = tempfile::tempdir().unwrap();
let core = open_core(dir.path(), roomy());
fill(&core, 0..300);
core.put(&key_of(5), b"newer").unwrap();
core.seal().unwrap();
assert!(core.seal_oldest().unwrap());
assert!(!core.seal_oldest().unwrap(), "nothing is left to seal");
let view = core.view();
assert!(view.frozen.is_empty());
assert_eq!(view.sealed.len(), 1);
assert_eq!(view.sealed[0].record().entries, 300);
assert_eq!(view.sealed[0].record().min_gsn, 1);
assert_eq!(view.sealed[0].record().max_gsn, 301);
assert_eq!(view.sealed[0].props().typ_hash, 0xcafe);
assert_eq!(view.sealed[0].props().value_version, 3);
drop(view);
assert!(!dir.path().join("gen/0000000000000001").exists());
assert_eq!(core.get(&key_of(5)).unwrap(), Some(b"newer".to_vec()));
for i in 6..300 {
assert_eq!(core.get(&key_of(i)).unwrap(), Some(value(i)));
}
assert_eq!(core.len(), 300);
let manifest = load(dir.path()).unwrap().unwrap();
assert_eq!(manifest.generations.len(), 1);
assert_eq!(manifest.segments.len(), 1);
assert_eq!(manifest.segments[0].segment_id, 1);
assert_eq!(manifest.next_segment_id, 2);
assert_eq!(manifest.gsn_floor, 301);
}
#[test]
fn tombstones_are_dropped_only_when_nothing_is_below() {
let dir = tempfile::tempdir().unwrap();
let core = open_core(dir.path(), roomy());
fill(&core, 0..100);
core.delete(&key_of(1)).unwrap();
core.seal().unwrap();
core.seal_oldest().unwrap();
assert_eq!(core.view().sealed[0].record().tombstones, 0);
assert_eq!(core.view().sealed[0].record().entries, 99);
core.delete(&key_of(2)).unwrap();
core.seal().unwrap();
core.seal_oldest().unwrap();
let view = core.view();
assert_eq!(
view.sealed[0].record().tombstones,
1,
"it masks the first segment"
);
assert_eq!(core.get(&key_of(1)).unwrap(), None);
assert_eq!(core.get(&key_of(2)).unwrap(), None);
assert_eq!(core.get(&key_of(3)).unwrap(), Some(value(3)));
}
#[test]
fn a_generation_that_leaves_nothing_leaves_no_segment() {
let dir = tempfile::tempdir().unwrap();
let core = open_core(dir.path(), roomy());
core.delete(&key_of(1)).unwrap();
core.delete(&key_of(2)).unwrap();
core.seal().unwrap();
assert!(core.seal_oldest().unwrap());
let view = core.view();
assert!(view.frozen.is_empty() && view.sealed.is_empty());
let manifest = load(dir.path()).unwrap().unwrap();
assert!(manifest.segments.is_empty());
assert_eq!(manifest.gsn_floor, 2);
assert_eq!(
std::fs::read_dir(dir.path().join("seg")).unwrap().count(),
0
);
}
#[test]
fn generations_are_sealed_oldest_first() {
let dir = tempfile::tempdir().unwrap();
let core = open_core(dir.path(), roomy());
fill(&core, 0..10);
core.seal().unwrap();
fill(&core, 10..20);
core.seal().unwrap();
assert_eq!(core.view().frozen.len(), 2);
core.seal_oldest().unwrap();
let view = core.view();
assert_eq!(view.frozen.len(), 1);
assert_eq!(view.frozen[0].id(), 2, "generation 1 went first");
assert_eq!(view.sealed[0].record().max_gsn, 10);
}
#[test]
fn a_failing_seal_blocks_the_generations_behind_it() {
let dir = tempfile::tempdir().unwrap();
let fs = TraceFs::new();
let core = try_open_core(dir.path(), roomy(), fs.clone(), &TestClock::new(1)).unwrap();
fill(&core, 0..10);
core.seal().unwrap();
fill(&core, 10..20);
core.seal().unwrap();
fs.fail_on(FsOpKind::CreateNew, ".sdat.tmp", 0);
assert!(core.seal_oldest().is_err());
let view = core.view();
assert_eq!(view.frozen.len(), 2);
assert!(view.sealed.is_empty());
let status = core.seal_status();
assert_eq!(status.frozen, 2);
assert!(status.last_error.is_some());
assert_eq!(status.in_flight, None);
assert!(core.seal_oldest().unwrap());
assert_eq!(core.view().frozen[0].id(), 2);
assert_eq!(core.view().sealed[0].id(), 2);
assert!(core.seal_status().last_error.is_none());
}
#[test]
fn a_reader_keeps_its_layers_across_a_seal() {
let dir = tempfile::tempdir().unwrap();
let core = open_core(dir.path(), roomy());
fill(&core, 0..50);
core.seal().unwrap();
let held = core.view();
let generation_dir = held.frozen[0].dir().to_path_buf();
core.seal_oldest().unwrap();
assert!(
generation_dir.exists(),
"a reader still holds the generation"
);
let key = key_of(7);
let h = crate::seg::hash::key_hash(&key, core.seed);
assert!(matches!(
held.frozen[0].lookup(h, &key).unwrap(),
crate::seg::generation::GenLookup::Live(_)
));
drop(held);
assert!(!generation_dir.exists());
}
#[test]
fn a_seal_that_was_not_published_leaves_no_file() {
let dir = tempfile::tempdir().unwrap();
let fs = TraceFs::new();
let core = try_open_core(dir.path(), roomy(), fs.clone(), &TestClock::new(1)).unwrap();
fill(&core, 0..50);
core.seal().unwrap();
fs.fail_on(FsOpKind::Rename, MANIFEST_NAME, 1);
assert!(core.seal_oldest().is_err());
assert_eq!(
std::fs::read_dir(dir.path().join("seg")).unwrap().count(),
0
);
assert_eq!(core.view().frozen.len(), 1);
assert!(dir.path().join("gen/0000000000000001").exists());
core.check_writable().unwrap();
assert!(core.seal_oldest().unwrap());
assert_eq!(core.view().sealed[0].id(), 2);
}
#[test]
fn an_unknown_outcome_at_seal_fences_and_keeps_everything() {
let dir = tempfile::tempdir().unwrap();
let fs = TraceFs::new();
let core = try_open_core(dir.path(), roomy(), fs.clone(), &TestClock::new(1)).unwrap();
fill(&core, 0..50);
core.seal().unwrap();
fs.fail_on(FsOpKind::SyncDir, "", 2);
assert!(matches!(
core.seal_oldest(),
Err(DbError::CollectionFenced { .. })
));
assert_eq!(
std::fs::read_dir(dir.path().join("seg")).unwrap().count(),
2
);
assert!(dir.path().join("gen/0000000000000001").exists());
assert!(core.seal_status().fenced.is_some());
for i in 0..50 {
assert_eq!(core.get(&key_of(i)).unwrap(), Some(value(i)));
}
drop(core);
let core = open_core(dir.path(), roomy());
assert_eq!(core.view().sealed.len(), 1);
for i in 0..50 {
assert_eq!(core.get(&key_of(i)).unwrap(), Some(value(i)));
}
}
#[test]
fn a_seal_makes_room_for_writers() {
let dir = tempfile::tempdir().unwrap();
let config = SegConfig {
seal_max_keys: 5,
max_frozen_generations: 1,
seal_wait_timeout: Duration::from_millis(50),
..SegConfig::test()
};
let fs = TraceFs::new();
let core = try_open_core(dir.path(), config, fs.clone(), &TestClock::new(1)).unwrap();
fill(&core, 0..10);
assert!(matches!(
core.put(&key_of(10), b"v"),
Err(DbError::SealBackpressure { .. })
));
for i in 10..13 {
fs.fail_on(FsOpKind::CreateNew, ".sdat.tmp", 0);
assert!(core.seal_oldest().is_err());
assert!(matches!(
core.put(&key_of(i), b"v"),
Err(DbError::SealBackpressure { .. })
));
assert_eq!(core.get(&key_of(i)).unwrap(), None);
assert_eq!(core.view().active.keys(), 5);
assert_eq!(core.view().frozen.len(), 1);
}
assert!(core.seal_oldest().unwrap());
core.put(&key_of(10), b"v").unwrap();
assert_eq!(core.get(&key_of(10)).unwrap(), Some(b"v".to_vec()));
}
#[test]
fn the_segment_is_durable_before_the_manifest_names_it() {
let dir = tempfile::tempdir().unwrap();
let fs = TraceFs::new();
let core = try_open_core(dir.path(), roomy(), fs.clone(), &TestClock::new(1)).unwrap();
fill(&core, 0..50);
core.seal().unwrap();
fs.clear();
core.seal_oldest().unwrap();
let ops = fs.ops();
let publications: Vec<usize> = ops
.iter()
.enumerate()
.filter(|(_, op)| op.kind() == FsOpKind::Rename && op.target().ends_with(MANIFEST_NAME))
.map(|(i, _)| i)
.collect();
assert_eq!(publications.len(), 2);
let first_file = fs.position(FsOpKind::CreateNew, ".sdat.tmp").unwrap();
let seg_dir = ops
.iter()
.position(|op| op.kind() == FsOpKind::SyncDir && op.target().ends_with("seg"))
.unwrap();
assert!(publications[0] < first_file, "the id is reserved first");
assert!(
seg_dir < publications[1],
"the files are durable before they are named"
);
}
#[test]
fn an_aged_generation_is_frozen_by_the_tick() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1_000_000);
let config = SegConfig {
seal_max_age: Duration::from_secs(3600),
maintenance_interval: Duration::from_secs(60),
..roomy()
};
let core = try_open_core(dir.path(), config, crate::seg::fs::real(), &clock).unwrap();
clock.advance(2 * 3600 * 1000);
core.tick().unwrap();
assert!(
core.view().frozen.is_empty(),
"an empty generation is never frozen"
);
core.put(&key_of(1), b"v").unwrap();
core.tick().unwrap();
assert!(core.view().frozen.is_empty(), "the interval has not passed");
clock.advance(61_000);
core.tick().unwrap();
assert_eq!(core.view().frozen.len(), 1);
}
#[test]
fn the_worker_seals_in_the_background() {
let dir = tempfile::tempdir().unwrap();
let core = open_core_with_worker(dir.path(), roomy());
fill(&core, 0..500);
core.seal().unwrap();
assert!(core.wait_sealed(Duration::from_secs(20)).unwrap());
let status = core.seal_status();
assert_eq!(status.frozen, 0);
assert!(status.last_sealed_at_ms.is_some());
assert_eq!(core.view().sealed.len(), 1);
for i in 0..500 {
assert_eq!(core.get(&key_of(i)).unwrap(), Some(value(i)));
}
}
#[test]
fn a_dropped_collection_can_be_opened_again_at_once() {
let dir = tempfile::tempdir().unwrap();
let core = open_core_with_worker(dir.path(), roomy());
fill(&core, 0..10);
drop(core);
let core = open_core_with_worker(dir.path(), roomy());
assert_eq!(core.get(&key_of(3)).unwrap(), Some(value(3)));
}
#[test]
fn waiting_for_a_seal_that_never_comes_times_out() {
let dir = tempfile::tempdir().unwrap();
let core = open_core(dir.path(), roomy());
fill(&core, 0..10);
core.seal().unwrap();
assert!(!core.wait_sealed(Duration::from_millis(50)).unwrap());
}
fn assert_shutdown_aborts(point: &'static str, output_named: bool) {
let dir = tempfile::tempdir().unwrap();
let fs = Arc::new(ShutdownFs {
inner: crate::seg::fs::real(),
core: std::sync::Mutex::new(None),
point,
gates: None,
writes: std::sync::atomic::AtomicUsize::new(0),
sdat_named: std::sync::atomic::AtomicBool::new(false),
});
let config = SegConfig {
shard_count: 1,
..roomy()
};
let core =
try_open_core(dir.path(), config.clone(), fs.clone(), &TestClock::new(1)).unwrap();
fill(&core, 0..3000);
core.seal().unwrap();
*fs.core.lock().unwrap() = Some(Arc::downgrade(&core));
assert!(core.seal_oldest().is_err());
assert!(fs.core.lock().unwrap().is_none(), "shutdown hook ran");
assert!(core.shut_down.load(Ordering::Acquire));
assert_eq!(
fs.sdat_named.load(Ordering::Relaxed),
output_named,
"a shutdown before finish aborts before any fsync or rename"
);
assert_eq!(core.view().frozen.len(), 1);
assert!(core.view().sealed.is_empty());
assert!(load(dir.path()).unwrap().unwrap().segments.is_empty());
assert_eq!(
std::fs::read_dir(dir.path().join("seg")).unwrap().count(),
0
);
assert!(dir.path().join("gen/0000000000000001").exists());
core.put(&key_of(4000), b"after-stop").unwrap();
assert_eq!(
core.get(&key_of(4000)).unwrap(),
Some(b"after-stop".to_vec())
);
drop(core);
let core = open_core(dir.path(), config);
assert_eq!(core.get(&key_of(2999)).unwrap(), Some(value(2999)));
assert_eq!(
core.get(&key_of(4000)).unwrap(),
Some(b"after-stop".to_vec())
);
assert!(core.seal_oldest().unwrap());
}
#[test]
fn shutdown_during_the_final_shard_preserves_the_frozen_input() {
assert_shutdown_aborts("last_shard", false);
}
#[test]
fn shutdown_during_finish_discards_only_unnamed_output() {
assert_shutdown_aborts("finish", true);
}
#[test]
fn overlapping_seal_attempts_reserve_and_publish_only_once() {
let dir = tempfile::tempdir().unwrap();
let entered = Arc::new(std::sync::Barrier::new(2));
let release = Arc::new(std::sync::Barrier::new(2));
let fs = Arc::new(ShutdownFs {
inner: crate::seg::fs::real(),
core: std::sync::Mutex::new(None),
point: "finish",
gates: Some((entered.clone(), release.clone())),
writes: std::sync::atomic::AtomicUsize::new(0),
sdat_named: std::sync::atomic::AtomicBool::new(false),
});
let core = try_open_core(dir.path(), roomy(), fs.clone(), &TestClock::new(1)).unwrap();
fill(&core, 0..50);
core.seal().unwrap();
*fs.core.lock().unwrap() = Some(Arc::downgrade(&core));
let first = core.clone();
let first = std::thread::spawn(move || first.seal_oldest());
entered.wait();
let attempt_held = sync::try_lock(&core.seal_attempt).is_none();
let manifest_free = sync::try_lock(&core.manifest).is_some();
let second = core.clone();
let second = std::thread::spawn(move || second.seal_oldest());
release.wait();
assert!(first.join().unwrap().unwrap());
let second_result = second.join().unwrap();
assert!(attempt_held, "attempt spans output I/O");
assert!(manifest_free);
assert!(!second_result.unwrap());
assert_eq!(core.view().sealed.len(), 1);
let manifest = load(dir.path()).unwrap().unwrap();
assert_eq!(manifest.next_segment_id, 2);
assert_eq!(manifest.segments.len(), 1);
}
fn three_segments(
dir: &std::path::Path,
clock: &TestClock,
retention: Option<Duration>,
) -> std::sync::Arc<SegCore<crate::seg::testkit::TestKey>> {
let config = SegConfig {
retention,
maintenance_interval: Duration::from_secs(1),
seal_max_age: Duration::from_secs(1_000_000),
..roomy()
};
let core = try_open_core(dir, config, crate::seg::fs::real(), clock).unwrap();
clock.set(1_000);
fill(&core, 0..10);
core.seal().unwrap();
core.seal_oldest().unwrap();
clock.set(2_000);
core.put(&key_of(1), b"newer").unwrap();
core.delete(&key_of(2)).unwrap();
core.seal().unwrap();
core.seal_oldest().unwrap();
clock.set(3_000);
fill(&core, 20..30);
core.seal().unwrap();
core.seal_oldest().unwrap();
assert_eq!(core.view().sealed.len(), 3);
core
}
fn sealed_ids(core: &SegCore<crate::seg::testkit::TestKey>) -> Vec<u64> {
core.view().sealed.iter().map(|s| s.id()).collect()
}
#[test]
fn a_segment_leaves_when_its_retention_has_passed() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let core = three_segments(dir.path(), &clock, Some(Duration::from_secs(10)));
clock.set(10_999);
core.tick().unwrap();
assert_eq!(
sealed_ids(&core),
[3, 2, 1],
"10 s have not passed since 1000"
);
clock.set(13_000);
core.tick().unwrap();
assert_eq!(
sealed_ids(&core),
[3, 2],
"one segment per tick, the oldest first"
);
assert!(!dir.path().join("seg").join(sdat_name(1, 1)).exists());
assert!(!dir.path().join("seg").join(sidx_name(1, 1)).exists());
clock.advance(1_000);
core.tick().unwrap();
assert_eq!(sealed_ids(&core), [3]);
clock.advance(1_000);
core.tick().unwrap();
assert!(sealed_ids(&core).is_empty());
let manifest = load(dir.path()).unwrap().unwrap();
assert!(manifest.segments.is_empty());
assert_eq!(manifest.next_segment_id, 4, "ids are never reused");
assert_eq!(manifest.gsn_floor, 22, "the floor never goes down");
}
#[test]
fn dropping_the_oldest_segment_uncovers_nothing() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let core = three_segments(dir.path(), &clock, None);
assert_eq!(core.get(&key_of(1)).unwrap(), Some(b"newer".to_vec()));
assert_eq!(core.get(&key_of(2)).unwrap(), None);
assert_eq!(core.get(&key_of(3)).unwrap(), Some(value(3)));
assert_eq!(core.drop_segments_before(1_001).unwrap(), 1);
assert_eq!(core.get(&key_of(1)).unwrap(), Some(b"newer".to_vec()));
assert_eq!(core.get(&key_of(2)).unwrap(), None);
assert_eq!(
core.get(&key_of(3)).unwrap(),
None,
"it lived in segment 1 only"
);
assert_eq!(core.get(&key_of(25)).unwrap(), Some(value(25)));
}
#[test]
fn segments_are_dropped_as_a_prefix() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let core = three_segments(dir.path(), &clock, None);
assert_eq!(core.drop_segments_before(1_000).unwrap(), 0);
assert_eq!(core.drop_segments_before(2_500).unwrap(), 2);
assert_eq!(sealed_ids(&core), [3]);
assert_eq!(core.drop_segments_before(u64::MAX).unwrap(), 1);
assert_eq!(core.drop_segments_before(u64::MAX).unwrap(), 0);
}
#[test]
fn without_a_retention_nothing_is_dropped() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let core = three_segments(dir.path(), &clock, None);
clock.set(u64::MAX / 2);
core.tick().unwrap();
assert_eq!(sealed_ids(&core), [3, 2, 1]);
}
#[test]
fn generations_are_never_dropped_by_retention() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let core = three_segments(dir.path(), &clock, None);
fill(&core, 40..50);
core.seal().unwrap();
fill(&core, 50..60);
assert_eq!(core.drop_segments_before(u64::MAX).unwrap(), 3);
assert_eq!(core.view().frozen.len(), 1);
assert_eq!(core.get(&key_of(45)).unwrap(), Some(value(45)));
assert_eq!(core.get(&key_of(55)).unwrap(), Some(value(55)));
}
#[test]
fn a_reader_keeps_a_dropped_segment_until_it_is_done() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let core = three_segments(dir.path(), &clock, None);
let held = core.view();
let oldest = held.sealed.last().unwrap().clone();
assert_eq!(oldest.id(), 1);
core.drop_segments_before(1_001).unwrap();
assert!(
oldest.sdat_path().exists(),
"a reader still holds the segment"
);
let key = key_of(3);
let h = crate::seg::hash::key_hash(&key, core.seed);
assert!(matches!(
oldest.lookup(h, &key, &core.cache).unwrap(),
crate::seg::segment::Lookup::Live(_)
));
let path = oldest.sdat_path().to_path_buf();
drop(oldest);
drop(held);
assert!(!path.exists());
}
#[test]
fn a_drop_that_was_not_published_is_tried_again() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let fs = TraceFs::new();
let config = SegConfig {
retention: Some(Duration::from_secs(1)),
maintenance_interval: Duration::from_secs(1),
..roomy()
};
let core = try_open_core(dir.path(), config, fs.clone(), &clock).unwrap();
clock.set(1_000);
fill(&core, 0..10);
core.seal().unwrap();
core.seal_oldest().unwrap();
clock.set(10_000);
fs.fail_on(FsOpKind::Rename, MANIFEST_NAME, 0);
assert!(core.tick().is_err());
assert_eq!(sealed_ids(&core), [1]);
assert!(dir.path().join("seg").join(sdat_name(1, 1)).exists());
core.check_writable().unwrap();
clock.advance(1_000);
core.tick().unwrap();
assert!(sealed_ids(&core).is_empty());
}
#[test]
fn a_fenced_collection_drops_nothing() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let core = three_segments(dir.path(), &clock, Some(Duration::from_secs(1)));
core.fence("test".into());
assert!(matches!(
core.drop_segments_before(u64::MAX),
Err(DbError::CollectionFenced { .. })
));
clock.set(1_000_000);
core.tick().unwrap();
assert_eq!(sealed_ids(&core), [3, 2, 1]);
}
#[test]
fn an_indeterminate_retention_drop_fences_and_preserves_the_old_view() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let fs = TraceFs::new();
let core = try_open_core(dir.path(), roomy(), fs.clone(), &clock).unwrap();
clock.set(1_000);
fill(&core, 0..10);
core.seal().unwrap();
core.seal_oldest().unwrap();
let held = core.view();
let oldest = &held.sealed[0];
let data = oldest.sdat_path().to_path_buf();
let index = dir.path().join("seg").join(sidx_name(1, 1));
fs.clear();
fs.fail_on(FsOpKind::SyncDir, "", 0);
let error = core.drop_segments_before(1_001).unwrap_err();
assert!(matches!(error, DbError::CollectionFenced { .. }));
assert!(error.to_string().contains("injected fault at SyncDir"));
let ops = fs.ops();
assert!(
ops.iter()
.any(|op| op.kind() == FsOpKind::Rename && op.target().ends_with(MANIFEST_NAME))
);
assert!(
!ops.iter().any(|op| op.kind() == FsOpKind::SyncDir),
"failed fsync is not recorded as successful"
);
assert!(
load(dir.path()).unwrap().unwrap().segments.is_empty(),
"rename made the new manifest visible"
);
assert_eq!(sealed_ids(&core), [1], "old View stays published");
assert!(matches!(
core.put(&key_of(99), b"fenced"),
Err(DbError::CollectionFenced { .. })
));
assert!(matches!(
core.delete(&key_of(3)),
Err(DbError::CollectionFenced { .. })
));
let key = key_of(3);
let h = crate::seg::hash::key_hash(&key, core.seed);
assert!(matches!(
oldest.lookup(h, &key, &core.cache).unwrap(),
crate::seg::segment::Lookup::Live(_)
));
assert_eq!(core.get(&key).unwrap(), Some(value(3)));
assert!(data.exists() && index.exists());
drop(held);
drop(core);
assert!(
data.exists() && index.exists(),
"indeterminate drop never dooms files"
);
let reopened = open_core(dir.path(), roomy());
assert!(reopened.view().sealed.is_empty());
assert_eq!(reopened.get(&key).unwrap(), None);
assert!(
!data.exists() && !index.exists(),
"recovery removes unnamed segment files"
);
reopened.put(&key_of(99), b"after-reopen").unwrap();
assert_eq!(
reopened.get(&key_of(99)).unwrap(),
Some(b"after-reopen".to_vec())
);
}
#[test]
fn a_retention_longer_than_the_clock_range_never_wraps() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let retention = Duration::from_secs((u64::MAX / 1_000) + 1);
let core = three_segments(dir.path(), &clock, Some(retention));
clock.set(13_000);
core.tick().unwrap();
assert_eq!(sealed_ids(&core), [3, 2, 1]);
clock.set(u64::MAX);
core.tick().unwrap();
assert_eq!(sealed_ids(&core), [3, 2, 1]);
}
#[test]
fn an_automatic_failure_records_retry_before_a_manual_success() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1_000);
let fs = TraceFs::new();
let core = try_open_core(dir.path(), roomy(), fs.clone(), &clock).unwrap();
fill(&core, 0..10);
core.seal().unwrap();
fs.fail_on(FsOpKind::CreateNew, ".sdat.tmp", 0);
assert!(core.seal_oldest_with_retry(Some(MIN_BACKOFF)).is_err());
let status = core.seal_status();
assert!(status.last_error.is_some());
assert_eq!(
status.retry_at_ms,
Some(2_000),
"retry is recorded inside the attempt"
);
assert!(core.seal_oldest().unwrap());
let status = core.seal_status();
assert!(status.last_error.is_none());
assert_eq!(
status.retry_at_ms, None,
"manual success clears the retry permanently"
);
}
#[test]
fn a_seal_age_longer_than_the_clock_range_never_wraps() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let config = SegConfig {
seal_max_age: Duration::from_secs((u64::MAX / 1_000) + 1),
maintenance_interval: Duration::from_secs(1),
..roomy()
};
let core = try_open_core(dir.path(), config, crate::seg::fs::real(), &clock).unwrap();
fill(&core, 0..10);
clock.set(13_000);
core.tick().unwrap();
assert!(core.view().frozen.is_empty());
}
#[test]
fn a_maintenance_interval_longer_than_the_clock_range_never_wraps() {
let dir = tempfile::tempdir().unwrap();
let clock = TestClock::new(1);
let config = SegConfig {
retention: Some(Duration::from_secs(1)),
maintenance_interval: Duration::from_secs((u64::MAX / 1_000) + 1),
..roomy()
};
let core = try_open_core(dir.path(), config, crate::seg::fs::real(), &clock).unwrap();
fill(&core, 0..10);
core.seal().unwrap();
core.seal_oldest().unwrap();
core.tick().unwrap();
clock.set(13_000);
core.tick().unwrap();
assert_eq!(sealed_ids(&core), [1]);
}
}