use std::fs;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use crate::sync::{self, Mutex, MutexGuard};
use crate::disk_loc::DiskLoc;
use crate::entry::{self, make_tombstone_gsn, serialize_entry};
use crate::error::DbResult;
use crate::file_id_floor::FileIdAllocator;
use crate::io::aligned_buf::AlignedBuf;
use crate::io::direct;
#[cfg(all(feature = "encryption", feature = "test-faults"))]
pub mod failpoints {
use std::sync::atomic::{AtomicU8, Ordering};
pub const AFTER_KEY_RECORD_SYNC: u8 = 1;
pub static STAGE: AtomicU8 = AtomicU8::new(0);
pub(crate) fn after_key_record_sync() {
if STAGE.load(Ordering::SeqCst) == AFTER_KEY_RECORD_SYNC {
std::process::exit(9);
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum MinSeq {
Empty,
Known(u64),
Unknown,
}
impl MinSeq {
pub(crate) const fn floor(self) -> u64 {
match self {
Self::Empty => u64::MAX,
Self::Known(sequence) => sequence,
Self::Unknown => 0,
}
}
pub(crate) fn observe(&mut self, sequence: u64) {
*self = match *self {
Self::Empty => Self::Known(sequence),
Self::Known(current) => Self::Known(current.min(sequence)),
Self::Unknown => Self::Unknown,
};
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TombSeq {
None,
Known { min: u64, max: u64 },
Unknown,
}
impl TombSeq {
pub(crate) const fn min(self) -> Option<u64> {
match self {
Self::Known { min, .. } => Some(min),
Self::None | Self::Unknown => Option::None,
}
}
pub(crate) const fn max(self) -> Option<u64> {
match self {
Self::Known { max, .. } => Some(max),
Self::None | Self::Unknown => Option::None,
}
}
fn observe(&mut self, sequence: u64) {
*self = match *self {
Self::None => Self::Known {
min: sequence,
max: sequence,
},
Self::Known { min, max } => Self::Known {
min: min.min(sequence),
max: max.max(sequence),
},
Self::Unknown => Self::Unknown,
};
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct FileSeqs {
pub(crate) min: MinSeq,
pub(crate) tombstones: TombSeq,
}
impl FileSeqs {
pub(crate) const EMPTY: Self = Self {
min: MinSeq::Empty,
tombstones: TombSeq::None,
};
pub(crate) const UNKNOWN: Self = Self {
min: MinSeq::Unknown,
tombstones: TombSeq::Unknown,
};
pub(crate) const fn floor(self) -> u64 {
self.min.floor()
}
pub(crate) fn observe(&mut self, sequence: u64, tombstone: bool) {
self.min.observe(sequence);
if tombstone {
self.tombstones.observe(sequence);
}
}
}
pub(crate) type FileSeqsMap = std::collections::HashMap<u32, FileSeqs>;
#[cfg(feature = "encryption")]
use crate::crypto::{FileKeys, KeyDomain};
#[cfg(feature = "encryption")]
use crate::io::keys::{
KeyFile, KeyRecord, keys_path_for_data, load_active_file_keys, load_file_keys,
};
#[cfg(feature = "encryption")]
use crate::io::tags::{self, TagFile};
#[cfg(feature = "encryption")]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum EncryptedFlushMode {
NonForce,
ForceAdvance,
}
pub(crate) struct WriteBuffer {
buf: AlignedBuf,
len: usize,
base_offset: u64,
pending_pad: u64,
}
impl WriteBuffer {
fn new(capacity: usize, base_offset: u64) -> Self {
Self {
buf: AlignedBuf::zeroed(capacity),
len: 0,
base_offset,
pending_pad: 0,
}
}
pub(crate) fn base_offset(&self) -> u64 {
self.base_offset
}
fn append(&mut self, data: &[u8]) -> u64 {
let offset = self.base_offset + self.len as u64;
self.buf[self.len..self.len + data.len()].copy_from_slice(data);
self.len += data.len();
offset
}
pub(crate) fn read(&self, file_offset: u64, len: usize) -> Option<&[u8]> {
if file_offset >= self.base_offset {
let start = (file_offset - self.base_offset) as usize;
if start + len <= self.len {
return Some(&self.buf[start..start + len]);
}
}
None
}
fn is_full(&self, needed: usize) -> bool {
self.buf.len() - self.len < needed
}
#[inline]
fn capacity(&self) -> usize {
self.buf.len()
}
fn data(&self) -> &[u8] {
&self.buf[..self.len]
}
fn reset(&mut self, new_base: u64) {
debug_assert_eq!(
self.pending_pad, 0,
"reset would discard pending padding and risk cross-file misattribution"
);
self.len = 0;
self.base_offset = new_base;
}
fn note_padding(&mut self, original: usize, target: usize) {
self.pending_pad += (target - original) as u64;
}
fn take_pending_pad(&mut self) -> u64 {
std::mem::take(&mut self.pending_pad)
}
fn compact(&mut self, flushed_bytes: usize) {
let remainder = self.len - flushed_bytes;
if remainder > 0 {
self.buf.copy_within(flushed_bytes..self.len, 0);
}
self.base_offset += flushed_bytes as u64;
self.len = remainder;
}
}
pub struct Shard {
pub id: u8,
dir: PathBuf,
gsn: Arc<AtomicU64>,
pub(crate) compacting: AtomicBool,
#[cfg(feature = "replication")]
replication_overflow: Arc<AtomicU64>,
durable_recovered_gsn: AtomicU64,
inner: Mutex<ShardInner>,
}
struct SyncClock {
opened_at: Instant,
#[cfg(test)]
test: Option<Arc<TestSyncClockState>>,
}
impl SyncClock {
fn new() -> Self {
Self {
opened_at: Instant::now(),
#[cfg(test)]
test: TEST_SYNC_CLOCK
.lock()
.expect("test sync clock poisoned")
.clone(),
}
}
fn now_millis(&self) -> u64 {
#[cfg(test)]
if let Some(test) = &self.test {
test.reads.fetch_add(1, Ordering::Relaxed);
return test.millis.load(Ordering::Relaxed);
}
self.opened_at
.elapsed()
.as_millis()
.min(u128::from(u64::MAX)) as u64
}
fn since(&self, stamp: u64) -> Duration {
Duration::from_millis(self.now_millis().saturating_sub(stamp))
}
}
#[cfg(test)]
struct TestSyncClockState {
millis: AtomicU64,
reads: AtomicU64,
}
#[cfg(test)]
static TEST_SYNC_CLOCK: std::sync::LazyLock<std::sync::Mutex<Option<Arc<TestSyncClockState>>>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
#[cfg(test)]
pub(crate) struct TestSyncClock {
state: Arc<TestSyncClockState>,
}
#[cfg(test)]
impl TestSyncClock {
pub(crate) fn install() -> Self {
let state = Arc::new(TestSyncClockState {
millis: AtomicU64::new(0),
reads: AtomicU64::new(0),
});
let mut active = TEST_SYNC_CLOCK.lock().expect("test sync clock poisoned");
assert!(active.is_none(), "a test sync clock is already installed");
*active = Some(Arc::clone(&state));
Self { state }
}
pub(crate) fn set_millis(&self, millis: u64) {
self.state.millis.store(millis, Ordering::Relaxed);
}
pub(crate) fn reads(&self) -> u64 {
self.state.reads.load(Ordering::Relaxed)
}
}
#[cfg(test)]
impl Drop for TestSyncClock {
fn drop(&mut self) {
let mut active = TEST_SYNC_CLOCK.lock().expect("test sync clock poisoned");
if active
.as_ref()
.is_some_and(|current| Arc::ptr_eq(current, &self.state))
{
*active = None;
}
}
}
pub struct ShardInner {
pub(crate) active: ActiveFile,
pub(crate) write_buf: WriteBuffer,
pub(crate) immutable: Vec<std::sync::Arc<ImmutableFile>>,
pub(crate) dead_bytes: std::collections::HashMap<u32, u64>,
pub(crate) key_len: Option<usize>,
pub(crate) hints: bool,
pub(crate) file_ids: FileIdAllocator,
pub(crate) max_file_size: u64,
pub(crate) compaction_max_files_per_pass: usize,
pub(crate) last_compaction_output_ids: Vec<u32>,
pub(crate) effective_direct_io: bool,
pub(crate) page_aligned: bool,
gsn: Arc<AtomicU64>,
pending_gsn: Option<u64>,
fenced: Option<u64>,
#[cfg(feature = "replication")]
pub(crate) replication_overflow: Arc<AtomicU64>,
write_gen: Arc<AtomicU64>,
synced_gen: Arc<AtomicU64>,
pub(crate) sync_batch_size: u32,
pub(crate) sync_interval: Option<Duration>,
last_synced_at: Arc<AtomicU64>,
clock: Arc<SyncClock>,
publication: Arc<Mutex<()>>,
#[cfg(all(target_os = "linux", feature = "io-uring"))]
uring_writer: Option<crate::io::uring::UringWriter>,
#[cfg(feature = "encryption")]
pub(crate) key_domain: Option<Arc<KeyDomain>>,
#[cfg(feature = "replication")]
pub(crate) replication_tx: Option<rtrb::Producer<crate::replication::ReplicationEntry>>,
pub(crate) hint_service: Option<Arc<crate::hint_worker::HintService>>,
#[cfg(test)]
pub(crate) test_force_hint_corruption: bool,
#[cfg(test)]
pub(crate) test_hint_gate: Option<Arc<HintGate>>,
}
#[cfg(test)]
pub(crate) struct HintGate {
open: std::sync::Mutex<bool>,
cv: std::sync::Condvar,
}
#[cfg(test)]
impl HintGate {
pub(crate) fn closed() -> Self {
Self {
open: std::sync::Mutex::new(false),
cv: std::sync::Condvar::new(),
}
}
pub(crate) fn wait(&self) {
let mut open = self.open.lock().expect("hint gate poisoned");
while !*open {
open = self.cv.wait(open).expect("hint gate poisoned");
}
}
pub(crate) fn open(&self) {
*self.open.lock().expect("hint gate poisoned") = true;
self.cv.notify_all();
}
}
#[cfg(feature = "encryption")]
fn create_file_keys(
domain: Option<&KeyDomain>,
path: &Path,
) -> DbResult<(Option<KeyFile>, Option<Arc<FileKeys>>)> {
let Some(domain) = domain else {
return Ok((None, None));
};
let (salt, cipher) = domain.new_range()?;
let file = KeyFile::create(
&keys_path_for_data(path),
KeyRecord {
first_page: 0,
salt,
},
)?;
Ok((
Some(file),
Some(Arc::new(FileKeys::single(0, Arc::new(cipher)))),
))
}
pub(crate) struct ActiveFile {
pub(crate) file: Arc<std::fs::File>,
pub(crate) read_file: Arc<std::fs::File>,
pub(crate) file_id: u32,
pub(crate) write_offset: u64,
pub(crate) path: PathBuf,
pub(crate) seqs: FileSeqs,
#[cfg(feature = "encryption")]
pub(crate) tag_file: Option<Arc<TagFile>>,
#[cfg(feature = "encryption")]
pub(crate) keys: Option<Arc<FileKeys>>,
#[cfg(feature = "encryption")]
pub(crate) key_file: Option<KeyFile>,
#[cfg(feature = "encryption")]
pub(crate) range_open: bool,
}
pub(crate) struct ImmutableFile {
pub(crate) file: std::fs::File,
pub(crate) file_id: u32,
#[cfg(feature = "encryption")]
pub(crate) path: PathBuf,
pub(crate) total_bytes: u64,
pub(crate) seqs: FileSeqs,
#[cfg(feature = "encryption")]
pub(crate) tag_file: Option<Arc<TagFile>>,
#[cfg(feature = "encryption")]
pub(crate) keys: Option<Arc<FileKeys>>,
}
#[derive(derive_more::Deref)]
pub struct Shards {
#[deref]
shards: Vec<Shard>,
_lock: fs::File,
}
impl Shards {
pub(crate) fn new(shards: Vec<Shard>, lock: fs::File) -> Self {
Self {
shards,
_lock: lock,
}
}
}
impl Shard {
#[allow(clippy::too_many_arguments)]
pub fn open(
id: u8,
dir: &Path,
max_file_size: u64,
write_buffer_size: usize,
hints: bool,
direct_io: bool,
effective_direct_io: bool,
io_backend: crate::config::IoBackend,
gsn: Arc<AtomicU64>,
) -> DbResult<Self> {
Self::open_with_sync_thresholds(
id,
dir,
max_file_size,
write_buffer_size,
hints,
direct_io,
effective_direct_io,
io_backend,
0,
None,
gsn,
)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn open_with_sync_thresholds(
id: u8,
dir: &Path,
max_file_size: u64,
write_buffer_size: usize,
hints: bool,
direct_io: bool,
effective_direct_io: bool,
io_backend: crate::config::IoBackend,
sync_batch_size: u32,
sync_interval: Option<Duration>,
gsn: Arc<AtomicU64>,
) -> DbResult<Self> {
Self::open_inner(
id,
dir,
max_file_size,
write_buffer_size,
hints,
direct_io,
effective_direct_io,
io_backend,
#[cfg(feature = "encryption")]
None,
sync_batch_size,
sync_interval,
gsn,
)
}
#[cfg(feature = "encryption")]
#[allow(clippy::too_many_arguments)]
pub fn open_encrypted(
id: u8,
dir: &Path,
max_file_size: u64,
write_buffer_size: usize,
hints: bool,
direct_io: bool,
effective_direct_io: bool,
io_backend: crate::config::IoBackend,
key_domain: Option<Arc<KeyDomain>>,
gsn: Arc<AtomicU64>,
) -> DbResult<Self> {
Self::open_encrypted_with_sync_thresholds(
id,
dir,
max_file_size,
write_buffer_size,
hints,
direct_io,
effective_direct_io,
io_backend,
key_domain,
0,
None,
gsn,
)
}
#[cfg(feature = "encryption")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn open_encrypted_with_sync_thresholds(
id: u8,
dir: &Path,
max_file_size: u64,
write_buffer_size: usize,
hints: bool,
direct_io: bool,
effective_direct_io: bool,
io_backend: crate::config::IoBackend,
key_domain: Option<Arc<KeyDomain>>,
sync_batch_size: u32,
sync_interval: Option<Duration>,
gsn: Arc<AtomicU64>,
) -> DbResult<Self> {
Self::open_inner(
id,
dir,
max_file_size,
write_buffer_size,
hints,
direct_io,
effective_direct_io,
io_backend,
key_domain,
sync_batch_size,
sync_interval,
gsn,
)
}
#[allow(clippy::too_many_arguments)]
fn open_inner(
id: u8,
dir: &Path,
max_file_size: u64,
write_buffer_size: usize,
hints: bool,
direct_io: bool,
effective_direct_io: bool,
#[cfg_attr(
not(all(target_os = "linux", feature = "io-uring")),
allow(unused_variables)
)]
io_backend: crate::config::IoBackend,
#[cfg(feature = "encryption")] key_domain: Option<Arc<KeyDomain>>,
sync_batch_size: u32,
sync_interval: Option<Duration>,
gsn: Arc<AtomicU64>,
) -> DbResult<Self> {
fs::create_dir_all(dir)?;
#[cfg(feature = "replication")]
let replication_overflow = Arc::new(AtomicU64::new(0));
let clock = Arc::new(SyncClock::new());
let last_synced_at = Arc::new(AtomicU64::new(0));
let publication = Arc::new(Mutex::new(()));
let mut file_ids: Vec<u32> = Vec::new();
for entry in fs::read_dir(dir)? {
let entry = entry?;
let name = entry.file_name();
let name = name.to_string_lossy();
if name.ends_with(".data")
&& let Ok(id) = name.trim_end_matches(".data").parse::<u32>()
{
file_ids.push(id);
}
}
file_ids.sort();
for entry in fs::read_dir(dir)? {
let entry = entry?;
let name = entry.file_name();
let name = name.to_string_lossy();
if name.ends_with(".data.tmp")
|| name.ends_with(".tags.tmp")
|| name.ends_with(".hint.tmp")
|| name.ends_with(".keys.tmp")
{
match fs::remove_file(entry.path()) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e.into()),
}
}
}
let mut immutable = Vec::new();
#[cfg(feature = "encryption")]
let has_key_domain = key_domain.is_some();
let page_aligned = {
#[cfg(feature = "encryption")]
{
direct_io || key_domain.is_some()
}
#[cfg(not(feature = "encryption"))]
{
direct_io
}
};
if file_ids.is_empty() {
let mut allocator = FileIdAllocator::open(dir, 0)?;
let file_id = allocator.allocate()?;
let path = dir.join(format!("{file_id:06}.data"));
#[cfg(feature = "encryption")]
let tag_file = if has_key_domain {
Some(Arc::new(TagFile::open_write(&tags::tags_path_for_data(
&path,
))?))
} else {
None
};
#[cfg(feature = "encryption")]
let (key_file, keys) = create_file_keys(key_domain.as_deref(), &path)?;
let file = direct::open_serving_write(&path, effective_direct_io)?;
let read_file = Arc::new(direct::open_serving_read(&path, effective_direct_io)?);
direct::sync_dir(dir)?;
let active = ActiveFile {
file: Arc::new(file),
read_file,
file_id,
write_offset: 0,
path,
seqs: FileSeqs::EMPTY,
#[cfg(feature = "encryption")]
tag_file,
#[cfg(feature = "encryption")]
keys,
#[cfg(feature = "encryption")]
key_file,
#[cfg(feature = "encryption")]
range_open: true,
};
#[cfg(all(target_os = "linux", feature = "io-uring"))]
let uring_writer = match io_backend {
crate::config::IoBackend::Uring { sqpoll_idle_ms } => {
use std::os::unix::io::AsRawFd;
let mut w = crate::io::uring::UringWriter::new(sqpoll_idle_ms)?;
w.set_file(active.file.as_raw_fd());
w.set_direct(effective_direct_io);
Some(w)
}
crate::config::IoBackend::Pwrite => None,
};
let write_gen = Arc::new(AtomicU64::new(0));
let synced_gen = Arc::new(AtomicU64::new(0));
return Ok(Self {
id,
dir: dir.to_path_buf(),
gsn: gsn.clone(),
compacting: AtomicBool::new(false),
#[cfg(feature = "replication")]
replication_overflow: replication_overflow.clone(),
durable_recovered_gsn: AtomicU64::new(0),
inner: Mutex::new(ShardInner {
active,
write_buf: WriteBuffer::new(write_buffer_size, 0),
immutable,
dead_bytes: std::collections::HashMap::new(),
key_len: None,
hints,
file_ids: allocator,
max_file_size,
compaction_max_files_per_pass:
crate::config::DEFAULT_COMPACTION_MAX_FILES_PER_PASS,
last_compaction_output_ids: Vec::new(),
effective_direct_io,
page_aligned,
gsn,
#[cfg(feature = "replication")]
replication_overflow: replication_overflow.clone(),
write_gen,
synced_gen,
sync_batch_size,
sync_interval,
last_synced_at: Arc::clone(&last_synced_at),
clock: Arc::clone(&clock),
publication: Arc::clone(&publication),
#[cfg(all(target_os = "linux", feature = "io-uring"))]
uring_writer,
#[cfg(feature = "encryption")]
key_domain,
#[cfg(feature = "replication")]
replication_tx: None,
pending_gsn: None,
fenced: None,
hint_service: None,
#[cfg(test)]
test_force_hint_corruption: false,
#[cfg(test)]
test_hint_gate: None,
}),
});
}
let active_id = file_ids.pop().expect("file_ids is not empty");
for &fid in &file_ids {
let path = dir.join(format!("{fid:06}.data"));
let file = direct::open_serving_read(&path, effective_direct_io)?;
let total_bytes = file.metadata()?.len();
#[cfg(feature = "encryption")]
let tag_file = if has_key_domain {
let tp = tags::tags_path_for_data(&path);
if tp.exists() {
Some(Arc::new(TagFile::open_read(&tp)?))
} else {
None
}
} else {
None
};
#[cfg(feature = "encryption")]
let keys = key_domain
.as_deref()
.map(|d| load_file_keys(d, &path))
.transpose()?
.flatten();
immutable.push(std::sync::Arc::new(ImmutableFile {
file,
file_id: fid,
#[cfg(feature = "encryption")]
path,
total_bytes,
seqs: FileSeqs::UNKNOWN,
#[cfg(feature = "encryption")]
tag_file,
#[cfg(feature = "encryption")]
keys,
}));
}
let active_path = dir.join(format!("{active_id:06}.data"));
let active_file = direct::open_serving_write(&active_path, effective_direct_io)?;
let write_offset = active_file.metadata()?.len();
let active_read = Arc::new(direct::open_serving_read(
&active_path,
effective_direct_io,
)?);
#[cfg(feature = "encryption")]
let tag_file = if has_key_domain {
Some(Arc::new(TagFile::open_write(&tags::tags_path_for_data(
&active_path,
))?))
} else {
None
};
#[cfg(feature = "encryption")]
let (key_file, keys) = match key_domain.as_deref() {
Some(d) => load_active_file_keys(d, &active_path)?,
None => (None, None),
};
let active = ActiveFile {
file: Arc::new(active_file),
read_file: active_read,
file_id: active_id,
write_offset,
path: active_path,
seqs: FileSeqs::UNKNOWN,
#[cfg(feature = "encryption")]
tag_file,
#[cfg(feature = "encryption")]
keys,
#[cfg(feature = "encryption")]
key_file,
#[cfg(feature = "encryption")]
range_open: false,
};
#[cfg(all(target_os = "linux", feature = "io-uring"))]
let uring_writer = match io_backend {
crate::config::IoBackend::Uring { sqpoll_idle_ms } => {
use std::os::unix::io::AsRawFd;
let mut w = crate::io::uring::UringWriter::new(sqpoll_idle_ms)?;
w.set_file(active.file.as_raw_fd());
w.set_direct(effective_direct_io);
Some(w)
}
crate::config::IoBackend::Pwrite => None,
};
let write_gen = Arc::new(AtomicU64::new(1));
let synced_gen = Arc::new(AtomicU64::new(0));
Ok(Self {
id,
dir: dir.to_path_buf(),
gsn: gsn.clone(),
compacting: AtomicBool::new(false),
#[cfg(feature = "replication")]
replication_overflow: replication_overflow.clone(),
durable_recovered_gsn: AtomicU64::new(0),
inner: Mutex::new(ShardInner {
active,
write_buf: WriteBuffer::new(write_buffer_size, write_offset),
immutable,
dead_bytes: std::collections::HashMap::new(),
key_len: None,
hints,
file_ids: FileIdAllocator::open(dir, active_id)?,
max_file_size,
compaction_max_files_per_pass: crate::config::DEFAULT_COMPACTION_MAX_FILES_PER_PASS,
last_compaction_output_ids: Vec::new(),
effective_direct_io,
page_aligned,
gsn,
#[cfg(feature = "replication")]
replication_overflow: replication_overflow.clone(),
write_gen,
synced_gen,
sync_batch_size,
sync_interval,
last_synced_at,
clock,
publication,
#[cfg(all(target_os = "linux", feature = "io-uring"))]
uring_writer,
#[cfg(feature = "encryption")]
key_domain,
#[cfg(feature = "replication")]
replication_tx: None,
pending_gsn: None,
fenced: None,
hint_service: None,
#[cfg(test)]
test_force_hint_corruption: false,
#[cfg(test)]
test_hint_gate: None,
}),
})
}
pub(crate) fn effective_direct_io(&self) -> bool {
self.lock().effective_direct_io
}
#[cfg_attr(not(feature = "replication"), allow(dead_code))]
pub(crate) fn hints_enabled(&self) -> bool {
self.lock().hints
}
pub fn gsn(&self) -> &AtomicU64 {
&self.gsn
}
pub(crate) fn set_hint_service(&self, svc: Arc<crate::hint_worker::HintService>) {
sync::lock(&self.inner).hint_service = Some(svc);
}
pub fn lock(&self) -> MutexGuard<'_, ShardInner> {
sync::lock(&self.inner)
}
#[cfg(test)]
pub(crate) fn test_try_lock(&self) -> bool {
sync::try_lock(&self.inner).is_some()
}
#[cfg(feature = "var-collections")]
pub fn read_block(
&self,
file_id: u32,
block_offset: u64,
) -> DbResult<(crate::io::aligned_buf::AlignedBuf, bool)> {
let refs = {
let inner = sync::lock(&self.inner);
inner.resolve_block_refs(file_id)?
};
pread_decrypt_block(file_id, block_offset, &refs)
}
#[cfg(feature = "var-collections")]
pub fn read_value(&self, file_id: u32, offset: u32, len: usize) -> DbResult<Vec<u8>> {
let refs = {
let inner = sync::lock(&self.inner);
inner.resolve_block_refs(file_id)?
};
let file: &std::fs::File = match (&refs.active_read, &refs.immutable_file) {
(Some(f), _) => f,
(None, Some(arc)) => &arc.file,
(None, None) => return Err(crate::error::DbError::StaleDiskLoc),
};
#[cfg(feature = "encryption")]
if let Some(cipher) = &refs.keys {
let tag_file = refs
.tag_file
.as_ref()
.ok_or(crate::error::DbError::StaleDiskLoc)?;
return crate::io::direct::pread_value_encrypted(
file,
tag_file,
cipher,
file_id,
offset as u64,
len,
);
}
crate::io::direct::pread_value(file, offset as u64, len)
}
pub fn dir(&self) -> &Path {
&self.dir
}
pub fn file_ids(&self) -> Vec<u32> {
let inner = sync::lock(&self.inner);
let mut ids: Vec<u32> = inner.immutable.iter().map(|f| f.file_id).collect();
ids.push(inner.active.file_id);
ids
}
pub fn file_sizes(&self) -> Vec<(u32, u64)> {
let inner = sync::lock(&self.inner);
let mut result: Vec<(u32, u64)> = inner
.immutable
.iter()
.map(|f| (f.file_id, f.total_bytes))
.collect();
result.push((inner.active.file_id, inner.active.write_offset));
result
}
pub fn write_active_hint(&self, key_len: usize) -> DbResult<()> {
let mut inner = sync::lock(&self.inner);
if inner.active.write_offset == 0 {
return Ok(()); }
inner.barrier()?;
let read_file = direct::open_serving_read(&inner.active.path, inner.effective_direct_io)?;
let file_len = inner.active.write_offset;
#[cfg(feature = "encryption")]
let hint_data = if let (Some(cipher), Some(tag_file)) =
(&inner.active.keys, &inner.active.tag_file)
{
match crate::hint::generate_hint_data_dyn_encrypted(
&read_file,
file_len,
key_len,
cipher,
tag_file.as_ref(),
inner.active.file_id,
) {
Ok(data) => data,
Err(crate::error::DbError::CorruptedEntry { offset }) => {
tracing::warn!(offset, "hint generation stopped early — skipping hint file");
return Ok(());
}
Err(e) => return Err(e),
}
} else {
match crate::hint::generate_hint_data_dyn(&read_file, file_len, key_len) {
Ok(data) => data,
Err(crate::error::DbError::CorruptedEntry { offset }) => {
tracing::warn!(offset, "hint generation stopped early — skipping hint file");
return Ok(());
}
Err(e) => return Err(e),
}
};
#[cfg(not(feature = "encryption"))]
let hint_data = match crate::hint::generate_hint_data_dyn(&read_file, file_len, key_len) {
Ok(data) => data,
Err(crate::error::DbError::CorruptedEntry { offset }) => {
tracing::warn!(offset, "hint generation stopped early — skipping hint file");
return Ok(());
}
Err(e) => return Err(e),
};
let hint_path = crate::hint::hint_path_for_data(&inner.active.path);
crate::hint::write_hint_file(&hint_path, &hint_data)?;
Ok(())
}
pub fn flush_buf(&self) -> DbResult<()> {
let mut inner = sync::lock(&self.inner);
inner.flush_write_buf()
}
pub fn flush_for_replication_catchup(&self) -> DbResult<()> {
let mut inner = sync::lock(&self.inner);
inner.flush_write_buf_final()
}
pub fn flush(&self) -> DbResult<()> {
let barrier = {
let mut inner = crate::sync::lock(&self.inner);
if inner.write_gen.load(Ordering::Relaxed) == inner.synced_gen.load(Ordering::Acquire) {
crate::flush_pass::record_skip();
return Ok(());
}
inner.barrier_prepare()?
};
barrier.complete()
}
pub(crate) fn is_dirty(&self) -> bool {
let inner = sync::lock(&self.inner);
inner.write_gen.load(Ordering::Relaxed) != inner.synced_gen.load(Ordering::Acquire)
}
#[cfg(test)]
pub(crate) fn prepare_barrier_for_test(&self) -> DbResult<Barrier> {
crate::sync::lock(&self.inner).barrier_prepare()
}
#[cfg(test)]
pub(crate) fn generations_for_test(&self) -> (u64, u64) {
let inner = crate::sync::lock(&self.inner);
(
inner.write_gen.load(Ordering::Relaxed),
inner.synced_gen.load(Ordering::Acquire),
)
}
#[cfg(test)]
pub(crate) fn last_synced_at_for_test(&self) -> u64 {
sync::lock(&self.inner)
.last_synced_at
.load(Ordering::Acquire)
}
#[cfg(test)]
pub(crate) fn set_last_synced_at_for_test(&self, stamp: u64) {
sync::lock(&self.inner)
.last_synced_at
.store(stamp, Ordering::Release);
}
}
impl Shard {
#[cfg(feature = "replication")]
pub(crate) fn catchup_snapshot(
&self,
) -> DbResult<crate::replication::snapshot::ShardCatchupSnapshot> {
use crate::replication::snapshot::{CatchupFile, ShardCatchupSnapshot};
let mut inner = sync::lock(&self.inner);
inner.barrier()?;
let max_gsn = self.gsn.load(Ordering::Relaxed).saturating_sub(1);
let overflow_baseline = self.replication_overflow.load(Ordering::Relaxed);
let mut files = Vec::with_capacity(inner.immutable.len() + 1);
files.extend(inner.immutable.iter().cloned().map(CatchupFile::immutable));
files.push(CatchupFile::active(
inner.active.file_id,
inner.active.write_offset,
inner.active.read_file.clone(),
#[cfg(feature = "encryption")]
inner.active.tag_file.clone(),
#[cfg(feature = "encryption")]
inner.active.keys.clone(),
));
Ok(ShardCatchupSnapshot::new(
self.id,
max_gsn,
overflow_baseline,
files,
))
}
pub fn active_file_id(&self) -> u32 {
sync::lock(&self.inner).active.file_id
}
#[cfg(feature = "replication")]
pub fn set_replication_producer(
&self,
producer: rtrb::Producer<crate::replication::ReplicationEntry>,
) {
sync::lock(&self.inner).replication_tx = Some(producer);
}
#[cfg(feature = "replication")]
pub fn replication_overflow(&self) -> u64 {
self.replication_overflow.load(Ordering::Relaxed)
}
pub fn set_durable_recovered_gsn(&self, gsn: u64) {
self.durable_recovered_gsn.store(gsn, Ordering::Relaxed);
}
pub fn durable_recovered_gsn(&self) -> u64 {
self.durable_recovered_gsn.load(Ordering::Relaxed)
}
}
impl ShardInner {
#[allow(dead_code)] pub(crate) fn arm_gsn(&mut self, gsn: u64) {
debug_assert!(self.pending_gsn.is_none(), "GSN arming is single-shot");
self.pending_gsn = Some(gsn);
}
pub(crate) fn clear_armed_gsn(&mut self) {
self.pending_gsn = None;
}
pub(crate) fn fence(&mut self, gsn: u64) -> u64 {
match self.fenced {
Some(current) if current <= gsn => current,
_ => {
self.fenced = Some(gsn);
gsn
}
}
}
#[cfg(test)]
pub(crate) fn armed_gsn(&self) -> Option<u64> {
self.pending_gsn
}
}
impl Shard {
pub(crate) fn set_key_len(&self, key_len: usize) {
sync::lock(&self.inner).key_len = Some(key_len);
}
#[cfg(feature = "segmented")]
pub(crate) fn clear_key_len(&self) {
sync::lock(&self.inner).key_len = None;
}
pub(crate) fn apply_recovery_tail(&self, tail: &crate::recovery::ActiveTail) -> DbResult<()> {
let mut inner = sync::lock(&self.inner);
if inner.active.file_id != tail.file_id {
return Err(crate::error::DbError::Client(
"apply_recovery_tail: file_id mismatch",
));
}
if inner.page_aligned {
const PAGE_SIZE: u64 = 4096;
#[cfg_attr(not(feature = "encryption"), allow(unused_mut))]
let mut sealed_pages = tail.last_valid_offset.div_ceil(PAGE_SIZE);
#[cfg(feature = "encryption")]
if inner.key_domain.is_some() {
let data_len = inner.active.file.metadata()?.len();
sealed_pages = sealed_pages.max(data_len.div_ceil(PAGE_SIZE));
if let Some(tag_file) = inner.active.tag_file.as_ref() {
let tag_len = tag_file.byte_len()?;
sealed_pages =
sealed_pages.max(tag_len.div_ceil(crate::crypto::TAG_LEN as u64));
}
}
let new_base = sealed_pages * PAGE_SIZE;
inner.write_buf.reset(new_base);
inner.active.write_offset = new_base;
return Ok(());
}
inner.active.file.set_len(tail.last_valid_offset)?;
inner.write_buf.reset(tail.last_valid_offset);
inner.active.write_offset = tail.last_valid_offset;
inner
.write_gen
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
Ok(())
}
pub(crate) fn install_dead_bytes(&self, dead_bytes: std::collections::HashMap<u32, u64>) {
let mut inner = sync::lock(&self.inner);
inner.dead_bytes = dead_bytes;
}
pub(crate) fn install_recovery_file_seqs(&self, file_seqs: FileSeqsMap) {
let mut inner = sync::lock(&self.inner);
inner.active.seqs = file_seqs
.get(&inner.active.file_id)
.copied()
.unwrap_or(FileSeqs::UNKNOWN);
for file in &mut inner.immutable {
debug_assert!(
Arc::get_mut(file).is_some(),
"recovery installs summaries before immutable files are shared"
);
if let Some(file) = Arc::get_mut(file) {
file.seqs = file_seqs
.get(&file.file_id)
.copied()
.unwrap_or(FileSeqs::UNKNOWN);
}
}
}
pub(crate) fn set_compaction_max_files_per_pass(&self, cap: usize) {
sync::lock(&self.inner).compaction_max_files_per_pass = cap;
}
}
impl Drop for Shard {
fn drop(&mut self) {
let inner = sync::lock(&self.inner);
let key_len = inner.key_len;
let hints = inner.hints;
drop(inner);
if hints && let Some(kl) = key_len {
if let Err(e) = self.write_active_hint(kl) {
tracing::error!(shard_id = self.id, "failed to write hint on drop: {e}");
if let Err(e2) = self.flush() {
tracing::error!(shard_id = self.id, "fallback flush also failed: {e2}");
}
}
return;
}
if let Err(e) = self.flush() {
tracing::error!(shard_id = self.id, "failed to flush shard on drop: {e}");
}
}
}
pub(crate) struct Barrier {
generation: u64,
file: Arc<std::fs::File>,
#[cfg(feature = "encryption")]
tag_file: Option<Arc<TagFile>>,
synced_gen: Arc<AtomicU64>,
last_synced_at: Arc<AtomicU64>,
clock: Arc<SyncClock>,
publication: Arc<Mutex<()>>,
}
impl Barrier {
pub(crate) fn complete(self) -> DbResult<()> {
#[cfg(test)]
crate::flush_test::wait_data_sync_gate();
#[cfg(test)]
crate::flush_test::note_data_attempt();
let result = (|| {
#[cfg(test)]
if crate::flush_test::take_fail_data() {
return Err(crate::flush_test::injected_sync_err("data"));
}
crate::io::direct::fsync(&self.file)?;
#[cfg(test)]
crate::flush_test::note_data_success();
#[cfg(feature = "encryption")]
if let Some(tag_file) = &self.tag_file {
tag_file.sync()?;
}
Ok(())
})();
match result {
Ok(()) => {
let _publication = sync::lock(&self.publication);
let previous = self.synced_gen.fetch_max(self.generation, Ordering::AcqRel);
if previous < self.generation {
#[cfg(test)]
crate::flush_test::wait_barrier_publication_gate(self.generation);
let stamp = self.clock.now_millis().saturating_add(1);
self.last_synced_at.store(stamp, Ordering::Release);
}
metrics::counter!("armdb.flush.barriers", "result" => "ok").increment(1);
Ok(())
}
Err(error) => {
crate::flush_pass::record_failure();
metrics::counter!("armdb.flush.barriers", "result" => "failed").increment(1);
Err(error)
}
}
}
}
impl ShardInner {
pub(crate) fn barrier_prepare(&mut self) -> DbResult<Barrier> {
crate::flush_pass::record_attempt();
if let Err(error) = self.flush_write_buf_final() {
crate::flush_pass::record_failure();
return Err(error);
}
Ok(Barrier {
generation: self.write_gen.load(Ordering::Relaxed),
file: Arc::clone(&self.active.file),
#[cfg(feature = "encryption")]
tag_file: self.active.tag_file.clone(),
synced_gen: Arc::clone(&self.synced_gen),
last_synced_at: Arc::clone(&self.last_synced_at),
clock: Arc::clone(&self.clock),
publication: Arc::clone(&self.publication),
})
}
pub(crate) fn should_sync(&self) -> bool {
#[cfg(feature = "ctx-gates")]
crate::ctx_gates_sync::record_should_sync_read();
let pending = self
.write_gen
.load(Ordering::Relaxed)
.saturating_sub(self.synced_gen.load(Ordering::Acquire));
if self.sync_batch_size > 0 && pending >= u64::from(self.sync_batch_size) {
return true;
}
self.sync_interval
.is_some_and(|interval| pending > 0 && self.since_last_success() >= interval)
}
fn since_last_success(&self) -> Duration {
let stamp = self.last_synced_at.load(Ordering::Acquire);
self.clock.since(stamp)
}
pub(crate) fn barrier(&mut self) -> DbResult<()> {
self.barrier_prepare()?.complete()
}
pub(crate) fn flush_write_buf(&mut self) -> DbResult<()> {
if self.write_buf.len == 0 {
return Ok(());
}
#[cfg(feature = "encryption")]
if self.key_domain.is_some() {
return self.flush_write_buf_encrypted(EncryptedFlushMode::NonForce);
}
self.flush_write_buf_plain(false)
}
pub(crate) fn flush_write_buf_final(&mut self) -> DbResult<()> {
if self.write_buf.len == 0 {
return Ok(());
}
#[cfg(feature = "encryption")]
if self.key_domain.is_some() {
return self.flush_write_buf_encrypted(EncryptedFlushMode::ForceAdvance);
}
self.flush_write_buf_plain(true)
}
fn flush_write_buf_plain(&mut self, force: bool) -> DbResult<()> {
let data_len = self.write_buf.len;
if data_len == 0 {
return Ok(());
}
if !self.page_aligned {
let data = self.write_buf.data();
let offset = self.write_buf.base_offset;
#[cfg(all(target_os = "linux", feature = "io-uring"))]
match &mut self.uring_writer {
Some(w) => w.write_at(data, offset)?,
None => direct::pwrite_at(&self.active.file, data, offset)?,
}
#[cfg(not(all(target_os = "linux", feature = "io-uring")))]
direct::pwrite_at(&self.active.file, data, offset)?;
let flushed = data.len() as u64;
self.write_buf.reset(offset + flushed);
metrics::counter!("armdb.flush.count").increment(1);
metrics::counter!("armdb.flush.bytes").increment(flushed);
return Ok(());
}
let complete_bytes = (data_len / 4096) * 4096;
let remainder = data_len % 4096;
let base_offset = self.write_buf.base_offset;
let flush_bytes = if force && remainder > 0 {
let target = complete_bytes + 4096;
let original = self.write_buf.len;
for i in original..target {
self.write_buf.buf[i] = 0;
}
self.write_buf.len = target;
self.write_buf.note_padding(original, target);
target
} else {
if complete_bytes == 0 {
return Ok(());
}
complete_bytes
};
let data = &self.write_buf.buf[..flush_bytes];
#[cfg(test)]
if crate::flush_test::take_fail_force_write() {
self.active.write_offset = base_offset + flush_bytes as u64;
return Err(crate::flush_test::injected_sync_err("force-write"));
}
#[cfg(all(target_os = "linux", feature = "io-uring"))]
let res = match &mut self.uring_writer {
Some(w) => w.write_at(data, base_offset),
None => direct::pwrite_at(&self.active.file, data, base_offset),
};
#[cfg(not(all(target_os = "linux", feature = "io-uring")))]
let res = direct::pwrite_at(&self.active.file, data, base_offset);
if let Err(e) = res {
self.active.write_offset = base_offset + flush_bytes as u64;
return Err(e);
}
metrics::counter!("armdb.flush.count").increment(1);
metrics::counter!("armdb.flush.bytes").increment(flush_bytes as u64);
let padding = self.write_buf.take_pending_pad();
if padding > 0 {
self.add_dead_bytes(self.active.file_id, padding);
}
if force {
self.write_buf.reset(base_offset + flush_bytes as u64);
} else {
self.write_buf.compact(complete_bytes);
}
Ok(())
}
#[cfg(feature = "encryption")]
fn check_key_file_writable(&self) -> DbResult<()> {
if self
.active
.key_file
.as_ref()
.is_some_and(KeyFile::is_poisoned)
{
return Err(crate::DbError::Internal("key file poisoned"));
}
Ok(())
}
#[cfg(feature = "encryption")]
pub(crate) fn ensure_write_range(&mut self, first_page: u64) -> DbResult<()> {
self.check_key_file_writable()?;
if self.active.range_open {
return Ok(());
}
let domain = self.key_domain.as_ref().expect("encrypted shard domain");
let first_page = if self.active.keys.as_ref().is_none_or(|keys| keys.is_empty()) {
0
} else {
first_page
};
let (salt, cipher) = domain.new_range()?;
let record = KeyRecord { first_page, salt };
if let Some(file) = self.active.key_file.as_mut() {
file.append(record)?;
} else {
let file = KeyFile::create(&keys_path_for_data(&self.active.path), record)?;
#[cfg(test)]
keyrange_tests::before_key_dir_sync()?;
direct::sync_dir(self.active.path.parent().expect("shard directory"))?;
self.active.key_file = Some(file);
}
#[cfg(feature = "test-faults")]
failpoints::after_key_record_sync();
let cipher = Arc::new(cipher);
self.active.keys = Some(Arc::new(match &self.active.keys {
Some(keys) => keys.with_range(first_page, cipher),
None => FileKeys::single(first_page, cipher),
}));
self.active.range_open = true;
Ok(())
}
#[cfg(feature = "encryption")]
fn flush_write_buf_encrypted(&mut self, mode: EncryptedFlushMode) -> DbResult<()> {
let data_len = self.write_buf.len;
let complete_bytes = (data_len / 4096) * 4096;
let remainder = data_len % 4096;
let force = mode == EncryptedFlushMode::ForceAdvance;
if !self.write_buf.base_offset.is_multiple_of(4096) {
return Err(crate::DbError::EncryptionError(
"unaligned encrypted sealing position".into(),
));
}
let flush_bytes = if force && remainder > 0 {
let target = complete_bytes + 4096;
let original = self.write_buf.len;
for i in original..target {
self.write_buf.buf[i] = 0;
}
self.write_buf.len = target;
self.write_buf.note_padding(original, target);
target
} else {
if complete_bytes == 0 {
return Ok(());
}
complete_bytes
};
let num_pages = flush_bytes / 4096;
let base_offset = self.write_buf.base_offset;
let start_page = base_offset / 4096;
let file_id = self.active.file_id;
self.ensure_write_range(start_page)?;
let keys = self.active.keys.as_ref().expect("write range installed");
let mut encrypted = AlignedBuf::zeroed(flush_bytes);
encrypted.copy_from_slice(&self.write_buf.buf[..flush_bytes]);
let mut tag_list = Vec::with_capacity(num_pages);
for i in 0..num_pages {
let page_start = i * 4096;
let page = &mut encrypted[page_start..page_start + 4096];
let page_number = start_page + i as u64;
let tag = keys
.cipher_for(page_number)?
.encrypt_page(file_id, page_number, page)?;
tag_list.push(tag);
}
#[cfg(test)]
if crate::flush_test::take_fail_force_write() {
self.active.write_offset = base_offset + flush_bytes as u64;
return Err(crate::flush_test::injected_sync_err("force-write"));
}
#[cfg(all(target_os = "linux", feature = "io-uring"))]
let res = match &mut self.uring_writer {
Some(w) => w.write_at(&encrypted, base_offset),
None => direct::pwrite_at(&self.active.file, &encrypted, base_offset),
};
#[cfg(not(all(target_os = "linux", feature = "io-uring")))]
let res = direct::pwrite_at(&self.active.file, &encrypted, base_offset);
if let Err(e) = res {
self.active.write_offset = base_offset + flush_bytes as u64;
return Err(e);
}
#[cfg(test)]
if crate::flush_test::take_fail_encrypted_data_write_after_pwrite() {
self.active.write_offset = base_offset + flush_bytes as u64;
return Err(crate::flush_test::injected_sync_err(
"encrypted-data-write-before-tags",
));
}
if let Some(ref tag_file) = self.active.tag_file
&& let Err(e) = tag_file.write_tags(start_page, &tag_list)
{
self.active.write_offset = base_offset + flush_bytes as u64;
return Err(e);
}
let padding = self.write_buf.take_pending_pad();
if padding > 0 {
self.add_dead_bytes(self.active.file_id, padding);
}
match mode {
EncryptedFlushMode::NonForce => {
self.write_buf.compact(complete_bytes);
}
EncryptedFlushMode::ForceAdvance => {
let new_base = base_offset + flush_bytes as u64;
self.write_buf.reset(new_base);
}
}
metrics::counter!("armdb.flush.count").increment(1);
metrics::counter!("armdb.flush.bytes").increment(flush_bytes as u64);
Ok(())
}
pub fn append_entry(
&mut self,
shard_id: u8,
key: &[u8],
value: &[u8],
tombstone: bool,
) -> DbResult<(DiskLoc, u64)> {
#[cfg(feature = "encryption")]
self.check_key_file_writable()?;
if let Some(gsn) = self.fenced {
return Err(crate::error::DbError::ShardFenced {
shard: shard_id as usize,
gsn,
});
}
let needed = crate::entry::entry_size(key.len(), value.len() as u32) as usize;
if needed > self.write_buf.capacity() {
return Err(crate::error::DbError::Client(
"entry exceeds write_buffer_size",
));
}
let next_offset = self.write_buf.base_offset + self.write_buf.len as u64;
if next_offset + (needed as u64) > self.max_file_size {
self.rotate(shard_id, key.len())?;
}
let gsn = match self.pending_gsn.take() {
Some(reserved) => {
self.gsn.fetch_max(reserved + 1, Ordering::Relaxed);
reserved
}
None => self.gsn.fetch_add(1, Ordering::Relaxed),
};
debug_assert!(
gsn & crate::entry::FLAGS_MASK == 0,
"GSN overflowed into reserved flag bits"
);
crate::entry::guard_gsn(gsn)?;
let buf = serialize_entry(gsn, key, value, tombstone);
if self.write_buf.is_full(buf.len()) {
self.flush_write_buf()?;
if self.write_buf.is_full(buf.len()) {
self.rotate(shard_id, key.len())?;
}
}
let entry_offset = self.write_buf.base_offset + self.write_buf.len as u64;
let header_and_key = size_of::<entry::EntryHeader>() + key.len();
let value_offset = entry_offset + header_and_key as u64;
if value_offset > u32::MAX as u64 {
return Err(crate::error::DbError::Client(
"value_offset exceeds u32::MAX after rotation check",
));
}
let loc = DiskLoc::new(self.active.file_id, value_offset as u32, value.len() as u32);
let appended_at = self.write_buf.append(&buf);
debug_assert_eq!(appended_at, entry_offset);
self.active.seqs.observe(gsn, tombstone);
#[cfg(feature = "replication")]
if let Some(tx) = &mut self.replication_tx
&& tx
.push(crate::replication::ReplicationEntry {
data: buf,
key_len: key.len() as u16,
})
.is_err()
{
self.replication_overflow.fetch_add(1, Ordering::Relaxed);
}
self.active.write_offset = self.write_buf.base_offset + self.write_buf.len as u64;
self.write_gen
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let actual_gsn = if tombstone {
make_tombstone_gsn(gsn)
} else {
gsn
};
Ok((loc, actual_gsn))
}
fn rotate(&mut self, _shard_id: u8, key_len: usize) -> DbResult<()> {
metrics::counter!("armdb.rotation").increment(1);
tracing::debug!(shard_id = _shard_id, "shard file rotation");
let logical_end = self.write_buf.base_offset + self.write_buf.len as u64;
self.barrier()?;
let old_file_id = self.active.file_id;
let old_path = self.active.path.clone();
let old_seqs = self.active.seqs;
let file_len = logical_end;
let imm_file = direct::open_serving_read(&old_path, self.effective_direct_io)?;
#[cfg(feature = "encryption")]
let imm_tag_file = if self.key_domain.is_some() {
let tp = tags::tags_path_for_data(&old_path);
if tp.exists() {
Some(Arc::new(TagFile::open_read(&tp)?))
} else {
None
}
} else {
None
};
let new_file_id = self.allocate_file_id()?;
let dir = old_path.parent().expect("active file has parent dir");
let new_path = dir.join(format!("{new_file_id:06}.data"));
#[cfg(feature = "encryption")]
let new_tag_file = if self.key_domain.is_some() {
Some(Arc::new(TagFile::open_write(&tags::tags_path_for_data(
&new_path,
))?))
} else {
None
};
#[cfg(feature = "encryption")]
let (new_key_file, new_keys) = create_file_keys(self.key_domain.as_deref(), &new_path)?;
let new_file = Arc::new(direct::open_serving_write(
&new_path,
self.effective_direct_io,
)?);
let new_read = Arc::new(direct::open_serving_read(
&new_path,
self.effective_direct_io,
)?);
direct::sync_dir(dir)?;
let old_file = std::mem::replace(&mut self.active.file, new_file);
self.active.path = new_path;
self.active.file_id = new_file_id;
self.active.write_offset = 0;
self.active.seqs = FileSeqs::EMPTY;
self.active.read_file = new_read;
self.write_buf.reset(0);
#[cfg(feature = "encryption")]
let old_tag_file = std::mem::replace(&mut self.active.tag_file, new_tag_file);
#[cfg(feature = "encryption")]
let old_keys = std::mem::replace(&mut self.active.keys, new_keys);
#[cfg(feature = "encryption")]
{
self.active.key_file = new_key_file;
self.active.range_open = true;
}
#[cfg(all(target_os = "linux", feature = "io-uring"))]
if let Some(w) = &mut self.uring_writer {
use std::os::unix::io::AsRawFd;
w.set_file(self.active.file.as_raw_fd());
w.set_direct(self.effective_direct_io);
}
self.immutable.push(std::sync::Arc::new(ImmutableFile {
file: imm_file,
file_id: old_file_id,
#[cfg(feature = "encryption")]
path: old_path.clone(),
total_bytes: file_len,
seqs: old_seqs,
#[cfg(feature = "encryption")]
tag_file: imm_tag_file,
#[cfg(feature = "encryption")]
keys: old_keys,
}));
drop(old_file);
#[cfg(feature = "encryption")]
drop(old_tag_file);
if self.hints
&& let Some(svc) = self.hint_service.clone()
{
let imm = self
.immutable
.last()
.expect("retired file was just pushed into immutable")
.clone();
svc.submit(crate::hint_worker::HintJob {
shard_idx: _shard_id as usize,
file_id: old_file_id,
file_len,
key_len,
data_path: old_path,
#[cfg(test)]
gate: self.test_hint_gate.clone(),
file: imm,
});
return Ok(());
}
if self.hints {
#[cfg(test)]
let force_corrupt = self.test_force_hint_corruption;
#[cfg(not(test))]
let force_corrupt = false;
let imm = self
.immutable
.last()
.expect("retired file was just pushed into immutable");
#[cfg(test)]
if let Some(gate) = self.test_hint_gate.clone() {
gate.wait();
}
let hint_result: DbResult<Vec<u8>> = if force_corrupt {
Err(crate::error::DbError::CorruptedEntry { offset: 0 })
} else {
#[cfg(feature = "encryption")]
{
if let (Some(cipher), Some(tag_file)) = (&imm.keys, &imm.tag_file) {
crate::hint::generate_hint_data_dyn_encrypted(
&imm.file,
file_len,
key_len,
cipher,
tag_file.as_ref(),
old_file_id,
)
} else {
crate::hint::generate_hint_data_dyn(&imm.file, file_len, key_len)
}
}
#[cfg(not(feature = "encryption"))]
{
crate::hint::generate_hint_data_dyn(&imm.file, file_len, key_len)
}
};
match hint_result {
Ok(hint_data) => {
let hint_path = crate::hint::hint_path_for_data(&old_path);
crate::hint::write_hint_file(&hint_path, &hint_data)?;
}
Err(crate::error::DbError::CorruptedEntry { offset }) => {
tracing::warn!(
offset,
shard_id = _shard_id,
"hint generation stopped early — skipping hint file (retired file retained in immutable)"
);
}
Err(e) => return Err(e),
}
}
Ok(())
}
pub fn add_dead_bytes(&mut self, file_id: u32, size: u64) {
*self.dead_bytes.entry(file_id).or_insert(0) += size;
}
pub(crate) fn allocate_file_id(&mut self) -> DbResult<u32> {
self.file_ids.allocate()
}
#[cfg(feature = "replication")]
pub fn append_raw_entry(
&mut self,
shard_id: u8,
key_len: u16,
data: &[u8],
) -> DbResult<(u32, u64)> {
#[cfg(feature = "encryption")]
self.check_key_file_writable()?;
if data.len() > self.write_buf.capacity() {
return Err(crate::error::DbError::Client(
"replicated entry exceeds write_buffer_size",
));
}
let next_offset = self.write_buf.base_offset + self.write_buf.len as u64;
if next_offset + (data.len() as u64) > self.max_file_size {
self.rotate(shard_id, key_len as usize)?;
}
if self.write_buf.is_full(data.len()) {
self.flush_write_buf()?;
if self.write_buf.is_full(data.len()) {
self.rotate(shard_id, key_len as usize)?;
}
}
let file_id = self.active.file_id;
let entry_offset = self.write_buf.append(data);
self.active.write_offset = self.write_buf.base_offset + self.write_buf.len as u64;
self.write_gen
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
Ok((file_id, entry_offset))
}
}
#[cfg(feature = "var-collections")]
struct BlockFileRefs {
active_read: Option<Arc<std::fs::File>>,
immutable_file: Option<Arc<ImmutableFile>>,
settled_end: u64,
#[cfg(feature = "encryption")]
keys: Option<Arc<FileKeys>>,
#[cfg(feature = "encryption")]
tag_file: Option<Arc<TagFile>>,
}
#[cfg(feature = "var-collections")]
fn pread_decrypt_block(
_file_id: u32,
block_offset: u64,
refs: &BlockFileRefs,
) -> DbResult<(AlignedBuf, bool)> {
#[allow(unused_mut)]
let (mut buf, _) = if let Some(file) = &refs.active_read {
direct::pread_block(file, block_offset)?
} else if let Some(arc) = &refs.immutable_file {
direct::pread_block(&arc.file, block_offset)?
} else {
unreachable!()
};
let is_full_block = block_offset + 4096 <= refs.settled_end;
#[cfg(feature = "encryption")]
if let Some(cipher) = &refs.keys {
let page_number = block_offset / 4096;
match &refs.tag_file {
Some(tf) => {
let tag = tf.read_tag(page_number)?;
cipher.cipher_for(page_number)?.decrypt_page(
_file_id,
page_number,
&mut buf,
&tag,
)?;
}
None => {
if refs.active_read.is_some() {
return Err(crate::error::DbError::EncryptionError(
"no tag file for active encrypted file_id".into(),
));
}
return Err(crate::error::DbError::StaleDiskLoc);
}
}
}
Ok((buf, is_full_block))
}
impl ShardInner {
#[cfg(feature = "var-collections")]
fn resolve_block_refs(&self, file_id: u32) -> DbResult<BlockFileRefs> {
if self.active.file_id == file_id {
Ok(BlockFileRefs {
active_read: Some(self.active.read_file.clone()),
immutable_file: None,
settled_end: self.write_buf.base_offset(),
#[cfg(feature = "encryption")]
keys: self.active.keys.clone(),
#[cfg(feature = "encryption")]
tag_file: self.active.tag_file.clone(),
})
} else {
let arc = self
.immutable
.iter()
.find(|f| f.file_id == file_id)
.ok_or(crate::error::DbError::StaleDiskLoc)?
.clone();
let total = arc.total_bytes;
#[cfg(feature = "encryption")]
let tag_file = arc.tag_file.clone();
#[cfg(feature = "encryption")]
let keys = arc.keys.clone();
Ok(BlockFileRefs {
active_read: None,
immutable_file: Some(arc),
settled_end: total,
#[cfg(feature = "encryption")]
keys,
#[cfg(feature = "encryption")]
tag_file,
})
}
}
#[cfg(feature = "var-collections")]
pub(crate) fn read_block_locked(
&self,
file_id: u32,
block_offset: u64,
) -> DbResult<(AlignedBuf, bool)> {
let refs = self.resolve_block_refs(file_id)?;
pread_decrypt_block(file_id, block_offset, &refs)
}
pub(crate) fn read_value_from_disk_locked(&self, loc: &DiskLoc) -> DbResult<Vec<u8>> {
let len = loc.len as usize;
let target_file_id = loc.file_id;
if self.active.file_id == target_file_id {
#[cfg(feature = "encryption")]
if let Some(cipher) = &self.active.keys {
let tag_file = self.active.tag_file.as_ref().ok_or_else(|| {
crate::error::DbError::EncryptionError(
"tag file missing for active encrypted file".to_string(),
)
})?;
return crate::io::direct::pread_value_encrypted(
&self.active.read_file,
tag_file.as_ref(),
cipher,
target_file_id,
loc.offset as u64,
len,
);
}
return crate::io::direct::pread_value(&self.active.read_file, loc.offset as u64, len);
}
let imm = self
.immutable
.iter()
.find(|f| f.file_id == target_file_id)
.ok_or(crate::error::DbError::StaleDiskLoc)?;
#[cfg(feature = "encryption")]
if let Some(cipher) = &imm.keys {
let tag_file = imm
.tag_file
.as_ref()
.ok_or(crate::error::DbError::StaleDiskLoc)?;
return crate::io::direct::pread_value_encrypted(
&imm.file,
tag_file.as_ref(),
cipher,
target_file_id,
loc.offset as u64,
len,
);
}
crate::io::direct::pread_value(&imm.file, loc.offset as u64, len)
}
pub(crate) fn read_value_bytes_locked(&self, loc: &DiskLoc) -> DbResult<Vec<u8>> {
if self.active.file_id == loc.file_id {
if let Some(bytes) = self.write_buf.read(loc.offset as u64, loc.len as usize) {
return Ok(bytes.to_vec());
}
if self.straddles_write_buf(loc) {
return self.read_straddling_value_locked(loc);
}
}
self.read_value_from_disk_locked(loc)
}
pub(crate) fn straddles_write_buf(&self, loc: &DiskLoc) -> bool {
let base = self.write_buf.base_offset;
let start = loc.offset as u64;
let end = start + loc.len as u64;
start < base && end > base
}
pub(crate) fn read_straddling_value_locked(&self, loc: &DiskLoc) -> DbResult<Vec<u8>> {
debug_assert_eq!(self.active.file_id, loc.file_id);
debug_assert!(self.straddles_write_buf(loc));
let base = self.write_buf.base_offset;
let prefix_len = (base - loc.offset as u64) as usize;
let tail_len = loc.len as usize - prefix_len;
let prefix_loc = DiskLoc::new(loc.file_id, loc.offset, prefix_len as u32);
let mut out = self.read_value_from_disk_locked(&prefix_loc)?;
let tail = self
.write_buf
.read(base, tail_len)
.ok_or(crate::error::DbError::StaleDiskLoc)?;
out.extend_from_slice(tail);
Ok(out)
}
}
#[cfg(test)]
#[allow(unused)]
impl Shard {
pub(crate) fn set_next_file_id(&self, id: u32) {
sync::lock(&self.inner).file_ids.set_next(id);
}
pub(crate) fn rotate_active_for_test(&self, key_len: usize) -> DbResult<()> {
sync::lock(&self.inner).rotate(self.id, key_len)
}
}
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use super::*;
use crate::entry::{EntryHeader, entry_size};
use crate::error::DbError;
use std::mem::size_of;
use std::sync::atomic::AtomicU64;
use tempfile::tempdir;
use zerocopy::FromBytes;
fn open_test_shard(dir: &std::path::Path) -> Shard {
let gsn = Arc::new(AtomicU64::new(0));
Shard::open(
0,
dir,
1 << 20,
64 * 1024,
false,
false,
false,
crate::config::IoBackend::default(),
gsn,
)
.expect("open test shard")
}
#[test]
fn file_id_reopen_after_highest_file_is_lost_does_not_reissue_its_id() {
let dir = tempdir().unwrap();
let shard = open_test_shard(dir.path());
shard.rotate_active_for_test(3).unwrap();
shard.rotate_active_for_test(3).unwrap();
assert_eq!(shard.active_file_id(), 3);
drop(shard);
fs::remove_file(dir.path().join("000003.data")).unwrap();
let shard = open_test_shard(dir.path());
shard.rotate_active_for_test(3).unwrap();
assert!(shard.active_file_id() > 3);
}
#[cfg(feature = "encryption")]
#[test]
fn file_id_reopen_after_highest_file_is_lost_does_not_reissue_its_id_encrypted() {
let dir = tempdir().unwrap();
let shard = open_encrypted_test_shard(dir.path());
shard.rotate_active_for_test(3).unwrap();
shard.rotate_active_for_test(3).unwrap();
assert_eq!(shard.active_file_id(), 3);
drop(shard);
fs::remove_file(dir.path().join("000003.data")).unwrap();
fs::remove_file(dir.path().join("000003.tags")).unwrap();
let shard = open_encrypted_test_shard(dir.path());
shard.rotate_active_for_test(3).unwrap();
assert!(shard.active_file_id() > 3);
}
#[test]
fn file_id_empty_shard_reserves_before_creating_first_file() {
let dir = tempdir().unwrap();
let shard = open_test_shard(dir.path());
assert_eq!(shard.active_file_id(), 1);
assert_eq!(crate::file_id_floor::read(dir.path()).unwrap(), 64);
}
fn file_id_open_plain(dir: &Path) -> DbResult<Shard> {
Shard::open(
0,
dir,
1 << 20,
64 * 1024,
false,
false,
false,
crate::config::IoBackend::default(),
Arc::new(AtomicU64::new(0)),
)
}
#[test]
fn file_id_empty_shard_open_fails_without_creating_a_file_when_exhausted() {
let dir = tempdir().unwrap();
crate::file_id_floor::write(dir.path(), u32::MAX - 1).unwrap();
assert!(file_id_open_plain(dir.path()).is_err());
assert!(fs::read_dir(dir.path()).unwrap().all(|entry| {
let path = entry.unwrap().path();
!matches!(
path.extension().and_then(|ext| ext.to_str()),
Some("data" | "tags")
)
}));
}
#[test]
fn file_id_existing_shard_with_exhausted_ceiling_refuses_rotation() {
let dir = tempdir().unwrap();
drop(open_test_shard(dir.path()));
crate::file_id_floor::write(dir.path(), u32::MAX - 1).unwrap();
let shard = open_test_shard(dir.path());
assert!(shard.rotate_active_for_test(3).is_err());
}
#[test]
fn file_id_corrupt_marker_fails_open() {
let dir = tempdir().unwrap();
drop(open_test_shard(dir.path()));
fs::write(dir.path().join("format.file-id"), b"garbage").unwrap();
assert!(matches!(
file_id_open_plain(dir.path()),
Err(DbError::FormatMismatch(_))
));
}
#[cfg(feature = "encryption")]
#[test]
fn file_id_corrupt_marker_fails_open_encrypted() {
let dir = tempdir().unwrap();
drop(open_encrypted_test_shard(dir.path()));
fs::write(dir.path().join("format.file-id"), b"garbage").unwrap();
let result = Shard::open_encrypted(
0,
dir.path(),
1 << 20,
64 * 1024,
false,
false,
false,
crate::config::IoBackend::default(),
Some(Arc::new(
KeyDomain::new(&[0x42; 32], &[0x24; 16], None, 0).unwrap(),
)),
Arc::new(AtomicU64::new(0)),
);
assert!(matches!(result, Err(DbError::FormatMismatch(_))));
}
#[test]
fn file_id_marker_that_is_a_directory_fails_open() {
let dir = tempdir().unwrap();
fs::create_dir(dir.path().join("format.file-id")).unwrap();
assert!(file_id_open_plain(dir.path()).is_err());
}
#[test]
fn file_id_reopen_without_allocation_publishes_nothing() {
let dir = tempdir().unwrap();
drop(open_test_shard(dir.path()));
let marker = dir.path().join("format.file-id");
let bytes = fs::read(&marker).unwrap();
let mtime = fs::metadata(&marker).unwrap().modified().unwrap();
for _ in 0..3 {
drop(open_test_shard(dir.path()));
}
assert_eq!(fs::read(&marker).unwrap(), bytes);
assert_eq!(fs::metadata(&marker).unwrap().modified().unwrap(), mtime);
assert!(!dir.path().join("format.file-id.tmp").exists());
}
#[cfg(feature = "replication")]
#[test]
fn file_id_follower_rotation_reserves_before_issuing() {
let dir = tempdir().unwrap();
let shard = Shard::open(
0,
dir.path(),
4096,
4096,
false,
false,
false,
crate::config::IoBackend::default(),
Arc::new(AtomicU64::new(0)),
)
.unwrap();
shard.set_next_file_id(65);
for k in 1..10_000u64 {
let (id, _) = crate::sync::lock(&shard.inner)
.append_raw_entry(
shard.id,
8,
&crate::entry::serialize_entry(k, &k.to_be_bytes(), &[0u8; 64], false),
)
.unwrap();
if id == 65 {
assert_eq!(crate::file_id_floor::read(dir.path()).unwrap(), 128);
return;
}
}
panic!("follower did not rotate");
}
#[test]
fn min_seq_observe_never_promotes_unknown() {
let mut state = MinSeq::Unknown;
state.observe(41);
assert_eq!(state, MinSeq::Unknown);
}
#[test]
fn tomb_seq_observe_never_promotes_unknown() {
let mut state = TombSeq::Unknown;
state.observe(41);
assert_eq!(state, TombSeq::Unknown);
}
#[cfg(feature = "var-collections")]
#[test]
fn append_records_only_tombstone_sequences() {
let dir = tempfile::tempdir().unwrap();
let shard = open_test_shard(dir.path());
let mut inner = shard.lock();
inner
.append_entry(0, &1u64.to_be_bytes(), &[7; 8], false)
.unwrap();
assert_eq!(inner.active.seqs.tombstones, TombSeq::None);
let (_, first) = inner
.append_entry(0, &2u64.to_be_bytes(), &[], true)
.unwrap();
let (_, last) = inner
.append_entry(0, &3u64.to_be_bytes(), &[], true)
.unwrap();
assert_eq!(
inner.active.seqs.tombstones,
TombSeq::Known {
min: first & crate::entry::SEQUENCE_MASK,
max: last & crate::entry::SEQUENCE_MASK,
}
);
}
#[cfg(feature = "var-collections")]
#[test]
fn append_updates_empty_min_and_preserves_unknown_min() {
let dir = tempfile::tempdir().unwrap();
let shard = open_test_shard(dir.path());
{
let mut inner = shard.lock();
assert_eq!(inner.active.seqs, FileSeqs::EMPTY);
let (_, gsn) = inner
.append_entry(0, &1u64.to_be_bytes(), &[7; 8], false)
.unwrap();
assert_eq!(inner.active.seqs.min, MinSeq::Known(gsn));
inner.active.seqs = FileSeqs::UNKNOWN;
inner
.append_entry(0, &2u64.to_be_bytes(), &[8; 8], false)
.unwrap();
assert_eq!(inner.active.seqs, FileSeqs::UNKNOWN);
}
}
#[cfg(feature = "var-collections")]
#[test]
fn rotation_moves_min_verbatim_and_starts_empty_active() {
let dir = tempfile::tempdir().unwrap();
let shard = open_test_shard(dir.path());
let expected = {
let mut inner = shard.lock();
let (_, gsn) = inner
.append_entry(0, &1u64.to_be_bytes(), &[7; 8], false)
.unwrap();
gsn
};
shard.rotate_active_for_test(8).unwrap();
let inner = shard.lock();
assert_eq!(
inner.immutable.last().unwrap().seqs.min,
MinSeq::Known(expected)
);
assert_eq!(inner.active.seqs, FileSeqs::EMPTY);
}
#[cfg(feature = "replication")]
fn append_test_entry(shard: &Shard, key: u64, value: &[u8]) {
crate::sync::lock(&shard.inner)
.append_entry(shard.id, &key.to_be_bytes(), value, false)
.unwrap();
}
#[cfg(feature = "replication")]
fn shard_with_rotated_entries() -> (tempfile::TempDir, Shard) {
let dir = tempfile::tempdir().unwrap();
let shard = open_test_shard(dir.path());
shard.gsn.store(1, Ordering::Relaxed);
append_test_entry(&shard, 1, b"one");
shard.rotate_active_for_test(8).unwrap();
(dir, shard)
}
#[cfg(feature = "replication")]
fn shard_with_entries_in_active_file() -> (tempfile::TempDir, Shard) {
let dir = tempfile::tempdir().unwrap();
let shard = open_test_shard(dir.path());
shard.gsn.store(1, Ordering::Relaxed);
append_test_entry(&shard, 1, b"one");
(dir, shard)
}
#[cfg(feature = "replication")]
fn rotate_and_append(shard: &Shard) {
shard.rotate_active_for_test(8).unwrap();
append_test_entry(shard, 2, b"two");
}
#[cfg(feature = "encryption")]
fn open_encrypted_test_shard(dir: &std::path::Path) -> Shard {
Shard::open_encrypted(
0,
dir,
1 << 20,
64 * 1024,
false,
false,
false,
crate::config::IoBackend::default(),
Some(Arc::new(
KeyDomain::new(&[0x42; 32], &[0x24; 16], None, 0).unwrap(),
)),
Arc::new(AtomicU64::new(1)),
)
.unwrap()
}
#[cfg(feature = "replication")]
#[test]
fn snapshot_reads_source_after_path_unlink() {
let (dir, shard) = shard_with_rotated_entries();
let snapshot = shard.catchup_snapshot().unwrap();
let source = snapshot.files.first().unwrap().clone();
std::fs::remove_file(dir.path().join("000001.data")).unwrap();
let header = source.read_exact_at(0, size_of::<EntryHeader>()).unwrap();
assert_eq!(EntryHeader::read_from_bytes(&header).unwrap().sequence(), 1);
}
#[cfg(feature = "replication")]
#[test]
fn snapshot_active_boundary_does_not_move_after_rotation() {
let (_dir, shard) = shard_with_entries_in_active_file();
let snapshot = shard.catchup_snapshot().unwrap();
let captured_end = snapshot.files.last().unwrap().readable_end;
rotate_and_append(&shard);
assert_eq!(snapshot.files.last().unwrap().readable_end, captured_end);
assert!(
snapshot
.files
.last()
.unwrap()
.read_exact_at(captured_end, 1)
.is_err()
);
}
#[cfg(all(feature = "encryption", feature = "replication"))]
#[test]
fn encrypted_snapshot_retains_data_and_tag_handles_after_unlink() {
let dir = tempfile::tempdir().unwrap();
let shard = open_encrypted_test_shard(dir.path());
append_test_entry(&shard, 1, b"encrypted");
shard.rotate_active_for_test(8).unwrap();
let snapshot = shard.catchup_snapshot().unwrap();
let source = snapshot.files.first().unwrap();
std::fs::remove_file(dir.path().join("000001.data")).unwrap();
std::fs::remove_file(dir.path().join("000001.tags")).unwrap();
let bytes = source.read_exact_at(0, entry_size(8, 9) as usize).unwrap();
assert_eq!(
EntryHeader::read_from_bytes(&bytes[..16])
.unwrap()
.sequence(),
1
);
}
#[cfg(feature = "var-collections")]
#[test]
fn read_block_returns_stale_for_unknown_file_id() {
let dir = tempdir().unwrap();
let shard = open_test_shard(dir.path());
match shard.read_block(9999, 0) {
Err(DbError::StaleDiskLoc) => {}
Err(e) => panic!("expected StaleDiskLoc, got Err({e})"),
Ok(_) => panic!("expected StaleDiskLoc, got Ok"),
}
}
#[cfg(feature = "var-collections")]
#[test]
fn read_value_returns_stale_for_unknown_file_id() {
let dir = tempdir().unwrap();
let shard = open_test_shard(dir.path());
match shard.read_value(9999, 0, 128) {
Err(DbError::StaleDiskLoc) => {}
Err(e) => panic!("expected StaleDiskLoc, got Err({e})"),
Ok(_) => panic!("expected StaleDiskLoc, got Ok"),
}
}
#[cfg(all(feature = "encryption", feature = "var-collections"))]
#[test]
fn read_block_returns_stale_when_immutable_missing_under_encryption() {
use crate::crypto::KeyDomain;
let dir = tempdir().unwrap();
let gsn = Arc::new(AtomicU64::new(0));
let cipher = Some(Arc::new(
KeyDomain::new(&[0x42; 32], &[0x24; 16], None, 0).expect("create cipher"),
));
let shard = Shard::open_encrypted(
0,
dir.path(),
1 << 20,
64 * 1024,
false,
false,
false,
crate::config::IoBackend::default(),
cipher,
gsn,
)
.expect("open encrypted shard");
match shard.read_block(42, 0) {
Err(DbError::StaleDiskLoc) => {}
Err(e) => panic!("expected StaleDiskLoc, got Err({e})"),
Ok(_) => panic!("expected StaleDiskLoc, got Ok"),
}
}
#[cfg(feature = "var-collections")]
#[test]
fn read_value_from_disk_locked_returns_stale_for_unknown_file_id() {
let dir = tempdir().unwrap();
let shard = open_test_shard(dir.path());
let fake = DiskLoc::new(9999, 0, 0);
let inner = shard.lock();
match inner.read_value_from_disk_locked(&fake) {
Err(DbError::StaleDiskLoc) => {}
Ok(_) => panic!("expected StaleDiskLoc, got Ok"),
Err(e) => panic!("expected StaleDiskLoc, got Err({e})"),
}
}
#[cfg(feature = "var-collections")]
#[test]
fn read_block_with_file_id_above_u16() {
let dir = tempdir().unwrap();
let shard = open_test_shard(dir.path());
shard.set_next_file_id(70_000);
shard.rotate_active_for_test(3).expect("first rotate");
assert!(
shard.active_file_id() >= 70_000,
"active_file_id should be >= 70_000 after rotation"
);
let key = b"abc";
let value = vec![0xABu8; 256];
let (disk, _gsn) = shard
.lock()
.append_entry(0, key, &value, false)
.expect("append entry");
assert!(
disk.file_id > u16::MAX as u32,
"DiskLoc.file_id must be above u16::MAX"
);
shard.flush().expect("flush");
shard.rotate_active_for_test(3).expect("second rotate");
let block_offset = disk.offset as u64 & !4095;
let (block, _) = shard
.read_block(disk.file_id, block_offset)
.expect("read_block");
let start = (disk.offset & 4095) as usize;
let end = start + value.len();
assert_eq!(&block[start..end], &value[..]);
}
#[cfg(feature = "var-collections")]
#[test]
fn write_buffer_read_with_file_id_above_u16() {
let dir = tempdir().unwrap();
let shard = open_test_shard(dir.path());
shard.set_next_file_id(70_000);
shard.rotate_active_for_test(1).expect("rotate");
assert!(shard.active_file_id() >= 70_000);
let value = vec![0x5Au8; 200];
let (disk, _gsn) = shard
.lock()
.append_entry(0, b"k", &value, false)
.expect("append");
assert!(
disk.file_id > u16::MAX as u32,
"DiskLoc.file_id must be above u16::MAX"
);
let inner = shard.lock();
let bytes = inner
.write_buf
.read(disk.offset as u64, disk.len as usize)
.expect("write-buf read must succeed for unflushed entry");
assert_eq!(bytes, &value[..]);
}
#[test]
fn rotate_errors_when_file_id_exhausted() {
let tmp = tempdir().unwrap();
let gsn = Arc::new(AtomicU64::new(1));
let shard = Shard::open(
0,
tmp.path(),
16 * 4096,
8192,
false,
false,
false,
crate::config::IoBackend::default(),
gsn,
)
.unwrap();
shard.set_next_file_id(u32::MAX);
let res = shard.rotate_active_for_test(4);
match res {
Err(DbError::Client(msg)) => {
assert!(
msg.contains("file_id"),
"expected file_id error, got: {msg}"
);
}
other => panic!("expected DbError::Client, got {other:?}"),
}
}
#[test]
fn open_errors_when_active_id_is_max() {
let tmp = tempdir().unwrap();
let path = tmp.path().join(format!("{}.data", u32::MAX));
std::fs::write(&path, b"").unwrap();
let gsn = Arc::new(AtomicU64::new(1));
let res = Shard::open(
0,
tmp.path(),
16 * 4096,
8192,
false,
false,
false,
crate::config::IoBackend::default(),
gsn,
);
match res {
Err(DbError::Client(msg)) => {
assert!(
msg.contains("file_id"),
"expected file_id error, got: {msg}"
);
}
Ok(_) => panic!("expected DbError::Client, got Ok"),
Err(e) => panic!("expected DbError::Client, got Err({e})"),
}
}
#[cfg(all(feature = "encryption", feature = "var-collections"))]
#[test]
fn flush_for_replication_catchup_pads_encrypted_trailing_page() {
use crate::crypto::KeyDomain;
let dir = tempdir().unwrap();
let gsn = Arc::new(AtomicU64::new(0));
let cipher = Some(Arc::new(
KeyDomain::new(&[0xAB; 32], &[0x24; 16], None, 0).expect("create cipher"),
));
let shard = Shard::open_encrypted(
0,
dir.path(),
1 << 20,
64 * 1024,
false,
false,
false,
crate::config::IoBackend::default(),
cipher,
gsn,
)
.expect("open encrypted shard");
{
let mut inner = sync::lock(&shard.inner);
inner
.append_entry(0, b"key", b"value", false)
.expect("append_entry");
}
let data_path = dir.path().join("000001.data");
shard.flush_buf().expect("flush_buf");
let len_after_nonforce = std::fs::metadata(&data_path).expect("metadata").len();
assert_eq!(
len_after_nonforce, 0,
"NonForce encrypted flush must leave partial page in buffer (no whole page yet), got {len_after_nonforce}"
);
shard
.flush_for_replication_catchup()
.expect("flush_for_replication_catchup");
let len_after_force = std::fs::metadata(&data_path).expect("metadata").len();
assert!(
len_after_force > 0,
"ForceAdvance must flush the padded page to disk, got 0 bytes"
);
assert_eq!(
len_after_force % 4096,
0,
"encrypted file length must be 4096-aligned after ForceAdvance, got {len_after_force}"
);
assert_eq!(
len_after_force, 4096,
"expected exactly one 4096-byte padded page, got {len_after_force}"
);
}
#[cfg(feature = "encryption")]
#[test]
fn apply_recovery_tail_encrypted_advances_to_high_watermark() {
use crate::crypto::KeyDomain;
let dir = tempdir().unwrap();
let gsn = Arc::new(AtomicU64::new(0));
let cipher = Some(Arc::new(
KeyDomain::new(&[0xCD; 32], &[0x24; 16], None, 0).unwrap(),
));
let shard = Shard::open_encrypted(
0,
dir.path(),
1 << 20,
64 * 1024,
false,
false,
false,
crate::config::IoBackend::default(),
cipher,
gsn,
)
.expect("open encrypted shard");
{
let mut inner = sync::lock(&shard.inner);
inner
.append_entry(0, b"k1", b"v1", false)
.expect("append k1");
}
shard.flush().expect("flush 1");
{
let mut inner = sync::lock(&shard.inner);
inner
.append_entry(0, b"k2", b"v2", false)
.expect("append k2");
}
shard.flush().expect("flush 2");
let data_len = std::fs::metadata(dir.path().join("000001.data"))
.unwrap()
.len();
assert_eq!(data_len, 8192, "expected two sealed 4096-byte pages");
let tail = crate::recovery::ActiveTail {
shard_idx: 0,
file_id: 1,
last_valid_offset: 4096,
};
shard
.apply_recovery_tail(&tail)
.expect("apply_recovery_tail");
let base = sync::lock(&shard.inner).active.write_offset;
assert_eq!(
base, 8192,
"F6: post-recovery base must clear all sealed pages"
);
}
#[test]
fn reopen_page_aligned_appends_at_page_boundary() {
use crate::recovery::recover_const_tree;
use crate::skiplist::SkipList;
use crate::skiplist::node::ConstNode;
let dir = tempdir().unwrap();
let gsn = Arc::new(AtomicU64::new(0));
const WB: usize = 8192;
{
let shard = Shard::open(
0,
dir.path(),
1 << 20,
WB,
false,
true,
false,
crate::config::IoBackend::default(),
gsn.clone(),
)
.expect("open page-aligned shard");
{
let mut inner = sync::lock(&shard.inner);
inner
.append_entry(0, b"key", b"value", false)
.expect("append");
}
shard.flush().expect("final flush");
}
{
let shard = Shard::open(
0,
dir.path(),
1 << 20,
WB,
false,
true,
false,
crate::config::IoBackend::default(),
gsn.clone(),
)
.expect("reopen page-aligned shard");
let index = SkipList::<ConstNode<[u8; 3], 5>>::new(false);
let outcome = recover_const_tree::<[u8; 3], 5>(
&[dir.path()],
&[0],
&index,
false,
#[cfg(feature = "encryption")]
None,
)
.expect("recover");
assert_eq!(outcome.active_tails.len(), 1);
let tail = &outcome.active_tails[0];
assert!(
tail.last_valid_offset < 4096,
"expected sub-page last_valid_offset, got {}",
tail.last_valid_offset
);
shard
.apply_recovery_tail(tail)
.expect("apply_recovery_tail");
{
let inner = sync::lock(&shard.inner);
assert!(inner.page_aligned);
assert_eq!(
inner.write_buf.base_offset % 4096,
0,
"base_offset must be page-aligned after apply_recovery_tail"
);
}
{
let mut inner = sync::lock(&shard.inner);
inner
.append_entry(0, b"k2!", b"val2!", false)
.expect("append after recovery");
}
shard.flush_buf().expect("flush_buf");
shard.flush().expect("final flush");
}
}
#[test]
fn plain_flush_is_page_granular_when_page_aligned() {
let dir = tempdir().unwrap();
let gsn = Arc::new(AtomicU64::new(0));
let shard = Shard::open(
0,
dir.path(),
1 << 20,
8192,
false,
true,
false,
crate::config::IoBackend::default(),
gsn,
)
.expect("open page-aligned shard");
{
let mut inner = sync::lock(&shard.inner);
inner
.append_entry(0, b"key", b"value", false)
.expect("append_entry");
}
let data_path = dir.path().join("000001.data");
shard.flush_buf().expect("flush_buf");
let len_after_nonforce = std::fs::metadata(&data_path).expect("metadata").len();
assert_eq!(
len_after_nonforce, 0,
"page-aligned NonForce flush must leave partial page in buffer, got {len_after_nonforce}"
);
shard.flush().expect("flush");
let len_after_force = std::fs::metadata(&data_path).expect("metadata").len();
assert_eq!(
len_after_force, 4096,
"page-aligned ForceAdvance must pad trailing page to 4096 bytes, got {len_after_force}"
);
}
#[cfg(feature = "encryption")]
#[test]
fn encrypted_direct_roundtrip() {
use crate::crypto::KeyDomain;
let dir = tempdir().unwrap();
let gsn = Arc::new(AtomicU64::new(0));
let cipher = Some(Arc::new(
KeyDomain::new(&[0xCD; 32], &[0x24; 16], None, 0).expect("create cipher"),
));
let value: Vec<u8> = (0..9000).map(|i| (i % 251) as u8).collect();
let expected = value.clone();
let disk = {
let shard = Shard::open_encrypted(
0,
dir.path(),
1 << 20,
1 << 20,
false,
true,
true,
crate::config::IoBackend::default(),
cipher.clone(),
gsn.clone(),
)
.expect("open encrypted direct shard");
assert!(shard.effective_direct_io());
let (disk, _gsn) = sync::lock(&shard.inner)
.append_entry(0, b"key", &value, false)
.expect("append_entry");
shard.flush().expect("flush");
disk
};
{
let shard = Shard::open_encrypted(
0,
dir.path(),
1 << 20,
1 << 20,
false,
true,
false,
crate::config::IoBackend::default(),
cipher,
gsn,
)
.expect("reopen encrypted direct shard");
let inner = sync::lock(&shard.inner);
let got = inner
.read_value_from_disk_locked(&disk)
.expect("read encrypted value after reopen");
assert_eq!(got, expected);
}
}
#[cfg(feature = "var-collections")]
#[test]
fn flush_for_replication_catchup_plain_mode_flushes() {
let dir = tempdir().unwrap();
let shard = open_test_shard(dir.path());
let expected_entry_size = crate::entry::entry_size(b"k".len(), b"v".len() as u32);
assert_eq!(expected_entry_size, 24);
{
let mut inner = sync::lock(&shard.inner);
inner
.append_entry(0, b"k", b"v", false)
.expect("append_entry");
}
shard
.flush_for_replication_catchup()
.expect("flush_for_replication_catchup plain");
let data_path = dir.path().join("000001.data");
let file_len = std::fs::metadata(&data_path).expect("metadata").len();
assert_eq!(
file_len, expected_entry_size,
"plain mode must write exactly entry bytes (no padding); expected {expected_entry_size}, got {file_len}"
);
}
#[test]
fn rotate_retains_retired_file_when_hint_gen_stops_early() {
let dir = tempdir().unwrap();
let gsn = Arc::new(AtomicU64::new(0));
let shard = Shard::open(
0,
dir.path(),
1 << 20,
64 * 1024,
true, false,
false,
crate::config::IoBackend::default(),
gsn,
)
.expect("open shard with hints");
let value = vec![0xABu8; 128];
let disk = {
let mut inner = sync::lock(&shard.inner);
let (disk, _gsn) = inner.append_entry(0, b"k", &value, false).expect("append");
disk
};
let old_file_id = disk.file_id;
sync::lock(&shard.inner).test_force_hint_corruption = true;
shard.rotate_active_for_test(1).expect("rotate");
let ids = shard.file_ids();
assert!(
ids.contains(&old_file_id),
"retired file {old_file_id} must be retained in immutable, got file_ids {ids:?}"
);
#[cfg(feature = "var-collections")]
{
let inner = shard.lock();
let got = inner
.read_value_from_disk_locked(&disk)
.expect("read retired-file value must succeed, not StaleDiskLoc");
assert_eq!(got, value);
}
}
#[test]
fn rotate_check_accounts_for_force_flush_padding_gap() {
let dir = tempdir().unwrap();
let gsn = Arc::new(AtomicU64::new(0));
const WB: usize = 8192;
const MAX: u64 = 8192; let shard = Shard::open(
0,
dir.path(),
MAX,
WB,
false,
true, false,
crate::config::IoBackend::default(),
gsn,
)
.expect("open page-aligned shard");
{
let mut inner = sync::lock(&shard.inner);
inner
.append_entry(0, b"k", &[0u8; 100], false)
.expect("append A");
}
shard.flush().expect("force flush");
{
let inner = sync::lock(&shard.inner);
assert_eq!(
inner.write_buf.base_offset, 4096,
"force flush must advance base_offset to the next page"
);
assert!(
inner.active.write_offset < 4096,
"logical write_offset must lag behind the padded base_offset (got {})",
inner.active.write_offset
);
}
let value_b = vec![0x11u8; 5000];
let loc_b = {
let mut inner = sync::lock(&shard.inner);
let (loc, _gsn) = inner
.append_entry(0, b"k", &value_b, false)
.expect("append B");
loc
};
assert_eq!(
loc_b.file_id, 2,
"append past max_file_size (accounting for padding gap) must rotate to a new file, \
got file_id {} (offset {})",
loc_b.file_id, loc_b.offset
);
assert!(
loc_b.offset < 4096,
"after rotation the value must land near the start of the new file, got {}",
loc_b.offset
);
}
#[test]
fn rotate_and_first_open_fsync_shard_dir() {
use crate::io::direct::SYNC_DIR_CALLS;
use std::sync::atomic::Ordering;
let dir = tempdir().unwrap();
let gsn = Arc::new(AtomicU64::new(0));
let before_open = SYNC_DIR_CALLS.load(Ordering::Relaxed);
let shard = Shard::open(
0,
dir.path(),
1 << 20,
64 * 1024,
false,
false,
false,
crate::config::IoBackend::default(),
gsn.clone(),
)
.expect("open shard (create branch)");
let after_open = SYNC_DIR_CALLS.load(Ordering::Relaxed);
assert!(
after_open > before_open,
"first open (create branch) must fsync the shard dir"
);
shard.rotate_active_for_test(1).expect("rotate");
let after_rotate = SYNC_DIR_CALLS.load(Ordering::Relaxed);
assert!(
after_rotate > after_open,
"rotate() must fsync the shard dir after creating the new data file"
);
let ids = shard.file_ids();
assert_eq!(
ids,
vec![1, 2],
"expected file 1 (immutable) + file 2 (active)"
);
drop(shard);
let reopened = Shard::open(
0,
dir.path(),
1 << 20,
64 * 1024,
false,
false,
false,
crate::config::IoBackend::default(),
gsn,
)
.expect("reopen shard");
let reopened_ids = reopened.file_ids();
assert!(
reopened_ids.contains(&2),
"reopen must see the rotated-in file 2, got {reopened_ids:?}"
);
}
#[test]
fn tx_armed_gsn_is_used_once_then_cleared() {
let dir = tempfile::tempdir().unwrap();
let shard = Shard::open(
0,
dir.path(),
1 << 20,
64 * 1024,
false,
false,
false,
crate::config::IoBackend::Pwrite,
std::sync::Arc::new(std::sync::atomic::AtomicU64::new(1)),
)
.unwrap();
let mut inner = shard.lock();
inner.arm_gsn(9_000);
let (_loc, gsn) = inner.append_entry(0, b"k0", b"v0", false).unwrap();
assert_eq!(gsn, 9_000, "the armed GSN must be used verbatim");
assert!(inner.armed_gsn().is_none(), "arming is single-shot");
let (_loc, next) = inner.append_entry(0, b"k1", b"v1", false).unwrap();
assert_ne!(next, 9_000, "the next append falls back to the counter");
}
}
#[cfg(test)]
mod append_offset_tests {
use super::*;
use std::sync::Arc;
use std::sync::atomic::AtomicU64;
#[test]
fn rotate_before_append_at_u32_max_boundary() {
let tmp = tempfile::tempdir().unwrap();
let gsn = Arc::new(AtomicU64::new(1));
let shard = Shard::open(
0,
tmp.path(),
u32::MAX as u64,
8192,
false,
false,
false,
crate::config::IoBackend::default(),
gsn,
)
.unwrap();
{
let mut inner = sync::lock(&shard.inner);
inner.active.write_offset = u32::MAX as u64 - 100;
inner.write_buf.reset(u32::MAX as u64 - 100);
}
let key = b"k";
let value = vec![0u8; 200];
let (loc, _gsn) = {
let mut inner = sync::lock(&shard.inner);
inner.append_entry(0, key, &value, false).unwrap()
};
assert_eq!(loc.file_id, 2, "expected rotation to new file");
assert!(loc.offset < 4096, "expected offset near start of new file");
}
}
#[cfg(test)]
#[cfg(feature = "encryption")]
mod keyrange_tests {
use super::*;
use crate::crypto::{FileKeys, KeyDomain};
use crate::io::keys::KeyFile;
std::thread_local! {
static KEY_DIR_SYNC: std::cell::Cell<Option<(bool, usize)>> = const {
std::cell::Cell::new(None)
};
}
pub(super) fn before_key_dir_sync() -> DbResult<()> {
KEY_DIR_SYNC.with(|state| {
let Some((fail_once, attempts)) = state.get() else {
return Ok(());
};
state.set(Some((false, attempts + 1)));
if fail_once {
return Err(std::io::Error::other("injected key directory sync failure").into());
}
Ok(())
})
}
fn domain() -> Arc<KeyDomain> {
Arc::new(KeyDomain::new(&[0x42; 32], &[0x24; 16], None, 0).unwrap())
}
fn open(dir: &Path) -> Shard {
let shard = Shard::open_encrypted(
0,
dir,
1 << 20,
65536,
true,
false,
false,
crate::config::IoBackend::default(),
Some(domain()),
Arc::new(AtomicU64::new(1)),
)
.unwrap();
let (file_id, len) = {
let i = shard.lock();
(i.active.file_id, i.active.file.metadata().unwrap().len())
};
shard
.apply_recovery_tail(&crate::recovery::ActiveTail {
shard_idx: 0,
file_id,
last_valid_offset: len,
})
.unwrap();
shard
}
fn write(shard: &Shard, key: u64) -> DiskLoc {
let mut i = shard.lock();
let loc = i
.append_entry(0, &key.to_be_bytes(), &key.to_le_bytes(), false)
.unwrap()
.0;
i.barrier().unwrap();
loc
}
fn read(shard: &Shard, loc: DiskLoc, value: u64) {
assert_eq!(
shard.lock().read_value_from_disk_locked(&loc).unwrap(),
value.to_le_bytes()
);
}
fn records(dir: &Path) -> Vec<crate::io::keys::KeyRecord> {
KeyFile::open(&dir.join("000001.keys")).unwrap().1
}
#[test]
fn keyrange_inherited_active_gets_one_new_range_and_same_file_id() {
let dir = tempfile::tempdir().unwrap();
let a = {
let s = open(dir.path());
write(&s, 1)
};
let prefix = fs::read(dir.path().join("000001.data")).unwrap();
let b = {
let s = open(dir.path());
assert_eq!(s.active_file_id(), 1);
write(&s, 2)
};
assert_eq!(records(dir.path()).len(), 2);
let bytes = fs::read(dir.path().join("000001.data")).unwrap();
assert_eq!(&bytes[..prefix.len()], &prefix);
assert_eq!(
records(dir.path())[1].first_page,
(prefix.len() / 4096) as u64
);
let s = open(dir.path());
read(&s, a, 1);
read(&s, b, 2);
}
#[test]
fn keyrange_ten_sessions_leave_one_file_and_no_extra_fds() {
let dir = tempfile::tempdir().unwrap();
#[cfg(target_os = "linux")]
let fd_count = || {
fs::read_dir("/proc/self/fd")
.unwrap()
.filter_map(|entry| fs::read_link(entry.ok()?.path()).ok())
.filter(|path| path.starts_with(dir.path()))
.count()
};
#[cfg(target_os = "linux")]
let before = fd_count();
for n in 0..10 {
let s = open(dir.path());
write(&s, n);
}
assert_eq!(records(dir.path()).len(), 10);
assert_eq!(
fs::read_dir(dir.path())
.unwrap()
.filter(|e| e
.as_ref()
.unwrap()
.path()
.extension()
.is_some_and(|e| e == "data"))
.count(),
1
);
#[cfg(target_os = "linux")]
assert_eq!(fd_count(), before);
}
#[test]
fn keyrange_reader_holding_old_keys_survives_range_opening() {
let dir = tempfile::tempdir().unwrap();
let a = {
let s = open(dir.path());
write(&s, 1)
};
let s = open(dir.path());
let old: Arc<FileKeys> = s.lock().active.keys.clone().unwrap();
let b = write(&s, 2);
let i = s.lock();
assert_eq!(
direct::pread_value_encrypted(
&i.active.read_file,
i.active.tag_file.as_ref().unwrap(),
&old,
a.file_id,
a.offset as u64,
a.len as usize
)
.unwrap(),
1u64.to_le_bytes()
);
drop(i);
read(&s, b, 2);
}
#[test]
fn keyrange_read_only_session_appends_nothing() {
let dir = tempfile::tempdir().unwrap();
let mut locs = Vec::new();
{
let s = open(dir.path());
for n in 0..3 {
locs.push(write(&s, n));
if n < 2 {
s.rotate_active_for_test(8).unwrap();
}
}
}
fs::remove_file(dir.path().join("000001.hint")).unwrap();
let before: Vec<_> = (1..=3)
.map(|id| fs::read(dir.path().join(format!("{id:06}.keys"))).unwrap())
.collect();
{
let s = open(dir.path());
for (n, l) in locs.into_iter().enumerate() {
read(&s, l, n as u64);
}
}
for (id, bytes) in (1..=3).zip(before) {
assert_eq!(
fs::read(dir.path().join(format!("{id:06}.keys"))).unwrap(),
bytes
);
}
assert_eq!(
fs::read_dir(dir.path())
.unwrap()
.filter(|e| e
.as_ref()
.unwrap()
.path()
.extension()
.is_some_and(|e| e == "data"))
.count(),
3
);
}
#[test]
fn keyrange_first_write_that_rotates_appends_no_range_to_the_inherited_file() {
let dir = tempfile::tempdir().unwrap();
{
let s = open(dir.path());
write(&s, 1);
}
let bytes = fs::read(dir.path().join("000001.keys")).unwrap();
{
let s = open(dir.path());
s.lock().max_file_size = 4096;
write(&s, 2);
assert_ne!(s.active_file_id(), 1);
}
assert_eq!(fs::read(dir.path().join("000001.keys")).unwrap(), bytes);
}
#[test]
fn keyrange_unaligned_sealing_position_refuses_writes() {
let dir = tempfile::tempdir().unwrap();
let s = open(dir.path());
let mut i = s.lock();
i.write_buf.reset(1);
i.append_entry(0, &1u64.to_be_bytes(), b"value", false)
.unwrap();
assert!(matches!(
i.barrier(),
Err(crate::DbError::EncryptionError(_))
));
assert_eq!(i.active.file.metadata().unwrap().len(), 0);
i.write_buf.reset(0);
}
#[test]
fn keyrange_failed_keys_append_poisons_encrypted_writes_until_reopen() {
let dir = tempfile::tempdir().unwrap();
let a = {
let s = open(dir.path());
write(&s, 1)
};
{
let s = open(dir.path());
s.lock()
.active
.key_file
.as_mut()
.unwrap()
.force_read_only_for_test(&dir.path().join("000001.keys"));
let mut i = s.lock();
i.append_entry(0, &2u64.to_be_bytes(), b"second", false)
.unwrap();
assert!(i.barrier().is_err());
assert!(i.active.key_file.as_ref().unwrap().is_poisoned());
assert!(!i.active.range_open);
i.pending_gsn = Some(100);
#[cfg(feature = "replication")]
let mut receiver = {
let (producer, consumer) = rtrb::RingBuffer::new(4);
i.replication_tx = Some(producer);
consumer
};
let before = (
i.write_buf.buf[..i.write_buf.len].to_vec(),
i.write_buf.base_offset,
i.write_buf.pending_pad,
i.active.write_offset,
i.active.file_id,
i.active.seqs,
i.gsn.load(Ordering::Relaxed),
i.write_gen.load(Ordering::Relaxed),
i.pending_gsn,
);
let append = i.append_entry(0, &3u64.to_be_bytes(), b"refused", false);
#[cfg(feature = "replication")]
let raw_append = {
let raw = serialize_entry(101, &4u64.to_be_bytes(), b"refused raw", false);
i.append_raw_entry(0, 8, &raw)
};
assert!(matches!(
append,
Err(crate::DbError::Internal("key file poisoned"))
));
#[cfg(feature = "replication")]
assert!(matches!(
raw_append,
Err(crate::DbError::Internal("key file poisoned"))
));
assert_eq!(
(
i.write_buf.buf[..i.write_buf.len].to_vec(),
i.write_buf.base_offset,
i.write_buf.pending_pad,
i.active.write_offset,
i.active.file_id,
i.active.seqs,
i.gsn.load(Ordering::Relaxed),
i.write_gen.load(Ordering::Relaxed),
i.pending_gsn,
),
before
);
#[cfg(feature = "replication")]
{
assert!(receiver.pop().is_err());
assert_eq!(i.replication_overflow.load(Ordering::Relaxed), 0);
}
assert!(i.barrier().is_err());
drop(i);
read(&s, a, 1);
}
let s = open(dir.path());
write(&s, 3);
read(&s, a, 1);
}
#[test]
fn keyrange_plaintext_shard_has_no_keys_and_appends_in_place() {
let dir = tempfile::tempdir().unwrap();
for n in 0u64..3 {
let s = Shard::open(
0,
dir.path(),
1 << 20,
65536,
false,
false,
false,
crate::config::IoBackend::default(),
Arc::new(AtomicU64::new(1)),
)
.unwrap();
write(&s, n);
assert_eq!(s.active_file_id(), 1);
}
assert!(!dir.path().join("000001.keys").exists());
}
fn damaged_tail(long_tags: bool) {
let dir = tempfile::tempdir().unwrap();
let a = {
let s = open(dir.path());
write(&s, 1)
};
let data = dir.path().join("000001.data");
if long_tags {
fs::OpenOptions::new()
.write(true)
.open(dir.path().join("000001.tags"))
.unwrap()
.set_len(16 * 4)
.unwrap();
} else {
fs::OpenOptions::new()
.write(true)
.open(&data)
.unwrap()
.set_len(4096 + 7)
.unwrap();
}
let expected = if long_tags { 4 } else { 2 };
let b = {
let s = open(dir.path());
let b = write(&s, 2);
assert_eq!(records(dir.path())[1].first_page, expected);
read(&s, a, 1);
b
};
let s = open(dir.path());
read(&s, a, 1);
read(&s, b, 2);
}
#[test]
fn keyrange_short_data_tail_starts_range_at_recovered_position() {
damaged_tail(false);
}
#[test]
fn keyrange_longer_tag_tail_starts_range_at_recovered_position() {
damaged_tail(true);
}
#[test]
fn keyrange_empty_inherited_data_without_keys_is_read_only_until_write() {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("000001.data"), []).unwrap();
{
let _s = open(dir.path());
assert!(!dir.path().join("000001.keys").exists());
}
{
let s = open(dir.path());
write(&s, 1);
}
assert_eq!(records(dir.path()).len(), 1);
}
#[test]
fn keyrange_empty_data_with_torn_keys_header_opens_and_recreates_it() {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("000001.data"), []).unwrap();
fs::write(dir.path().join("000001.keys"), b"AKY").unwrap();
let loc = {
let s = open(dir.path());
write(&s, 1)
};
assert_eq!(records(dir.path()).len(), 1);
read(&open(dir.path()), loc, 1);
}
#[test]
fn keyrange_first_range_of_empty_data_covers_page_zero_past_a_tag_tail() {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("000001.data"), []).unwrap();
fs::write(dir.path().join("000001.tags"), [0u8; 32]).unwrap();
let loc = {
let s = open(dir.path());
write(&s, 1)
};
assert_eq!(loc.offset / 4096, 2);
assert_eq!(records(dir.path())[0].first_page, 0);
read(&open(dir.path()), loc, 1);
}
#[test]
fn keyrange_lazy_keys_creation_retries_failed_directory_sync_before_seal() {
let dir = tempfile::tempdir().unwrap();
let data_path = dir.path().join("000001.data");
fs::write(&data_path, []).unwrap();
let loc = {
let s = open(dir.path());
let mut i = s.lock();
let loc = i
.append_entry(0, &1u64.to_be_bytes(), &1u64.to_le_bytes(), false)
.unwrap()
.0;
assert!(
i.active.key_file.is_none(),
"buffering must not open a range"
);
KEY_DIR_SYNC.with(|state| state.set(Some((true, 0))));
assert!(i.barrier().is_err());
assert!(!i.active.range_open);
assert!(
i.active.keys.is_none(),
"no cipher installed before publication"
);
assert_eq!(fs::metadata(&data_path).unwrap().len(), 0);
assert_eq!(
fs::metadata(dir.path().join("000001.tags")).unwrap().len(),
0
);
i.barrier().unwrap();
assert_eq!(
KEY_DIR_SYNC.with(|state| state.replace(None)),
Some((false, 2)),
"retry must repeat directory publication before any seal"
);
assert!(i.active.range_open);
assert_eq!(fs::metadata(&data_path).unwrap().len(), 4096);
loc
};
assert_eq!(records(dir.path()).len(), 1);
let s = open(dir.path());
read(&s, loc, 1);
}
#[test]
fn keyrange_sealed_data_missing_keys_or_page_zero_is_refused() {
let dir = tempfile::tempdir().unwrap();
{
let s = open(dir.path());
write(&s, 1);
}
fs::remove_file(dir.path().join("000001.keys")).unwrap();
let try_open = || {
Shard::open_encrypted(
0,
dir.path(),
1 << 20,
65536,
false,
false,
false,
crate::config::IoBackend::default(),
Some(domain()),
Arc::new(AtomicU64::new(1)),
)
};
assert!(matches!(try_open(), Err(crate::DbError::FormatMismatch(_))));
KeyFile::create(
&dir.path().join("000001.keys"),
crate::io::keys::KeyRecord {
first_page: 1,
salt: [3; 16],
},
)
.unwrap();
assert!(matches!(try_open(), Err(crate::DbError::FormatMismatch(_))));
}
#[cfg(feature = "replication")]
#[test]
fn keyrange_follower_raw_append_opens_a_range_without_rotating() {
let dir = tempfile::tempdir().unwrap();
{
let s = open(dir.path());
write(&s, 1);
}
let s = open(dir.path());
let raw = serialize_entry(2, &2u64.to_be_bytes(), b"raw", false);
{
let mut i = s.lock();
i.append_raw_entry(0, 8, &raw).unwrap();
i.barrier().unwrap();
}
assert_eq!(s.active_file_id(), 1);
assert_eq!(records(dir.path()).len(), 2);
}
#[test]
fn keyrange_substituted_shard_salt_and_context_fail_authentication() {
let dir = tempfile::tempdir().unwrap();
let s = open(dir.path());
let loc = write(&s, 1);
let records = records(dir.path());
let context = crate::EncryptionContext::new(b"another-domain").unwrap();
let wrong_domains = [
KeyDomain::new(&[0x42; 32], &[0x24; 16], None, 1).unwrap(),
KeyDomain::new(&[0x42; 32], &[0x25; 16], None, 0).unwrap(),
KeyDomain::new(&[0x42; 32], &[0x24; 16], Some(&context), 0).unwrap(),
];
let i = s.lock();
for wrong in wrong_domains {
let keys = FileKeys::from_records(&wrong, &records).unwrap();
assert!(matches!(
direct::pread_value_encrypted(
&i.active.read_file,
i.active.tag_file.as_ref().unwrap(),
&keys,
loc.file_id,
loc.offset as u64,
loc.len as usize
),
Err(crate::DbError::EncryptionError(_))
));
}
}
#[cfg(feature = "var-collections")]
#[test]
fn keyrange_mixed_range_recovery_with_and_without_hints() {
use crate::VarTree;
let dir = tempfile::tempdir().unwrap();
let mut config = crate::Config::test();
config.shard_count = 1;
config.hints = Some(true);
config.encryption_key = Some([0x42; 32]);
for n in 0u64..3 {
let tree = VarTree::<[u8; 8]>::open(dir.path(), config.clone()).unwrap();
tree.put(&n.to_be_bytes(), &n.to_le_bytes()).unwrap();
tree.close().unwrap();
}
let shard_dir = dir.path().join("shard_000");
assert_eq!(records(&shard_dir).len(), 3);
for hints in [true, false] {
if !hints {
fs::remove_file(shard_dir.join("000001.hint")).unwrap();
}
for cached in [true, false] {
let mut cfg = config.clone();
cfg.block_cache.max_size = if cached { 1024 * 1024 } else { 0 };
let tree = VarTree::<[u8; 8]>::open(dir.path(), cfg).unwrap();
for _ in 0..2 {
for n in 0u64..3 {
assert_eq!(
tree.get(&n.to_be_bytes()).unwrap().as_ref(),
&n.to_le_bytes()
);
}
}
tree.close().unwrap();
if !hints {
fs::remove_file(shard_dir.join("000001.hint")).unwrap();
}
}
}
let salt = crate::salt::read(dir.path()).unwrap();
let domain = Arc::new(KeyDomain::new(&[0x42; 32], &salt, None, 0).unwrap());
let shard = Shard::open_encrypted(
0,
&shard_dir,
1 << 20,
65536,
true,
false,
false,
crate::config::IoBackend::default(),
Some(domain),
Arc::new(AtomicU64::new(4)),
)
.unwrap();
let svc = Arc::new(crate::hint_worker::HintService::new());
let shards = Arc::new(Shards::new(
vec![shard],
fs::File::open(&shard_dir).unwrap(),
));
svc.attach_shards(Arc::downgrade(&shards));
shards[0].lock().hint_service = Some(svc.clone());
shards[0].rotate_active_for_test(8).unwrap();
svc.drain_and_stop();
let hint = fs::read(shard_dir.join("000001.hint")).unwrap();
assert!(crate::hint::parse_hint_header(&hint).is_some());
drop(shards);
let tree = VarTree::<[u8; 8]>::open(dir.path(), config).unwrap();
for n in 0u64..3 {
assert_eq!(
tree.get(&n.to_be_bytes()).unwrap().as_ref(),
&n.to_le_bytes()
);
}
}
}