mod files;
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use crate::StoreMode;
static NEXT: AtomicU64 = AtomicU64::new(0);
pub const MAX_KEY_BYTES: usize = 64 * 1024 * 1024;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Limits {
pub key_bytes: usize,
pub entries: u64,
pub probe_steps: u64,
pub summary_pairs: u64,
}
impl Default for Limits {
fn default() -> Self {
Self {
key_bytes: MAX_KEY_BYTES,
entries: 16 * 1024 * 1024,
probe_steps: 32 * 1024 * 1024,
summary_pairs: 1_000_000,
}
}
}
impl Limits {
pub fn validate(self) -> Result<(), SeenError> {
if self.key_bytes > MAX_KEY_BYTES
|| self.key_bytes < 8
|| self.entries == 0
|| self.entries > u64::from(u32::MAX)
|| self.probe_steps == 0
{
return Err(error(
Operation::Admission,
"invalid disk seen-state limits",
));
}
Ok(())
}
}
#[derive(Clone, Copy, Debug)]
pub enum Key<'a> {
Fingerprint(u64),
Exact(&'a [u8]),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum InsertOutcome {
Present,
Inserted,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Operation {
Admission,
Create,
Read,
Seek,
Write,
Allocate,
Rehash,
Corruption,
Summary,
Cleanup,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SeenError {
pub operation: Operation,
pub message: String,
pub cleanup_failure: Option<String>,
}
impl SeenError {
#[must_use]
pub const fn code(&self) -> &'static str {
"DIVE-S0700"
}
}
impl std::fmt::Display for SeenError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}: {:?}: {}", self.code(), self.operation, self.message)?;
if let Some(cleanup) = &self.cleanup_failure {
write!(f, "; cleanup also failed: {cleanup}")?;
}
Ok(())
}
}
impl std::error::Error for SeenError {}
fn error(operation: Operation, cause: impl std::fmt::Display) -> SeenError {
SeenError {
operation,
message: cause.to_string(),
cleanup_failure: None,
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Stats {
pub physical_entries: u64,
pub disk_bytes: u64,
pub resident_key_bytes: u64,
pub poisoned: bool,
pub cleanup_completed: bool,
}
#[derive(Debug)]
pub struct DiskSeen {
directory: PathBuf,
mode: StoreMode,
limits: Limits,
capacity: u64,
count: u64,
data_end: u64,
first: Option<SeenError>,
closed: bool,
audited: bool,
#[cfg(test)]
fault: Option<(Operation, u64)>,
#[cfg(test)]
forced_hash: Option<u64>,
}
impl DiskSeen {
pub fn create(mode: StoreMode, parent: &Path, limits: Limits) -> Result<Self, SeenError> {
limits.validate()?;
let directory = parent.join(format!(
"dive-seen-{}-{}",
std::process::id(),
NEXT.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir(&directory).map_err(|e| error(Operation::Create, e))?;
let mut set = Self {
directory,
mode,
limits,
capacity: 8,
count: 0,
data_end: files::HEADER,
first: None,
closed: false,
audited: false,
#[cfg(test)]
fault: None,
#[cfg(test)]
forced_hash: None,
};
if let Err(mut failure) = set.initialize() {
if let Err(cleanup) = set.remove_owned() {
failure.cleanup_failure = Some(cleanup.message);
}
return Err(failure);
}
Ok(set)
}
#[must_use]
pub fn stats(&self) -> Stats {
Stats {
physical_entries: self.count,
disk_bytes: if self.closed {
0
} else {
self.data_end + files::HEADER + self.capacity * files::SLOT
},
resident_key_bytes: 0,
poisoned: self.first.is_some(),
cleanup_completed: self.closed,
}
}
pub fn contains(&mut self, key: Key<'_>) -> Result<bool, SeenError> {
self.available()?;
let result = self.find_key(key).map(|(_, found)| found);
self.retain(result)
}
pub fn insert(&mut self, key: Key<'_>) -> Result<InsertOutcome, SeenError> {
self.available()?;
let result = self.insert_inner(key);
self.retain(result)
}
pub fn tlc_min_gap(&mut self, fp_bits: u32) -> Result<u64, SeenError> {
self.available()?;
let result = self.gap_inner(fp_bits);
self.retain(result)
}
pub fn close(&mut self) -> Result<Stats, SeenError> {
let (stats, audited) = self.close_controlled(true, &|| true)?;
if audited {
Ok(stats)
} else {
Err(error(
Operation::Admission,
"store was discarded without a complete audit",
))
}
}
pub fn close_controlled(
&mut self,
validate: bool,
proceed: &dyn Fn() -> bool,
) -> Result<(Stats, bool), SeenError> {
if !self.closed {
if validate && self.first.is_none() {
match self.audit_inner(proceed) {
Ok(audited) => self.audited = audited,
Err(failure) => self.first = Some(failure),
}
}
if let Err(cleanup) = self.remove_owned() {
if let Some(first) = &mut self.first {
first.cleanup_failure.get_or_insert(cleanup.message);
} else {
self.first = Some(cleanup);
}
} else {
self.closed = true;
}
}
self.first
.clone()
.map_or_else(|| Ok((self.stats(), self.audited)), Err)
}
fn remove_owned(&mut self) -> Result<(), SeenError> {
self.check_fault(Operation::Cleanup)?;
for name in ["data", "index", "index-next"] {
match fs::remove_file(self.directory.join(name)) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(error(Operation::Cleanup, e)),
}
}
fs::remove_dir(&self.directory).map_err(|e| error(Operation::Cleanup, e))
}
fn available(&self) -> Result<(), SeenError> {
if let Some(first) = &self.first {
return Err(first.clone());
}
if self.closed {
return Err(error(Operation::Admission, "store is closed"));
}
Ok(())
}
fn retain<T>(&mut self, result: Result<T, SeenError>) -> Result<T, SeenError> {
if let Err(failure) = &result {
self.first = Some(failure.clone());
}
result
}
#[allow(
clippy::unused_self,
clippy::unnecessary_wraps,
reason = "test-only fault injection uses the production I/O boundaries"
)]
fn check_fault(&mut self, operation: Operation) -> Result<(), SeenError> {
#[cfg(test)]
if let Some((wanted, remaining)) = &mut self.fault
&& *wanted == operation
{
if *remaining == 0 {
self.fault = None;
return Err(error(
operation,
if operation == Operation::Write {
std::io::Error::new(
std::io::ErrorKind::StorageFull,
"injected storage exhaustion",
)
.to_string()
} else {
"injected storage failure".to_owned()
},
));
}
*remaining -= 1;
}
let _ = operation;
Ok(())
}
}
#[cfg(test)]
mod tests;
impl Drop for DiskSeen {
fn drop(&mut self) {
if !self.closed {
let _ = self.remove_owned();
}
}
}