mod envelope_keys;
mod segment;
use std::collections::{BTreeMap, HashMap};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use anyhow::{Context, Result, anyhow};
use sha2::{Digest, Sha256};
pub use crate::keys::KeyId;
use uuid::Uuid;
pub use zeroize::Zeroizing;
pub use envelope_keys::EnvelopeArchiveKeys;
pub use segment::{BlockEntry, ColdRecord, SegmentManifest, SegmentSummary};
use segment::{OpenSegment, seal};
pub const COLD_AFTER_DAYS: i64 = 30;
pub const SEGMENT_BYTES: usize = 8 << 20;
pub const BLOCK_BYTES: usize = 256 << 10;
pub const MAX_RECORD_BYTES: usize = 32 << 20;
const STUB_MARKER: &str = "\u{1e}sha256:";
const SEGMENT_SUFFIX: &str = "seg";
const MANIFEST_CACHE: usize = 32;
pub fn dir_for_db(db_path: &Path) -> PathBuf {
db_path.with_extension("cold")
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct KekLabel {
pub epoch: u32,
pub key_id: KeyId,
}
pub struct Kek {
pub label: KekLabel,
pub key: Zeroizing<[u8; 32]>,
}
pub trait ArchiveKeys: Send + Sync {
fn current(&self) -> Result<Option<Kek>>;
fn by_label(&self, label: &KekLabel) -> Result<Option<Zeroizing<[u8; 32]>>>;
}
pub fn stub_for(content: &str) -> String {
let lead: String = content
.chars()
.take(crate::store::CONTENT_PREFIX_CHARS)
.collect();
format!("{lead}{STUB_MARKER}{}", content_digest(content))
}
pub(crate) fn stub_lead_sql(column: &str) -> String {
format!(
"substr({column}, 1, length({column}) - {})",
STUB_MARKER.chars().count() + 64
)
}
pub fn stub_matches(stub: &str, content: &str) -> bool {
stub_digest(stub) == Some(content_digest(content).as_str())
}
pub fn content_digest(content: &str) -> String {
crate::keys::hex(&Sha256::digest(content.as_bytes()))
}
pub fn stub_digest(stub: &str) -> Option<&str> {
let (_, digest) = stub.rsplit_once(STUB_MARKER)?;
(digest.len() == 64 && digest.bytes().all(|b| b.is_ascii_hexdigit())).then_some(digest)
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct UnreadableSegment {
pub segment: String,
pub error: String,
}
pub struct ColdTier {
dir: PathBuf,
keys: Arc<dyn ArchiveKeys>,
manifests: Mutex<HashMap<String, Arc<SegmentManifest>>>,
summaries: Mutex<HashMap<String, SegmentSummary>>,
last_block: Mutex<Option<(String, u32, Arc<Vec<ColdRecord>>)>>,
unreadable: Mutex<BTreeMap<(String, Option<u32>), String>>,
pass: Mutex<()>,
}
fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
m.lock().unwrap_or_else(PoisonError::into_inner)
}
impl ColdTier {
pub fn open(dir: impl Into<PathBuf>, keys: Arc<dyn ArchiveKeys>) -> Result<Self> {
let dir = dir.into();
create_private_dir(&dir).with_context(|| format!("creating {}", dir.display()))?;
Ok(Self {
dir,
keys,
manifests: Mutex::new(HashMap::new()),
summaries: Mutex::new(HashMap::new()),
last_block: Mutex::new(None),
unreadable: Mutex::new(BTreeMap::new()),
pass: Mutex::new(()),
})
}
pub fn dir(&self) -> &Path {
&self.dir
}
pub(crate) fn pass_guard(&self) -> MutexGuard<'_, ()> {
lock(&self.pass)
}
fn path(&self, segment: &str) -> PathBuf {
self.dir.join(format!("{segment}.{SEGMENT_SUFFIX}"))
}
pub fn write_segment(&self, records: &[ColdRecord]) -> Result<Option<(String, String)>> {
let Some(kek) = self.keys.current()? else {
return Ok(None);
};
let segment = Uuid::new_v4();
let (bytes, manifest) = seal(segment, records, &kek, BLOCK_BYTES)?;
let id = segment.to_string();
let path = self.path(&id);
write_private_atomic(&path, &bytes)
.with_context(|| format!("writing {}", path.display()))?;
Ok(Some((id, serde_json::to_string(&manifest)?)))
}
pub fn read(
&self,
segment: &str,
memory: Uuid,
manifest: impl FnOnce() -> Result<Option<String>>,
) -> Result<ColdRecord> {
self.read_inner(segment, memory, manifest)
.with_context(|| format!("cold segment {segment}"))
}
fn read_inner(
&self,
segment: &str,
memory: Uuid,
load: impl FnOnce() -> Result<Option<String>>,
) -> Result<ColdRecord> {
let manifest = self.failed(segment, None, self.manifest(segment, load))?;
let block = self.failed(
segment,
None,
manifest
.records
.get(&memory)
.copied()
.ok_or_else(|| anyhow!("memory {memory} is not in the segment's manifest")),
)?;
let records = self.block(segment, &manifest, block)?;
let found = records
.iter()
.find(|r| r.id == memory)
.cloned()
.ok_or_else(|| anyhow!("memory {memory} is not in block {block}"));
self.noted(segment, Some(block), found)
}
pub fn check(
&self,
segment: &str,
manifest: impl FnOnce() -> Result<Option<String>>,
) -> Result<()> {
let result = self
.summary(segment, manifest)
.and_then(|s| self.open_segment(segment, &s).map(|_| ()));
self.noted(segment, None, result)
.with_context(|| format!("cold segment {segment}"))
}
fn failed<T>(&self, segment: &str, block: Option<u32>, result: Result<T>) -> Result<T> {
if let Err(e) = &result {
lock(&self.unreadable).insert((segment.to_string(), block), format!("{e:#}"));
}
result
}
fn noted<T>(&self, segment: &str, block: Option<u32>, result: Result<T>) -> Result<T> {
let key = (segment.to_string(), block);
let mut unreadable = lock(&self.unreadable);
match &result {
Ok(_) => {
unreadable.remove(&key);
}
Err(e) => {
unreadable.insert(key, format!("{e:#}"));
}
}
result
}
pub fn record_count(
&self,
segment: &str,
manifest: impl FnOnce() -> Result<Option<String>>,
) -> Result<usize> {
Ok(self.summary(segment, manifest)?.records)
}
fn summary(
&self,
segment: &str,
manifest: impl FnOnce() -> Result<Option<String>>,
) -> Result<SegmentSummary> {
if let Some(s) = lock(&self.summaries).get(segment) {
return Ok(*s);
}
let summary = self.manifest(segment, manifest)?.summary()?;
lock(&self.summaries).insert(segment.to_string(), summary);
Ok(summary)
}
pub fn record_ids(
&self,
segment: &str,
manifest: impl FnOnce() -> Result<Option<String>>,
) -> Result<Vec<Uuid>> {
let manifest = self.manifest(segment, manifest)?;
let mut ids: Vec<(u32, Uuid)> = manifest.records.iter().map(|(id, b)| (*b, *id)).collect();
ids.sort();
Ok(ids.into_iter().map(|(_, id)| id).collect())
}
fn manifest(
&self,
segment: &str,
load: impl FnOnce() -> Result<Option<String>>,
) -> Result<Arc<SegmentManifest>> {
if let Some(m) = lock(&self.manifests).get(segment) {
return Ok(m.clone());
}
let text = load()?.ok_or_else(|| anyhow!("the store holds no manifest for it"))?;
let parsed: Arc<SegmentManifest> =
Arc::new(serde_json::from_str(&text).context("parsing its manifest")?);
let mut cache = lock(&self.manifests);
if cache.len() >= MANIFEST_CACHE {
cache.clear();
}
cache.insert(segment.to_string(), parsed.clone());
Ok(parsed)
}
fn open_segment(&self, segment: &str, summary: &SegmentSummary) -> Result<OpenSegment> {
let id = Uuid::parse_str(segment).context("segment ids are UUIDs")?;
let label = summary.label;
let key = self.keys.by_label(&label)?.ok_or_else(|| {
anyhow!(
"this device holds no archive key for epoch {} (key id {})",
label.epoch,
label.key_id
)
})?;
OpenSegment::open(&self.path(segment), id, summary, &key)
}
fn block(
&self,
segment: &str,
manifest: &SegmentManifest,
index: u32,
) -> Result<Arc<Vec<ColdRecord>>> {
if let Some((s, i, records)) = &*lock(&self.last_block)
&& s == segment
&& *i == index
{
return Ok(records.clone());
}
let mut open = self.noted(
segment,
None,
manifest
.summary()
.and_then(|s| self.open_segment(segment, &s)),
)?;
let records = Arc::new(self.noted(segment, Some(index), open.block(manifest, index))?);
*lock(&self.last_block) = Some((segment.to_string(), index, records.clone()));
Ok(records)
}
pub fn segment_len(&self, segment: &str) -> Option<u64> {
std::fs::metadata(self.path(segment)).ok().map(|m| m.len())
}
pub fn remove_segment(&self, segment: &str) -> Result<()> {
lock(&self.manifests).remove(segment);
lock(&self.summaries).remove(segment);
lock(&self.unreadable).retain(|(s, _), _| s != segment);
{
let mut last = lock(&self.last_block);
if last.as_ref().is_some_and(|(s, _, _)| s == segment) {
*last = None;
}
}
match std::fs::remove_file(self.path(segment)) {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e).with_context(|| format!("removing cold segment {segment}")),
}
}
pub fn files(&self) -> Result<(Vec<String>, Vec<PathBuf>)> {
let mut segments = Vec::new();
let mut partial = Vec::new();
let suffix = format!(".{SEGMENT_SUFFIX}");
for entry in std::fs::read_dir(&self.dir)? {
let path = entry?.path();
let name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or_default();
if let Some(id) = name.strip_suffix(&suffix) {
segments.push(id.to_string());
} else if name.ends_with(".tmp") {
partial.push(path);
}
}
segments.sort();
Ok((segments, partial))
}
pub fn unreadable(&self) -> Vec<UnreadableSegment> {
let mut listed: Vec<UnreadableSegment> = Vec::new();
for ((segment, _), error) in lock(&self.unreadable).iter() {
if listed.last().is_some_and(|u| u.segment == *segment) {
continue;
}
listed.push(UnreadableSegment {
segment: segment.clone(),
error: error.clone(),
});
}
listed
}
}
fn create_private_dir(dir: &Path) -> std::io::Result<()> {
let mut builder = std::fs::DirBuilder::new();
builder.recursive(true);
#[cfg(unix)]
std::os::unix::fs::DirBuilderExt::mode(&mut builder, crate::private_files::PRIVATE_DIR_MODE);
builder.create(dir)
}
fn write_private_atomic(target: &Path, bytes: &[u8]) -> std::io::Result<()> {
use std::io::Write;
let mut tmp_name = target.file_name().unwrap_or_default().to_os_string();
tmp_name.push(".tmp");
let tmp = target.with_file_name(tmp_name);
let mut options = std::fs::OpenOptions::new();
options.write(true).create(true).truncate(true);
#[cfg(unix)]
std::os::unix::fs::OpenOptionsExt::mode(&mut options, crate::private_files::PRIVATE_FILE_MODE);
let mut file = options.open(&tmp)?;
file.write_all(bytes)?;
file.sync_all()?;
std::fs::rename(&tmp, target)?;
match target.parent() {
Some(dir) => std::fs::File::open(dir)?.sync_all(),
None => Ok(()),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_stub_carries_the_digest_of_the_text_it_stands_for() {
let stub = stub_for("the daemon port is 7200");
assert_eq!(
stub_digest(&stub),
Some(content_digest("the daemon port is 7200").as_str())
);
assert!(stub_matches(&stub, "the daemon port is 7200"));
assert!(!stub_matches(&stub, "the daemon port is 7300"));
assert_eq!(stub_digest("the daemon port is 7200"), None);
assert_eq!(stub_digest("text\u{1e}sha256:abc"), None);
}
#[test]
fn a_stub_keeps_the_leading_characters_ingest_dedup_compares() {
let long: String = "é".repeat(crate::store::CONTENT_PREFIX_CHARS + 50);
let stub = stub_for(&long);
let lead: String = long
.chars()
.take(crate::store::CONTENT_PREFIX_CHARS)
.collect();
assert!(stub.starts_with(&lead));
assert!(!stub.starts_with(&long));
assert!(stub_for("short").starts_with("short"));
}
}