pub mod container_store;
pub mod read;
pub mod refold;
pub mod wal;
pub use container_store::ContainerStore;
use crate::fold::{Fold, FoldCfg, FoldTail, Loc};
use crate::part::cache::SectionCache;
use crate::part::{self, Part};
use crate::read_limits::ReadLimits;
use crate::types::{AttrValue, BodyOp, Content, ContentHash, PieceHash, Record, BODY_CONTENT};
use anyhow::{bail, Context, Result};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::Read;
use std::path::{Component, Path, PathBuf};
use std::sync::Arc;
use wal::Wal;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct WriteLimits {
pub max_record_bytes: u64,
pub max_batch_bytes: u64,
pub max_batch_records: usize,
pub max_identifier_bytes: usize,
}
pub const DEFAULT_MAX_RECORD_BYTES: u64 = 64 << 20;
pub const DEFAULT_MAX_BATCH_BYTES: u64 = 256 << 20;
pub const DEFAULT_MAX_BATCH_RECORDS: usize = 4_096;
pub const DEFAULT_MAX_IDENTIFIER_BYTES: usize = 4 << 10;
pub const MAX_MANIFEST_BYTES: u64 = 64 << 20;
impl Default for WriteLimits {
fn default() -> Self {
WriteLimits {
max_record_bytes: DEFAULT_MAX_RECORD_BYTES,
max_batch_bytes: DEFAULT_MAX_BATCH_BYTES,
max_batch_records: DEFAULT_MAX_BATCH_RECORDS,
max_identifier_bytes: DEFAULT_MAX_IDENTIFIER_BYTES,
}
}
}
#[derive(Clone, Copy, Debug)]
pub struct StoreOptions {
pub fold: FoldCfg,
pub write_limits: WriteLimits,
pub read_limits: ReadLimits,
pub part_cache_bytes: usize,
}
impl Default for StoreOptions {
fn default() -> Self {
StoreOptions {
fold: FoldCfg::default(),
write_limits: WriteLimits::default(),
read_limits: ReadLimits::default(),
part_cache_bytes: crate::part::cache::BUDGET_DEFAULT,
}
}
}
impl WriteLimits {
pub fn validate(self) -> std::result::Result<Self, WriteAdmissionError> {
if self.max_record_bytes == 0 {
return Err(WriteAdmissionError::InvalidLimits(
"max_record_bytes must be greater than zero",
));
}
if self.max_batch_bytes == 0 {
return Err(WriteAdmissionError::InvalidLimits(
"max_batch_bytes must be greater than zero",
));
}
if self.max_batch_records == 0 {
return Err(WriteAdmissionError::InvalidLimits(
"max_batch_records must be greater than zero",
));
}
if self.max_identifier_bytes == 0 {
return Err(WriteAdmissionError::InvalidLimits(
"max_identifier_bytes must be greater than zero",
));
}
Ok(self)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum WriteAdmissionError {
InvalidLimits(&'static str),
EmptyIdentifier { kind: &'static str, item: Option<usize> },
IdentifierTooLong { kind: &'static str, item: Option<usize>, actual: usize, allowed: usize },
DuplicateContentName { item: Option<usize>, name: String },
RecordTooLarge { item: Option<usize>, actual: u64, allowed: u64 },
BatchTooLarge { actual: u64, allowed: u64 },
TooManyBatchRecords { actual: usize, allowed: usize },
}
impl std::fmt::Display for WriteAdmissionError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let where_ = |item: &Option<usize>| {
item.map_or_else(String::new, |item| format!(" in batch item {item}"))
};
match self {
WriteAdmissionError::InvalidLimits(reason) => write!(f, "invalid write limits: {reason}"),
WriteAdmissionError::EmptyIdentifier { kind, item } => {
write!(f, "{kind}{} must not be empty", where_(item))
}
WriteAdmissionError::IdentifierTooLong { kind, item, actual, allowed } => write!(
f,
"{kind}{} is {actual} UTF-8 bytes, exceeding the configured limit of {allowed}",
where_(item)
),
WriteAdmissionError::DuplicateContentName { item, name } => {
write!(f, "duplicate content name {name:?}{}", where_(item))
}
WriteAdmissionError::RecordTooLarge { item, actual, allowed } => write!(
f,
"record{} has a worst-case WAL frame of {actual} bytes, exceeding the configured limit of {allowed}",
where_(item)
),
WriteAdmissionError::BatchTooLarge { actual, allowed } => write!(
f,
"atomic batch has a worst-case WAL representation of {actual} bytes, exceeding the configured limit of {allowed}"
),
WriteAdmissionError::TooManyBatchRecords { actual, allowed } => write!(
f,
"atomic batch has {actual} records, exceeding the configured limit of {allowed}"
),
}
}
}
impl std::error::Error for WriteAdmissionError {}
#[derive(Clone, Copy)]
pub enum Span<'a> {
Lit(&'a [u8]),
Piece(&'a [u8]),
}
pub struct ContentSpans<'a> {
pub name: &'a str,
pub spans: Vec<Span<'a>>,
}
impl<'a> ContentSpans<'a> {
pub fn new(name: &'a str, spans: Vec<Span<'a>>) -> ContentSpans<'a> {
ContentSpans { name, spans }
}
pub fn carve(name: &'a str, bytes: &'a [u8], carve: &crate::carve::Carve) -> ContentSpans<'a> {
ContentSpans { name, spans: carve.carve(bytes) }
}
}
#[derive(Default)]
pub struct Batch {
items: Vec<BatchItem>,
}
enum BatchItem {
Put { id: String, contents: Vec<OwnedContent>, attrs: Vec<(String, AttrValue)> },
Delete { id: String },
}
struct OwnedContent {
name: String,
spans: Vec<OwnedSpan>,
}
enum OwnedSpan {
Lit(Vec<u8>),
Piece(Vec<u8>),
}
impl Batch {
pub fn new() -> Batch {
Batch::default()
}
pub fn put(&mut self, id: &str, spans: &[Span], attrs: Vec<(String, AttrValue)>) {
let spans = own_spans(spans);
self.items.push(BatchItem::Put {
id: id.to_string(),
contents: vec![OwnedContent { name: BODY_CONTENT.to_string(), spans }],
attrs,
});
}
pub fn put_record(
&mut self,
id: &str,
contents: &[ContentSpans<'_>],
attrs: Vec<(String, AttrValue)>,
) -> Result<()> {
let item = Some(self.items.len());
validate_content_inputs(id, contents, item)?;
if attrs.iter().any(|(name, _)| name.is_empty()) {
return Err(
WriteAdmissionError::EmptyIdentifier { kind: "attribute name", item }.into()
);
}
let contents = contents
.iter()
.map(|content| OwnedContent {
name: content.name.to_string(),
spans: own_spans(&content.spans),
})
.collect();
self.items.push(BatchItem::Put { id: id.to_string(), contents, attrs });
Ok(())
}
pub fn put_body(&mut self, id: &str, body: &[u8], attrs: Vec<(String, AttrValue)>) {
self.put(id, &crate::carve::Carve::default().carve(body), attrs);
}
pub fn delete(&mut self, id: &str) {
self.items.push(BatchItem::Delete { id: id.to_string() });
}
pub fn len(&self) -> usize {
self.items.len()
}
pub fn is_empty(&self) -> bool {
self.items.is_empty()
}
}
fn own_spans(spans: &[Span<'_>]) -> Vec<OwnedSpan> {
spans
.iter()
.map(|s| match s {
Span::Lit(b) => OwnedSpan::Lit(b.to_vec()),
Span::Piece(b) => OwnedSpan::Piece(b.to_vec()),
})
.collect()
}
const WAL_FRAME_OVERHEAD: u64 = 1 + 8 + 4 + 4;
fn varint_bytes(mut value: u64) -> u64 {
let mut bytes = 1;
while value >= 0x80 {
value >>= 7;
bytes += 1;
}
bytes
}
fn add_size(total: &mut u64, value: u64) {
*total = total.saturating_add(value);
}
fn bytes_field_size(len: usize) -> u64 {
let len = len as u64;
varint_bytes(len).saturating_add(len)
}
fn attr_encoded_size(name: &str, value: &AttrValue) -> u64 {
let value = match value {
AttrValue::Str(value) => bytes_field_size(value.len()),
AttrValue::Int(_)
| AttrValue::Float(_)
| AttrValue::UInt(_)
| AttrValue::TimestampNs(_) => 8,
AttrValue::Bool(_) => 1,
AttrValue::Bytes(value) => bytes_field_size(value.len()),
AttrValue::Null => 0,
};
bytes_field_size(name.len()).saturating_add(1).saturating_add(value)
}
fn validate_identifier(
kind: &'static str,
value: &str,
limits: WriteLimits,
item: Option<usize>,
) -> Result<()> {
if value.is_empty() {
return Err(WriteAdmissionError::EmptyIdentifier { kind, item }.into());
}
if value.len() > limits.max_identifier_bytes {
return Err(WriteAdmissionError::IdentifierTooLong {
kind,
item,
actual: value.len(),
allowed: limits.max_identifier_bytes,
}
.into());
}
Ok(())
}
fn validate_attr_names(
attrs: &[(String, AttrValue)],
limits: WriteLimits,
item: Option<usize>,
) -> Result<()> {
for (name, _) in attrs {
validate_identifier("attribute name", name, limits, item)?;
}
Ok(())
}
fn input_record_admission_bytes(
id: &str,
contents: &[ContentSpans<'_>],
attrs: &[(String, AttrValue)],
limits: WriteLimits,
read_limits: ReadLimits,
item: Option<usize>,
) -> Result<u64> {
validate_identifier("record id", id, limits, item)?;
validate_attr_names(attrs, limits, item)?;
let mut names = std::collections::BTreeSet::new();
let mut size = WAL_FRAME_OVERHEAD;
add_size(&mut size, bytes_field_size(id.len()));
add_size(&mut size, varint_bytes(contents.len() as u64));
for content in contents {
validate_identifier("content name", content.name, limits, item)?;
if !names.insert(content.name) {
return Err(WriteAdmissionError::DuplicateContentName {
item,
name: content.name.to_string(),
}
.into());
}
validate_spans(&content.spans)?;
add_size(&mut size, bytes_field_size(content.name.len()));
add_size(&mut size, 1 + 32); add_size(&mut size, varint_bytes(content.spans.len() as u64));
let mut novel = 0u64;
for span in &content.spans {
match span {
Span::Lit(bytes) => {
add_size(&mut size, 1u64.saturating_add(bytes_field_size(bytes.len())))
}
Span::Piece(bytes) => {
read_limits.admit("new fold block", bytes.len() as u64, bytes.len() as u64)?;
add_size(
&mut size,
1u64.saturating_add(32).saturating_add(varint_bytes(bytes.len() as u64)),
);
add_size(&mut novel, 32u64.saturating_add(bytes_field_size(bytes.len())));
}
}
}
add_size(&mut size, novel);
}
add_size(&mut size, varint_bytes(attrs.len() as u64));
for (name, value) in attrs {
add_size(&mut size, attr_encoded_size(name, value));
}
let piece_count = contents
.iter()
.flat_map(|content| content.spans.iter())
.filter(|span| matches!(span, Span::Piece(_)))
.count() as u64;
add_size(&mut size, varint_bytes(piece_count));
if size > limits.max_record_bytes {
return Err(WriteAdmissionError::RecordTooLarge {
item,
actual: size,
allowed: limits.max_record_bytes,
}
.into());
}
read_limits.admit("new WAL frame", size, size)?;
Ok(size)
}
fn validate_content_inputs(
id: &str,
contents: &[ContentSpans<'_>],
item: Option<usize>,
) -> Result<()> {
if id.is_empty() {
return Err(WriteAdmissionError::EmptyIdentifier { kind: "record id", item }.into());
}
let mut names = std::collections::BTreeSet::new();
for content in contents {
if content.name.is_empty() {
return Err(WriteAdmissionError::EmptyIdentifier { kind: "content name", item }.into());
}
if !names.insert(content.name) {
return Err(WriteAdmissionError::DuplicateContentName {
item,
name: content.name.to_string(),
}
.into());
}
validate_spans(&content.spans)?;
}
Ok(())
}
fn validate_spans(spans: &[Span<'_>]) -> Result<()> {
for span in spans {
if let Span::Piece(bytes) = span {
u32::try_from(bytes.len())
.context("one folded piece exceeds the format's u32 length")?;
}
}
Ok(())
}
fn owned_record_admission_bytes(
id: &str,
contents: &[OwnedContent],
attrs: &[(String, AttrValue)],
limits: WriteLimits,
read_limits: ReadLimits,
item: Option<usize>,
) -> Result<u64> {
validate_identifier("record id", id, limits, item)?;
validate_attr_names(attrs, limits, item)?;
let mut names = std::collections::BTreeSet::new();
let mut size = WAL_FRAME_OVERHEAD;
add_size(&mut size, bytes_field_size(id.len()));
add_size(&mut size, varint_bytes(contents.len() as u64));
let mut piece_count = 0u64;
let mut novel = 0u64;
for content in contents {
validate_identifier("content name", &content.name, limits, item)?;
if !names.insert(content.name.as_str()) {
return Err(WriteAdmissionError::DuplicateContentName {
item,
name: content.name.clone(),
}
.into());
}
add_size(&mut size, bytes_field_size(content.name.len()));
add_size(&mut size, 1 + 32);
add_size(&mut size, varint_bytes(content.spans.len() as u64));
for span in &content.spans {
match span {
OwnedSpan::Lit(bytes) => {
add_size(&mut size, 1u64.saturating_add(bytes_field_size(bytes.len())))
}
OwnedSpan::Piece(bytes) => {
read_limits.admit("new fold block", bytes.len() as u64, bytes.len() as u64)?;
u32::try_from(bytes.len())
.context("one folded piece exceeds the format's u32 length")?;
piece_count = piece_count.saturating_add(1);
add_size(
&mut size,
1u64.saturating_add(32).saturating_add(varint_bytes(bytes.len() as u64)),
);
add_size(&mut novel, 32u64.saturating_add(bytes_field_size(bytes.len())));
}
}
}
}
add_size(&mut size, varint_bytes(attrs.len() as u64));
for (name, value) in attrs {
add_size(&mut size, attr_encoded_size(name, value));
}
add_size(&mut size, varint_bytes(piece_count));
add_size(&mut size, novel);
if size > limits.max_record_bytes {
return Err(WriteAdmissionError::RecordTooLarge {
item,
actual: size,
allowed: limits.max_record_bytes,
}
.into());
}
read_limits.admit("new WAL frame", size, size)?;
Ok(size)
}
fn delete_admission_bytes(
id: &str,
limits: WriteLimits,
read_limits: ReadLimits,
item: Option<usize>,
) -> Result<u64> {
validate_identifier("record id", id, limits, item)?;
let size = WAL_FRAME_OVERHEAD.saturating_add(id.len() as u64);
if size > limits.max_record_bytes {
return Err(WriteAdmissionError::RecordTooLarge {
item,
actual: size,
allowed: limits.max_record_bytes,
}
.into());
}
read_limits.admit("new WAL frame", size, size)?;
Ok(size)
}
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
pub struct PartRef {
pub file: String,
pub seq_lo: u64,
pub seq_hi: u64,
pub records: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub b3: Option<String>,
}
pub const MANIFEST_RETAIN: usize = 4;
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
pub struct Manifest {
pub parts: Vec<PartRef>,
#[serde(default)]
pub fold_gen: u32,
pub fold_seg: u32,
pub fold_off: u32,
pub next_seq: u64,
#[serde(default)]
pub commit: u64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub punched: Vec<(u32, u32)>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prev: Option<String>,
}
fn safe_part_file_name(name: &str) -> bool {
if name.is_empty() || name.contains('\\') {
return false;
}
let mut components = Path::new(name).components();
matches!(components.next(), Some(Component::Normal(_))) && components.next().is_none()
}
fn valid_blake3_hex(value: &str) -> bool {
value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit())
}
pub(crate) fn read_manifest_file(path: &Path) -> Result<Vec<u8>> {
let file = File::open(path)?;
let announced = file.metadata()?.len();
if announced > MAX_MANIFEST_BYTES {
bail!(
"MANIFEST at {} is {announced} bytes, exceeding the supported {MAX_MANIFEST_BYTES}-byte limit",
path.display()
);
}
let capacity =
usize::try_from(announced).context("MANIFEST length does not fit this platform")?;
let mut bytes = Vec::new();
bytes.try_reserve_exact(capacity).context("reserve MANIFEST buffer")?;
file.take(MAX_MANIFEST_BYTES + 1).read_to_end(&mut bytes)?;
if bytes.len() as u64 > MAX_MANIFEST_BYTES {
bail!(
"MANIFEST at {} grew past the supported {MAX_MANIFEST_BYTES}-byte limit while reading",
path.display()
);
}
Ok(bytes)
}
impl Manifest {
#[cfg(test)]
fn load(dir: &Path) -> Result<Manifest> {
Self::load_with_limits(dir, ReadLimits::default())
}
fn load_with_limits(dir: &Path, read_limits: ReadLimits) -> Result<Manifest> {
let path = dir.join("MANIFEST");
match read_manifest_file(&path) {
Ok(b) => Manifest::parse(&b),
Err(e)
if e.downcast_ref::<std::io::Error>()
.is_some_and(|error| error.kind() == std::io::ErrorKind::NotFound) =>
{
let retained = list_retained_with_limits(dir, read_limits)?;
if retained.is_empty() {
Ok(Manifest::default())
} else {
bail!(
"MANIFEST is missing but {} retained commits exist at {} — a damaged \
store, not a new one; recover_manifest() can promote the newest intact copy",
retained.len(),
dir.display()
)
}
}
Err(e) => Err(e.context(format!(
"cannot read {} — refusing to treat an unreadable manifest as an empty store",
path.display()
))),
}
}
fn parse(bytes: &[u8]) -> Result<Manifest> {
let payload = match checksum_trailer(bytes) {
Some((payload, want)) => {
let got = crc32fast::hash(payload);
if got != want {
bail!(
"MANIFEST fails its checksum (crc32 {got:08x}, recorded {want:08x}) — \
refusing to open from a corrupt commit point"
);
}
payload
}
None => bytes,
};
if payload.len() as u64 > MAX_MANIFEST_BYTES {
bail!(
"MANIFEST is {} bytes, exceeding the supported {MAX_MANIFEST_BYTES}-byte limit",
payload.len()
);
}
let manifest: Manifest = serde_json::from_slice(payload).context("corrupt MANIFEST")?;
manifest.validate()
}
fn validate(self) -> Result<Manifest> {
let mut files = HashSet::with_capacity(self.parts.len());
for part in &self.parts {
if !safe_part_file_name(&part.file) {
bail!("MANIFEST part file {:?} is not one store-local path component", part.file);
}
if !files.insert(part.file.as_str()) {
bail!("MANIFEST names part file {:?} more than once", part.file);
}
if part.seq_lo > part.seq_hi {
bail!(
"MANIFEST part {:?} has inverted sequence range {}..{}",
part.file,
part.seq_lo,
part.seq_hi
);
}
if part.b3.as_deref().is_some_and(|digest| !valid_blake3_hex(digest)) {
bail!("MANIFEST part {:?} carries an invalid BLAKE3 digest", part.file);
}
}
if self.prev.as_deref().is_some_and(|digest| !valid_blake3_hex(digest)) {
bail!("MANIFEST carries an invalid previous-manifest BLAKE3 digest");
}
let mut previous_hi = None;
for &(lo, hi) in &self.punched {
if lo > hi || previous_hi.is_some_and(|previous| lo <= previous) {
bail!("MANIFEST punched ranges must be ascending, disjoint, and non-empty");
}
previous_hi = Some(hi);
}
Ok(self)
}
fn encode(&self) -> Result<Vec<u8>> {
let mut buf = serde_json::to_vec(self)?;
let crc = crc32fast::hash(&buf);
buf.extend_from_slice(format!("\ncrc32={crc:08x}").as_bytes());
Ok(buf)
}
#[cfg(test)]
fn commit(&mut self, dir: &Path) -> Result<()> {
self.commit_with_limits(dir, ReadLimits::default())
}
fn commit_with_limits(&mut self, dir: &Path, read_limits: ReadLimits) -> Result<()> {
let retained_before = list_retained_with_limits(dir, read_limits)?;
let directory_entries = count_directory_entries(dir, read_limits, "store directory")?;
read_limits.admit_directory_entries(
"store directory during manifest commit",
directory_entries.saturating_add(2),
)?;
self.commit += 1;
self.prev = read_manifest_file(&dir.join("MANIFEST"))
.ok()
.map(|b| blake3::hash(&b).to_hex().to_string());
let bytes = self.encode()?;
{
let p = retained_path(dir, self.commit);
let f = crate::vfs::create(&p)?;
crate::vfs::write_all_at(&f, &p, &bytes, 0)?;
crate::vfs::sync_file(&f, &p)?;
}
let tmp = dir.join("MANIFEST.tmp");
let f = crate::vfs::create(&tmp)?;
crate::vfs::write_all_at(&f, &tmp, &bytes, 0)?;
crate::vfs::sync_file(&f, &tmp)?;
drop(f);
crate::vfs::rename(&tmp, &dir.join("MANIFEST"))?;
crate::vfs::sync_dir(dir)?;
for c in retained_before {
if c + (MANIFEST_RETAIN as u64) <= self.commit {
let _ = crate::vfs::unlink(&retained_path(dir, c));
}
}
Ok(())
}
fn fold_tail(&self) -> Option<FoldTail> {
if self.next_seq == 0 && self.parts.is_empty() && self.fold_off == 0 {
None
} else {
Some(FoldTail { seg: self.fold_seg, off: self.fold_off })
}
}
}
fn checksum_trailer(bytes: &[u8]) -> Option<(&[u8], u32)> {
let pos = bytes.iter().rposition(|&b| b == b'\n')?;
let tail = &bytes[pos + 1..];
if tail.len() != 14 || !tail.starts_with(b"crc32=") {
return None;
}
let hex = std::str::from_utf8(&tail[6..]).ok()?;
let want = u32::from_str_radix(hex, 16).ok()?;
Some((&bytes[..pos], want))
}
fn retained_path(dir: &Path, commit: u64) -> PathBuf {
dir.join(format!("MANIFEST.{commit:08}"))
}
fn list_retained(dir: &Path) -> Result<Vec<u64>> {
list_retained_with_limits(dir, ReadLimits::default())
}
fn list_retained_with_limits(dir: &Path, read_limits: ReadLimits) -> Result<Vec<u64>> {
let read_limits = read_limits.validate()?;
let mut out = Vec::new();
let rd = std::fs::read_dir(dir).with_context(|| {
format!("read store directory {} for retained manifests", dir.display())
})?;
let mut visited = 0u64;
for e in rd {
visited = visited.saturating_add(1);
read_limits.admit_directory_entries("store directory", visited)?;
let e = e?;
let name = e.file_name().to_string_lossy().to_string();
if let Some(rest) = name.strip_prefix("MANIFEST.") {
if !rest.is_empty() && rest.bytes().all(|b| b.is_ascii_digit()) {
if let Ok(n) = rest.parse::<u64>() {
out.push(n);
}
}
}
}
out.sort_unstable();
Ok(out)
}
fn count_directory_entries(
dir: &Path,
read_limits: ReadLimits,
label: &'static str,
) -> Result<u64> {
let mut visited = 0u64;
for entry in std::fs::read_dir(dir)? {
visited = visited.saturating_add(1);
read_limits.admit_directory_entries(label, visited)?;
entry?;
}
Ok(visited)
}
pub fn retained_commits(dir: &Path) -> Result<Vec<u64>> {
list_retained(dir)
}
pub fn retained_commits_with_limits(dir: &Path, read_limits: ReadLimits) -> Result<Vec<u64>> {
list_retained_with_limits(dir, read_limits)
}
pub(crate) fn manifest_from_bytes(b: &[u8]) -> Result<Manifest> {
Manifest::parse(b)
}
pub fn open_read_pack(path: &Path, cfg: FoldCfg) -> Result<ReadStore> {
open_read_pack_with_limits(path, cfg, ReadLimits::default())
}
pub fn open_read_pack_with_limits(
path: &Path,
cfg: FoldCfg,
read_limits: ReadLimits,
) -> Result<ReadStore> {
let read_limits = read_limits.validate()?;
let pack = crate::pack::Pack::open(path)?;
let manifest = Manifest::parse(&pack.read_file_bounded("MANIFEST", MAX_MANIFEST_BYTES)?)?;
let fold_rel = if manifest.fold_gen == 0 {
"fold".to_string()
} else {
format!("fold-{:04}", manifest.fold_gen)
};
let mut segs = Vec::new();
let mut dict_files = Vec::new();
for name in pack.names().map(String::from).collect::<Vec<_>>() {
let Some(rest) = name.strip_prefix(&format!("{fold_rel}/")) else { continue };
if let Some(n) = crate::fold::segment::parse_seg_name(rest) {
let extent = pack.file(&name).expect("name came from this pack's TOC");
let segment_len = crate::readat::ReadAt::len(&extent)?;
segs.push(crate::fold::SegmentInput {
seg: n,
reader: Arc::new(extent) as Arc<dyn crate::readat::ReadAt>,
sidecar: pack
.read_file_bounded(
&format!("{fold_rel}/seg-{n:08}.dir"),
crate::fold::segment::max_dir_sidecar_bytes(segment_len)
.min(read_limits.max_stored_frame_bytes),
)
.ok(),
});
} else if rest.starts_with("zdict-") && rest.ends_with(".zd") {
dict_files.push(pack.read_file_bounded(&name, crate::fold::MAX_DICTIONARY_BYTES)?);
}
}
let fold = Fold::open_read_from_with_limits(
segs,
dict_files,
cfg,
path,
&manifest.punched,
read_limits,
)?;
let pcache = SectionCache::shared();
let mut parts = Vec::with_capacity(manifest.parts.len());
for p in &manifest.parts {
let ext = pack.file(&p.file).ok_or_else(|| {
anyhow::anyhow!("pack manifest names {} but the pack does not hold it", p.file)
})?;
parts.push(Arc::new(Part::open_reader_with_limits(
Box::new(ext),
pcache.clone(),
read_limits,
)?));
}
Ok(ReadStore { fold: Arc::new(fold), parts, manifest, read_limits })
}
pub fn open_read_container(path: &Path, cfg: FoldCfg) -> Result<ReadStore> {
open_read_container_with_limits(path, cfg, ReadLimits::default())
}
pub fn open_read_container_with_limits(
path: &Path,
cfg: FoldCfg,
read_limits: ReadLimits,
) -> Result<ReadStore> {
let read_limits = read_limits.validate()?;
let container = crate::container::Container::open(path)?;
let manifest = Manifest::parse(&container.read_file_bounded("MANIFEST", MAX_MANIFEST_BYTES)?)?;
let fold_rel = if manifest.fold_gen == 0 {
"fold".to_string()
} else {
format!("fold-{:04}", manifest.fold_gen)
};
let mut segs = Vec::new();
let mut dict_files = Vec::new();
for name in container.names().map(String::from).collect::<Vec<_>>() {
let Some(rest) = name.strip_prefix(&format!("{fold_rel}/")) else { continue };
if let Some(n) = crate::fold::segment::parse_seg_name(rest) {
let extent = container.extent(&name).expect("name came from this directory");
let segment_len = crate::readat::ReadAt::len(&extent)?;
segs.push(crate::fold::SegmentInput {
seg: n,
reader: Arc::new(extent) as Arc<dyn crate::readat::ReadAt>,
sidecar: container
.read_file_bounded(
&format!("{fold_rel}/seg-{n:08}.dir"),
crate::fold::segment::max_dir_sidecar_bytes(segment_len)
.min(read_limits.max_stored_frame_bytes),
)
.ok(),
});
} else if rest.starts_with("zdict-") && rest.ends_with(".zd") {
dict_files.push(container.read_file_bounded(&name, crate::fold::MAX_DICTIONARY_BYTES)?);
}
}
let fold = Fold::open_read_from_with_limits(
segs,
dict_files,
cfg,
path,
&manifest.punched,
read_limits,
)?;
let pcache = SectionCache::shared();
let mut parts = Vec::with_capacity(manifest.parts.len());
for p in &manifest.parts {
let ext = container.extent(&p.file).ok_or_else(|| {
anyhow::anyhow!(
"container manifest names {} but the container does not hold it",
p.file
)
})?;
parts.push(Arc::new(Part::open_reader_with_limits(
Box::new(ext),
pcache.clone(),
read_limits,
)?));
}
Ok(ReadStore { fold: Arc::new(fold), parts, manifest, read_limits })
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SingleFileKind {
Pack,
Container,
}
pub fn single_file_kind(path: &Path) -> Option<SingleFileKind> {
if !path.is_file() {
return None;
}
let f = std::fs::File::open(path).ok()?;
let len = f.metadata().ok()?.len();
let mut magic = [0u8; 8];
if len >= crate::container::REGION_START
&& crate::sys::read_exact_at(&f, &mut magic, 0).is_ok()
&& &magic == crate::container::MAGIC
{
return Some(SingleFileKind::Container);
}
if len >= crate::pack::FOOTER_LEN
&& crate::sys::read_exact_at(&f, &mut magic, len - crate::pack::FOOTER_LEN).is_ok()
&& &magic == crate::pack::MAGIC
{
return Some(SingleFileKind::Pack);
}
None
}
pub fn open_read_file(path: &Path, cfg: FoldCfg) -> Result<ReadStore> {
open_read_file_with_limits(path, cfg, ReadLimits::default())
}
pub fn open_read_file_with_limits(
path: &Path,
cfg: FoldCfg,
read_limits: ReadLimits,
) -> Result<ReadStore> {
match single_file_kind(path) {
Some(SingleFileKind::Container) => open_read_container_with_limits(path, cfg, read_limits),
Some(SingleFileKind::Pack) => open_read_pack_with_limits(path, cfg, read_limits),
None => bail!("{} is not a turndb pack or container (no recognised magic)", path.display()),
}
}
pub fn checkpoint_into_container(dir: &Path, out: &Path) -> Result<CheckpointStats> {
let wal = dir.join("WAL");
if wal.metadata().map(|m| m.len()).unwrap_or(0) != 0 {
bail!("checkpoint refuses a store with a non-empty WAL: settle it with sync and flush");
}
let manifest = Manifest::load_with_limits(dir, ReadLimits::default())?;
let mut container = if out.exists() {
crate::container::Container::open(out)?
} else {
crate::container::Container::create(out)?
};
let mut names: Vec<String> = vec!["MANIFEST".to_string()];
for p in &manifest.parts {
names.push(p.file.clone());
}
let fold_rel = if manifest.fold_gen == 0 {
"fold".to_string()
} else {
format!("fold-{:04}", manifest.fold_gen)
};
let fold_path = refold::fold_dir(dir, manifest.fold_gen);
let mut fold_names: Vec<String> = Vec::new();
if let Ok(entries) = std::fs::read_dir(&fold_path) {
for entry in entries.flatten() {
let name = entry.file_name().to_string_lossy().into_owned();
let keep = name.ends_with(".fold")
|| name.ends_with(".dir")
|| (name.starts_with("zdict-") && name.ends_with(".zd"));
if keep {
fold_names.push(format!("{fold_rel}/{name}"));
}
}
}
fold_names.sort();
names.extend(fold_names);
let mut ingested = 0u64;
let mut skipped = 0usize;
for name in &names {
let src = dir.join(name);
let src_len = std::fs::metadata(&src).map(|m| m.len()).unwrap_or(0);
let immutable = name != "MANIFEST" && !name.ends_with(".dir");
if immutable {
if let Some(existing) = container.extent(name) {
if crate::readat::ReadAt::len(&existing)? == src_len {
skipped += 1;
continue;
}
}
}
ingested += container.ingest(name, &src)?;
}
let seq = container.commit()?;
Ok(CheckpointStats {
members: container.len(),
ingested_bytes: ingested,
skipped_members: skipped,
commit_seq: seq,
free_bytes: container.free_bytes(),
})
}
#[derive(Clone, Copy, Debug)]
pub struct CheckpointStats {
pub members: usize,
pub ingested_bytes: u64,
pub skipped_members: usize,
pub commit_seq: u64,
pub free_bytes: u64,
}
fn load_retained(dir: &Path, commit: u64) -> Result<Manifest> {
let p = retained_path(dir, commit);
let b = read_manifest_file(&p).with_context(|| {
format!("no retained manifest {} — the retention window has moved past it", p.display())
})?;
Manifest::parse(&b).with_context(|| format!("retained manifest {} is corrupt", p.display()))
}
fn sweep_unreachable_with_limits(dir: &Path, read_limits: ReadLimits) -> Result<()> {
let mut keep: Vec<Manifest> = vec![Manifest::load_with_limits(dir, read_limits)?];
for c in list_retained_with_limits(dir, read_limits)? {
if let Ok(m) = load_retained(dir, c) {
keep.push(m);
}
}
let live_parts: HashSet<&str> =
keep.iter().flat_map(|m| m.parts.iter().map(|p| p.file.as_str())).collect();
let live_gens: HashSet<u32> = keep.iter().map(|m| m.fold_gen).collect();
let rd = std::fs::read_dir(dir)
.with_context(|| format!("read store directory {} for orphan cleanup", dir.display()))?;
let mut visited = 0u64;
for e in rd {
visited = visited.saturating_add(1);
read_limits.admit_directory_entries("store directory", visited)?;
let e = e?;
let n = e.file_name().to_string_lossy().to_string();
if n.starts_with("part-") && n.ends_with(".part") && !live_parts.contains(n.as_str()) {
let _ = crate::vfs::unlink(&e.path());
}
if e.path().is_dir() {
if let Some(g) = refold::parse_fold_gen(&n) {
if !live_gens.contains(&g) {
let _ = crate::vfs::remove_tree(&e.path());
}
}
}
}
Ok(())
}
fn cleanup_refold_stage(dir: &Path, generation: u32, built: &[refold::RefoldedPart]) {
let fold = refold::fold_dir(dir, generation);
if fold.exists() {
let _ = crate::vfs::remove_tree(&fold);
}
for (file, ..) in built {
let path = dir.join(file);
if path.exists() {
let _ = crate::vfs::unlink(&path);
}
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct ChainReport {
pub retained_manifests: usize,
pub links: usize,
pub part_digests: usize,
pub undigested: usize,
}
#[derive(Clone, Copy, Debug, Default)]
pub struct StoreVerification {
pub chain: ChainReport,
pub fold: crate::fold::FoldScrub,
pub parts: usize,
pub part_sections: usize,
pub records: usize,
pub content_values: usize,
pub content_bytes: u64,
pub content_identities: usize,
pub unidentified_content_values: usize,
}
pub fn verify_chain(dir: &Path) -> Result<ChainReport> {
verify_chain_with_control(dir, &crate::control::OperationControl::default())
}
pub fn verify_chain_with_control(
dir: &Path,
control: &crate::control::OperationControl,
) -> Result<ChainReport> {
verify_chain_with_limits_and_control(dir, ReadLimits::default(), control)
}
pub fn verify_chain_with_limits_and_control(
dir: &Path,
read_limits: ReadLimits,
control: &crate::control::OperationControl,
) -> Result<ChainReport> {
let mut report = ChainReport::default();
let commits = list_retained_with_limits(dir, read_limits)?;
report.retained_manifests = commits.len();
let mut prev_bytes: Option<Vec<u8>> = None;
for &c in &commits {
control.check("manifest verification")?;
let bytes = read_manifest_file(&retained_path(dir, c))?;
let m =
Manifest::parse(&bytes).with_context(|| format!("retained manifest {c} is corrupt"))?;
if let (Some(want), Some(pb)) = (&m.prev, &prev_bytes) {
let got = blake3::hash(pb).to_hex().to_string();
if *want != got {
bail!("manifest chain broken: commit {c} names prev {want} but commit {} hashes to {got}", c - 1);
}
report.links += 1;
}
for p in &m.parts {
control.check("manifest verification")?;
match &p.b3 {
Some(want) => {
let got = hash_file_with_control(
&dir.join(&p.file),
control,
"manifest verification",
)
.with_context(|| format!("part {} named by commit {c}", p.file))?
.to_hex()
.to_string();
if *want != got {
bail!("part {} drifted from the digest commit {c} pinned", p.file);
}
report.part_digests += 1;
}
None => report.undigested += 1,
}
}
prev_bytes = Some(bytes);
}
if let (Some(&newest), Some(pb)) = (commits.last(), &prev_bytes) {
let live = read_manifest_file(&dir.join("MANIFEST"))?;
if live != *pb {
bail!("MANIFEST diverges from its retained copy at commit {newest}");
}
report.links += 1;
}
Ok(report)
}
fn verification_integrity<T>(context: &'static str, result: Result<T>) -> Result<T> {
result.map_err(|error| {
if crate::error::classify(&error) == crate::error::ErrorClass::Internal {
crate::error::IntegrityError::new(context, error).into()
} else {
error
}
})
}
#[cfg(test)]
fn complete_first_commit(dir: &Path) -> Result<u64> {
complete_first_commit_with_limits(dir, ReadLimits::default())
}
fn complete_first_commit_with_limits(dir: &Path, read_limits: ReadLimits) -> Result<u64> {
if Manifest::load_with_limits(dir, read_limits).is_ok() {
bail!("MANIFEST at {} is intact — refusing to roll back a healthy store", dir.display());
}
for c in list_retained_with_limits(dir, read_limits)?.into_iter().rev() {
if load_retained(dir, c).is_err() {
continue;
}
let bytes = read_manifest_file(&retained_path(dir, c))?;
promote_manifest_with_limits(dir, c, &bytes, read_limits)?;
return Ok(c);
}
bail!("MANIFEST at {} is damaged and no retained manifest is intact", dir.display());
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct RecoveryOptions {
pub max_rollback_commits: u64,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct RecoveryReport {
pub commit: u64,
pub rollback_commits: u64,
pub records: usize,
pub content_values: usize,
pub parts: usize,
pub part_sections: usize,
pub fold_segments: u32,
pub fold_blocks: usize,
pub fold_bytes: u64,
}
#[derive(Debug)]
pub enum RecoveryError {
Healthy(PathBuf),
RollbackLimit { needed: u64, allowed: u64 },
NoUsableCandidate { examined: usize, reason: String },
}
impl std::fmt::Display for RecoveryError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
RecoveryError::Healthy(path) => write!(
f,
"MANIFEST at {} is intact; refusing recovery of a healthy store",
path.display()
),
RecoveryError::RollbackLimit { needed, allowed } => write!(
f,
"recovery needs to abandon {needed} newer retained commits but only {allowed} were authorized"
),
RecoveryError::NoUsableCandidate { examined, reason } => write!(
f,
"none of {examined} retained manifests is a fully readable recovery candidate: {reason}"
),
}
}
}
impl std::error::Error for RecoveryError {}
pub fn recover_manifest(
dir: &Path,
cfg: FoldCfg,
options: RecoveryOptions,
) -> Result<RecoveryReport> {
recover_manifest_with_control(dir, cfg, options, &crate::control::OperationControl::default())
}
pub fn recover_manifest_with_control(
dir: &Path,
cfg: FoldCfg,
options: RecoveryOptions,
control: &crate::control::OperationControl,
) -> Result<RecoveryReport> {
recover_manifest_with_limits_and_control(dir, cfg, options, ReadLimits::default(), control)
}
pub fn recover_manifest_with_limits_and_control(
dir: &Path,
cfg: FoldCfg,
options: RecoveryOptions,
read_limits: ReadLimits,
control: &crate::control::OperationControl,
) -> Result<RecoveryReport> {
let read_limits = read_limits.validate()?;
control.check("manifest recovery")?;
let _locks = recovery_locks(dir, read_limits, control)?;
control.check("manifest recovery")?;
if Manifest::load_with_limits(dir, read_limits).is_ok() {
return Err(RecoveryError::Healthy(dir.to_path_buf()).into());
}
let commits = list_retained_with_limits(dir, read_limits)?;
let newest = commits.last().copied().unwrap_or(0);
let mut examined = 0usize;
let mut last_reason = "no retained manifests exist".to_string();
for c in commits.into_iter().rev() {
control.check("manifest recovery validation")?;
examined += 1;
let manifest = match load_retained(dir, c) {
Ok(manifest) => manifest,
Err(error) => {
last_reason = error.to_string();
continue;
}
};
match validate_recovery_candidate(dir, cfg, manifest, read_limits, control) {
Ok(mut report) => {
let rollback_commits = newest.saturating_sub(c);
if rollback_commits > options.max_rollback_commits {
return Err(RecoveryError::RollbackLimit {
needed: rollback_commits,
allowed: options.max_rollback_commits,
}
.into());
}
let bytes = read_manifest_file(&retained_path(dir, c))?;
control.check("manifest recovery publication")?;
promote_manifest_with_limits(dir, c, &bytes, read_limits)?;
report.commit = c;
report.rollback_commits = rollback_commits;
return Ok(report);
}
Err(error) => {
if matches!(
crate::error::classify(&error),
crate::error::ErrorClass::Cancelled
| crate::error::ErrorClass::ResourceExhausted
) {
return Err(error);
}
last_reason = error.to_string();
}
}
}
Err(RecoveryError::NoUsableCandidate { examined, reason: last_reason }.into())
}
fn recovery_locks(
dir: &Path,
read_limits: ReadLimits,
control: &crate::control::OperationControl,
) -> Result<Vec<File>> {
let mut folds = Vec::new();
let mut visited = 0u64;
for entry in std::fs::read_dir(dir)
.with_context(|| format!("read store directory {} for recovery", dir.display()))?
{
visited = visited.saturating_add(1);
read_limits.admit_directory_entries("store directory", visited)?;
control.check("manifest recovery locking")?;
let entry = entry?;
if entry.path().is_dir()
&& refold::parse_fold_gen(&entry.file_name().to_string_lossy()).is_some()
{
folds.push(entry.path());
}
}
folds.sort();
folds
.into_iter()
.map(|path| {
control.check("manifest recovery locking")?;
crate::fold::acquire_writer_lock(&path)
})
.collect()
}
fn validate_recovery_candidate(
dir: &Path,
cfg: FoldCfg,
manifest: Manifest,
read_limits: ReadLimits,
control: &crate::control::OperationControl,
) -> Result<RecoveryReport> {
control.check("manifest recovery validation")?;
let fold_dir = refold::fold_dir(dir, manifest.fold_gen);
let fold = match manifest.fold_tail() {
Some(tail) => {
Fold::open_read_at_with_limits(&fold_dir, cfg, tail, &manifest.punched, read_limits)?
}
None => Fold::open_read_with_limits(&fold_dir, cfg, &manifest.punched, read_limits)?,
};
let scrub = fold.scrub_with_control(control)?;
let pcache = SectionCache::shared();
let mut parts = Vec::with_capacity(manifest.parts.len());
let mut part_sections = 0usize;
for part_ref in &manifest.parts {
control.check("manifest recovery validation")?;
let path = dir.join(&part_ref.file);
if let Some(want) = &part_ref.b3 {
let got = hash_file_with_control(&path, control, "manifest recovery validation")?
.to_hex()
.to_string();
if &got != want {
bail!(
"candidate commit {} names part {} with the wrong digest",
manifest.commit,
part_ref.file
);
}
}
let part = Arc::new(Part::open_in_with_limits(&path, pcache.clone(), read_limits)?);
part_sections += part.verify_sections_with_control(control)?;
parts.push(part);
}
let reader = ReadStore { fold: Arc::new(fold), parts, manifest, read_limits };
let ids = reader.ids()?;
let mut content_values = 0usize;
for id in &ids {
control.check("manifest recovery validation")?;
let record = reader.get(id)?.expect("ids returns visible records");
for content in record.contents {
control.check("manifest recovery validation")?;
let mut identity = blake3::Hasher::new();
for op in content.ops {
control.check("manifest recovery validation")?;
match op {
BodyOp::Lit(bytes) => {
identity.update(&bytes);
}
BodyOp::Piece { hash, len } => {
let mut loc = None;
for part in reader.parts.iter().rev() {
if let Some(found) = part.lookup_piece(&hash)? {
loc = Some(found);
break;
}
}
let loc = loc.ok_or_else(|| {
anyhow::anyhow!(
"candidate record {id:?} references absent piece {hash}"
)
})?;
let bytes = reader.fold.read_verified(loc, hash)?;
if bytes.len() != len as usize {
bail!(
"candidate record {id:?} says piece {hash} is {len} bytes but it is {}",
bytes.len()
);
}
identity.update(&bytes);
}
}
}
if let Some(want) = content.identity {
let got = ContentHash(identity.finalize().into());
if got != want {
bail!(
"candidate record {id:?} content {:?} has identity {got}, expected {want}",
content.name
);
}
}
content_values += 1;
}
}
Ok(RecoveryReport {
records: ids.len(),
content_values,
parts: reader.parts.len(),
part_sections,
fold_segments: scrub.segments,
fold_blocks: scrub.blocks,
fold_bytes: scrub.bytes,
..RecoveryReport::default()
})
}
fn hash_file_with_control(
path: &Path,
control: &crate::control::OperationControl,
operation: &'static str,
) -> Result<blake3::Hash> {
use std::io::Read;
let mut file = File::open(path)?;
let mut hasher = blake3::Hasher::new();
let mut buf = vec![0u8; 1 << 20];
loop {
control.check(operation)?;
let read = file.read(&mut buf)?;
if read == 0 {
break;
}
hasher.update(&buf[..read]);
}
Ok(hasher.finalize())
}
fn promote_manifest_with_limits(
dir: &Path,
commit: u64,
bytes: &[u8],
read_limits: ReadLimits,
) -> Result<()> {
let directory_entries = count_directory_entries(dir, read_limits, "store directory")?;
read_limits.admit_directory_entries(
"store directory during manifest recovery",
directory_entries.saturating_add(1),
)?;
let mut abandoned = false;
for retained in list_retained_with_limits(dir, read_limits)? {
if retained > commit {
crate::vfs::unlink(&retained_path(dir, retained))?;
abandoned = true;
}
}
if abandoned {
crate::vfs::sync_dir(dir)?;
}
let tmp = dir.join("MANIFEST.tmp");
let f = crate::vfs::create(&tmp)?;
crate::vfs::write_all_at(&f, &tmp, bytes, 0)?;
crate::vfs::sync_file(&f, &tmp)?;
drop(f);
crate::vfs::rename(&tmp, &dir.join("MANIFEST"))?;
crate::vfs::sync_dir(dir)?;
Ok(())
}
pub struct Store {
dir: PathBuf,
fold: Fold,
parts: Vec<Arc<Part>>,
manifest: Manifest,
mem: BTreeMap<String, Option<Record>>,
mem_bytes: usize,
wal: Wal,
cfg: FoldCfg,
write_limits: WriteLimits,
read_limits: ReadLimits,
retained_commit_count: usize,
pcache: Arc<SectionCache>,
metrics: crate::observability::StoreMetrics,
events: crate::observability::EventJournal,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct StoreHealth {
pub commit: u64,
pub fold_generation: u32,
pub parts: usize,
pub part_rows: u64,
pub memtable_entries: usize,
pub memtable_bytes: usize,
pub wal_bytes: u64,
pub wal_frames: u64,
pub fold_disk_bytes: u64,
pub fold_segments: u32,
pub fold_cache_hits: u64,
pub fold_cache_misses: u64,
pub fold_cache_bytes: usize,
pub fold_cache_budget: usize,
pub fold_block_target_bytes: usize,
pub fold_segment_max_bytes: u32,
pub fold_compression_level: i32,
pub fold_compression_threads: usize,
pub part_cache_bytes: usize,
pub part_cache_budget: usize,
pub max_stored_frame_bytes: u64,
pub max_decoded_frame_bytes: u64,
pub max_directory_entries: u64,
pub max_wal_frames: u64,
pub max_fold_blocks: u64,
pub dedup_window_entries: usize,
pub retained_commits: usize,
pub punched_blocks: u64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct SpaceAmount {
pub files: usize,
pub logical_bytes: u64,
pub allocated_bytes: Option<u64>,
}
impl Default for SpaceAmount {
fn default() -> Self {
SpaceAmount {
files: 0,
logical_bytes: 0,
allocated_bytes: if cfg!(unix) { Some(0) } else { None },
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct StoreSpaceUsage {
pub live: SpaceAmount,
pub retained_only: SpaceAmount,
pub unclassified: SpaceAmount,
pub total: SpaceAmount,
pub filesystem_available_bytes: Option<u64>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RefoldSpaceEstimate {
pub source_fold_logical_bytes: u64,
pub source_part_bytes: u64,
pub source_part_sections: usize,
pub source_part_raw_section_bytes: u64,
pub retained_only_bytes_before: u64,
pub estimated_stage_bytes: u64,
pub estimate_is_hard_bound: bool,
pub filesystem_available_bytes: Option<u64>,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct FormatMigrationStatus {
pub target_part_version: u8,
pub live_parts: usize,
pub current_parts: usize,
pub legacy_parts: usize,
pub legacy_rows: u64,
pub legacy_bytes: u64,
pub retained_legacy_parts: usize,
pub retained_legacy_rows: u64,
pub retained_legacy_bytes: u64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct FormatMigrationPlan {
pub part_index: usize,
pub source_part_version: u8,
pub seq_lo: u64,
pub seq_hi: u64,
pub input_rows: u64,
pub input_bytes: u64,
pub input_sections: usize,
pub input_raw_section_bytes: u64,
pub estimated_stage_bytes: u64,
pub estimate_is_hard_bound: bool,
pub retained_input_bytes_after_commit: u64,
pub filesystem_available_bytes: Option<u64>,
}
#[derive(Clone, Copy, Debug)]
pub struct FormatMigrationStep {
pub plan: FormatMigrationPlan,
pub output_bytes: u64,
pub remaining_legacy_parts: usize,
pub rewrite: crate::part::merge::MergeStats,
}
fn check_range(from: Option<&str>, to: Option<&str>) -> Result<()> {
if let (Some(f), Some(t)) = (from, to) {
if f > t {
bail!("scan range is inverted: from {f:?} sorts after to {t:?}");
}
}
Ok(())
}
impl Store {
pub const AUTO_COMPACT_K: usize = 8;
pub fn open(dir: &Path, cfg: FoldCfg) -> Result<Store> {
Self::open_with_options(dir, StoreOptions { fold: cfg, ..StoreOptions::default() })
}
pub fn open_with_limits(dir: &Path, cfg: FoldCfg, write_limits: WriteLimits) -> Result<Store> {
Self::open_with_options(
dir,
StoreOptions { fold: cfg, write_limits, ..StoreOptions::default() },
)
}
pub fn open_with_options(dir: &Path, options: StoreOptions) -> Result<Store> {
let recovery_started = std::time::Instant::now();
let StoreOptions { fold: cfg, write_limits, read_limits, part_cache_bytes } = options;
let write_limits = write_limits.validate()?;
let read_limits = read_limits.validate()?;
if part_cache_bytes < crate::part::cache::BUDGET_MIN {
bail!("part_cache_bytes must be at least {}", crate::part::cache::BUDGET_MIN);
}
crate::vfs::mkdir_all(dir)?;
let manifest = match Manifest::load_with_limits(dir, read_limits) {
Ok(m) => m,
Err(e) => {
let retained = list_retained_with_limits(dir, read_limits)?;
if !dir.join("MANIFEST").exists() && retained == [1] {
if load_retained(dir, 1).is_ok() {
complete_first_commit_with_limits(dir, read_limits)?;
} else {
crate::vfs::unlink(&retained_path(dir, 1))?;
crate::vfs::sync_dir(dir)?;
}
verification_integrity(
"open committed manifest",
Manifest::load_with_limits(dir, read_limits),
)?
} else {
return verification_integrity("open committed manifest", Err(e));
}
}
};
let mut reconciled = false;
for c in list_retained_with_limits(dir, read_limits)? {
let stale_timeline = c > manifest.commit;
let stale_generation = !stale_timeline
&& load_retained(dir, c).map(|m| m.fold_gen != manifest.fold_gen).unwrap_or(false);
if stale_timeline || stale_generation {
crate::vfs::unlink(&retained_path(dir, c))?;
reconciled = true;
}
}
if reconciled {
crate::vfs::sync_dir(dir)?;
}
let root_entries = count_directory_entries(dir, read_limits, "store directory")?;
let fold_path = refold::fold_dir(dir, manifest.fold_gen);
let additions = u64::from(!fold_path.exists()) + u64::from(!dir.join("WAL").exists());
read_limits.admit_directory_entries(
"store directory during writer open",
root_entries.saturating_add(additions),
)?;
let mut fold = verification_integrity(
"open committed fold",
Fold::open_at_with_limits(
&fold_path,
cfg,
manifest.fold_tail(),
&manifest.punched,
read_limits,
),
)?;
let pcache = Arc::new(SectionCache::new(part_cache_bytes));
let mut parts = Vec::with_capacity(manifest.parts.len());
for p in &manifest.parts {
parts.push(Arc::new(verification_integrity(
"open committed part",
Part::open_in_with_limits(&dir.join(&p.file), pcache.clone(), read_limits),
)?));
}
sweep_unreachable_with_limits(dir, read_limits)?;
let rd = std::fs::read_dir(dir)
.with_context(|| format!("read store directory {} for temp cleanup", dir.display()))?;
let mut visited = 0u64;
for e in rd {
visited = visited.saturating_add(1);
read_limits.admit_directory_entries("store directory", visited)?;
let e = e?;
if e.file_name().to_string_lossy().ends_with(".tmp") && e.path().is_file() {
let _ = crate::vfs::unlink(&e.path());
}
}
let wal_path = dir.join("WAL");
let retained_commit_count = list_retained_with_limits(dir, read_limits)?.len();
let replay = Wal::replay_state_with_limits(&wal_path, read_limits)?;
let recovered_wal_frames = u64::try_from(replay.frames.len()).unwrap_or(u64::MAX);
let physical_wal_frames = replay.physical_frames;
let valid_wal_bytes = replay.valid_bytes;
let mut mem: BTreeMap<String, Option<Record>> = BTreeMap::new();
let mut mem_bytes = 0usize;
for f in replay.frames {
for (h, bytes) in &f.novel {
let put = fold.put_hashed(bytes, *h)?;
debug_assert_eq!(put.hash, *h);
}
mem_bytes += approx_bytes(&f.record);
if f.tomb {
mem.insert(f.record.id, None);
} else {
mem.insert(f.record.id.clone(), Some(f.record));
}
}
let wal =
Wal::open_recovered(&wal_path, read_limits, physical_wal_frames, valid_wal_bytes)?;
let mut metrics = crate::observability::StoreMetrics {
recovered_wal_frames,
..crate::observability::StoreMetrics::default()
};
let recovery_duration = recovery_started.elapsed();
metrics.open_recovery.observe_success(recovery_duration);
let mut events = crate::observability::EventJournal::default();
let recovery_result: Result<()> = Ok(());
events.observe(
crate::observability::LifecycleOperation::OpenRecovery,
recovery_duration,
&recovery_result,
);
Ok(Store {
dir: dir.to_path_buf(),
fold,
parts,
manifest,
mem,
mem_bytes,
wal,
cfg,
write_limits,
read_limits,
retained_commit_count,
pcache,
metrics,
events,
})
}
pub fn write_limits(&self) -> WriteLimits {
self.write_limits
}
pub fn read_limits(&self) -> ReadLimits {
self.read_limits
}
pub fn fold_cfg(&self) -> FoldCfg {
self.cfg
}
fn note_manifest_commit(&mut self) {
self.retained_commit_count =
self.retained_commit_count.saturating_add(1).min(MANIFEST_RETAIN);
}
fn admit_store_directory_growth(&self, additional: u64, operation: &str) -> Result<()> {
let current = count_directory_entries(&self.dir, self.read_limits, "store directory")?;
self.read_limits.admit_directory_entries(
format!("store directory during {operation}"),
current.saturating_add(additional),
)?;
Ok(())
}
pub fn metrics(&self) -> crate::observability::StoreMetrics {
self.metrics
}
pub fn lifecycle_events_after(
&self,
after_sequence: u64,
limit: usize,
) -> crate::observability::LifecycleEventBatch {
self.events.read_after(after_sequence, limit)
}
fn observe_lifecycle<T>(
&mut self,
operation: crate::observability::LifecycleOperation,
duration: std::time::Duration,
result: &Result<T>,
) {
use crate::observability::LifecycleOperation;
match operation {
LifecycleOperation::OpenRecovery => {
self.metrics.open_recovery.observe(duration, result)
}
LifecycleOperation::Sync => self.metrics.sync.observe(duration, result),
LifecycleOperation::Flush => self.metrics.flush.observe(duration, result),
LifecycleOperation::Compaction => self.metrics.compaction.observe(duration, result),
LifecycleOperation::Backup => self.metrics.backup.observe(duration, result),
LifecycleOperation::Verification => self.metrics.verification.observe(duration, result),
LifecycleOperation::Punch => self.metrics.punch.observe(duration, result),
LifecycleOperation::Refold => self.metrics.refold.observe(duration, result),
LifecycleOperation::Erase => self.metrics.erase.observe(duration, result),
LifecycleOperation::FormatMigration => {
self.metrics.format_migration.observe(duration, result)
}
}
self.events.observe(operation, duration, result);
}
pub fn verify(&mut self) -> Result<StoreVerification> {
self.verify_with_control(&crate::control::OperationControl::default())
}
pub fn verify_with_control(
&mut self,
control: &crate::control::OperationControl,
) -> Result<StoreVerification> {
let started = std::time::Instant::now();
let result = self.verify_inner_with_control(control);
self.observe_lifecycle(
crate::observability::LifecycleOperation::Verification,
started.elapsed(),
&result,
);
if result.as_ref().err().is_some_and(|error| {
crate::error::classify(error) == crate::error::ErrorClass::Corruption
}) {
self.metrics.verification_corruption_failures =
self.metrics.verification_corruption_failures.saturating_add(1);
}
result
}
fn verify_inner_with_control(
&self,
control: &crate::control::OperationControl,
) -> Result<StoreVerification> {
control.check("store verification")?;
let chain = verification_integrity(
"verify retained manifest chain",
verify_chain_with_limits_and_control(&self.dir, self.read_limits, control),
)?;
let fold =
verification_integrity("verify fold frames", self.fold.scrub_with_control(control))?;
let mut part_sections = 0usize;
for part in &self.parts {
control.check("store verification")?;
let sections = verification_integrity(
"verify immutable part sections",
part.verify_sections_with_control(control),
)?;
part_sections = part_sections
.checked_add(sections)
.ok_or_else(|| anyhow::anyhow!("verified part section count overflow"))?;
}
let ids = verification_integrity("enumerate committed records", read::ids(&self.parts))?;
let mut content_values = 0usize;
let mut content_bytes = 0u64;
let mut content_identities = 0usize;
let mut unidentified_content_values = 0usize;
for id in &ids {
control.check("store verification")?;
let record =
verification_integrity("decode committed record", read::get(&self.parts, id))?
.ok_or_else(|| {
anyhow::Error::new(crate::error::IntegrityError::new(
"decode committed record",
anyhow::anyhow!("live id {id:?} disappeared during verification"),
))
})?;
for content in &record.contents {
control.check("store verification")?;
let bytes = verification_integrity(
"reconstruct committed content",
read::reconstruct_content(&self.parts, &self.fold, id, &content.name),
)?
.ok_or_else(|| {
anyhow::Error::new(crate::error::IntegrityError::new(
"reconstruct committed content",
anyhow::anyhow!(
"record {id:?} lost named content {:?} during verification",
content.name
),
))
})?;
content_values = content_values
.checked_add(1)
.ok_or_else(|| anyhow::anyhow!("verified content value count overflow"))?;
content_bytes = content_bytes
.checked_add(bytes.len() as u64)
.ok_or_else(|| anyhow::anyhow!("verified content byte count overflow"))?;
match content.identity {
Some(want) => {
let got = crate::types::ContentHash::of(&bytes);
if got != want {
return Err(crate::error::IntegrityError::new(
"verify committed content identity",
anyhow::anyhow!(
"record {id:?} content {:?} hashes to {got} but claims {want}",
content.name
),
)
.into());
}
content_identities =
content_identities.checked_add(1).ok_or_else(|| {
anyhow::anyhow!("verified content identity count overflow")
})?;
}
None => {
unidentified_content_values =
unidentified_content_values.checked_add(1).ok_or_else(|| {
anyhow::anyhow!("unidentified content value count overflow")
})?;
}
}
}
}
Ok(StoreVerification {
chain,
fold,
parts: self.parts.len(),
part_sections,
records: ids.len(),
content_values,
content_bytes,
content_identities,
unidentified_content_values,
})
}
pub fn part_distribution(&self) -> Result<crate::observability::PartDistribution> {
self.part_distribution_with_control(&crate::control::OperationControl::default())
}
pub fn part_distribution_with_control(
&self,
control: &crate::control::OperationControl,
) -> Result<crate::observability::PartDistribution> {
control.check("part distribution")?;
let mut bytes = Vec::with_capacity(self.manifest.parts.len());
let mut rows = Vec::with_capacity(self.manifest.parts.len());
let mut total_bytes = 0u64;
let mut total_rows = 0u64;
for part in &self.manifest.parts {
control.check("part distribution")?;
let size = std::fs::metadata(self.dir.join(&part.file))
.with_context(|| format!("measure live part {}", part.file))?
.len();
let part_rows = u64::from(part.records);
total_bytes = total_bytes
.checked_add(size)
.ok_or_else(|| anyhow::anyhow!("part distribution byte count overflow"))?;
total_rows = total_rows
.checked_add(part_rows)
.ok_or_else(|| anyhow::anyhow!("part distribution row count overflow"))?;
bytes.push(size);
rows.push(part_rows);
}
bytes.sort_unstable();
rows.sort_unstable();
let percentile = |values: &[u64], percent: usize| -> u64 {
let rank = values
.len()
.saturating_mul(percent)
.saturating_add(99)
.checked_div(100)
.unwrap_or(0);
values
.get(rank.saturating_sub(1).min(values.len().saturating_sub(1)))
.copied()
.unwrap_or(0)
};
Ok(crate::observability::PartDistribution {
parts: bytes.len(),
total_bytes,
min_bytes: bytes.first().copied().unwrap_or(0),
p50_bytes: percentile(&bytes, 50),
p95_bytes: percentile(&bytes, 95),
max_bytes: bytes.last().copied().unwrap_or(0),
total_rows,
min_rows: rows.first().copied().unwrap_or(0),
p50_rows: percentile(&rows, 50),
p95_rows: percentile(&rows, 95),
max_rows: rows.last().copied().unwrap_or(0),
})
}
pub fn content_liveness(&self) -> Result<crate::observability::ContentLiveness> {
self.content_liveness_with_control(&crate::control::OperationControl::default())
}
pub fn content_liveness_with_control(
&self,
control: &crate::control::OperationControl,
) -> Result<crate::observability::ContentLiveness> {
control.check("content liveness")?;
if !self.mem.is_empty() {
bail!("content liveness requires a flushed memtable; call sync() and flush() first");
}
let live_pieces = self.live_fold_pieces_with_control(control)?;
let live_block_ids: HashSet<u32> =
live_pieces.values().map(|location| location.block_id).collect();
let mut report = crate::observability::ContentLiveness {
live_pieces: u64::try_from(live_pieces.len())
.map_err(|_| anyhow::anyhow!("live piece count exceeds u64"))?,
..crate::observability::ContentLiveness::default()
};
for location in live_pieces.values() {
report.live_logical_bytes = report
.live_logical_bytes
.checked_add(u64::from(location.raw))
.ok_or_else(|| anyhow::anyhow!("live content byte count overflow"))?;
}
let inventory = self.fold.block_inventory_with_control(control)?;
let punched = |block_id: u32| {
self.manifest.punched.iter().any(|&(lo, hi)| (lo..=hi).contains(&block_id))
};
let mut observed_live_blocks = HashSet::new();
for block in inventory {
control.check("content liveness")?;
if punched(block.block_id) {
continue;
}
if live_block_ids.contains(&block.block_id) {
report
.live_blocks
.checked_observe(block.raw_bytes, block.stored_bytes)
.ok_or_else(|| anyhow::anyhow!("live block space count overflow"))?;
observed_live_blocks.insert(block.block_id);
} else {
report
.reclaimable_blocks
.checked_observe(block.raw_bytes, block.stored_bytes)
.ok_or_else(|| anyhow::anyhow!("reclaimable block space count overflow"))?;
}
}
if observed_live_blocks != live_block_ids {
let missing = live_block_ids.difference(&observed_live_blocks).next().copied().unwrap();
bail!("live content references block {missing}, which is absent or declared punched");
}
report.stranded_dead_logical_bytes =
report.live_blocks.raw_bytes.checked_sub(report.live_logical_bytes).ok_or_else(
|| anyhow::anyhow!("live piece bytes exceed their containing block bytes"),
)?;
report.dead_logical_bytes = report
.reclaimable_blocks
.raw_bytes
.checked_add(report.stranded_dead_logical_bytes)
.ok_or_else(|| anyhow::anyhow!("dead content byte count overflow"))?;
Ok(report)
}
pub fn open_read(dir: &Path, cfg: FoldCfg) -> Result<ReadStore> {
Self::open_read_with_limits(dir, cfg, ReadLimits::default())
}
pub fn open_read_with_limits(
dir: &Path,
cfg: FoldCfg,
read_limits: ReadLimits,
) -> Result<ReadStore> {
let read_limits = read_limits.validate()?;
let mut last: Option<anyhow::Error> = None;
for _ in 0..8 {
let manifest = Manifest::load_with_limits(dir, read_limits)?;
let fold_path = refold::fold_dir(dir, manifest.fold_gen);
let fold = match manifest.fold_tail().map_or_else(
|| Fold::open_read_with_limits(&fold_path, cfg, &manifest.punched, read_limits),
|tail| {
Fold::open_read_at_with_limits(
&fold_path,
cfg,
tail,
&manifest.punched,
read_limits,
)
},
) {
Ok(fold) => fold,
Err(e) => {
let gone = e
.downcast_ref::<std::io::Error>()
.map(|io| io.kind() == std::io::ErrorKind::NotFound)
.unwrap_or(false)
|| !fold_path.exists();
if !gone {
return Err(e);
}
last = Some(e);
continue;
}
};
let pcache = SectionCache::shared();
let mut parts = Vec::with_capacity(manifest.parts.len());
let mut missed = false;
for p in &manifest.parts {
match Part::open_in_with_limits(&dir.join(&p.file), pcache.clone(), read_limits) {
Ok(part) => parts.push(Arc::new(part)),
Err(e) => {
let gone = e
.downcast_ref::<std::io::Error>()
.map(|io| io.kind() == std::io::ErrorKind::NotFound)
.unwrap_or(false)
|| !dir.join(&p.file).exists();
if !gone {
return Err(e);
}
last = Some(e);
missed = true;
break;
}
}
}
if !missed {
if Manifest::load_with_limits(dir, read_limits)?.fold_gen != manifest.fold_gen {
last = Some(anyhow::anyhow!(
"fold generation changed while opening a reader snapshot"
));
continue;
}
return Ok(ReadStore { fold: Arc::new(fold), parts, manifest, read_limits });
}
}
Err(last.unwrap_or_else(|| {
anyhow::anyhow!("manifest snapshot names storage that does not exist")
}))
}
pub fn open_read_at(dir: &Path, cfg: FoldCfg, commit: u64) -> Result<ReadStore> {
Self::open_read_at_with_limits(dir, cfg, commit, ReadLimits::default())
}
pub fn open_read_at_with_limits(
dir: &Path,
cfg: FoldCfg,
commit: u64,
read_limits: ReadLimits,
) -> Result<ReadStore> {
let read_limits = read_limits.validate()?;
let manifest = load_retained(dir, commit)?;
let fold_dir = refold::fold_dir(dir, manifest.fold_gen);
let live = Manifest::load_with_limits(dir, read_limits).with_context(|| {
format!(
"retained snapshot at commit {commit} needs the live manifest in {} to tell erased \
blocks from damaged ones, and it could not be read",
dir.display()
)
})?;
if live.fold_gen != manifest.fold_gen {
bail!(
"retained snapshot at commit {commit} is fold generation {} but the live manifest is \
{}, so no erasure declaration covers it",
manifest.fold_gen,
live.fold_gen
);
}
let fold = match manifest.fold_tail() {
Some(tail) => {
Fold::open_read_at_with_limits(&fold_dir, cfg, tail, &live.punched, read_limits)?
}
None => Fold::open_read_with_limits(&fold_dir, cfg, &live.punched, read_limits)?,
};
let pcache = SectionCache::shared();
let mut parts = Vec::with_capacity(manifest.parts.len());
for p in &manifest.parts {
parts.push(Arc::new(Part::open_in_with_limits(
&dir.join(&p.file),
pcache.clone(),
read_limits,
)?));
}
Ok(ReadStore { fold: Arc::new(fold), parts, manifest, read_limits })
}
fn fold_piece(&mut self, b: &[u8]) -> Result<crate::fold::Put> {
let hash = PieceHash::of(b);
let result = if let Some(loc) = self.locate(&hash)? {
self.fold.note(hash, loc);
crate::fold::Put { hash, loc, deduped: true }
} else {
self.fold.put_hashed(b, hash)?
};
self.metrics.folded_content.observe(b.len(), result.deduped);
Ok(result)
}
pub fn put(&mut self, id: &str, spans: &[Span], attrs: Vec<(String, AttrValue)>) -> Result<()> {
let input = [ContentSpans::new(BODY_CONTENT, spans.to_vec())];
input_record_admission_bytes(
id,
&input,
&attrs,
self.write_limits,
self.read_limits,
None,
)?;
self.wal.admit_additional_frames(1)?;
let mut novel = Vec::new();
let body = self.fold_spans(BODY_CONTENT, spans, &mut novel)?;
let rec = Record::new(id, vec![body], attrs)?;
self.stage_record(rec, novel)
}
pub fn put_record(
&mut self,
id: &str,
contents: &[ContentSpans<'_>],
attrs: Vec<(String, AttrValue)>,
) -> Result<()> {
input_record_admission_bytes(
id,
contents,
&attrs,
self.write_limits,
self.read_limits,
None,
)?;
self.wal.admit_additional_frames(1)?;
let mut novel = Vec::new();
let mut carved = Vec::with_capacity(contents.len());
for content in contents {
carved.push(self.fold_spans(content.name, &content.spans, &mut novel)?);
}
let rec = Record::new(id, carved, attrs)?;
self.stage_record(rec, novel)
}
fn fold_spans(
&mut self,
name: &str,
spans: &[Span<'_>],
novel: &mut Vec<(PieceHash, Vec<u8>)>,
) -> Result<Content> {
let mut ops = Vec::with_capacity(spans.len());
let mut identity = blake3::Hasher::new();
for s in spans {
match s {
Span::Lit(b) => {
identity.update(b);
ops.push(BodyOp::Lit(b.to_vec()));
}
Span::Piece(b) => {
identity.update(b);
let put = self.fold_piece(b)?;
if !put.deduped {
novel.push((put.hash, b.to_vec()));
}
ops.push(BodyOp::Piece { hash: put.hash, len: b.len() as u32 });
}
}
}
Ok(Content::identified(name, ops, ContentHash(identity.finalize().into())))
}
fn stage_record(&mut self, rec: Record, novel: Vec<(PieceHash, Vec<u8>)>) -> Result<()> {
self.wal.append(self.manifest.next_seq, &rec, &novel)?;
self.mem_bytes += approx_bytes(&rec);
self.mem.insert(rec.id.clone(), Some(rec));
Ok(())
}
pub fn put_body(
&mut self,
id: &str,
body: &[u8],
attrs: Vec<(String, AttrValue)>,
) -> Result<()> {
self.put_body_with(id, body, attrs, &crate::carve::Carve::default())
}
pub fn put_body_with(
&mut self,
id: &str,
body: &[u8],
attrs: Vec<(String, AttrValue)>,
carve: &crate::carve::Carve,
) -> Result<()> {
self.put(id, &carve.carve(body), attrs)
}
pub fn apply(&mut self, batch: Batch) -> Result<()> {
if batch.items.is_empty() {
return Ok(());
}
if batch.items.len() > self.write_limits.max_batch_records {
return Err(WriteAdmissionError::TooManyBatchRecords {
actual: batch.items.len(),
allowed: self.write_limits.max_batch_records,
}
.into());
}
let mut batch_bytes =
WAL_FRAME_OVERHEAD.saturating_add(varint_bytes(batch.items.len() as u64));
for (index, item) in batch.items.iter().enumerate() {
let item_bytes = match item {
BatchItem::Put { id, contents, attrs } => owned_record_admission_bytes(
id,
contents,
attrs,
self.write_limits,
self.read_limits,
Some(index),
)?,
BatchItem::Delete { id } => {
delete_admission_bytes(id, self.write_limits, self.read_limits, Some(index))?
}
};
add_size(&mut batch_bytes, item_bytes);
}
if batch_bytes > self.write_limits.max_batch_bytes {
return Err(WriteAdmissionError::BatchTooLarge {
actual: batch_bytes,
allowed: self.write_limits.max_batch_bytes,
}
.into());
}
self.wal.admit_additional_frames(batch.items.len() as u64 + 1)?;
let mut framed: Vec<crate::store::wal::FramedRecord> =
Vec::with_capacity(batch.items.len());
for item in &batch.items {
match item {
BatchItem::Put { id, contents, attrs } => {
let mut novel = Vec::new();
let mut carved = Vec::with_capacity(contents.len());
for content in contents {
let mut ops = Vec::with_capacity(content.spans.len());
let mut identity = blake3::Hasher::new();
for s in &content.spans {
match s {
OwnedSpan::Lit(b) => {
identity.update(b);
ops.push(BodyOp::Lit(b.clone()));
}
OwnedSpan::Piece(b) => {
identity.update(b);
let put = self.fold_piece(b)?;
if !put.deduped {
novel.push((put.hash, b.clone()));
}
ops.push(BodyOp::Piece { hash: put.hash, len: b.len() as u32 });
}
}
}
carved.push(Content::identified(
&content.name,
ops,
ContentHash(identity.finalize().into()),
));
}
framed.push((Record::new(id, carved, attrs.clone())?, novel, false));
}
BatchItem::Delete { id } => {
framed.push((
Record { id: id.clone(), contents: Vec::new(), attrs: Vec::new() },
Vec::new(),
true,
));
}
}
}
self.wal.append_batch(self.manifest.next_seq, &framed)?;
for (rec, _, tomb) in framed {
if tomb {
self.mem_bytes += rec.id.len() + 32;
self.mem.insert(rec.id, None);
} else {
self.mem_bytes += approx_bytes(&rec);
self.mem.insert(rec.id.clone(), Some(rec));
}
}
Ok(())
}
pub fn delete(&mut self, id: &str) -> Result<()> {
delete_admission_bytes(id, self.write_limits, self.read_limits, None)?;
self.wal.admit_additional_frames(1)?;
self.wal.append_tomb(self.manifest.next_seq, id)?;
self.mem_bytes += id.len() + 32;
self.mem.insert(id.to_string(), None);
Ok(())
}
pub fn sync(&mut self) -> Result<()> {
self.sync_with_control(&crate::control::OperationControl::default())
}
pub fn sync_with_control(&mut self, control: &crate::control::OperationControl) -> Result<()> {
let started = std::time::Instant::now();
let result = (|| {
control.check("store sync")?;
self.wal.sync()
})();
self.observe_lifecycle(
crate::observability::LifecycleOperation::Sync,
started.elapsed(),
&result,
);
result
}
pub fn backup(&mut self, out: &Path) -> Result<crate::pack::BackupStats> {
self.backup_with_control(out, &crate::control::OperationControl::default())
}
pub fn backup_with_control(
&mut self,
out: &Path,
control: &crate::control::OperationControl,
) -> Result<crate::pack::BackupStats> {
let started = std::time::Instant::now();
let result = self.backup_inner_with_control(out, control);
self.observe_lifecycle(
crate::observability::LifecycleOperation::Backup,
started.elapsed(),
&result,
);
result
}
fn backup_inner_with_control(
&mut self,
out: &Path,
control: &crate::control::OperationControl,
) -> Result<crate::pack::BackupStats> {
control.check("backup")?;
crate::pack::ensure_destination_available(out)?;
self.sync_with_control(control)?;
self.flush_with_control(control)?;
control.check("backup")?;
let stats =
crate::pack::write_committed_with_control(&self.dir, out, self.read_limits, control)?;
Ok(crate::pack::BackupStats {
files: stats.files,
bytes: stats.bytes,
commit: self.manifest.commit,
})
}
pub fn flush(&mut self) -> Result<Option<PartRef>> {
self.flush_with_control(&crate::control::OperationControl::default())
}
pub fn flush_with_control(
&mut self,
control: &crate::control::OperationControl,
) -> Result<Option<PartRef>> {
let started = std::time::Instant::now();
let result = self.flush_inner_with_control(control);
self.observe_lifecycle(
crate::observability::LifecycleOperation::Flush,
started.elapsed(),
&result,
);
result
}
fn flush_inner_with_control(
&mut self,
control: &crate::control::OperationControl,
) -> Result<Option<PartRef>> {
control.check("memtable flush")?;
if self.mem.is_empty() {
return Ok(None);
}
self.admit_store_directory_growth(3, "memtable flush")?;
let tail = self.fold.sync()?;
let seq = self.manifest.next_seq + 1;
let file = format!("part-{seq:08}.part");
let path = self.dir.join(&file);
let mut recs: Vec<Record> = Vec::with_capacity(self.mem.len());
let mut tombs: Vec<bool> = Vec::with_capacity(self.mem.len());
for (id, v) in &self.mem {
control.check("memtable flush planning")?;
match v {
Some(r) => {
recs.push(r.clone());
tombs.push(false);
}
None => {
recs.push(Record { id: id.clone(), contents: Vec::new(), attrs: Vec::new() });
tombs.push(true);
}
}
}
let mut locs: HashMap<PieceHash, Loc> = HashMap::new();
for r in &recs {
control.check("memtable flush planning")?;
for content in &r.contents {
for op in &content.ops {
control.check("memtable flush planning")?;
let BodyOp::Piece { hash, .. } = op else { continue };
if locs.contains_key(hash) {
continue;
}
let loc = self.locate(hash)?.ok_or_else(|| {
anyhow::anyhow!(
"staged piece {hash} is in neither the fold window nor any live part"
)
})?;
locs.insert(*hash, loc);
}
}
}
let meta = match part::build_full_with_limits(
&path,
&recs,
&tombs,
seq,
seq,
self.cfg.level,
|h| locs.get(h).copied(),
&HashMap::new(),
self.read_limits,
) {
Ok(meta) => meta,
Err(error) => {
let _ = crate::vfs::unlink(&path);
return Err(error);
}
};
if let Err(error) = control.check("memtable flush") {
let _ = crate::vfs::unlink(&path);
return Err(error.into());
}
let digest = match hash_file_with_control(&path, control, "memtable flush") {
Ok(hash) => hash.to_hex().to_string(),
Err(error) => {
let _ = crate::vfs::unlink(&path);
return Err(error);
}
};
let mut m = self.manifest.clone();
m.parts.push(PartRef {
file: file.clone(),
seq_lo: seq,
seq_hi: seq,
records: meta.n_records,
b3: Some(digest),
});
m.fold_seg = tail.seg;
m.fold_off = tail.off;
m.next_seq = seq;
if let Err(error) = control.check("memtable flush publication") {
let _ = crate::vfs::unlink(&path);
return Err(error.into());
}
m.commit_with_limits(&self.dir, self.read_limits)?;
self.parts.push(Arc::new(Part::open_in_with_limits(
&path,
self.pcache.clone(),
self.read_limits,
)?));
self.manifest = m;
self.note_manifest_commit();
sweep_unreachable_with_limits(&self.dir, self.read_limits)?;
self.mem.clear();
self.mem_bytes = 0;
self.fold.seal_window();
self.wal.truncate()?;
Ok(self.manifest.parts.last().cloned())
}
pub fn merge_range(
&mut self,
lo: usize,
len: usize,
) -> Result<Option<crate::part::merge::MergeStats>> {
self.merge_range_with_control(lo, len, &crate::control::OperationControl::default())
}
pub fn merge_range_with_control(
&mut self,
lo: usize,
len: usize,
control: &crate::control::OperationControl,
) -> Result<Option<crate::part::merge::MergeStats>> {
let started = std::time::Instant::now();
let result = self.merge_range_inner_with_control(lo, len, control);
self.observe_lifecycle(
crate::observability::LifecycleOperation::Compaction,
started.elapsed(),
&result,
);
result
}
fn merge_range_inner_with_control(
&mut self,
lo: usize,
len: usize,
control: &crate::control::OperationControl,
) -> Result<Option<crate::part::merge::MergeStats>> {
control.check("part compaction")?;
if len < 2 || lo + len > self.parts.len() {
return Ok(None);
}
self.admit_store_directory_growth(3, "part compaction")?;
let inputs: Vec<Arc<Part>> = self.parts[lo..lo + len].to_vec();
let seq_lo = self.manifest.parts[lo].seq_lo;
let seq_hi = self.manifest.parts[lo + len - 1].seq_hi;
let file = format!("part-{seq_lo:08}-{seq_hi:08}.part");
debug_assert!(
!self.manifest.parts.iter().any(|p| p.file == file),
"merge output {file} collides with a live part"
);
let path = self.dir.join(&file);
let total = lo == 0 && len == self.parts.len();
let (meta, stats) = match crate::part::merge::merge_opts_with_control_and_limits(
&path,
&inputs,
self.cfg.level,
total,
control,
self.read_limits,
) {
Ok(built) => built,
Err(error) => {
let _ = crate::vfs::unlink(&path);
return Err(error);
}
};
let digest = match hash_file_with_control(&path, control, "part compaction output hashing")
.map(|hash| hash.to_hex().to_string())
{
Ok(digest) => digest,
Err(error) => {
let _ = crate::vfs::unlink(&path);
return Err(error);
}
};
if let Err(error) = control.check("part compaction") {
let _ = crate::vfs::unlink(&path);
return Err(error.into());
}
let mut m = self.manifest.clone();
m.parts.splice(
lo..lo + len,
[PartRef {
file: file.clone(),
seq_lo: meta.seq_lo,
seq_hi: meta.seq_hi,
records: meta.n_records,
b3: Some(digest),
}],
);
m.commit_with_limits(&self.dir, self.read_limits)?;
self.parts.splice(
lo..lo + len,
[Arc::new(Part::open_in_with_limits(&path, self.pcache.clone(), self.read_limits)?)],
);
self.manifest = m;
self.note_manifest_commit();
sweep_unreachable_with_limits(&self.dir, self.read_limits)?;
Ok(Some(stats))
}
pub fn maybe_compact(
&mut self,
trigger: usize,
run: usize,
) -> Result<Option<crate::part::merge::MergeStats>> {
self.maybe_compact_with_control(trigger, run, &crate::control::OperationControl::default())
}
pub fn maybe_compact_with_control(
&mut self,
trigger: usize,
run: usize,
control: &crate::control::OperationControl,
) -> Result<Option<crate::part::merge::MergeStats>> {
control.check("part compaction")?;
if self.parts.len() < trigger {
return Ok(None);
}
self.merge_range_with_control(0, run.min(self.parts.len()), control)
}
pub fn auto_compact(&mut self) -> Result<Option<crate::part::merge::MergeStats>> {
self.auto_compact_with_control(&crate::control::OperationControl::default())
}
pub fn auto_compact_with_control(
&mut self,
control: &crate::control::OperationControl,
) -> Result<Option<crate::part::merge::MergeStats>> {
control.check("part compaction")?;
if self.parts.len() < Self::AUTO_COMPACT_K {
return Ok(None);
}
self.merge_range_with_control(0, self.parts.len(), control)
}
pub fn plan_compaction(&self, budget: CompactionBudget) -> Result<Option<CompactionPlan>> {
budget.validate()?;
if self.manifest.parts.len() < 2 {
return Ok(None);
}
let costs: Vec<(u64, u64)> = self
.manifest
.parts
.iter()
.map(|part| {
let bytes = std::fs::metadata(self.dir.join(&part.file))
.with_context(|| format!("measure compaction input {}", part.file))?
.len();
Ok((u64::from(part.records), bytes))
})
.collect::<Result<_>>()?;
let mut best: Option<CompactionPlan> = None;
for start in 0..costs.len() - 1 {
let mut rows = 0u64;
let mut bytes = 0u64;
for (offset, &(part_rows, part_bytes)) in
costs[start..].iter().take(budget.max_input_parts).enumerate()
{
rows = rows.saturating_add(part_rows);
bytes = bytes.saturating_add(part_bytes);
if rows > budget.max_input_rows || bytes > budget.max_input_bytes {
break;
}
let parts = offset + 1;
if parts < 2 {
continue;
}
let plan = CompactionPlan {
start_part: start,
input_parts: parts,
input_rows: rows,
input_bytes: bytes,
drops_tombstones: start == 0 && parts == costs.len(),
};
if best.as_ref().is_none_or(|current| plan.input_parts > current.input_parts) {
best = Some(plan);
}
}
}
if best.is_some() {
return Ok(best);
}
let (start_part, input_rows, input_bytes) = costs
.windows(2)
.enumerate()
.map(|(start, pair)| {
(start, pair[0].0.saturating_add(pair[1].0), pair[0].1.saturating_add(pair[1].1))
})
.min_by_key(|&(start, rows, bytes)| (bytes, rows, start))
.expect("at least two parts were checked above");
Err(CompactionError::BudgetTooSmall { start_part, input_rows, input_bytes, budget }.into())
}
pub fn estimate_compaction_space(
&self,
budget: CompactionBudget,
) -> Result<Option<CompactionSpaceEstimate>> {
self.estimate_compaction_space_with_control(
budget,
&crate::control::OperationControl::default(),
)
}
pub fn estimate_compaction_space_with_control(
&self,
budget: CompactionBudget,
control: &crate::control::OperationControl,
) -> Result<Option<CompactionSpaceEstimate>> {
control.check("compaction space preflight")?;
let Some(plan) = self.plan_compaction(budget)? else {
return Ok(None);
};
let end = plan.start_part + plan.input_parts;
let mut input_sections = 0usize;
let mut input_raw_section_bytes = 0u64;
for part in &self.parts[plan.start_part..end] {
control.check("compaction space preflight")?;
for (_, _, raw, _) in part.sections() {
control.check("compaction space preflight")?;
input_sections = input_sections
.checked_add(1)
.ok_or_else(|| anyhow::anyhow!("compaction section count overflow"))?;
input_raw_section_bytes = input_raw_section_bytes
.checked_add(u64::from(raw))
.ok_or_else(|| anyhow::anyhow!("compaction raw section byte count overflow"))?;
}
}
let row_allowance = plan
.input_rows
.checked_mul(64)
.ok_or_else(|| anyhow::anyhow!("compaction row framing estimate overflow"))?;
let section_allowance = u64::try_from(input_sections)
.map_err(|_| anyhow::anyhow!("compaction section count exceeds u64"))?
.checked_mul(256)
.ok_or_else(|| anyhow::anyhow!("compaction section framing estimate overflow"))?;
let estimated_stage_bytes = input_raw_section_bytes
.checked_add(row_allowance)
.and_then(|bytes| bytes.checked_add(section_allowance))
.and_then(|bytes| bytes.checked_add(1 << 20))
.ok_or_else(|| anyhow::anyhow!("compaction stage estimate overflow"))?;
Ok(Some(CompactionSpaceEstimate {
plan,
input_sections,
input_raw_section_bytes,
estimated_stage_bytes,
estimate_is_hard_bound: false,
retained_input_bytes_after_commit: plan.input_bytes,
filesystem_available_bytes: crate::sys::filesystem_available_bytes(&self.dir)
.with_context(|| {
format!("measure available filesystem bytes at {}", self.dir.display())
})?,
}))
}
pub fn compact_bounded(
&mut self,
budget: CompactionBudget,
) -> Result<Option<BoundedCompaction>> {
self.compact_bounded_with_control(budget, &crate::control::OperationControl::default())
}
pub fn compact_bounded_with_control(
&mut self,
budget: CompactionBudget,
control: &crate::control::OperationControl,
) -> Result<Option<BoundedCompaction>> {
control.check("bounded compaction")?;
let Some(plan) = self.plan_compaction(budget)? else {
return Ok(None);
};
let merge = self
.merge_range_with_control(plan.start_part, plan.input_parts, control)?
.expect("a compaction plan always contains at least two parts");
let output = &self.manifest.parts[plan.start_part];
let output_bytes = std::fs::metadata(self.dir.join(&output.file))
.with_context(|| format!("measure compaction output {}", output.file))?
.len();
Ok(Some(BoundedCompaction { plan, output_bytes, merge }))
}
pub fn format_migration_status(&self) -> Result<FormatMigrationStatus> {
self.format_migration_status_with_control(&crate::control::OperationControl::default())
}
pub fn format_migration_status_with_control(
&self,
control: &crate::control::OperationControl,
) -> Result<FormatMigrationStatus> {
control.check("format migration status")?;
let mut status = FormatMigrationStatus {
target_part_version: crate::part::PART_VERSION,
live_parts: self.parts.len(),
..FormatMigrationStatus::default()
};
for (part, part_ref) in self.parts.iter().zip(&self.manifest.parts) {
control.check("format migration status")?;
if part.format_version() == crate::part::PART_VERSION {
status.current_parts = status
.current_parts
.checked_add(1)
.ok_or_else(|| anyhow::anyhow!("current format part count overflow"))?;
continue;
}
status.legacy_parts = status
.legacy_parts
.checked_add(1)
.ok_or_else(|| anyhow::anyhow!("legacy format part count overflow"))?;
status.legacy_rows = status
.legacy_rows
.checked_add(u64::from(part_ref.records))
.ok_or_else(|| anyhow::anyhow!("legacy format row count overflow"))?;
status.legacy_bytes = status
.legacy_bytes
.checked_add(std::fs::metadata(self.dir.join(&part_ref.file))?.len())
.ok_or_else(|| anyhow::anyhow!("legacy format byte count overflow"))?;
}
let live_files: HashSet<&str> =
self.manifest.parts.iter().map(|part| part.file.as_str()).collect();
let mut retained_seen = HashSet::new();
for commit in list_retained_with_limits(&self.dir, self.read_limits)? {
control.check("format migration status")?;
let manifest = load_retained(&self.dir, commit)
.with_context(|| format!("inspect migration state at retained commit {commit}"))?;
for part_ref in manifest.parts {
control.check("format migration status")?;
if live_files.contains(part_ref.file.as_str())
|| !retained_seen.insert(part_ref.file.clone())
{
continue;
}
let path = self.dir.join(&part_ref.file);
let part = Part::open_in_with_limits(&path, self.pcache.clone(), self.read_limits)?;
if part.format_version() == crate::part::PART_VERSION {
continue;
}
status.retained_legacy_parts = status
.retained_legacy_parts
.checked_add(1)
.ok_or_else(|| anyhow::anyhow!("retained legacy part count overflow"))?;
status.retained_legacy_rows = status
.retained_legacy_rows
.checked_add(u64::from(part_ref.records))
.ok_or_else(|| anyhow::anyhow!("retained legacy row count overflow"))?;
status.retained_legacy_bytes = status
.retained_legacy_bytes
.checked_add(std::fs::metadata(path)?.len())
.ok_or_else(|| anyhow::anyhow!("retained legacy byte count overflow"))?;
}
}
Ok(status)
}
pub fn estimate_format_migration_space(&self) -> Result<Option<FormatMigrationPlan>> {
self.estimate_format_migration_space_with_control(
&crate::control::OperationControl::default(),
)
}
pub fn estimate_format_migration_space_with_control(
&self,
control: &crate::control::OperationControl,
) -> Result<Option<FormatMigrationPlan>> {
control.check("format migration preflight")?;
if !self.mem.is_empty() {
bail!(
"format migration preflight requires a flushed memtable; call sync() and flush() first"
);
}
let Some(part_index) =
self.parts.iter().position(|part| part.format_version() < crate::part::PART_VERSION)
else {
return Ok(None);
};
let part = &self.parts[part_index];
let part_ref = &self.manifest.parts[part_index];
let input_bytes = std::fs::metadata(self.dir.join(&part_ref.file))?.len();
let mut input_sections = 0usize;
let mut input_raw_section_bytes = 0u64;
for (_, _, raw, _) in part.sections() {
control.check("format migration preflight")?;
input_sections = input_sections
.checked_add(1)
.ok_or_else(|| anyhow::anyhow!("migration section count overflow"))?;
input_raw_section_bytes = input_raw_section_bytes
.checked_add(u64::from(raw))
.ok_or_else(|| anyhow::anyhow!("migration raw section byte count overflow"))?;
}
let input_rows = u64::from(part_ref.records);
let row_allowance = input_rows
.checked_mul(64)
.ok_or_else(|| anyhow::anyhow!("migration row framing estimate overflow"))?;
let section_allowance = u64::try_from(input_sections)
.map_err(|_| anyhow::anyhow!("migration section count exceeds u64"))?
.checked_mul(256)
.ok_or_else(|| anyhow::anyhow!("migration section framing estimate overflow"))?;
let estimated_stage_bytes = input_raw_section_bytes
.checked_add(row_allowance)
.and_then(|bytes| bytes.checked_add(section_allowance))
.and_then(|bytes| bytes.checked_add(1 << 20))
.ok_or_else(|| anyhow::anyhow!("migration stage estimate overflow"))?;
Ok(Some(FormatMigrationPlan {
part_index,
source_part_version: part.format_version(),
seq_lo: part_ref.seq_lo,
seq_hi: part_ref.seq_hi,
input_rows,
input_bytes,
input_sections,
input_raw_section_bytes,
estimated_stage_bytes,
estimate_is_hard_bound: false,
retained_input_bytes_after_commit: input_bytes,
filesystem_available_bytes: crate::sys::filesystem_available_bytes(&self.dir)
.with_context(|| {
format!("measure available filesystem bytes at {}", self.dir.display())
})?,
}))
}
pub fn migrate_format_step(&mut self) -> Result<Option<FormatMigrationStep>> {
self.migrate_format_step_with_control(&crate::control::OperationControl::default())
}
pub fn migrate_format_step_with_control(
&mut self,
control: &crate::control::OperationControl,
) -> Result<Option<FormatMigrationStep>> {
let started = std::time::Instant::now();
let result = self.migrate_format_step_inner_with_control(control);
self.observe_lifecycle(
crate::observability::LifecycleOperation::FormatMigration,
started.elapsed(),
&result,
);
result
}
fn migrate_format_step_inner_with_control(
&mut self,
control: &crate::control::OperationControl,
) -> Result<Option<FormatMigrationStep>> {
let Some(plan) = self.estimate_format_migration_space_with_control(control)? else {
return Ok(None);
};
self.admit_store_directory_growth(3, "format migration")?;
let input = self.parts[plan.part_index].clone();
let file = format!(
"part-mv{}-{:08}-{:08}.part",
crate::part::PART_VERSION,
plan.seq_lo,
plan.seq_hi
);
let path = self.dir.join(&file);
if path.exists() {
bail!("format migration staging path already exists: {}", path.display());
}
let (meta, rewrite) = match crate::part::merge::merge_opts_with_control_for_operation(
&path,
&[input],
self.cfg.level,
false,
control,
"format migration",
self.read_limits,
) {
Ok(built) => built,
Err(error) => {
let _ = crate::vfs::unlink(&path);
return Err(error);
}
};
let digest = match hash_file_with_control(&path, control, "format migration") {
Ok(hash) => hash.to_hex().to_string(),
Err(error) => {
let _ = crate::vfs::unlink(&path);
return Err(error);
}
};
if let Err(error) = control.check("format migration publication") {
let _ = crate::vfs::unlink(&path);
return Err(error.into());
}
let output_bytes = match std::fs::metadata(&path) {
Ok(metadata) => metadata.len(),
Err(error) => {
let _ = crate::vfs::unlink(&path);
return Err(error.into());
}
};
let mut manifest = self.manifest.clone();
manifest.parts[plan.part_index] = PartRef {
file,
seq_lo: meta.seq_lo,
seq_hi: meta.seq_hi,
records: meta.n_records,
b3: Some(digest),
};
manifest.commit_with_limits(&self.dir, self.read_limits)?;
self.parts[plan.part_index] =
Arc::new(Part::open_in_with_limits(&path, self.pcache.clone(), self.read_limits)?);
self.manifest = manifest;
self.note_manifest_commit();
sweep_unreachable_with_limits(&self.dir, self.read_limits)?;
let remaining_legacy_parts = self
.parts
.iter()
.filter(|part| part.format_version() < crate::part::PART_VERSION)
.count();
Ok(Some(FormatMigrationStep { plan, output_bytes, remaining_legacy_parts, rewrite }))
}
pub fn get(&self, id: &str) -> Result<Option<Record>> {
if let Some(v) = self.mem.get(id) {
return Ok(v.clone());
}
verification_integrity("read committed record", read::get(&self.parts, id))
}
pub(crate) fn project_candidates(
&self,
candidates: &[crate::scan::ScanCandidate],
attrs: &HashSet<&str>,
contents: &HashSet<&str>,
) -> Result<Vec<Record>> {
let mut committed = Vec::new();
let mut committed_outputs = Vec::new();
let mut out: Vec<Option<Record>> = vec![None; candidates.len()];
for (output, candidate) in candidates.iter().enumerate() {
match candidate {
crate::scan::ScanCandidate::Committed(row) => {
committed.push(row);
committed_outputs.push(output);
}
crate::scan::ScanCandidate::Memtable(id) => {
let record = self.mem.get(id).and_then(Option::as_ref).ok_or_else(|| {
anyhow::anyhow!("resolved memtable row {id:?} is no longer live")
})?;
out[output] = Some(Record {
id: record.id.clone(),
contents: record
.contents
.iter()
.filter(|content| contents.contains(content.name.as_str()))
.cloned()
.collect(),
attrs: record
.attrs
.iter()
.filter(|(name, _)| attrs.contains(name.as_str()))
.cloned()
.collect(),
});
}
}
}
for (output, record) in committed_outputs.into_iter().zip(read::project_rows(
&self.parts,
&committed,
attrs,
contents,
)?) {
out[output] = Some(record);
}
out.into_iter()
.map(|record| record.ok_or_else(|| anyhow::anyhow!("projected row was not produced")))
.collect()
}
pub(crate) fn reconstruct_candidate_content(
&self,
candidate: &crate::scan::ScanCandidate,
content: &Content,
) -> Result<Vec<u8>> {
match candidate {
crate::scan::ScanCandidate::Committed(row) => {
read::reconstruct_projected_content(&self.parts, &self.fold, row, content)
}
crate::scan::ScanCandidate::Memtable(id) => {
self.mem.get(id).and_then(Option::as_ref).ok_or_else(|| {
anyhow::anyhow!("resolved memtable row {id:?} is no longer live")
})?;
self.rebuild_projected_content(content)
}
}
}
pub fn reconstruct(&self, id: &str) -> Result<Option<Vec<u8>>> {
self.reconstruct_content(id, BODY_CONTENT)
}
pub fn reconstruct_content(&self, id: &str, name: &str) -> Result<Option<Vec<u8>>> {
if let Some(v) = self.mem.get(id) {
return match v {
Some(r) => self.rebuild_content(r, name),
None => Ok(None), };
}
verification_integrity(
"reconstruct committed content",
read::reconstruct_content(&self.parts, &self.fold, id, name),
)
}
fn locate(&self, h: &PieceHash) -> Result<Option<Loc>> {
if let Some(l) = self.fold.lookup(*h) {
return Ok(Some(l));
}
for p in self.parts.iter().rev() {
if let Some(l) = p.lookup_piece(h)? {
return Ok(Some(l));
}
}
Ok(None)
}
fn rebuild_content(&self, r: &Record, name: &str) -> Result<Option<Vec<u8>>> {
let Some(content) = r.content(name) else {
return Ok(None);
};
Ok(Some(self.rebuild_projected_content(content)?))
}
fn rebuild_projected_content(&self, content: &Content) -> Result<Vec<u8>> {
let mut out = Vec::new();
for op in &content.ops {
match op {
BodyOp::Lit(b) => out.extend_from_slice(b),
BodyOp::Piece { hash, .. } => {
let loc = self
.locate(hash)?
.ok_or_else(|| anyhow::anyhow!("piece {hash} not resolvable"))?;
self.fold.read_verified_into(loc, *hash, &mut out)?;
}
}
}
if let Some(expected) = content.identity {
let got = ContentHash::of(&out);
if got != expected {
bail!(
"content {:?} reconstructed as {got} but its identity is {expected}",
content.name
);
}
}
Ok(out)
}
pub fn memtable_len(&self) -> usize {
self.mem.len()
}
pub fn memtable_bytes(&self) -> usize {
self.mem_bytes
}
pub fn into_parts(self) -> (Arc<Fold>, Vec<Arc<Part>>) {
(Arc::new(self.fold), self.parts)
}
pub fn part_count(&self) -> usize {
self.parts.len()
}
pub fn parts(&self) -> &[Arc<Part>] {
&self.parts
}
pub fn scan_ids(
&self,
from: Option<&str>,
to: Option<&str>,
limit: usize,
reverse: bool,
) -> Result<Vec<String>> {
Ok(self
.scan_candidates(from, to, limit, reverse, usize::MAX, true)?
.candidates
.into_iter()
.map(crate::scan::ScanCandidate::into_id)
.collect())
}
pub(crate) fn scan_candidates(
&self,
from: Option<&str>,
to: Option<&str>,
limit: usize,
reverse: bool,
max_resolution_entries: usize,
allow_oversized_group: bool,
) -> Result<crate::scan::CandidateBatch> {
check_range(from, to)?;
if limit == 0 {
return Ok(crate::scan::CandidateBatch::default());
}
let range = (
from.map_or(std::ops::Bound::Unbounded, std::ops::Bound::Included),
to.map_or(std::ops::Bound::Unbounded, std::ops::Bound::Excluded),
);
let overlay: Box<dyn Iterator<Item = (&str, bool)> + '_> = if reverse {
Box::new(
self.mem
.range::<str, _>(range)
.rev()
.map(|(id, value)| (id.as_str(), value.is_some())),
)
} else {
Box::new(
self.mem.range::<str, _>(range).map(|(id, value)| (id.as_str(), value.is_some())),
)
};
let resolved = read::scan_rows(
&self.parts,
overlay,
read::RowScan {
from,
to,
limit,
reverse,
max_resolution_entries,
allow_oversized_group,
},
)?;
Ok(crate::scan::CandidateBatch {
candidates: resolved
.rows
.into_iter()
.map(|row| match row.origin {
read::RowOrigin::Part { .. } => crate::scan::ScanCandidate::Committed(row),
read::RowOrigin::Memtable => crate::scan::ScanCandidate::Memtable(row.id),
})
.collect(),
resolution: crate::scan::ScanResolutionStats {
physical_rows: resolved.physical_rows,
superseded_rows: resolved.superseded_rows,
tombstones: resolved.tombstones,
memtable_entries: resolved.memtable_entries,
budget_exhausted: resolved.budget_exhausted,
},
resolved_through: resolved.resolved_through,
has_more: resolved.has_more,
})
}
pub fn ids(&self) -> Result<Vec<String>> {
let mut all = read::ids(&self.parts)?;
all.retain(|id| !matches!(self.mem.get(id), Some(None)));
for (id, v) in &self.mem {
if v.is_some() && !all.contains(id) {
all.push(id.clone());
}
}
all.sort();
all.dedup();
Ok(all)
}
pub fn scan(&self, request: &crate::scan::ScanRequest) -> Result<crate::scan::ScanPage> {
crate::scan::scan_store(self, request)
}
pub fn explain_scan(
&self,
request: &crate::scan::ScanRequest,
) -> Result<crate::scan::ScanExplanation> {
crate::scan::explain_store(self, request)
}
pub(crate) fn scan_physical_scope(
&self,
from: Option<&str>,
to: Option<&str>,
) -> Result<crate::scan::ScanPhysicalScope> {
let mut scope = read::scan_physical_scope(&self.parts, from, to)?;
let range = (
from.map_or(std::ops::Bound::Unbounded, std::ops::Bound::Included),
to.map_or(std::ops::Bound::Unbounded, std::ops::Bound::Excluded),
);
scope.memtable_entries_in_bounds = self.mem.range::<str, _>(range).count();
Ok(scope)
}
pub fn erase_ids(&mut self, ids: &[String]) -> Result<ErasureStats> {
self.erase_ids_with_control(ids, &crate::control::OperationControl::default())
}
pub fn erase_ids_with_control(
&mut self,
ids: &[String],
control: &crate::control::OperationControl,
) -> Result<ErasureStats> {
let started = std::time::Instant::now();
let result = self.erase_ids_inner_with_control(ids, control);
self.observe_lifecycle(
crate::observability::LifecycleOperation::Erase,
started.elapsed(),
&result,
);
result
}
fn erase_ids_inner_with_control(
&mut self,
ids: &[String],
control: &crate::control::OperationControl,
) -> Result<ErasureStats> {
let mut tombstoned = 0usize;
let mut absent = 0usize;
let mut delete = Vec::new();
let mut seen = HashSet::new();
for id in ids {
control.check("record erasure")?;
if seen.insert(id) && self.get(id)?.is_some() {
delete.push(id);
tombstoned += 1;
} else {
absent += 1;
}
}
if tombstoned == 0 {
return Ok(ErasureStats {
requested: ids.len(),
tombstoned,
absent,
remaining: self.ids()?.len(),
refold: None,
});
}
control.check("record erasure")?;
let mut batch = Batch::new();
for id in delete {
batch.delete(id);
}
self.apply(batch)?;
self.sync()?;
self.flush()?;
if self.parts.len() > 1 {
self.merge_range(0, self.parts.len())?;
}
let refold = self.refold()?;
Ok(ErasureStats {
requested: ids.len(),
tombstoned,
absent,
remaining: self.ids()?.len(),
refold: Some(refold),
})
}
fn live_fold_pieces_with_control(
&self,
control: &crate::control::OperationControl,
) -> Result<HashMap<PieceHash, Loc>> {
let visible = read::visibility(&self.parts)?;
let mut pieces: HashMap<PieceHash, Loc> = HashMap::new();
for (part_index, rows) in visible.rows.iter().enumerate() {
for &row in rows {
control.check("content reachability")?;
for content in self.parts[part_index].record(row)?.contents {
for operation in content.ops {
let BodyOp::Piece { hash, len } = operation else { continue };
if let Some(existing) = pieces.get(&hash) {
if existing.raw != len {
bail!(
"live piece {hash} has inconsistent lengths {} and {len}",
existing.raw
);
}
continue;
}
let location = self.locate(&hash)?.ok_or_else(|| {
anyhow::anyhow!("live record references absent piece {hash}")
})?;
if location.raw != len {
bail!(
"live piece {hash} is {} bytes but its record says {len}",
location.raw
);
}
pieces.insert(hash, location);
}
}
}
}
Ok(pieces)
}
pub fn punch_unreferenced(&mut self) -> Result<PunchStats> {
self.punch_unreferenced_with_control(&crate::control::OperationControl::default())
}
pub fn punch_unreferenced_with_control(
&mut self,
control: &crate::control::OperationControl,
) -> Result<PunchStats> {
let started = std::time::Instant::now();
let result = self.punch_unreferenced_inner_with_control(control);
self.observe_lifecycle(
crate::observability::LifecycleOperation::Punch,
started.elapsed(),
&result,
);
result
}
fn punch_unreferenced_inner_with_control(
&mut self,
control: &crate::control::OperationControl,
) -> Result<PunchStats> {
control.check("content punching")?;
if !self.mem.is_empty() {
bail!("punching requires a flushed memtable; call sync() and flush() first");
}
let live_blocks: HashSet<u32> = self
.live_fold_pieces_with_control(control)?
.values()
.map(|location| location.block_id)
.collect();
let mut dead: Vec<u32> =
self.fold.block_ids().into_iter().filter(|b| !live_blocks.contains(b)).collect();
dead.sort_unstable();
let already: HashSet<u32> =
self.manifest.punched.iter().flat_map(|&(lo, hi)| lo..=hi).collect();
if dead.is_empty() {
return Ok(PunchStats::default());
}
if dead.iter().any(|block| !already.contains(block)) {
control.check("content punching")?;
self.admit_store_directory_growth(2, "content punching")?;
let mut m = self.manifest.clone();
let mut all: Vec<u32> = already.into_iter().chain(dead.iter().copied()).collect();
all.sort_unstable();
m.punched = to_ranges(&all);
m.commit_with_limits(&self.dir, self.read_limits)?;
self.manifest = m;
self.note_manifest_commit();
}
self.fold.declare_punched(&self.manifest.punched);
let punched = self.fold.punch_blocks_with_control(&dead, control)?;
Ok(PunchStats { blocks_punched: punched.len(), blocks_examined: dead.len() })
}
pub fn refold(&mut self) -> Result<refold::RefoldStats> {
self.refold_with_control(&crate::control::OperationControl::default())
}
pub fn estimate_refold_space(&self) -> Result<Option<RefoldSpaceEstimate>> {
self.estimate_refold_space_with_control(&crate::control::OperationControl::default())
}
pub fn estimate_refold_space_with_control(
&self,
control: &crate::control::OperationControl,
) -> Result<Option<RefoldSpaceEstimate>> {
control.check("refold space preflight")?;
if !self.mem.is_empty() {
bail!("refold space estimation requires a flushed memtable; call sync() and flush() first");
}
if self.parts.is_empty() {
return Ok(None);
}
let source_fold_logical_bytes = self.fold.disk_bytes();
let mut source_part_bytes = 0u64;
let mut source_part_sections = 0usize;
let mut source_part_raw_section_bytes = 0u64;
for (part, part_ref) in self.parts.iter().zip(&self.manifest.parts) {
control.check("refold space preflight")?;
source_part_bytes = source_part_bytes
.checked_add(std::fs::metadata(self.dir.join(&part_ref.file))?.len())
.ok_or_else(|| anyhow::anyhow!("refold source part byte count overflow"))?;
for (_, _, raw, _) in part.sections() {
control.check("refold space preflight")?;
source_part_sections = source_part_sections
.checked_add(1)
.ok_or_else(|| anyhow::anyhow!("refold section count overflow"))?;
source_part_raw_section_bytes = source_part_raw_section_bytes
.checked_add(u64::from(raw))
.ok_or_else(|| anyhow::anyhow!("refold raw section byte count overflow"))?;
}
}
let rows = self.manifest.parts.iter().try_fold(0u64, |rows, part| {
rows.checked_add(u64::from(part.records))
.ok_or_else(|| anyhow::anyhow!("refold row count overflow"))
})?;
let row_allowance = rows
.checked_mul(64)
.ok_or_else(|| anyhow::anyhow!("refold row framing estimate overflow"))?;
let section_allowance = u64::try_from(source_part_sections)
.map_err(|_| anyhow::anyhow!("refold section count exceeds u64"))?
.checked_mul(256)
.ok_or_else(|| anyhow::anyhow!("refold section framing estimate overflow"))?;
let estimated_stage_bytes = source_fold_logical_bytes
.checked_add(source_part_raw_section_bytes)
.and_then(|bytes| bytes.checked_add(row_allowance))
.and_then(|bytes| bytes.checked_add(section_allowance))
.and_then(|bytes| bytes.checked_add(1 << 20))
.ok_or_else(|| anyhow::anyhow!("refold stage estimate overflow"))?;
let usage = self.space_usage_with_control(control)?;
Ok(Some(RefoldSpaceEstimate {
source_fold_logical_bytes,
source_part_bytes,
source_part_sections,
source_part_raw_section_bytes,
retained_only_bytes_before: usage.retained_only.logical_bytes,
estimated_stage_bytes,
estimate_is_hard_bound: false,
filesystem_available_bytes: usage.filesystem_available_bytes,
}))
}
pub fn refold_with_control(
&mut self,
control: &crate::control::OperationControl,
) -> Result<refold::RefoldStats> {
let started = std::time::Instant::now();
let result = self.refold_inner_with_control(control);
self.observe_lifecycle(
crate::observability::LifecycleOperation::Refold,
started.elapsed(),
&result,
);
result
}
fn refold_inner_with_control(
&mut self,
control: &crate::control::OperationControl,
) -> Result<refold::RefoldStats> {
control.check("content refold")?;
if !self.mem.is_empty() {
bail!("refold requires a flushed memtable; call sync() and flush() first");
}
if self.parts.is_empty() {
return Ok(refold::RefoldStats::default());
}
self.admit_store_directory_growth(self.parts.len() as u64 + 3, "content refold")?;
let seqs: Vec<(u64, u64)> =
self.manifest.parts.iter().map(|p| (p.seq_lo, p.seq_hi)).collect();
let (new_gen, built, mut stats) = refold::refold_with_control_and_limits(
&self.dir,
&self.parts,
&seqs,
&self.fold,
self.manifest.fold_gen,
self.cfg,
control,
self.read_limits,
)?;
let new_dir = refold::fold_dir(&self.dir, new_gen);
let prepared = (|| -> Result<Manifest> {
let mut m = self.manifest.clone();
m.parts = built
.iter()
.map(|(file, lo, hi, n)| {
control.check("content refold")?;
Ok(PartRef {
file: file.clone(),
seq_lo: *lo,
seq_hi: *hi,
records: *n,
b3: Some(
hash_file_with_control(
&self.dir.join(file),
control,
"content refold output hashing",
)?
.to_hex()
.to_string(),
),
})
})
.collect::<Result<_>>()?;
m.fold_gen = new_gen;
m.punched.clear();
let f = Fold::open_with_limits(&new_dir, self.cfg, self.read_limits)?;
let t = f.tail();
m.fold_seg = t.seg;
m.fold_off = t.off;
control.check("content refold")?;
Ok(m)
})();
let mut m = match prepared {
Ok(manifest) => manifest,
Err(error) => {
cleanup_refold_stage(&self.dir, new_gen, &built);
return Err(error);
}
};
m.commit_with_limits(&self.dir, self.read_limits)?;
let old_gen = self.manifest.fold_gen;
self.manifest = m;
for c in list_retained_with_limits(&self.dir, self.read_limits)? {
if c != self.manifest.commit {
crate::vfs::unlink(&retained_path(&self.dir, c)).with_context(|| {
format!(
"re-fold committed, but purging retained manifest {c} failed — reopen \
the store to complete the purge"
)
})?;
}
}
crate::vfs::sync_dir(&self.dir)?;
self.retained_commit_count = 1;
let part_cache_budget = self.pcache.budget();
self.pcache = Arc::new(SectionCache::new(part_cache_budget));
self.parts.clear();
for p in &self.manifest.parts {
self.parts.push(Arc::new(Part::open_in_with_limits(
&self.dir.join(&p.file),
self.pcache.clone(),
self.read_limits,
)?));
}
self.fold = Fold::open_at_with_limits(
&new_dir,
self.cfg,
self.manifest.fold_tail(),
&self.manifest.punched,
self.read_limits,
)?;
sweep_unreachable_with_limits(&self.dir, self.read_limits)?;
if refold::fold_dir(&self.dir, old_gen).exists() {
stats.stale_generation_left = true;
}
Ok(stats)
}
pub fn part_cache_bytes(&self) -> (usize, usize) {
(self.pcache.bytes(), self.pcache.budget())
}
pub fn dedup_window_len(&self) -> usize {
self.fold.window_len()
}
pub fn manifest(&self) -> &Manifest {
&self.manifest
}
pub fn fold(&self) -> &Fold {
&self.fold
}
pub fn wal_bytes(&self) -> u64 {
self.wal.bytes()
}
pub fn health(&self) -> StoreHealth {
let fold_cache = self.fold.cache_stats();
let (part_cache_bytes, part_cache_budget) = self.part_cache_bytes();
let punched_blocks =
self.manifest.punched.iter().map(|&(lo, hi)| u64::from(hi) - u64::from(lo) + 1).sum();
StoreHealth {
commit: self.manifest.commit,
fold_generation: self.manifest.fold_gen,
parts: self.parts.len(),
part_rows: self.manifest.parts.iter().map(|part| u64::from(part.records)).sum(),
memtable_entries: self.mem.len(),
memtable_bytes: self.mem_bytes,
wal_bytes: self.wal.bytes(),
wal_frames: self.wal.frame_count(),
fold_disk_bytes: self.fold.disk_bytes(),
fold_segments: self.fold.segment_count(),
fold_cache_hits: fold_cache.hits,
fold_cache_misses: fold_cache.misses,
fold_cache_bytes: fold_cache.bytes,
fold_cache_budget: fold_cache.budget,
fold_block_target_bytes: self.cfg.block_target,
fold_segment_max_bytes: self.cfg.seg_max,
fold_compression_level: self.cfg.level,
fold_compression_threads: self.cfg.compress_threads,
part_cache_bytes,
part_cache_budget,
max_stored_frame_bytes: self.read_limits.max_stored_frame_bytes,
max_decoded_frame_bytes: self.read_limits.max_decoded_frame_bytes,
max_directory_entries: self.read_limits.max_directory_entries,
max_wal_frames: self.read_limits.max_wal_frames,
max_fold_blocks: self.read_limits.max_fold_blocks,
dedup_window_entries: self.fold.window_len(),
retained_commits: self.retained_commit_count,
punched_blocks,
}
}
pub fn space_usage(&self) -> Result<StoreSpaceUsage> {
self.space_usage_with_control(&crate::control::OperationControl::default())
}
pub fn space_usage_with_control(
&self,
control: &crate::control::OperationControl,
) -> Result<StoreSpaceUsage> {
store_space_usage(&self.dir, &self.manifest, self.read_limits, control)
}
pub fn schema(&self) -> Result<crate::schema::Schema> {
let mut schema = crate::schema::Builder::default();
schema.add_parts(&self.parts)?;
for record in self.mem.values().flatten() {
schema.add_record(record);
}
Ok(schema.finish())
}
}
fn store_space_usage(
dir: &Path,
live_manifest: &Manifest,
read_limits: ReadLimits,
control: &crate::control::OperationControl,
) -> Result<StoreSpaceUsage> {
control.check("store space inventory")?;
let mut live_files: HashSet<PathBuf> =
[PathBuf::from("MANIFEST"), PathBuf::from("WAL")].into_iter().collect();
live_files.extend(live_manifest.parts.iter().map(|part| PathBuf::from(&part.file)));
let live_fold = fold_relative_dir(dir, live_manifest.fold_gen)?;
let mut retained_files = HashSet::new();
let mut retained_folds = HashSet::new();
for commit in list_retained_with_limits(dir, read_limits)? {
control.check("store space inventory")?;
let retained_path = retained_path(dir, commit);
retained_files.insert(relative_store_path(dir, &retained_path)?);
let manifest = load_retained(dir, commit)
.with_context(|| format!("account retained manifest {commit}"))?;
retained_files.extend(manifest.parts.iter().map(|part| PathBuf::from(&part.file)));
retained_folds.insert(fold_relative_dir(dir, manifest.fold_gen)?);
}
let mut usage = StoreSpaceUsage {
filesystem_available_bytes: crate::sys::filesystem_available_bytes(dir)
.with_context(|| format!("measure available filesystem bytes at {}", dir.display()))?,
..StoreSpaceUsage::default()
};
let reachability = SpaceReachability {
live_files: &live_files,
live_fold: &live_fold,
retained_files: &retained_files,
retained_folds: &retained_folds,
};
account_store_files(dir, dir, &reachability, &mut usage, read_limits, control)?;
Ok(usage)
}
fn fold_relative_dir(dir: &Path, generation: u32) -> Result<PathBuf> {
relative_store_path(dir, &refold::fold_dir(dir, generation))
}
fn relative_store_path(dir: &Path, path: &Path) -> Result<PathBuf> {
path.strip_prefix(dir)
.map(Path::to_path_buf)
.with_context(|| format!("{} is outside store {}", path.display(), dir.display()))
}
struct SpaceReachability<'a> {
live_files: &'a HashSet<PathBuf>,
live_fold: &'a Path,
retained_files: &'a HashSet<PathBuf>,
retained_folds: &'a HashSet<PathBuf>,
}
fn account_store_files(
root: &Path,
start: &Path,
reachability: &SpaceReachability<'_>,
usage: &mut StoreSpaceUsage,
read_limits: ReadLimits,
control: &crate::control::OperationControl,
) -> Result<()> {
let mut directories = vec![start.to_path_buf()];
let mut visited = 0u64;
while let Some(dir) = directories.pop() {
for entry in std::fs::read_dir(&dir)
.with_context(|| format!("read store space directory {}", dir.display()))?
{
visited = visited.saturating_add(1);
read_limits.admit_directory_entries("store filesystem inventory", visited)?;
control.check("store space inventory")?;
let entry = entry?;
let file_type = entry.file_type()?;
if file_type.is_dir() {
directories.push(entry.path());
} else if file_type.is_file() {
let relative = relative_store_path(root, &entry.path())?;
let metadata = entry.metadata()?;
add_space(&mut usage.total, &metadata)?;
let live = reachability.live_files.contains(&relative)
|| relative.starts_with(reachability.live_fold);
let retained = reachability.retained_files.contains(&relative)
|| reachability.retained_folds.iter().any(|fold| relative.starts_with(fold));
if live {
add_space(&mut usage.live, &metadata)?;
} else if retained {
add_space(&mut usage.retained_only, &metadata)?;
} else {
add_space(&mut usage.unclassified, &metadata)?;
}
}
}
}
Ok(())
}
fn add_space(amount: &mut SpaceAmount, metadata: &std::fs::Metadata) -> Result<()> {
amount.files =
amount.files.checked_add(1).ok_or_else(|| anyhow::anyhow!("store file count overflow"))?;
amount.logical_bytes = amount
.logical_bytes
.checked_add(metadata.len())
.ok_or_else(|| anyhow::anyhow!("store logical byte count overflow"))?;
amount.allocated_bytes = match (amount.allocated_bytes, crate::sys::allocated_bytes(metadata)) {
(Some(total), Some(bytes)) => Some(
total
.checked_add(bytes)
.ok_or_else(|| anyhow::anyhow!("store allocated byte count overflow"))?,
),
_ => None,
};
Ok(())
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct CompactionBudget {
pub max_input_parts: usize,
pub max_input_rows: u64,
pub max_input_bytes: u64,
}
impl CompactionBudget {
pub fn validate(self) -> std::result::Result<(), CompactionError> {
if self.max_input_parts < 2 {
return Err(CompactionError::InvalidBudget(
"max_input_parts must be at least 2".into(),
));
}
if self.max_input_rows == 0 {
return Err(CompactionError::InvalidBudget(
"max_input_rows must be greater than zero".into(),
));
}
if self.max_input_bytes == 0 {
return Err(CompactionError::InvalidBudget(
"max_input_bytes must be greater than zero".into(),
));
}
Ok(())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct CompactionPlan {
pub start_part: usize,
pub input_parts: usize,
pub input_rows: u64,
pub input_bytes: u64,
pub drops_tombstones: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct CompactionSpaceEstimate {
pub plan: CompactionPlan,
pub input_sections: usize,
pub input_raw_section_bytes: u64,
pub estimated_stage_bytes: u64,
pub estimate_is_hard_bound: bool,
pub retained_input_bytes_after_commit: u64,
pub filesystem_available_bytes: Option<u64>,
}
#[derive(Clone, Copy, Debug)]
pub struct BoundedCompaction {
pub plan: CompactionPlan,
pub output_bytes: u64,
pub merge: crate::part::merge::MergeStats,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum CompactionError {
InvalidBudget(String),
BudgetTooSmall {
start_part: usize,
input_rows: u64,
input_bytes: u64,
budget: CompactionBudget,
},
}
impl std::fmt::Display for CompactionError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
CompactionError::InvalidBudget(reason) => write!(f, "invalid compaction budget: {reason}"),
CompactionError::BudgetTooSmall {
start_part,
input_rows,
input_bytes,
budget,
} => write!(
f,
"no adjacent part pair fits the compaction budget; the smallest-byte pair starts at part {start_part} and needs {input_rows} rows / {input_bytes} bytes, limits are {} rows / {} bytes",
budget.max_input_rows,
budget.max_input_bytes
),
}
}
}
impl std::error::Error for CompactionError {}
#[derive(Clone, Copy, Debug, Default)]
pub struct PunchStats {
pub blocks_examined: usize,
pub blocks_punched: usize,
}
fn to_ranges(ids: &[u32]) -> Vec<(u32, u32)> {
let mut out: Vec<(u32, u32)> = Vec::new();
for &id in ids {
match out.last_mut() {
Some(last) if last.1 + 1 == id => last.1 = id,
Some(last) if last.1 >= id => {}
_ => out.push((id, id)),
}
}
out
}
#[derive(Clone, Copy, Debug)]
pub struct ErasureStats {
pub requested: usize,
pub tombstoned: usize,
pub absent: usize,
pub remaining: usize,
pub refold: Option<refold::RefoldStats>,
}
#[derive(Clone)]
pub struct ReadStore {
fold: Arc<Fold>,
parts: Vec<Arc<Part>>,
manifest: Manifest,
read_limits: ReadLimits,
}
impl ReadStore {
pub fn read_limits(&self) -> ReadLimits {
self.read_limits
}
pub fn get(&self, id: &str) -> Result<Option<Record>> {
verification_integrity("read committed record", read::get(&self.parts, id))
}
pub(crate) fn project_candidates(
&self,
candidates: &[crate::scan::ScanCandidate],
attrs: &HashSet<&str>,
contents: &HashSet<&str>,
) -> Result<Vec<Record>> {
let committed: Result<Vec<_>> = candidates
.iter()
.map(|candidate| match candidate {
crate::scan::ScanCandidate::Committed(row) => Ok(row),
crate::scan::ScanCandidate::Memtable(id) => {
bail!("immutable snapshot received memtable row {id:?}")
}
})
.collect();
read::project_rows(&self.parts, &committed?, attrs, contents)
}
pub(crate) fn reconstruct_candidate_content(
&self,
candidate: &crate::scan::ScanCandidate,
content: &Content,
) -> Result<Vec<u8>> {
match candidate {
crate::scan::ScanCandidate::Committed(row) => {
read::reconstruct_projected_content(&self.parts, &self.fold, row, content)
}
crate::scan::ScanCandidate::Memtable(id) => {
bail!("immutable snapshot received memtable row {id:?}")
}
}
}
pub fn reconstruct(&self, id: &str) -> Result<Option<Vec<u8>>> {
verification_integrity(
"reconstruct committed content",
read::reconstruct(&self.parts, &self.fold, id),
)
}
pub fn reconstruct_content(&self, id: &str, name: &str) -> Result<Option<Vec<u8>>> {
verification_integrity(
"reconstruct committed content",
read::reconstruct_content(&self.parts, &self.fold, id, name),
)
}
pub fn ids(&self) -> Result<Vec<String>> {
read::ids(&self.parts)
}
pub fn scan(&self, request: &crate::scan::ScanRequest) -> Result<crate::scan::ScanPage> {
crate::scan::scan_read_store(self, request)
}
pub fn explain_scan(
&self,
request: &crate::scan::ScanRequest,
) -> Result<crate::scan::ScanExplanation> {
crate::scan::explain_read_store(self, request)
}
pub(crate) fn scan_physical_scope(
&self,
from: Option<&str>,
to: Option<&str>,
) -> Result<crate::scan::ScanPhysicalScope> {
read::scan_physical_scope(&self.parts, from, to)
}
pub fn scan_ids(
&self,
from: Option<&str>,
to: Option<&str>,
limit: usize,
reverse: bool,
) -> Result<Vec<String>> {
Ok(self
.scan_candidates(from, to, limit, reverse, usize::MAX, true)?
.candidates
.into_iter()
.map(crate::scan::ScanCandidate::into_id)
.collect())
}
pub(crate) fn scan_candidates(
&self,
from: Option<&str>,
to: Option<&str>,
limit: usize,
reverse: bool,
max_resolution_entries: usize,
allow_oversized_group: bool,
) -> Result<crate::scan::CandidateBatch> {
check_range(from, to)?;
let resolved = read::scan_rows(
&self.parts,
std::iter::empty::<(&str, bool)>(),
read::RowScan {
from,
to,
limit,
reverse,
max_resolution_entries,
allow_oversized_group,
},
)?;
Ok(crate::scan::CandidateBatch {
candidates: resolved
.rows
.into_iter()
.map(crate::scan::ScanCandidate::Committed)
.collect(),
resolution: crate::scan::ScanResolutionStats {
physical_rows: resolved.physical_rows,
superseded_rows: resolved.superseded_rows,
tombstones: resolved.tombstones,
memtable_entries: resolved.memtable_entries,
budget_exhausted: resolved.budget_exhausted,
},
resolved_through: resolved.resolved_through,
has_more: resolved.has_more,
})
}
pub fn into_parts(self) -> (Arc<Fold>, Vec<Arc<Part>>) {
(self.fold, self.parts)
}
pub fn part_count(&self) -> usize {
self.parts.len()
}
pub fn parts(&self) -> &[Arc<Part>] {
&self.parts
}
pub fn fold(&self) -> &Fold {
&self.fold
}
pub fn manifest(&self) -> &Manifest {
&self.manifest
}
pub fn schema(&self) -> Result<crate::schema::Schema> {
let mut schema = crate::schema::Builder::default();
schema.add_parts(&self.parts)?;
Ok(schema.finish())
}
}
fn approx_bytes(r: &Record) -> usize {
r.id.len()
+ r.contents
.iter()
.map(|content| {
content.name.len()
+ content
.ops
.iter()
.map(|o| match o {
BodyOp::Lit(b) => b.len() + 8,
BodyOp::Piece { .. } => 40,
})
.sum::<usize>()
})
.sum::<usize>()
+ r.attrs.iter().map(|(k, _)| k.len() + 24).sum::<usize>()
}
pub fn looks_like_store(dir: &Path) -> bool {
dir.join("MANIFEST").exists() || dir.join("fold").exists()
}
#[cfg(not(any(unix, target_os = "wasi")))]
compile_error!(
"turndb needs positioned file reads (pread). Unix and WASI provide them; \
wasm32-unknown-unknown has no filesystem at all — build for wasm32-wasip1 instead."
);
#[cfg(test)]
mod tests {
fn manifest_bytes_with_part(file: &str) -> Vec<u8> {
serde_json::to_vec(&super::Manifest {
parts: vec![super::PartRef {
file: file.into(),
seq_lo: 1,
seq_hi: 1,
records: 1,
b3: None,
}],
next_seq: 2,
..Default::default()
})
.unwrap()
}
#[test]
fn manifest_part_names_cannot_escape_the_store_root() {
for hostile in
["../secret.part", "/absolute/secret.part", "nested/secret.part", "..\\secret.part"]
{
let error =
super::Manifest::parse(&manifest_bytes_with_part(hostile)).unwrap_err().to_string();
assert!(error.contains("store-local path component"), "{hostile:?}: {error}");
}
assert!(
super::Manifest::parse(&manifest_bytes_with_part("part-00000001.part")).is_ok(),
"an ordinary legacy manifest remains accepted"
);
}
#[test]
fn manifest_semantics_reject_ambiguous_or_malformed_authority() {
let part = super::PartRef {
file: "part-00000001.part".into(),
seq_lo: 2,
seq_hi: 1,
records: 1,
b3: None,
};
let inverted = super::Manifest { parts: vec![part.clone()], ..Default::default() };
assert!(super::Manifest::parse(&serde_json::to_vec(&inverted).unwrap()).is_err());
let duplicate = super::Manifest {
parts: vec![
super::PartRef { seq_lo: 1, seq_hi: 1, ..part.clone() },
super::PartRef { seq_lo: 2, seq_hi: 2, ..part.clone() },
],
..Default::default()
};
assert!(super::Manifest::parse(&serde_json::to_vec(&duplicate).unwrap()).is_err());
let bad_digest = super::Manifest {
parts: vec![super::PartRef {
seq_lo: 1,
seq_hi: 1,
b3: Some("not-a-blake3-digest".into()),
..part
}],
..Default::default()
};
assert!(super::Manifest::parse(&serde_json::to_vec(&bad_digest).unwrap()).is_err());
let punched = super::Manifest { punched: vec![(5, 9), (9, 12)], ..Default::default() };
assert!(super::Manifest::parse(&serde_json::to_vec(&punched).unwrap()).is_err());
}
#[test]
fn manifest_size_is_refused_before_reading_a_sparse_body() {
let d = std::env::temp_dir().join(format!("turndb-manifest-limit-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
let path = d.join("MANIFEST");
std::fs::File::create(&path).unwrap().set_len(super::MAX_MANIFEST_BYTES + 1).unwrap();
let error = format!("{:#}", super::Manifest::load(&d).unwrap_err());
std::fs::remove_dir_all(d).ok();
assert!(error.contains("exceeding") && error.contains("limit"), "{error}");
}
#[test]
fn a_flipped_byte_that_still_parses_is_refused() {
let d = std::env::temp_dir().join(format!("turndb-mancrc-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
let mut m = super::Manifest { fold_off: 4096, next_seq: 9, ..Default::default() };
m.commit(&d).unwrap();
let mut b = std::fs::read(d.join("MANIFEST")).unwrap();
let at = b.windows(4).position(|w| w == b"4096").expect("fold_off literal in the JSON");
b[at] = b'1'; std::fs::write(d.join("MANIFEST"), &b).unwrap();
let err = super::Manifest::load(&d).unwrap_err().to_string();
assert!(err.contains("checksum"), "must refuse via the checksum, got: {err}");
std::fs::remove_dir_all(&d).ok();
}
#[test]
fn a_damaged_trailer_is_not_read_as_legacy() {
let d = std::env::temp_dir().join(format!("turndb-mantrail-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
let mut m = super::Manifest::default();
m.commit(&d).unwrap();
let mut b = std::fs::read(d.join("MANIFEST")).unwrap();
let at = b.len() - 14; b[at] = b'x';
std::fs::write(d.join("MANIFEST"), &b).unwrap();
assert!(
super::Manifest::load(&d).is_err(),
"trailing bytes must fail JSON parsing, not be ignored"
);
std::fs::remove_dir_all(&d).ok();
}
#[test]
fn the_commit_log_retains_prunes_and_recovers() {
let d = std::env::temp_dir().join(format!("turndb-manlog-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
let mut m = super::Manifest::default();
for i in 1..=6u32 {
m.fold_off = i * 100; m.commit(&d).unwrap();
}
assert_eq!(m.commit, 6);
assert_eq!(
super::list_retained(&d).unwrap(),
vec![3, 4, 5, 6],
"window of {} commits",
super::MANIFEST_RETAIN
);
let mut b = std::fs::read(d.join("MANIFEST")).unwrap();
b[10] ^= 0xFF;
std::fs::write(d.join("MANIFEST"), &b).unwrap();
assert!(super::Manifest::load(&d).is_err());
assert_eq!(super::complete_first_commit(&d).unwrap(), 6);
assert_eq!(super::Manifest::load(&d).unwrap().fold_off, 600);
let mut b = std::fs::read(d.join("MANIFEST")).unwrap();
b[10] ^= 0xFF;
std::fs::write(d.join("MANIFEST"), &b).unwrap();
std::fs::write(super::retained_path(&d, 6), b"garbage").unwrap();
std::fs::write(super::retained_path(&d, 5), b"garbage").unwrap();
assert_eq!(super::complete_first_commit(&d).unwrap(), 4);
assert_eq!(super::Manifest::load(&d).unwrap().fold_off, 400);
assert_eq!(
super::list_retained(&d).unwrap(),
vec![3, 4],
"the abandoned timeline is cleared"
);
assert!(
super::complete_first_commit(&d).is_err(),
"recovery of a healthy store must refuse"
);
std::fs::remove_file(d.join("MANIFEST")).unwrap();
assert!(super::Manifest::load(&d).is_err(), "missing MANIFEST + commit log must refuse");
assert_eq!(super::complete_first_commit(&d).unwrap(), 4);
assert_eq!(super::Manifest::load(&d).unwrap().fold_off, 400);
std::fs::remove_dir_all(&d).ok();
}
#[test]
fn manifest_roundtrips() {
let d = std::env::temp_dir().join(format!("turndb-man-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
let mut m = super::Manifest {
parts: vec![super::PartRef {
file: "p.part".into(),
seq_lo: 1,
seq_hi: 1,
records: 7,
b3: None,
}],
fold_seg: 2,
fold_off: 4096,
next_seq: 9,
fold_gen: 3,
commit: 0,
prev: None,
punched: Vec::new(),
};
m.commit(&d).unwrap();
let got = super::Manifest::load(&d).unwrap();
assert_eq!(got.parts.len(), 1);
assert_eq!(got.fold_off, 4096);
assert_eq!(got.fold_gen, 3);
assert_eq!(got.next_seq, 9);
assert_eq!(got.commit, 1, "commit() must advance the commit counter");
assert!(!d.join("MANIFEST.tmp").exists(), "staging file must not survive a commit");
std::fs::remove_dir_all(&d).ok();
}
#[test]
fn format_migration_is_one_part_atomic_resumable_and_content_preserving() {
let d = std::env::temp_dir().join(format!(
"turndb-migration-{}-{}",
std::process::id(),
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
));
std::fs::create_dir_all(&d).unwrap();
let mut fold =
crate::fold::Fold::open(&d.join("fold"), crate::fold::FoldCfg::default()).unwrap();
fold.sync().unwrap();
let tail = fold.tail();
drop(fold);
let mut part_refs = Vec::new();
for (seq, id) in [(1, "legacy-one"), (2, "legacy-two")] {
let file = format!("legacy-{seq}.part");
let meta = crate::part::build_revision_one_fixture(&d.join(&file), seq, id).unwrap();
part_refs.push(super::PartRef {
file,
seq_lo: meta.seq_lo,
seq_hi: meta.seq_hi,
records: meta.n_records,
b3: None,
});
}
let mut manifest = super::Manifest {
parts: part_refs,
fold_seg: tail.seg,
fold_off: tail.off,
next_seq: 2,
..super::Manifest::default()
};
manifest.commit(&d).unwrap();
let mut store = super::Store::open(&d, crate::fold::FoldCfg::default()).unwrap();
let before = store.format_migration_status().unwrap();
assert_eq!(before.legacy_parts, 2);
assert_eq!(before.current_parts, 0);
let plan = store.estimate_format_migration_space().unwrap().unwrap();
assert_eq!(plan.source_part_version, 1);
assert_eq!(plan.input_rows, 1);
assert!(!plan.estimate_is_hard_bound);
let cancellation = crate::control::CancellationToken::new();
cancellation.cancel();
let error = store
.migrate_format_step_with_control(&crate::control::OperationControl {
deadline: None,
cancellation: Some(cancellation),
})
.unwrap_err();
assert!(error.downcast_ref::<crate::control::OperationInterrupted>().is_some());
assert_eq!(store.format_migration_status().unwrap().legacy_parts, 2);
let step = store.migrate_format_step().unwrap().unwrap();
assert_eq!(step.plan.part_index, plan.part_index);
assert_eq!(step.plan.source_part_version, plan.source_part_version);
assert_eq!((step.plan.seq_lo, step.plan.seq_hi), (plan.seq_lo, plan.seq_hi));
assert_eq!(step.plan.input_bytes, plan.input_bytes);
assert_eq!(step.rewrite.inputs, 1);
assert_eq!(step.remaining_legacy_parts, 1);
assert!(step.output_bytes <= plan.estimated_stage_bytes);
let record = store.get("legacy-one").unwrap().unwrap();
assert_eq!(record.contents[0].name, crate::types::BODY_CONTENT);
assert_eq!(record.contents[0].identity, None, "migration must not invent identity");
assert_eq!(record.contents[0].ops, [crate::BodyOp::Lit(b"legacy".to_vec())]);
drop(store);
let mut reopened = super::Store::open(&d, crate::fold::FoldCfg::default()).unwrap();
let midway = reopened.format_migration_status().unwrap();
assert_eq!(midway.legacy_parts, 1);
assert_eq!(midway.current_parts, 1);
assert_eq!(midway.retained_legacy_parts, 1);
let resumed = reopened.migrate_format_step().unwrap().unwrap();
assert_eq!(resumed.remaining_legacy_parts, 0);
assert!(reopened.migrate_format_step().unwrap().is_none());
let after = reopened.format_migration_status().unwrap();
assert_eq!(after.legacy_parts, 0);
assert_eq!(after.current_parts, 2);
assert_eq!(after.retained_legacy_parts, 2);
assert!(after.retained_legacy_bytes > 0);
assert_eq!(reopened.get("legacy-one").unwrap().unwrap(), record);
assert!(reopened.get("legacy-two").unwrap().is_some());
std::fs::remove_dir_all(&d).ok();
}
#[cfg(target_os = "linux")]
#[test]
fn punching_retries_blocks_already_declared_before_a_crash() {
let d = std::env::temp_dir().join(format!(
"turndb-punch-resume-{}-{}",
std::process::id(),
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos()
));
let cfg = crate::fold::FoldCfg { block_target: 1, ..Default::default() };
let mut store = super::Store::open(&d, cfg).unwrap();
let old = vec![0x41; 64 * 1024];
let live = vec![0x42; 64 * 1024];
store.put("k", &[super::Span::Piece(&old)], vec![]).unwrap();
store.sync().unwrap();
store.flush().unwrap();
store.put("k", &[super::Span::Piece(&live)], vec![]).unwrap();
store.sync().unwrap();
store.flush().unwrap();
let live_block = store.locate(&crate::PieceHash::of(&live)).unwrap().unwrap().block_id;
let dead: Vec<u32> =
store.fold.block_ids().into_iter().filter(|&block| block != live_block).collect();
assert!(!dead.is_empty());
let mut manifest = store.manifest.clone();
manifest.punched = super::to_ranges(&dead);
manifest.commit(&d).unwrap();
store.manifest = manifest;
store.fold.declare_punched(&store.manifest.punched);
let stats = store.punch_unreferenced().unwrap();
assert_eq!(stats.blocks_examined, dead.len());
assert_eq!(stats.blocks_punched, dead.len(), "declared blocks must be retried");
assert_eq!(store.reconstruct("k").unwrap().unwrap(), live);
std::fs::remove_dir_all(&d).ok();
}
}
#[cfg(test)]
mod compat_tests {
#[test]
fn a_manifest_without_fold_gen_reads_as_generation_zero() {
let d = std::env::temp_dir().join(format!("turndb-oldman-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
std::fs::write(
d.join("MANIFEST"),
br#"{"parts":[],"fold_seg":0,"fold_off":48,"next_seq":4}"#,
)
.unwrap();
let m = super::Manifest::load(&d).unwrap();
assert_eq!(m.fold_gen, 0, "a pre-generation manifest must mean the original fold/");
assert_eq!(m.next_seq, 4);
assert_eq!(super::refold::fold_dir(&d, m.fold_gen), d.join("fold"));
std::fs::remove_dir_all(&d).ok();
}
}