use super::block::{self, BLOCK_HDR_LEN, BLOCK_XSUM_LEN};
use crate::readat::ReadAt;
use anyhow::{bail, Context, Result};
use std::fs::{File, OpenOptions};
use std::io::Read;
use std::path::{Path, PathBuf};
pub const SEG_HDR_LEN: u64 = 48;
pub const MAGIC: &[u8; 8] = b"TURNFOLD";
pub const SEG_FLAG_ENCRYPTED: u32 = 1 << 0;
pub const SEG_MAX_DEFAULT: u32 = 1 << 30;
pub const SEG_MAX_LIMIT: u64 = 1 << 32;
pub type BlockDirectory = Vec<(u32, u32)>;
pub type DirectorySidecar = (u32, BlockDirectory);
pub fn seg_name(n: u32) -> String {
format!("seg-{n:08}.fold")
}
pub fn parse_seg_name(name: &str) -> Option<u32> {
let rest = name.strip_prefix("seg-")?.strip_suffix(".fold")?;
if rest.is_empty() || !rest.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
rest.parse::<u32>().ok()
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct SegHeader {
pub seg: u32,
pub flags: u32,
pub dict_id: [u8; 32],
}
impl SegHeader {
pub fn has_dict(&self) -> bool {
self.dict_id != [0u8; 32]
}
pub fn encode(&self) -> [u8; SEG_HDR_LEN as usize] {
let mut b = [0u8; SEG_HDR_LEN as usize];
b[0..8].copy_from_slice(MAGIC);
b[8..12].copy_from_slice(&self.seg.to_le_bytes());
b[12..16].copy_from_slice(&self.flags.to_le_bytes());
b[16..48].copy_from_slice(&self.dict_id);
b
}
pub fn decode(b: &[u8], expect_seg: u32) -> Result<SegHeader> {
if b.len() < SEG_HDR_LEN as usize {
bail!("segment header truncated: {} bytes", b.len());
}
if &b[0..8] != MAGIC {
bail!("not a turndb fold segment (bad magic)");
}
let seg = u32::from_le_bytes(b[8..12].try_into().unwrap());
if seg != expect_seg {
bail!("segment header says seg {seg} but the file is named for {expect_seg}");
}
let flags = u32::from_le_bytes(b[12..16].try_into().unwrap());
if flags & SEG_FLAG_ENCRYPTED != 0 {
bail!(
"segment {expect_seg} is ENCRYPTED and this build has no decryption path — \
refusing rather than serving ciphertext as content"
);
}
if flags != 0 {
bail!("segment flags {flags:#x} unknown — refusing (no compatibility negotiation)");
}
let mut dict_id = [0u8; 32];
dict_id.copy_from_slice(&b[16..48]);
Ok(SegHeader { seg, flags, dict_id })
}
}
pub fn fsync_dir(dir: &Path) -> Result<()> {
crate::vfs::sync_dir(dir).with_context(|| format!("fsync dir {}", dir.display()))
}
pub fn create(dir: &Path, n: u32, dict_id: [u8; 32]) -> Result<File> {
create_flagged(dir, n, dict_id, 0)
}
pub fn create_flagged(dir: &Path, n: u32, dict_id: [u8; 32], flags: u32) -> Result<File> {
let path = dir.join(seg_name(n));
let f = crate::vfs::create_new(&path)
.with_context(|| format!("create segment {}", path.display()))?;
crate::vfs::write_all_at(&f, &path, &SegHeader { seg: n, flags, dict_id }.encode(), 0)?;
crate::vfs::sync_file(&f, &path)?;
fsync_dir(dir)?;
Ok(f)
}
pub fn open_read(dir: &Path, n: u32) -> Result<File> {
let path = dir.join(seg_name(n));
File::open(&path).with_context(|| format!("open segment {} read-only", path.display()))
}
pub fn open_rw(dir: &Path, n: u32) -> Result<File> {
let path = dir.join(seg_name(n));
OpenOptions::new()
.read(true)
.write(true)
.open(&path)
.with_context(|| format!("open segment {}", path.display()))
}
pub fn punch(f: &File, path: &Path, off: u64, len: u64) -> Result<()> {
if let Err(e) = crate::vfs::punch_hole(f, path, off, len) {
bail!("punching {len} bytes at {off} failed ({e}) — this filesystem may not support hole punching; re-fold instead");
}
crate::vfs::sync_file(f, path)?;
Ok(())
}
pub fn scan_tail(f: &dyn ReadAt, file_len: u64, has_dict: bool) -> Result<(u64, Vec<(u32, u32)>)> {
scan_tail_with_limits(f, file_len, has_dict, &[], crate::read_limits::ReadLimits::default())
}
pub fn scan_tail_with_limits(
f: &dyn ReadAt,
file_len: u64,
has_dict: bool,
punched: &[(u32, u32)],
read_limits: crate::read_limits::ReadLimits,
) -> Result<(u64, Vec<(u32, u32)>)> {
scan_tail_controlled_with_limits(
f,
file_len,
has_dict,
punched,
&crate::control::OperationControl::default(),
"fold scan",
read_limits,
)
}
pub fn scan_tail_controlled(
f: &dyn ReadAt,
file_len: u64,
has_dict: bool,
control: &crate::control::OperationControl,
operation: &'static str,
) -> Result<(u64, Vec<(u32, u32)>)> {
scan_tail_controlled_with_limits(
f,
file_len,
has_dict,
&[],
control,
operation,
crate::read_limits::ReadLimits::default(),
)
}
pub fn scan_tail_controlled_with_limits(
f: &dyn ReadAt,
file_len: u64,
has_dict: bool,
punched: &[(u32, u32)],
control: &crate::control::OperationControl,
operation: &'static str,
read_limits: crate::read_limits::ReadLimits,
) -> Result<(u64, Vec<(u32, u32)>)> {
let read_limits = read_limits.validate()?;
let mut off = SEG_HDR_LEN;
let mut hdr = [0u8; BLOCK_HDR_LEN];
let mut payload = Vec::new();
let mut dir: Vec<(u32, u32)> = Vec::new();
loop {
control.check(operation)?;
if off + (block::BLOCK_OVERHEAD as u64) > file_len {
break; }
if f.read_exact_at(&mut hdr, off).is_err() {
break;
}
let h = match block::parse_hdr(&hdr, has_dict) {
Ok(h) => h,
Err(_) => break,
};
read_limits.admit(
format!("fold block {}", h.block_id),
u64::from(h.stored),
u64::from(h.raw),
)?;
let end =
match off.checked_add(BLOCK_HDR_LEN as u64 + h.stored as u64 + BLOCK_XSUM_LEN as u64) {
Some(e) => e,
None => break,
};
if end > file_len {
break; }
let span = (BLOCK_HDR_LEN + h.stored as usize + BLOCK_XSUM_LEN) as usize;
payload.resize(span, 0);
if f.read_exact_at(&mut payload[..span], off).is_err() {
break;
}
control.check(operation)?;
if block::verify_frame_bytes(&payload[..span], has_dict).is_err() {
let declared = punched.iter().any(|&(lo, hi)| (lo..=hi).contains(&h.block_id));
let body = &payload[BLOCK_HDR_LEN..span - BLOCK_XSUM_LEN];
if declared || (h.stored > 0 && body.iter().all(|&b| b == 0)) {
read_limits.admit_fold_blocks(dir.len() as u64 + 1)?;
read_limits.admit_fold_blocks(u64::from(h.block_id) + 1)?;
dir.push((h.block_id, off as u32));
off = end;
continue;
}
break;
}
read_limits.admit_fold_blocks(dir.len() as u64 + 1)?;
read_limits.admit_fold_blocks(u64::from(h.block_id) + 1)?;
dir.push((h.block_id, off as u32));
off = end;
}
Ok((off, dir))
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Default)]
pub struct FoldTail {
pub seg: u32,
pub off: u32,
}
pub const DIR_MAGIC: &[u8; 8] = b"TURNSDIR";
pub fn dir_path(dir: &Path, n: u32) -> PathBuf {
dir.join(format!("seg-{n:08}.dir"))
}
pub fn write_dir_sidecar(dir: &Path, n: u32, tail: u32, entries: &[(u32, u32)]) -> Result<()> {
let mut b = Vec::with_capacity(24 + entries.len() * 8);
b.extend_from_slice(DIR_MAGIC);
b.extend_from_slice(&n.to_le_bytes());
b.extend_from_slice(&tail.to_le_bytes());
b.extend_from_slice(&(entries.len() as u32).to_le_bytes());
for (id, off) in entries {
b.extend_from_slice(&id.to_le_bytes());
b.extend_from_slice(&off.to_le_bytes());
}
let crc = crc32fast::hash(&b);
b.extend_from_slice(&crc.to_le_bytes());
let tmp = dir.join(format!("seg-{n:08}.dir.tmp"));
crate::vfs::write_file(&tmp, &b)?;
crate::vfs::rename(&tmp, &dir_path(dir, n))?;
Ok(())
}
pub fn read_dir_sidecar(dir: &Path, n: u32, file_len: u64) -> Option<DirectorySidecar> {
read_dir_sidecar_with_limits(dir, n, file_len, crate::read_limits::ReadLimits::default())
.ok()
.flatten()
}
pub fn read_dir_sidecar_with_limits(
dir: &Path,
n: u32,
file_len: u64,
read_limits: crate::read_limits::ReadLimits,
) -> Result<Option<DirectorySidecar>> {
let Some(bytes) = read_dir_sidecar_bytes_with_limits(dir, n, file_len, read_limits)? else {
return Ok(None);
};
parse_dir_sidecar_with_limits(&bytes, n, file_len, read_limits)
}
pub fn max_dir_sidecar_bytes(file_len: u64) -> u64 {
let frames = file_len.saturating_sub(SEG_HDR_LEN).div_ceil(super::block::BLOCK_OVERHEAD as u64);
24u64.saturating_add(frames.saturating_mul(8))
}
pub fn read_dir_sidecar_bytes(dir: &Path, n: u32, file_len: u64) -> Option<Vec<u8>> {
read_dir_sidecar_bytes_with_limits(dir, n, file_len, crate::read_limits::ReadLimits::default())
.ok()
.flatten()
}
pub fn read_dir_sidecar_bytes_with_limits(
dir: &Path,
n: u32,
file_len: u64,
read_limits: crate::read_limits::ReadLimits,
) -> Result<Option<Vec<u8>>> {
let file = match File::open(dir_path(dir, n)) {
Ok(file) => file,
Err(_) => return Ok(None),
};
let max = max_dir_sidecar_bytes(file_len);
let len = match file.metadata() {
Ok(metadata) => metadata.len(),
Err(_) => return Ok(None),
};
if len > max {
return Ok(None);
}
read_limits.admit_stored(format!("fold segment {n} directory sidecar"), len)?;
let capacity = match usize::try_from(len) {
Ok(capacity) => capacity,
Err(_) => return Ok(None),
};
let mut bytes = Vec::new();
if bytes.try_reserve_exact(capacity).is_err()
|| file.take(max.saturating_add(1)).read_to_end(&mut bytes).is_err()
{
return Ok(None);
}
Ok((bytes.len() as u64 <= max).then_some(bytes))
}
pub fn parse_dir_sidecar(b: &[u8], n: u32, file_len: u64) -> Option<DirectorySidecar> {
parse_dir_sidecar_with_limits(b, n, file_len, crate::read_limits::ReadLimits::default())
.ok()
.flatten()
}
pub fn parse_dir_sidecar_with_limits(
b: &[u8],
n: u32,
file_len: u64,
read_limits: crate::read_limits::ReadLimits,
) -> Result<Option<DirectorySidecar>> {
if b.len() < 24 || &b[0..8] != DIR_MAGIC {
return Ok(None);
}
let crc = u32::from_le_bytes(b[b.len() - 4..].try_into().unwrap());
if crc32fast::hash(&b[..b.len() - 4]) != crc {
return Ok(None);
}
let seg = u32::from_le_bytes(b[8..12].try_into().unwrap());
let tail = u32::from_le_bytes(b[12..16].try_into().unwrap());
let n_entries = u32::from_le_bytes(b[16..20].try_into().unwrap()) as usize;
let expected_len = n_entries.checked_mul(8).and_then(|bytes| 24usize.checked_add(bytes));
if seg != n || tail as u64 != file_len || expected_len != Some(b.len()) {
return Ok(None);
}
read_limits.admit_fold_blocks(n_entries as u64)?;
let mut entries = Vec::with_capacity(n_entries);
for i in 0..n_entries {
let at = 20 + i * 8;
entries.push((
u32::from_le_bytes(b[at..at + 4].try_into().unwrap()),
u32::from_le_bytes(b[at + 4..at + 8].try_into().unwrap()),
));
}
Ok(Some((tail, entries)))
}
pub fn seg_path(dir: &Path, n: u32) -> PathBuf {
dir.join(seg_name(n))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn seg_names_parse_numerically() {
assert_eq!(seg_name(0), "seg-00000000.fold");
assert_eq!(parse_seg_name("seg-00000000.fold"), Some(0));
assert_eq!(parse_seg_name("seg-00000042.fold"), Some(42));
assert_eq!(parse_seg_name("seg-123456789.fold"), Some(123_456_789));
assert_eq!(parse_seg_name("seg-.fold"), None);
assert_eq!(parse_seg_name("seg-00x0.fold"), None);
assert_eq!(parse_seg_name("nope.fold"), None);
}
#[test]
fn header_roundtrips_and_validates() {
let h = SegHeader { seg: 5, flags: 0, dict_id: [0u8; 32] };
let b = h.encode();
assert_eq!(b.len(), SEG_HDR_LEN as usize);
assert_eq!(SegHeader::decode(&b, 5).unwrap(), h);
assert!(!h.has_dict());
assert!(SegHeader::decode(&b, 6).is_err());
let mut bad = b;
bad[0] = b'X';
assert!(SegHeader::decode(&bad, 5).is_err());
let mut flagged = b;
flagged[12] = SEG_FLAG_ENCRYPTED as u8;
assert!(SegHeader::decode(&flagged, 5).is_err(), "the reserved encryption bit must refuse");
let mut future = b;
future[13] = 0x04;
assert!(SegHeader::decode(&future, 5).is_err(), "unknown flags must refuse");
}
#[test]
fn advisory_sidecar_size_is_bounded_by_the_segment_it_describes() {
let dir = std::env::temp_dir().join(format!("turndb-sidecar-limit-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let file_len = SEG_HDR_LEN + crate::fold::block::BLOCK_OVERHEAD as u64;
let max = max_dir_sidecar_bytes(file_len);
std::fs::File::create(dir_path(&dir, 0)).unwrap().set_len(max + 1).unwrap();
let got = read_dir_sidecar_bytes(&dir, 0, file_len);
std::fs::remove_dir_all(dir).ok();
assert!(got.is_none(), "an impossible sparse sidecar must be ignored before allocation");
}
}