use std::collections::HashMap;
use std::fs::File;
use std::io::{Read, Seek, SeekFrom};
use std::path::Path;
use anyhow::{Context, Result, anyhow, bail, ensure};
use chacha20poly1305::aead::{Aead, KeyInit, OsRng, Payload, rand_core::RngCore};
use chacha20poly1305::{XChaCha20Poly1305, XNonce};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use zeroize::Zeroizing;
use super::{Kek, KekLabel};
const MAGIC: &[u8; 8] = b"SBCOLD\x00\x01";
const NONCE_LEN: usize = 24;
const TAG_LEN: usize = 16;
const KEY_LEN: usize = 32;
const BOUND_LEN: usize = 8 + 4 + 8 + 16;
const HEADER_LEN: usize = BOUND_LEN + NONCE_LEN + KEY_LEN + TAG_LEN;
const MANIFEST_VERSION: u32 = 1;
const MAX_BLOCK_PLAIN: u64 = 64 << 20;
const ZSTD_LEVEL: i32 = 3;
#[derive(Debug, Clone, PartialEq)]
pub struct ColdRecord {
pub id: Uuid,
pub content: String,
pub embedding: Vec<f32>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct BlockEntry {
pub offset: u64,
pub len: u64,
pub plain: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SegmentManifest {
pub v: u32,
pub epoch: u32,
pub key_id: String,
pub bytes: u64,
pub blocks: Vec<BlockEntry>,
pub records: HashMap<Uuid, u32>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SegmentSummary {
pub v: u32,
pub label: KekLabel,
pub bytes: u64,
pub records: usize,
}
impl SegmentManifest {
pub fn summary(&self) -> Result<SegmentSummary> {
Ok(SegmentSummary {
v: self.v,
label: KekLabel {
epoch: self.epoch,
key_id: self
.key_id
.parse()
.with_context(|| format!("manifest key id {:?}", self.key_id))?,
},
bytes: self.bytes,
records: self.records.len(),
})
}
}
fn bound(segment: Uuid, label: &KekLabel) -> [u8; BOUND_LEN] {
let mut out = [0u8; BOUND_LEN];
out[..8].copy_from_slice(MAGIC);
out[8..12].copy_from_slice(&label.epoch.to_be_bytes());
out[12..20].copy_from_slice(&label.key_id.0);
out[20..].copy_from_slice(segment.as_bytes());
out
}
fn block_aad(bound: &[u8; BOUND_LEN], index: u32) -> [u8; BOUND_LEN + 4] {
let mut out = [0u8; BOUND_LEN + 4];
out[..BOUND_LEN].copy_from_slice(bound);
out[BOUND_LEN..].copy_from_slice(&index.to_be_bytes());
out
}
fn random_nonce() -> [u8; NONCE_LEN] {
let mut nonce = [0u8; NONCE_LEN];
OsRng.fill_bytes(&mut nonce);
nonce
}
fn encode_records(records: &[&ColdRecord]) -> Result<Vec<u8>> {
let mut out = Vec::new();
for record in records {
out.extend_from_slice(record.id.as_bytes());
let content = record.content.as_bytes();
out.extend_from_slice(&u32::try_from(content.len())?.to_le_bytes());
out.extend_from_slice(content);
out.extend_from_slice(&u32::try_from(record.embedding.len())?.to_le_bytes());
for value in &record.embedding {
out.extend_from_slice(&value.to_le_bytes());
}
}
Ok(out)
}
struct Cursor<'a>(&'a [u8]);
impl<'a> Cursor<'a> {
fn take(&mut self, n: usize) -> Result<&'a [u8]> {
ensure!(
self.0.len() >= n,
"a cold record runs past the end of its block"
);
let (head, tail) = self.0.split_at(n);
self.0 = tail;
Ok(head)
}
fn u32(&mut self) -> Result<usize> {
let bytes: [u8; 4] = self.take(4)?.try_into()?;
Ok(u32::from_le_bytes(bytes) as usize)
}
}
fn decode_records(mut bytes: &[u8]) -> Result<Vec<ColdRecord>> {
let mut records = Vec::new();
while !bytes.is_empty() {
let mut cursor = Cursor(bytes);
let id = Uuid::from_slice(cursor.take(16)?)?;
let len = cursor.u32()?;
let content = std::str::from_utf8(cursor.take(len)?)
.context("a cold record's content is not UTF-8")?
.to_string();
let dims = cursor.u32()?;
let floats = cursor.take(dims.checked_mul(4).context("embedding length overflows")?)?;
let embedding = floats
.chunks_exact(4)
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect();
records.push(ColdRecord {
id,
content,
embedding,
});
bytes = cursor.0;
}
Ok(records)
}
pub fn seal(
segment: Uuid,
records: &[ColdRecord],
kek: &Kek,
block_bytes: usize,
) -> Result<(Vec<u8>, SegmentManifest)> {
let bound = bound(segment, &kek.label);
let segment_key = Zeroizing::new({
let mut key = [0u8; KEY_LEN];
OsRng.fill_bytes(&mut key);
key
});
let key_nonce = random_nonce();
let wrapped = XChaCha20Poly1305::new((&*kek.key).into())
.encrypt(
XNonce::from_slice(&key_nonce),
Payload {
msg: segment_key.as_slice(),
aad: &bound,
},
)
.map_err(|_| anyhow!("wrapping a segment key"))?;
let mut out = Vec::with_capacity(HEADER_LEN);
out.extend_from_slice(&bound);
out.extend_from_slice(&key_nonce);
out.extend_from_slice(&wrapped);
debug_assert_eq!(out.len(), HEADER_LEN);
let cipher = XChaCha20Poly1305::new((&*segment_key).into());
let mut blocks = Vec::new();
let mut index = HashMap::new();
let mut start = 0;
while start < records.len() {
let mut end = start;
let mut plain_estimate = 0;
while end < records.len() && (end == start || plain_estimate < block_bytes) {
plain_estimate += 24 + records[end].content.len() + 4 * records[end].embedding.len();
end += 1;
}
let group: Vec<&ColdRecord> = records[start..end].iter().collect();
let block_index = u32::try_from(blocks.len())?;
let plain = Zeroizing::new(encode_records(&group)?);
ensure!(
plain.len() as u64 <= MAX_BLOCK_PLAIN,
"a cold block of {} bytes is larger than a reader accepts",
plain.len()
);
let compressed = Zeroizing::new(zstd::bulk::compress(&plain, ZSTD_LEVEL)?);
let nonce = random_nonce();
let sealed = cipher
.encrypt(
XNonce::from_slice(&nonce),
Payload {
msg: &compressed,
aad: &block_aad(&bound, block_index),
},
)
.map_err(|_| anyhow!("sealing a cold block"))?;
let offset = out.len() as u64;
out.extend_from_slice(&nonce);
out.extend_from_slice(&sealed);
blocks.push(BlockEntry {
offset,
len: out.len() as u64 - offset,
plain: plain.len() as u64,
});
for record in group {
index.insert(record.id, block_index);
}
start = end;
}
let manifest = SegmentManifest {
v: MANIFEST_VERSION,
epoch: kek.label.epoch,
key_id: kek.label.key_id.to_string(),
bytes: out.len() as u64,
blocks,
records: index,
};
Ok((out, manifest))
}
pub struct OpenSegment {
file: File,
bound: [u8; BOUND_LEN],
cipher: XChaCha20Poly1305,
}
impl OpenSegment {
pub fn open(
path: &Path,
segment: Uuid,
summary: &SegmentSummary,
kek: &[u8; KEY_LEN],
) -> Result<Self> {
ensure!(
summary.v == MANIFEST_VERSION,
"manifest version {} is not one this build reads",
summary.v
);
let label = summary.label;
let mut file = File::open(path).with_context(|| format!("opening {}", path.display()))?;
let length = file.metadata()?.len();
ensure!(
length == summary.bytes,
"{} is {length} bytes, its manifest says {}",
path.display(),
summary.bytes
);
let mut header = [0u8; HEADER_LEN];
file.read_exact(&mut header)
.with_context(|| format!("reading the header of {}", path.display()))?;
let bound = bound(segment, &label);
ensure!(
header[..BOUND_LEN] == bound,
"{} does not carry the header its manifest names",
path.display()
);
let nonce = &header[BOUND_LEN..BOUND_LEN + NONCE_LEN];
let key = Zeroizing::new(
XChaCha20Poly1305::new(kek.into())
.decrypt(
XNonce::from_slice(nonce),
Payload {
msg: &header[BOUND_LEN + NONCE_LEN..],
aad: &bound,
},
)
.map_err(|_| anyhow!("the segment key of {} does not open", path.display()))?,
);
ensure!(key.len() == KEY_LEN, "a segment key is {} bytes", key.len());
Ok(Self {
file,
bound,
cipher: XChaCha20Poly1305::new(key.as_slice().into()),
})
}
pub fn block(&mut self, manifest: &SegmentManifest, index: u32) -> Result<Vec<ColdRecord>> {
let entry = manifest
.blocks
.get(index as usize)
.ok_or_else(|| anyhow!("block {index} is not in the manifest"))?;
if entry.plain > MAX_BLOCK_PLAIN {
bail!("block {index} claims {} bytes of plaintext", entry.plain);
}
ensure!(
entry.len > (NONCE_LEN + TAG_LEN) as u64
&& entry.offset >= HEADER_LEN as u64
&& entry.offset.saturating_add(entry.len) <= manifest.bytes,
"block {index} lies outside the segment"
);
let mut sealed = vec![0u8; usize::try_from(entry.len)?];
self.file.seek(SeekFrom::Start(entry.offset))?;
self.file.read_exact(&mut sealed)?;
let (nonce, body) = sealed.split_at(NONCE_LEN);
let compressed = Zeroizing::new(
self.cipher
.decrypt(
XNonce::from_slice(nonce),
Payload {
msg: body,
aad: &block_aad(&self.bound, index),
},
)
.map_err(|_| anyhow!("block {index} does not open"))?,
);
let plain = Zeroizing::new(zstd::bulk::decompress(
&compressed,
usize::try_from(entry.plain)?,
)?);
decode_records(&plain)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn kek(byte: u8) -> Kek {
Kek {
label: KekLabel {
epoch: 1,
key_id: crate::keys::KeyId([byte; 8]),
},
key: Zeroizing::new([byte; 32]),
}
}
fn records(n: usize) -> Vec<ColdRecord> {
(0..n)
.map(|i| ColdRecord {
id: Uuid::new_v4(),
content: format!("archived chunk {i} about the deploy port {}", i * 7),
embedding: (0..384).map(|d| (d * i) as f32 / 1000.0).collect(),
})
.collect()
}
fn written(bytes: &[u8]) -> (tempfile::TempDir, std::path::PathBuf) {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("s.seg");
std::fs::write(&path, bytes).unwrap();
(dir, path)
}
fn read_all(
path: &Path,
segment: Uuid,
manifest: &SegmentManifest,
key: &[u8; 32],
) -> Result<Vec<ColdRecord>> {
let mut open = OpenSegment::open(path, segment, &manifest.summary()?, key)?;
let mut out = Vec::new();
for i in 0..manifest.blocks.len() {
out.extend(open.block(manifest, i as u32)?);
}
Ok(out)
}
#[test]
fn every_record_reads_back_whole_from_the_block_the_manifest_names() {
let segment = Uuid::new_v4();
let input = records(40);
let (bytes, manifest) = seal(segment, &input, &kek(1), 4096).unwrap();
assert!(manifest.blocks.len() > 1, "{}", manifest.blocks.len());
let (_dir, path) = written(&bytes);
let mut open =
OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[1; 32]).unwrap();
for record in &input {
let block = open.block(&manifest, manifest.records[&record.id]).unwrap();
assert_eq!(block.iter().find(|r| r.id == record.id), Some(record));
}
assert_eq!(
read_all(&path, segment, &manifest, &[1; 32]).unwrap(),
input
);
}
#[test]
fn an_empty_embedding_and_unicode_text_survive() {
let segment = Uuid::new_v4();
let input = vec![ColdRecord {
id: Uuid::new_v4(),
content: "naïve café, 端口 7200".to_string(),
embedding: Vec::new(),
}];
let (bytes, manifest) = seal(segment, &input, &kek(2), 1 << 18).unwrap();
let (_dir, path) = written(&bytes);
assert_eq!(
read_all(&path, segment, &manifest, &[2; 32]).unwrap(),
input
);
}
#[test]
fn another_key_does_not_open_the_segment() {
let segment = Uuid::new_v4();
let (bytes, manifest) = seal(segment, &records(3), &kek(3), 1 << 18).unwrap();
let (_dir, path) = written(&bytes);
let err = OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[4; 32])
.err()
.unwrap();
assert!(err.to_string().contains("does not open"), "{err}");
}
#[test]
fn the_plaintext_is_not_in_the_file() {
let segment = Uuid::new_v4();
let (bytes, _) = seal(segment, &records(5), &kek(5), 1 << 18).unwrap();
let needle = b"deploy port";
assert!(!bytes.windows(needle.len()).any(|w| w == needle));
}
#[test]
fn a_flipped_byte_in_a_block_is_refused() {
let segment = Uuid::new_v4();
let (mut bytes, manifest) = seal(segment, &records(3), &kek(6), 1 << 18).unwrap();
let at = (manifest.blocks[0].offset + manifest.blocks[0].len - 1) as usize;
bytes[at] ^= 1;
let (_dir, path) = written(&bytes);
let mut open =
OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[6; 32]).unwrap();
let err = open.block(&manifest, 0).unwrap_err();
assert!(err.to_string().contains("does not open"), "{err}");
}
#[test]
fn a_segment_renamed_to_another_id_is_refused() {
let segment = Uuid::new_v4();
let (bytes, manifest) = seal(segment, &records(3), &kek(7), 1 << 18).unwrap();
let (_dir, path) = written(&bytes);
assert!(
OpenSegment::open(
&path,
Uuid::new_v4(),
&manifest.summary().unwrap(),
&[7; 32]
)
.is_err()
);
}
#[test]
fn blocks_swapped_within_a_segment_are_refused() {
let segment = Uuid::new_v4();
let (bytes, mut manifest) = seal(segment, &records(40), &kek(8), 2048).unwrap();
manifest.blocks.swap(0, 1);
let (_dir, path) = written(&bytes);
let mut open =
OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[8; 32]).unwrap();
assert!(open.block(&manifest, 0).is_err());
}
#[test]
fn a_truncated_file_is_refused_at_open() {
let segment = Uuid::new_v4();
let (bytes, manifest) = seal(segment, &records(3), &kek(9), 1 << 18).unwrap();
let (_dir, path) = written(&bytes[..bytes.len() - 1]);
let err = OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[9; 32])
.err()
.unwrap();
assert!(err.to_string().contains("bytes"), "{err}");
}
#[test]
fn a_manifest_that_overstates_a_block_is_refused_before_allocating() {
let segment = Uuid::new_v4();
let (bytes, mut manifest) = seal(segment, &records(3), &kek(10), 1 << 18).unwrap();
manifest.blocks[0].plain = u64::MAX;
let (_dir, path) = written(&bytes);
let mut open =
OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[10; 32]).unwrap();
assert!(open.block(&manifest, 0).is_err());
}
}