1use std::collections::HashMap;
2use std::fs::File;
3use std::io::{Read, Seek, SeekFrom};
4use std::path::Path;
5
6use anyhow::{Context, Result, anyhow, bail, ensure};
7use chacha20poly1305::aead::{Aead, KeyInit, OsRng, Payload, rand_core::RngCore};
8use chacha20poly1305::{XChaCha20Poly1305, XNonce};
9use serde::{Deserialize, Serialize};
10use uuid::Uuid;
11use zeroize::Zeroizing;
12
13use super::{Kek, KekLabel};
14
15const MAGIC: &[u8; 8] = b"SBCOLD\x00\x01";
16const NONCE_LEN: usize = 24;
17const TAG_LEN: usize = 16;
18const KEY_LEN: usize = 32;
19const BOUND_LEN: usize = 8 + 4 + 8 + 16;
21const HEADER_LEN: usize = BOUND_LEN + NONCE_LEN + KEY_LEN + TAG_LEN;
22const MANIFEST_VERSION: u32 = 1;
23const MAX_BLOCK_PLAIN: u64 = 64 << 20;
26const ZSTD_LEVEL: i32 = 3;
27
28#[derive(Debug, Clone, PartialEq)]
29pub struct ColdRecord {
30 pub id: Uuid,
31 pub content: String,
32 pub embedding: Vec<f32>,
33}
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
36pub struct BlockEntry {
37 pub offset: u64,
38 pub len: u64,
39 pub plain: u64,
40}
41
42#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
44pub struct SegmentManifest {
45 pub v: u32,
46 pub epoch: u32,
47 pub key_id: String,
48 pub bytes: u64,
49 pub blocks: Vec<BlockEntry>,
50 pub records: HashMap<Uuid, u32>,
51}
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
56pub struct SegmentSummary {
57 pub v: u32,
58 pub label: KekLabel,
59 pub bytes: u64,
60 pub records: usize,
61}
62
63impl SegmentManifest {
64 pub fn summary(&self) -> Result<SegmentSummary> {
65 Ok(SegmentSummary {
66 v: self.v,
67 label: KekLabel {
68 epoch: self.epoch,
69 key_id: self
70 .key_id
71 .parse()
72 .with_context(|| format!("manifest key id {:?}", self.key_id))?,
73 },
74 bytes: self.bytes,
75 records: self.records.len(),
76 })
77 }
78}
79
80fn bound(segment: Uuid, label: &KekLabel) -> [u8; BOUND_LEN] {
81 let mut out = [0u8; BOUND_LEN];
82 out[..8].copy_from_slice(MAGIC);
83 out[8..12].copy_from_slice(&label.epoch.to_be_bytes());
84 out[12..20].copy_from_slice(&label.key_id.0);
85 out[20..].copy_from_slice(segment.as_bytes());
86 out
87}
88
89fn block_aad(bound: &[u8; BOUND_LEN], index: u32) -> [u8; BOUND_LEN + 4] {
90 let mut out = [0u8; BOUND_LEN + 4];
91 out[..BOUND_LEN].copy_from_slice(bound);
92 out[BOUND_LEN..].copy_from_slice(&index.to_be_bytes());
93 out
94}
95
96fn random_nonce() -> [u8; NONCE_LEN] {
97 let mut nonce = [0u8; NONCE_LEN];
98 OsRng.fill_bytes(&mut nonce);
99 nonce
100}
101
102fn encode_records(records: &[&ColdRecord]) -> Result<Vec<u8>> {
103 let mut out = Vec::new();
104 for record in records {
105 out.extend_from_slice(record.id.as_bytes());
106 let content = record.content.as_bytes();
107 out.extend_from_slice(&u32::try_from(content.len())?.to_le_bytes());
108 out.extend_from_slice(content);
109 out.extend_from_slice(&u32::try_from(record.embedding.len())?.to_le_bytes());
110 for value in &record.embedding {
111 out.extend_from_slice(&value.to_le_bytes());
112 }
113 }
114 Ok(out)
115}
116
117struct Cursor<'a>(&'a [u8]);
118
119impl<'a> Cursor<'a> {
120 fn take(&mut self, n: usize) -> Result<&'a [u8]> {
121 ensure!(
122 self.0.len() >= n,
123 "a cold record runs past the end of its block"
124 );
125 let (head, tail) = self.0.split_at(n);
126 self.0 = tail;
127 Ok(head)
128 }
129
130 fn u32(&mut self) -> Result<usize> {
131 let bytes: [u8; 4] = self.take(4)?.try_into()?;
132 Ok(u32::from_le_bytes(bytes) as usize)
133 }
134}
135
136fn decode_records(mut bytes: &[u8]) -> Result<Vec<ColdRecord>> {
137 let mut records = Vec::new();
138 while !bytes.is_empty() {
139 let mut cursor = Cursor(bytes);
140 let id = Uuid::from_slice(cursor.take(16)?)?;
141 let len = cursor.u32()?;
142 let content = std::str::from_utf8(cursor.take(len)?)
143 .context("a cold record's content is not UTF-8")?
144 .to_string();
145 let dims = cursor.u32()?;
146 let floats = cursor.take(dims.checked_mul(4).context("embedding length overflows")?)?;
147 let embedding = floats
148 .chunks_exact(4)
149 .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
150 .collect();
151 records.push(ColdRecord {
152 id,
153 content,
154 embedding,
155 });
156 bytes = cursor.0;
157 }
158 Ok(records)
159}
160
161pub fn seal(
165 segment: Uuid,
166 records: &[ColdRecord],
167 kek: &Kek,
168 block_bytes: usize,
169) -> Result<(Vec<u8>, SegmentManifest)> {
170 let bound = bound(segment, &kek.label);
171 let segment_key = Zeroizing::new({
172 let mut key = [0u8; KEY_LEN];
173 OsRng.fill_bytes(&mut key);
174 key
175 });
176 let key_nonce = random_nonce();
177 let wrapped = XChaCha20Poly1305::new((&*kek.key).into())
178 .encrypt(
179 XNonce::from_slice(&key_nonce),
180 Payload {
181 msg: segment_key.as_slice(),
182 aad: &bound,
183 },
184 )
185 .map_err(|_| anyhow!("wrapping a segment key"))?;
186 let mut out = Vec::with_capacity(HEADER_LEN);
187 out.extend_from_slice(&bound);
188 out.extend_from_slice(&key_nonce);
189 out.extend_from_slice(&wrapped);
190 debug_assert_eq!(out.len(), HEADER_LEN);
191
192 let cipher = XChaCha20Poly1305::new((&*segment_key).into());
193 let mut blocks = Vec::new();
194 let mut index = HashMap::new();
195 let mut start = 0;
196 while start < records.len() {
197 let mut end = start;
198 let mut plain_estimate = 0;
199 while end < records.len() && (end == start || plain_estimate < block_bytes) {
200 plain_estimate += 24 + records[end].content.len() + 4 * records[end].embedding.len();
201 end += 1;
202 }
203 let group: Vec<&ColdRecord> = records[start..end].iter().collect();
204 let block_index = u32::try_from(blocks.len())?;
205 let plain = Zeroizing::new(encode_records(&group)?);
206 ensure!(
207 plain.len() as u64 <= MAX_BLOCK_PLAIN,
208 "a cold block of {} bytes is larger than a reader accepts",
209 plain.len()
210 );
211 let compressed = Zeroizing::new(zstd::bulk::compress(&plain, ZSTD_LEVEL)?);
212 let nonce = random_nonce();
213 let sealed = cipher
214 .encrypt(
215 XNonce::from_slice(&nonce),
216 Payload {
217 msg: &compressed,
218 aad: &block_aad(&bound, block_index),
219 },
220 )
221 .map_err(|_| anyhow!("sealing a cold block"))?;
222 let offset = out.len() as u64;
223 out.extend_from_slice(&nonce);
224 out.extend_from_slice(&sealed);
225 blocks.push(BlockEntry {
226 offset,
227 len: out.len() as u64 - offset,
228 plain: plain.len() as u64,
229 });
230 for record in group {
231 index.insert(record.id, block_index);
232 }
233 start = end;
234 }
235 let manifest = SegmentManifest {
236 v: MANIFEST_VERSION,
237 epoch: kek.label.epoch,
238 key_id: kek.label.key_id.to_string(),
239 bytes: out.len() as u64,
240 blocks,
241 records: index,
242 };
243 Ok((out, manifest))
244}
245
246pub struct OpenSegment {
248 file: File,
249 bound: [u8; BOUND_LEN],
250 cipher: XChaCha20Poly1305,
251}
252
253impl OpenSegment {
254 pub fn open(
255 path: &Path,
256 segment: Uuid,
257 summary: &SegmentSummary,
258 kek: &[u8; KEY_LEN],
259 ) -> Result<Self> {
260 ensure!(
261 summary.v == MANIFEST_VERSION,
262 "manifest version {} is not one this build reads",
263 summary.v
264 );
265 let label = summary.label;
266 let mut file = File::open(path).with_context(|| format!("opening {}", path.display()))?;
267 let length = file.metadata()?.len();
268 ensure!(
269 length == summary.bytes,
270 "{} is {length} bytes, its manifest says {}",
271 path.display(),
272 summary.bytes
273 );
274 let mut header = [0u8; HEADER_LEN];
275 file.read_exact(&mut header)
276 .with_context(|| format!("reading the header of {}", path.display()))?;
277 let bound = bound(segment, &label);
278 ensure!(
279 header[..BOUND_LEN] == bound,
280 "{} does not carry the header its manifest names",
281 path.display()
282 );
283 let nonce = &header[BOUND_LEN..BOUND_LEN + NONCE_LEN];
284 let key = Zeroizing::new(
285 XChaCha20Poly1305::new(kek.into())
286 .decrypt(
287 XNonce::from_slice(nonce),
288 Payload {
289 msg: &header[BOUND_LEN + NONCE_LEN..],
290 aad: &bound,
291 },
292 )
293 .map_err(|_| anyhow!("the segment key of {} does not open", path.display()))?,
294 );
295 ensure!(key.len() == KEY_LEN, "a segment key is {} bytes", key.len());
296 Ok(Self {
297 file,
298 bound,
299 cipher: XChaCha20Poly1305::new(key.as_slice().into()),
300 })
301 }
302
303 pub fn block(&mut self, manifest: &SegmentManifest, index: u32) -> Result<Vec<ColdRecord>> {
304 let entry = manifest
305 .blocks
306 .get(index as usize)
307 .ok_or_else(|| anyhow!("block {index} is not in the manifest"))?;
308 if entry.plain > MAX_BLOCK_PLAIN {
309 bail!("block {index} claims {} bytes of plaintext", entry.plain);
310 }
311 ensure!(
312 entry.len > (NONCE_LEN + TAG_LEN) as u64
313 && entry.offset >= HEADER_LEN as u64
314 && entry.offset.saturating_add(entry.len) <= manifest.bytes,
315 "block {index} lies outside the segment"
316 );
317 let mut sealed = vec![0u8; usize::try_from(entry.len)?];
318 self.file.seek(SeekFrom::Start(entry.offset))?;
319 self.file.read_exact(&mut sealed)?;
320 let (nonce, body) = sealed.split_at(NONCE_LEN);
321 let compressed = Zeroizing::new(
322 self.cipher
323 .decrypt(
324 XNonce::from_slice(nonce),
325 Payload {
326 msg: body,
327 aad: &block_aad(&self.bound, index),
328 },
329 )
330 .map_err(|_| anyhow!("block {index} does not open"))?,
331 );
332 let plain = Zeroizing::new(zstd::bulk::decompress(
333 &compressed,
334 usize::try_from(entry.plain)?,
335 )?);
336 decode_records(&plain)
337 }
338}
339
340#[cfg(test)]
341mod tests {
342 use super::*;
343
344 fn kek(byte: u8) -> Kek {
345 Kek {
346 label: KekLabel {
347 epoch: 1,
348 key_id: crate::keys::KeyId([byte; 8]),
349 },
350 key: Zeroizing::new([byte; 32]),
351 }
352 }
353
354 fn records(n: usize) -> Vec<ColdRecord> {
355 (0..n)
356 .map(|i| ColdRecord {
357 id: Uuid::new_v4(),
358 content: format!("archived chunk {i} about the deploy port {}", i * 7),
359 embedding: (0..384).map(|d| (d * i) as f32 / 1000.0).collect(),
360 })
361 .collect()
362 }
363
364 fn written(bytes: &[u8]) -> (tempfile::TempDir, std::path::PathBuf) {
365 let dir = tempfile::tempdir().unwrap();
366 let path = dir.path().join("s.seg");
367 std::fs::write(&path, bytes).unwrap();
368 (dir, path)
369 }
370
371 fn read_all(
372 path: &Path,
373 segment: Uuid,
374 manifest: &SegmentManifest,
375 key: &[u8; 32],
376 ) -> Result<Vec<ColdRecord>> {
377 let mut open = OpenSegment::open(path, segment, &manifest.summary()?, key)?;
378 let mut out = Vec::new();
379 for i in 0..manifest.blocks.len() {
380 out.extend(open.block(manifest, i as u32)?);
381 }
382 Ok(out)
383 }
384
385 #[test]
386 fn every_record_reads_back_whole_from_the_block_the_manifest_names() {
387 let segment = Uuid::new_v4();
388 let input = records(40);
389 let (bytes, manifest) = seal(segment, &input, &kek(1), 4096).unwrap();
390 assert!(manifest.blocks.len() > 1, "{}", manifest.blocks.len());
391 let (_dir, path) = written(&bytes);
392 let mut open =
393 OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[1; 32]).unwrap();
394 for record in &input {
395 let block = open.block(&manifest, manifest.records[&record.id]).unwrap();
396 assert_eq!(block.iter().find(|r| r.id == record.id), Some(record));
397 }
398 assert_eq!(
399 read_all(&path, segment, &manifest, &[1; 32]).unwrap(),
400 input
401 );
402 }
403
404 #[test]
405 fn an_empty_embedding_and_unicode_text_survive() {
406 let segment = Uuid::new_v4();
407 let input = vec![ColdRecord {
408 id: Uuid::new_v4(),
409 content: "naïve café, 端口 7200".to_string(),
410 embedding: Vec::new(),
411 }];
412 let (bytes, manifest) = seal(segment, &input, &kek(2), 1 << 18).unwrap();
413 let (_dir, path) = written(&bytes);
414 assert_eq!(
415 read_all(&path, segment, &manifest, &[2; 32]).unwrap(),
416 input
417 );
418 }
419
420 #[test]
421 fn another_key_does_not_open_the_segment() {
422 let segment = Uuid::new_v4();
423 let (bytes, manifest) = seal(segment, &records(3), &kek(3), 1 << 18).unwrap();
424 let (_dir, path) = written(&bytes);
425 let err = OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[4; 32])
426 .err()
427 .unwrap();
428 assert!(err.to_string().contains("does not open"), "{err}");
429 }
430
431 #[test]
432 fn the_plaintext_is_not_in_the_file() {
433 let segment = Uuid::new_v4();
434 let (bytes, _) = seal(segment, &records(5), &kek(5), 1 << 18).unwrap();
435 let needle = b"deploy port";
436 assert!(!bytes.windows(needle.len()).any(|w| w == needle));
437 }
438
439 #[test]
440 fn a_flipped_byte_in_a_block_is_refused() {
441 let segment = Uuid::new_v4();
442 let (mut bytes, manifest) = seal(segment, &records(3), &kek(6), 1 << 18).unwrap();
443 let at = (manifest.blocks[0].offset + manifest.blocks[0].len - 1) as usize;
444 bytes[at] ^= 1;
445 let (_dir, path) = written(&bytes);
446 let mut open =
447 OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[6; 32]).unwrap();
448 let err = open.block(&manifest, 0).unwrap_err();
449 assert!(err.to_string().contains("does not open"), "{err}");
450 }
451
452 #[test]
453 fn a_segment_renamed_to_another_id_is_refused() {
454 let segment = Uuid::new_v4();
455 let (bytes, manifest) = seal(segment, &records(3), &kek(7), 1 << 18).unwrap();
456 let (_dir, path) = written(&bytes);
457 assert!(
458 OpenSegment::open(
459 &path,
460 Uuid::new_v4(),
461 &manifest.summary().unwrap(),
462 &[7; 32]
463 )
464 .is_err()
465 );
466 }
467
468 #[test]
469 fn blocks_swapped_within_a_segment_are_refused() {
470 let segment = Uuid::new_v4();
471 let (bytes, mut manifest) = seal(segment, &records(40), &kek(8), 2048).unwrap();
472 manifest.blocks.swap(0, 1);
473 let (_dir, path) = written(&bytes);
474 let mut open =
475 OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[8; 32]).unwrap();
476 assert!(open.block(&manifest, 0).is_err());
477 }
478
479 #[test]
480 fn a_truncated_file_is_refused_at_open() {
481 let segment = Uuid::new_v4();
482 let (bytes, manifest) = seal(segment, &records(3), &kek(9), 1 << 18).unwrap();
483 let (_dir, path) = written(&bytes[..bytes.len() - 1]);
484 let err = OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[9; 32])
485 .err()
486 .unwrap();
487 assert!(err.to_string().contains("bytes"), "{err}");
488 }
489
490 #[test]
491 fn a_manifest_that_overstates_a_block_is_refused_before_allocating() {
492 let segment = Uuid::new_v4();
493 let (bytes, mut manifest) = seal(segment, &records(3), &kek(10), 1 << 18).unwrap();
494 manifest.blocks[0].plain = u64::MAX;
495 let (_dir, path) = written(&bytes);
496 let mut open =
497 OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[10; 32]).unwrap();
498 assert!(open.block(&manifest, 0).is_err());
499 }
500}