Skip to main content

second_brain_core/cold/
segment.rs

1use std::collections::HashMap;
2use std::fs::File;
3use std::io::{Read, Seek, SeekFrom};
4use std::path::Path;
5
6use anyhow::{Context, Result, anyhow, bail, ensure};
7use chacha20poly1305::aead::{Aead, KeyInit, OsRng, Payload, rand_core::RngCore};
8use chacha20poly1305::{XChaCha20Poly1305, XNonce};
9use serde::{Deserialize, Serialize};
10use uuid::Uuid;
11use zeroize::Zeroizing;
12
13use super::{Kek, KekLabel};
14
15const MAGIC: &[u8; 8] = b"SBCOLD\x00\x01";
16const NONCE_LEN: usize = 24;
17const TAG_LEN: usize = 16;
18const KEY_LEN: usize = 32;
19/// Magic, epoch, key id and segment id: authenticated with every sealed part.
20const BOUND_LEN: usize = 8 + 4 + 8 + 16;
21const HEADER_LEN: usize = BOUND_LEN + NONCE_LEN + KEY_LEN + TAG_LEN;
22const MANIFEST_VERSION: u32 = 1;
23/// A block whose stated plaintext is larger than this is refused before
24/// anything is allocated for it.
25const MAX_BLOCK_PLAIN: u64 = 64 << 20;
26const ZSTD_LEVEL: i32 = 3;
27
28#[derive(Debug, Clone, PartialEq)]
29pub struct ColdRecord {
30    pub id: Uuid,
31    pub content: String,
32    pub embedding: Vec<f32>,
33}
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
36pub struct BlockEntry {
37    pub offset: u64,
38    pub len: u64,
39    pub plain: u64,
40}
41
42/// What the store keeps about a segment, as JSON in `archive_manifest`.
43#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
44pub struct SegmentManifest {
45    pub v: u32,
46    pub epoch: u32,
47    pub key_id: String,
48    pub bytes: u64,
49    pub blocks: Vec<BlockEntry>,
50    pub records: HashMap<Uuid, u32>,
51}
52
53/// What opening a segment's header needs from its manifest, small enough to
54/// keep for every segment.
55#[derive(Debug, Clone, Copy, PartialEq, Eq)]
56pub struct SegmentSummary {
57    pub v: u32,
58    pub label: KekLabel,
59    pub bytes: u64,
60    pub records: usize,
61}
62
63impl SegmentManifest {
64    pub fn summary(&self) -> Result<SegmentSummary> {
65        Ok(SegmentSummary {
66            v: self.v,
67            label: KekLabel {
68                epoch: self.epoch,
69                key_id: self
70                    .key_id
71                    .parse()
72                    .with_context(|| format!("manifest key id {:?}", self.key_id))?,
73            },
74            bytes: self.bytes,
75            records: self.records.len(),
76        })
77    }
78}
79
80fn bound(segment: Uuid, label: &KekLabel) -> [u8; BOUND_LEN] {
81    let mut out = [0u8; BOUND_LEN];
82    out[..8].copy_from_slice(MAGIC);
83    out[8..12].copy_from_slice(&label.epoch.to_be_bytes());
84    out[12..20].copy_from_slice(&label.key_id.0);
85    out[20..].copy_from_slice(segment.as_bytes());
86    out
87}
88
89fn block_aad(bound: &[u8; BOUND_LEN], index: u32) -> [u8; BOUND_LEN + 4] {
90    let mut out = [0u8; BOUND_LEN + 4];
91    out[..BOUND_LEN].copy_from_slice(bound);
92    out[BOUND_LEN..].copy_from_slice(&index.to_be_bytes());
93    out
94}
95
96fn random_nonce() -> [u8; NONCE_LEN] {
97    let mut nonce = [0u8; NONCE_LEN];
98    OsRng.fill_bytes(&mut nonce);
99    nonce
100}
101
102fn encode_records(records: &[&ColdRecord]) -> Result<Vec<u8>> {
103    let mut out = Vec::new();
104    for record in records {
105        out.extend_from_slice(record.id.as_bytes());
106        let content = record.content.as_bytes();
107        out.extend_from_slice(&u32::try_from(content.len())?.to_le_bytes());
108        out.extend_from_slice(content);
109        out.extend_from_slice(&u32::try_from(record.embedding.len())?.to_le_bytes());
110        for value in &record.embedding {
111            out.extend_from_slice(&value.to_le_bytes());
112        }
113    }
114    Ok(out)
115}
116
117struct Cursor<'a>(&'a [u8]);
118
119impl<'a> Cursor<'a> {
120    fn take(&mut self, n: usize) -> Result<&'a [u8]> {
121        ensure!(
122            self.0.len() >= n,
123            "a cold record runs past the end of its block"
124        );
125        let (head, tail) = self.0.split_at(n);
126        self.0 = tail;
127        Ok(head)
128    }
129
130    fn u32(&mut self) -> Result<usize> {
131        let bytes: [u8; 4] = self.take(4)?.try_into()?;
132        Ok(u32::from_le_bytes(bytes) as usize)
133    }
134}
135
136fn decode_records(mut bytes: &[u8]) -> Result<Vec<ColdRecord>> {
137    let mut records = Vec::new();
138    while !bytes.is_empty() {
139        let mut cursor = Cursor(bytes);
140        let id = Uuid::from_slice(cursor.take(16)?)?;
141        let len = cursor.u32()?;
142        let content = std::str::from_utf8(cursor.take(len)?)
143            .context("a cold record's content is not UTF-8")?
144            .to_string();
145        let dims = cursor.u32()?;
146        let floats = cursor.take(dims.checked_mul(4).context("embedding length overflows")?)?;
147        let embedding = floats
148            .chunks_exact(4)
149            .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
150            .collect();
151        records.push(ColdRecord {
152            id,
153            content,
154            embedding,
155        });
156        bytes = cursor.0;
157    }
158    Ok(records)
159}
160
161/// Seals `records` as one segment under a fresh random key, wrapped under
162/// `kek`. Records are packed into blocks of about `block_bytes` of plaintext,
163/// so reading one record decrypts one block.
164pub fn seal(
165    segment: Uuid,
166    records: &[ColdRecord],
167    kek: &Kek,
168    block_bytes: usize,
169) -> Result<(Vec<u8>, SegmentManifest)> {
170    let bound = bound(segment, &kek.label);
171    let segment_key = Zeroizing::new({
172        let mut key = [0u8; KEY_LEN];
173        OsRng.fill_bytes(&mut key);
174        key
175    });
176    let key_nonce = random_nonce();
177    let wrapped = XChaCha20Poly1305::new((&*kek.key).into())
178        .encrypt(
179            XNonce::from_slice(&key_nonce),
180            Payload {
181                msg: segment_key.as_slice(),
182                aad: &bound,
183            },
184        )
185        .map_err(|_| anyhow!("wrapping a segment key"))?;
186    let mut out = Vec::with_capacity(HEADER_LEN);
187    out.extend_from_slice(&bound);
188    out.extend_from_slice(&key_nonce);
189    out.extend_from_slice(&wrapped);
190    debug_assert_eq!(out.len(), HEADER_LEN);
191
192    let cipher = XChaCha20Poly1305::new((&*segment_key).into());
193    let mut blocks = Vec::new();
194    let mut index = HashMap::new();
195    let mut start = 0;
196    while start < records.len() {
197        let mut end = start;
198        let mut plain_estimate = 0;
199        while end < records.len() && (end == start || plain_estimate < block_bytes) {
200            plain_estimate += 24 + records[end].content.len() + 4 * records[end].embedding.len();
201            end += 1;
202        }
203        let group: Vec<&ColdRecord> = records[start..end].iter().collect();
204        let block_index = u32::try_from(blocks.len())?;
205        let plain = Zeroizing::new(encode_records(&group)?);
206        ensure!(
207            plain.len() as u64 <= MAX_BLOCK_PLAIN,
208            "a cold block of {} bytes is larger than a reader accepts",
209            plain.len()
210        );
211        let compressed = Zeroizing::new(zstd::bulk::compress(&plain, ZSTD_LEVEL)?);
212        let nonce = random_nonce();
213        let sealed = cipher
214            .encrypt(
215                XNonce::from_slice(&nonce),
216                Payload {
217                    msg: &compressed,
218                    aad: &block_aad(&bound, block_index),
219                },
220            )
221            .map_err(|_| anyhow!("sealing a cold block"))?;
222        let offset = out.len() as u64;
223        out.extend_from_slice(&nonce);
224        out.extend_from_slice(&sealed);
225        blocks.push(BlockEntry {
226            offset,
227            len: out.len() as u64 - offset,
228            plain: plain.len() as u64,
229        });
230        for record in group {
231            index.insert(record.id, block_index);
232        }
233        start = end;
234    }
235    let manifest = SegmentManifest {
236        v: MANIFEST_VERSION,
237        epoch: kek.label.epoch,
238        key_id: kek.label.key_id.to_string(),
239        bytes: out.len() as u64,
240        blocks,
241        records: index,
242    };
243    Ok((out, manifest))
244}
245
246/// An opened segment: its header checked and its key unwrapped.
247pub struct OpenSegment {
248    file: File,
249    bound: [u8; BOUND_LEN],
250    cipher: XChaCha20Poly1305,
251}
252
253impl OpenSegment {
254    pub fn open(
255        path: &Path,
256        segment: Uuid,
257        summary: &SegmentSummary,
258        kek: &[u8; KEY_LEN],
259    ) -> Result<Self> {
260        ensure!(
261            summary.v == MANIFEST_VERSION,
262            "manifest version {} is not one this build reads",
263            summary.v
264        );
265        let label = summary.label;
266        let mut file = File::open(path).with_context(|| format!("opening {}", path.display()))?;
267        let length = file.metadata()?.len();
268        ensure!(
269            length == summary.bytes,
270            "{} is {length} bytes, its manifest says {}",
271            path.display(),
272            summary.bytes
273        );
274        let mut header = [0u8; HEADER_LEN];
275        file.read_exact(&mut header)
276            .with_context(|| format!("reading the header of {}", path.display()))?;
277        let bound = bound(segment, &label);
278        ensure!(
279            header[..BOUND_LEN] == bound,
280            "{} does not carry the header its manifest names",
281            path.display()
282        );
283        let nonce = &header[BOUND_LEN..BOUND_LEN + NONCE_LEN];
284        let key = Zeroizing::new(
285            XChaCha20Poly1305::new(kek.into())
286                .decrypt(
287                    XNonce::from_slice(nonce),
288                    Payload {
289                        msg: &header[BOUND_LEN + NONCE_LEN..],
290                        aad: &bound,
291                    },
292                )
293                .map_err(|_| anyhow!("the segment key of {} does not open", path.display()))?,
294        );
295        ensure!(key.len() == KEY_LEN, "a segment key is {} bytes", key.len());
296        Ok(Self {
297            file,
298            bound,
299            cipher: XChaCha20Poly1305::new(key.as_slice().into()),
300        })
301    }
302
303    pub fn block(&mut self, manifest: &SegmentManifest, index: u32) -> Result<Vec<ColdRecord>> {
304        let entry = manifest
305            .blocks
306            .get(index as usize)
307            .ok_or_else(|| anyhow!("block {index} is not in the manifest"))?;
308        if entry.plain > MAX_BLOCK_PLAIN {
309            bail!("block {index} claims {} bytes of plaintext", entry.plain);
310        }
311        ensure!(
312            entry.len > (NONCE_LEN + TAG_LEN) as u64
313                && entry.offset >= HEADER_LEN as u64
314                && entry.offset.saturating_add(entry.len) <= manifest.bytes,
315            "block {index} lies outside the segment"
316        );
317        let mut sealed = vec![0u8; usize::try_from(entry.len)?];
318        self.file.seek(SeekFrom::Start(entry.offset))?;
319        self.file.read_exact(&mut sealed)?;
320        let (nonce, body) = sealed.split_at(NONCE_LEN);
321        let compressed = Zeroizing::new(
322            self.cipher
323                .decrypt(
324                    XNonce::from_slice(nonce),
325                    Payload {
326                        msg: body,
327                        aad: &block_aad(&self.bound, index),
328                    },
329                )
330                .map_err(|_| anyhow!("block {index} does not open"))?,
331        );
332        let plain = Zeroizing::new(zstd::bulk::decompress(
333            &compressed,
334            usize::try_from(entry.plain)?,
335        )?);
336        decode_records(&plain)
337    }
338}
339
340#[cfg(test)]
341mod tests {
342    use super::*;
343
344    fn kek(byte: u8) -> Kek {
345        Kek {
346            label: KekLabel {
347                epoch: 1,
348                key_id: crate::keys::KeyId([byte; 8]),
349            },
350            key: Zeroizing::new([byte; 32]),
351        }
352    }
353
354    fn records(n: usize) -> Vec<ColdRecord> {
355        (0..n)
356            .map(|i| ColdRecord {
357                id: Uuid::new_v4(),
358                content: format!("archived chunk {i} about the deploy port {}", i * 7),
359                embedding: (0..384).map(|d| (d * i) as f32 / 1000.0).collect(),
360            })
361            .collect()
362    }
363
364    fn written(bytes: &[u8]) -> (tempfile::TempDir, std::path::PathBuf) {
365        let dir = tempfile::tempdir().unwrap();
366        let path = dir.path().join("s.seg");
367        std::fs::write(&path, bytes).unwrap();
368        (dir, path)
369    }
370
371    fn read_all(
372        path: &Path,
373        segment: Uuid,
374        manifest: &SegmentManifest,
375        key: &[u8; 32],
376    ) -> Result<Vec<ColdRecord>> {
377        let mut open = OpenSegment::open(path, segment, &manifest.summary()?, key)?;
378        let mut out = Vec::new();
379        for i in 0..manifest.blocks.len() {
380            out.extend(open.block(manifest, i as u32)?);
381        }
382        Ok(out)
383    }
384
385    #[test]
386    fn every_record_reads_back_whole_from_the_block_the_manifest_names() {
387        let segment = Uuid::new_v4();
388        let input = records(40);
389        let (bytes, manifest) = seal(segment, &input, &kek(1), 4096).unwrap();
390        assert!(manifest.blocks.len() > 1, "{}", manifest.blocks.len());
391        let (_dir, path) = written(&bytes);
392        let mut open =
393            OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[1; 32]).unwrap();
394        for record in &input {
395            let block = open.block(&manifest, manifest.records[&record.id]).unwrap();
396            assert_eq!(block.iter().find(|r| r.id == record.id), Some(record));
397        }
398        assert_eq!(
399            read_all(&path, segment, &manifest, &[1; 32]).unwrap(),
400            input
401        );
402    }
403
404    #[test]
405    fn an_empty_embedding_and_unicode_text_survive() {
406        let segment = Uuid::new_v4();
407        let input = vec![ColdRecord {
408            id: Uuid::new_v4(),
409            content: "naïve café, 端口 7200".to_string(),
410            embedding: Vec::new(),
411        }];
412        let (bytes, manifest) = seal(segment, &input, &kek(2), 1 << 18).unwrap();
413        let (_dir, path) = written(&bytes);
414        assert_eq!(
415            read_all(&path, segment, &manifest, &[2; 32]).unwrap(),
416            input
417        );
418    }
419
420    #[test]
421    fn another_key_does_not_open_the_segment() {
422        let segment = Uuid::new_v4();
423        let (bytes, manifest) = seal(segment, &records(3), &kek(3), 1 << 18).unwrap();
424        let (_dir, path) = written(&bytes);
425        let err = OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[4; 32])
426            .err()
427            .unwrap();
428        assert!(err.to_string().contains("does not open"), "{err}");
429    }
430
431    #[test]
432    fn the_plaintext_is_not_in_the_file() {
433        let segment = Uuid::new_v4();
434        let (bytes, _) = seal(segment, &records(5), &kek(5), 1 << 18).unwrap();
435        let needle = b"deploy port";
436        assert!(!bytes.windows(needle.len()).any(|w| w == needle));
437    }
438
439    #[test]
440    fn a_flipped_byte_in_a_block_is_refused() {
441        let segment = Uuid::new_v4();
442        let (mut bytes, manifest) = seal(segment, &records(3), &kek(6), 1 << 18).unwrap();
443        let at = (manifest.blocks[0].offset + manifest.blocks[0].len - 1) as usize;
444        bytes[at] ^= 1;
445        let (_dir, path) = written(&bytes);
446        let mut open =
447            OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[6; 32]).unwrap();
448        let err = open.block(&manifest, 0).unwrap_err();
449        assert!(err.to_string().contains("does not open"), "{err}");
450    }
451
452    #[test]
453    fn a_segment_renamed_to_another_id_is_refused() {
454        let segment = Uuid::new_v4();
455        let (bytes, manifest) = seal(segment, &records(3), &kek(7), 1 << 18).unwrap();
456        let (_dir, path) = written(&bytes);
457        assert!(
458            OpenSegment::open(
459                &path,
460                Uuid::new_v4(),
461                &manifest.summary().unwrap(),
462                &[7; 32]
463            )
464            .is_err()
465        );
466    }
467
468    #[test]
469    fn blocks_swapped_within_a_segment_are_refused() {
470        let segment = Uuid::new_v4();
471        let (bytes, mut manifest) = seal(segment, &records(40), &kek(8), 2048).unwrap();
472        manifest.blocks.swap(0, 1);
473        let (_dir, path) = written(&bytes);
474        let mut open =
475            OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[8; 32]).unwrap();
476        assert!(open.block(&manifest, 0).is_err());
477    }
478
479    #[test]
480    fn a_truncated_file_is_refused_at_open() {
481        let segment = Uuid::new_v4();
482        let (bytes, manifest) = seal(segment, &records(3), &kek(9), 1 << 18).unwrap();
483        let (_dir, path) = written(&bytes[..bytes.len() - 1]);
484        let err = OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[9; 32])
485            .err()
486            .unwrap();
487        assert!(err.to_string().contains("bytes"), "{err}");
488    }
489
490    #[test]
491    fn a_manifest_that_overstates_a_block_is_refused_before_allocating() {
492        let segment = Uuid::new_v4();
493        let (bytes, mut manifest) = seal(segment, &records(3), &kek(10), 1 << 18).unwrap();
494        manifest.blocks[0].plain = u64::MAX;
495        let (_dir, path) = written(&bytes);
496        let mut open =
497            OpenSegment::open(&path, segment, &manifest.summary().unwrap(), &[10; 32]).unwrap();
498        assert!(open.block(&manifest, 0).is_err());
499    }
500}