1use std::fmt;
2use std::fs::{self, File};
3use std::io::{BufRead, BufReader, BufWriter, Read, Write};
4use std::path::{Path, PathBuf};
5
6use anyhow::{Context, Result, anyhow, bail, ensure};
7use chacha20poly1305::aead::{Aead, KeyInit, OsRng, Payload};
8use chacha20poly1305::{XChaCha20Poly1305, XNonce};
9use hkdf::Hkdf;
10use rand_core::RngCore;
11use second_brain_core::keys::{Epoch, IdentityKey, KeyGuard, Subkey, derive_subkey};
12use second_brain_core::sqlite_store::{SnapshotRow, SnapshotState};
13use serde::{Deserialize, Serialize};
14use sha2::{Digest, Sha256};
15
16use crate::protocol::HandshakeKeys;
17
18pub const SNAPSHOT_FORMAT: u32 = 1;
19
20pub const FRAME_BYTES: usize = 4 * 1024 * 1024;
23
24const MAX_ROW_BYTES: usize = 48 * 1024 * 1024;
27
28const MAX_FRAME_BYTES: usize = MAX_ROW_BYTES + 16 * 1024 * 1024;
32
33const HEADER_PREFIX: &str = "second-brain-snapshot";
34const AAD_DOMAIN: &[u8] = b"second-brain snapshot frame v1";
35const EPOCH_ZERO_INFO: &[u8] = b"second-brain snapshot key over the epoch 0 wire key v1";
36const NONCE_LEN: usize = 24;
37const KIND_ROWS: u8 = 1;
38const KIND_MANIFEST: u8 = 2;
39const ZSTD_LEVEL: i32 = 3;
40
41#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
44pub struct SnapshotHeader {
45 pub format: u32,
46 pub snapshot_id: String,
47 pub exporter: String,
48 pub epoch: Epoch,
49 pub sync_protocol: u32,
50 pub schema_version: i64,
51 pub created_at: i64,
52}
53
54#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
57pub struct Manifest {
58 pub state: SnapshotState,
59 pub frames: u64,
60 pub hash: String,
62 pub keys: Option<HandshakeKeys>,
65}
66
67pub struct SnapshotKey {
71 epoch: Epoch,
72 key: KeyGuard<[u8; 32]>,
73}
74
75impl fmt::Debug for SnapshotKey {
76 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
77 f.debug_struct("SnapshotKey")
78 .field("epoch", &self.epoch)
79 .finish_non_exhaustive()
80 }
81}
82
83impl SnapshotKey {
84 pub fn derive(identity: &IdentityKey, epoch: &Epoch) -> Result<Self> {
85 if epoch.number != 0 {
86 return Ok(Self {
87 epoch: *epoch,
88 key: derive_subkey(identity, epoch, Subkey::Snapshot)?,
89 });
90 }
91 let wire = derive_subkey(identity, epoch, Subkey::Wire)?;
92 let mut key = KeyGuard::new([0u8; 32]);
93 Hkdf::<Sha256>::new(None, wire.expose())
94 .expand(EPOCH_ZERO_INFO, key.expose_mut())
95 .map_err(|e| anyhow!("deriving the epoch 0 snapshot key: {e}"))?;
96 Ok(Self { epoch: *epoch, key })
97 }
98
99 pub fn epoch(&self) -> &Epoch {
100 &self.epoch
101 }
102
103 fn cipher(&self) -> XChaCha20Poly1305 {
104 XChaCha20Poly1305::new(self.key.expose().into())
105 }
106}
107
108fn aad(header: &SnapshotHeader, kind: u8, index: u64) -> Vec<u8> {
109 let mut aad = Vec::with_capacity(AAD_DOMAIN.len() + 13 + header.snapshot_id.len() + 9);
110 aad.extend_from_slice(AAD_DOMAIN);
111 aad.extend_from_slice(&header.epoch.number.to_be_bytes());
112 aad.extend_from_slice(&header.epoch.key_id.0);
113 aad.extend_from_slice(header.snapshot_id.as_bytes());
114 aad.push(kind);
115 aad.extend_from_slice(&index.to_be_bytes());
116 aad
117}
118
119fn seal(
120 key: &SnapshotKey,
121 header: &SnapshotHeader,
122 kind: u8,
123 index: u64,
124 plain: &[u8],
125) -> Result<Vec<u8>> {
126 let mut nonce = [0u8; NONCE_LEN];
127 OsRng.fill_bytes(&mut nonce);
128 let ct = key
129 .cipher()
130 .encrypt(
131 XNonce::from_slice(&nonce),
132 Payload {
133 msg: plain,
134 aad: &aad(header, kind, index),
135 },
136 )
137 .map_err(|e| anyhow!("sealing a snapshot frame: {e}"))?;
138 let mut body = Vec::with_capacity(NONCE_LEN + ct.len());
139 body.extend_from_slice(&nonce);
140 body.extend_from_slice(&ct);
141 Ok(body)
142}
143
144fn open(
145 key: &SnapshotKey,
146 header: &SnapshotHeader,
147 kind: u8,
148 index: u64,
149 body: &[u8],
150) -> Result<Vec<u8>> {
151 ensure!(
152 body.len() > NONCE_LEN,
153 "snapshot frame {index} is too short"
154 );
155 let (nonce, ct) = body.split_at(NONCE_LEN);
156 key.cipher()
157 .decrypt(
158 XNonce::from_slice(nonce),
159 Payload {
160 msg: ct,
161 aad: &aad(header, kind, index),
162 },
163 )
164 .map_err(|e| anyhow!("snapshot frame {index} does not open (tampered or reordered): {e}"))
165}
166
167fn write_frame<W: Write>(out: &mut W, kind: u8, body: &[u8]) -> Result<()> {
168 out.write_all(&[kind])?;
169 out.write_all(&u32::try_from(body.len())?.to_be_bytes())?;
170 out.write_all(body)?;
171 Ok(())
172}
173
174fn read_frame<R: Read>(input: &mut R) -> Result<Option<(u8, Vec<u8>)>> {
176 let mut kind = [0u8; 1];
177 match input.read_exact(&mut kind) {
178 Ok(()) => {}
179 Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None),
180 Err(e) => return Err(e.into()),
181 }
182 let mut len = [0u8; 4];
183 input
184 .read_exact(&mut len)
185 .context("the snapshot stream ended inside a frame")?;
186 let len = u32::from_be_bytes(len) as usize;
187 ensure!(
188 len <= MAX_FRAME_BYTES,
189 "snapshot frame of {len} bytes is too large"
190 );
191 let mut body = vec![0u8; len];
192 input
193 .read_exact(&mut body)
194 .context("the snapshot stream ended inside a frame")?;
195 Ok(Some((kind[0], body)))
196}
197
198pub fn write_header<W: Write>(out: &mut W, header: &SnapshotHeader) -> Result<()> {
199 writeln!(out, "{HEADER_PREFIX} {}", serde_json::to_string(header)?)?;
200 Ok(())
201}
202
203pub fn read_header<R: BufRead>(input: &mut R) -> Result<SnapshotHeader> {
204 let mut line = String::new();
205 if input.read_line(&mut line)? == 0 {
206 bail!("the snapshot stream is empty");
207 }
208 let json = line
209 .trim()
210 .strip_prefix(HEADER_PREFIX)
211 .context("the stream is not a second-brain snapshot")?;
212 let header: SnapshotHeader =
213 serde_json::from_str(json.trim()).context("parsing the snapshot header")?;
214 ensure!(
215 header.format == SNAPSHOT_FORMAT,
216 "snapshot format {} is not the {SNAPSHOT_FORMAT} this binary reads",
217 header.format
218 );
219 Ok(header)
220}
221
222pub struct SnapshotWriter<W: Write> {
224 out: W,
225 key: SnapshotKey,
226 header: SnapshotHeader,
227 buf: Vec<u8>,
228 frames: u64,
229 hash: Sha256,
230}
231
232impl<W: Write> SnapshotWriter<W> {
233 pub fn new(mut out: W, key: SnapshotKey, header: SnapshotHeader) -> Result<Self> {
234 ensure!(
235 key.epoch == header.epoch,
236 "the snapshot key is not the header's epoch"
237 );
238 write_header(&mut out, &header)?;
239 Ok(Self {
240 out,
241 key,
242 header,
243 buf: Vec::with_capacity(FRAME_BYTES),
244 frames: 0,
245 hash: Sha256::new(),
246 })
247 }
248
249 pub fn row(&mut self, row: &SnapshotRow) -> Result<()> {
250 let mut line = serde_json::to_vec(row)?;
251 line.push(b'\n');
252 if line.len() > MAX_ROW_BYTES {
253 tracing::warn!(
254 section = row.section(),
255 bytes = line.len(),
256 "a snapshot row larger than sync carries was left out"
257 );
258 return Ok(());
259 }
260 if !self.buf.is_empty() && self.buf.len() + line.len() > FRAME_BYTES {
261 self.flush_rows()?;
262 }
263 self.buf.extend_from_slice(&line);
264 if self.buf.len() >= FRAME_BYTES {
265 self.flush_rows()?;
266 }
267 Ok(())
268 }
269
270 fn flush_rows(&mut self) -> Result<()> {
271 if self.buf.is_empty() {
272 return Ok(());
273 }
274 let compressed = zstd::encode_all(self.buf.as_slice(), ZSTD_LEVEL)?;
275 let body = seal(&self.key, &self.header, KIND_ROWS, self.frames, &compressed)?;
276 self.hash.update(&body);
277 write_frame(&mut self.out, KIND_ROWS, &body)?;
278 self.out.flush()?;
279 self.frames += 1;
280 self.buf.clear();
281 Ok(())
282 }
283
284 pub fn finish(mut self, state: SnapshotState, keys: Option<HandshakeKeys>) -> Result<W> {
285 self.flush_rows()?;
286 let manifest = Manifest {
287 state,
288 frames: self.frames,
289 hash: hex(&self.hash.clone().finalize()),
290 keys,
291 };
292 let body = seal(
293 &self.key,
294 &self.header,
295 KIND_MANIFEST,
296 self.frames,
297 &serde_json::to_vec(&manifest)?,
298 )?;
299 write_frame(&mut self.out, KIND_MANIFEST, &body)?;
300 self.out.flush()?;
301 Ok(self.out)
302 }
303}
304
305fn hex(bytes: &[u8]) -> String {
306 bytes.iter().map(|b| format!("{b:02x}")).collect()
307}
308
309pub fn spool<R: Read>(
314 input: R,
315 path: &Path,
316 key_for: &dyn Fn(&SnapshotHeader) -> Result<SnapshotKey>,
317) -> Result<(SnapshotHeader, Manifest)> {
318 let part = part_path(path);
319 let checked = copy_to(input, &part).and_then(|()| verify(&part, key_for));
320 match checked {
321 Ok(checked) => {
322 fs::rename(&part, path)
323 .with_context(|| format!("moving the spool to {}", path.display()))?;
324 Ok(checked)
325 }
326 Err(e) => {
327 if let Err(removed) = fs::remove_file(&part)
328 && removed.kind() != std::io::ErrorKind::NotFound
329 {
330 tracing::warn!("removing the snapshot spool {}: {removed}", part.display());
331 }
332 Err(e)
333 }
334 }
335}
336
337fn copy_to<R: Read>(input: R, part: &Path) -> Result<()> {
338 let mut input = BufReader::new(input);
339 let header = read_header(&mut input)?;
340 let mut out =
341 BufWriter::new(File::create(part).with_context(|| format!("creating {}", part.display()))?);
342 write_header(&mut out, &header)?;
343 let mut ended = false;
344 while let Some((kind, body)) = read_frame(&mut input)? {
345 ensure!(
346 !ended,
347 "the snapshot stream carries frames after its manifest"
348 );
349 write_frame(&mut out, kind, &body)?;
350 ended = kind == KIND_MANIFEST;
351 }
352 ensure!(ended, "the snapshot stream ended before its manifest");
353 out.into_inner()
354 .map_err(|e| anyhow!("writing the snapshot spool: {e}"))?
355 .sync_all()?;
356 Ok(())
357}
358
359fn part_path(path: &Path) -> PathBuf {
360 let mut name = path.file_name().unwrap_or_default().to_os_string();
361 name.push(".part");
362 path.with_file_name(name)
363}
364
365pub fn verify(
368 path: &Path,
369 key_for: &dyn Fn(&SnapshotHeader) -> Result<SnapshotKey>,
370) -> Result<(SnapshotHeader, Manifest)> {
371 let mut input = BufReader::new(File::open(path)?);
372 let header = read_header(&mut input)?;
373 let key = key_for(&header)?;
374 let mut hash = Sha256::new();
375 let mut frames = 0u64;
376 while let Some((kind, body)) = read_frame(&mut input)? {
377 match kind {
378 KIND_ROWS => {
379 hash.update(&body);
380 frames += 1;
381 }
382 KIND_MANIFEST => {
383 let plain = open(&key, &header, KIND_MANIFEST, frames, &body).context(
384 "the snapshot's manifest does not open at its place: it was cut or altered",
385 )?;
386 let manifest: Manifest =
387 serde_json::from_slice(&plain).context("parsing the snapshot manifest")?;
388 ensure!(
389 manifest.frames == frames && manifest.hash == hex(&hash.finalize()),
390 "the snapshot does not match its manifest: it was cut or altered"
391 );
392 return Ok((header, manifest));
393 }
394 other => bail!("unknown snapshot frame kind {other}"),
395 }
396 }
397 bail!("the snapshot spool has no manifest")
398}
399
400pub fn spool_rows(
402 path: &Path,
403 key_for: &dyn Fn(&SnapshotHeader) -> Result<SnapshotKey>,
404 each: &mut dyn FnMut(SnapshotRow) -> Result<()>,
405) -> Result<()> {
406 let mut input = BufReader::new(File::open(path)?);
407 let header = read_header(&mut input)?;
408 let key = key_for(&header)?;
409 let mut index = 0u64;
410 while let Some((kind, body)) = read_frame(&mut input)? {
411 if kind != KIND_ROWS {
412 break;
413 }
414 let compressed = open(&key, &header, KIND_ROWS, index, &body)?;
415 let jsonl = zstd::decode_all(compressed.as_slice())?;
416 for line in jsonl.split(|b| *b == b'\n').filter(|l| !l.is_empty()) {
417 each(serde_json::from_slice(line).context("parsing a snapshot row")?)?;
418 }
419 index += 1;
420 }
421 Ok(())
422}
423
424#[cfg(test)]
426pub(crate) fn snapshot_format() -> String {
427 format!(
428 "{HEADER_PREFIX} <json header> \\n, then frames of kind u8 (1 rows, 2 manifest) | len u32 be \
429 | nonce[24] | xchacha20poly1305(zstd(jsonl rows) or json manifest), aad = {:?} | epoch \
430 u32 be | key id[8] | snapshot id | kind | frame index u64 be",
431 String::from_utf8_lossy(AAD_DOMAIN)
432 )
433}
434
435#[cfg(test)]
436mod tests {
437 use second_brain_core::keys::KeyId;
438
439 use super::*;
440
441 fn key(number: u32) -> (IdentityKey, Epoch) {
442 let identity = IdentityKey::generate().unwrap();
443 let epoch = Epoch {
444 number,
445 key_id: identity.key_id(),
446 };
447 (identity, epoch)
448 }
449
450 fn header(epoch: Epoch) -> SnapshotHeader {
451 SnapshotHeader {
452 format: SNAPSHOT_FORMAT,
453 snapshot_id: "s1".to_string(),
454 exporter: "a".to_string(),
455 epoch,
456 sync_protocol: crate::protocol::SYNC_PROTOCOL_VERSION,
457 schema_version: 12,
458 created_at: 0,
459 }
460 }
461
462 fn rows(n: usize) -> Vec<SnapshotRow> {
463 (0..n)
464 .map(|i| SnapshotRow::Dedup {
465 origin: format!("origin-{i}"),
466 lo: i as i64,
467 hi: i as i64 + 10,
468 })
469 .collect()
470 }
471
472 fn stream(identity: &IdentityKey, epoch: Epoch, rows: &[SnapshotRow]) -> Vec<u8> {
473 let mut writer = SnapshotWriter::new(
474 Vec::new(),
475 SnapshotKey::derive(identity, &epoch).unwrap(),
476 header(epoch),
477 )
478 .unwrap();
479 for row in rows {
480 writer.row(row).unwrap();
481 }
482 writer
483 .finish(
484 SnapshotState {
485 seq: 9,
486 ..SnapshotState::default()
487 },
488 None,
489 )
490 .unwrap()
491 }
492
493 fn noise(len: usize) -> String {
494 let mut state = 0x9e37_79b9_7f4a_7c15u64;
495 (0..len)
496 .map(|_| {
497 state ^= state << 13;
498 state ^= state >> 7;
499 state ^= state << 17;
500 char::from(b'!' + (state % 90) as u8)
501 })
502 .collect()
503 }
504
505 fn large(origin: String) -> SnapshotRow {
506 SnapshotRow::Dedup {
507 origin,
508 lo: 0,
509 hi: 1,
510 }
511 }
512
513 #[test]
514 fn a_row_too_large_for_a_shared_frame_travels_in_one_of_its_own() {
515 let (identity, epoch) = key(1);
516 let dir = tempfile::tempdir().unwrap();
517 let path = dir.path().join("snapshot.spool");
518 let big = noise(3 * FRAME_BYTES);
519 let mut sent = rows(2);
520 sent.insert(1, large(big.clone()));
521 let bytes = stream(&identity, epoch, &sent);
522
523 spool(bytes.as_slice(), &path, &|h| {
524 SnapshotKey::derive(&identity, &h.epoch)
525 })
526 .unwrap();
527
528 let got = origins(&path, &identity);
529 assert_eq!(
530 got,
531 vec!["origin-0".to_string(), big, "origin-1".to_string()]
532 );
533 }
534
535 #[test]
536 fn a_row_larger_than_sync_carries_is_left_out_and_the_rest_arrive() {
537 let (identity, epoch) = key(1);
538 let dir = tempfile::tempdir().unwrap();
539 let path = dir.path().join("snapshot.spool");
540 let mut sent = rows(2);
541 sent.insert(1, large("x".repeat(MAX_ROW_BYTES)));
542 let bytes = stream(&identity, epoch, &sent);
543
544 spool(bytes.as_slice(), &path, &|h| {
545 SnapshotKey::derive(&identity, &h.epoch)
546 })
547 .unwrap();
548
549 assert_eq!(origins(&path, &identity), vec!["origin-0", "origin-1"]);
550 }
551
552 fn origins(path: &Path, identity: &IdentityKey) -> Vec<String> {
553 let mut seen = Vec::new();
554 spool_rows(
555 path,
556 &|h| SnapshotKey::derive(identity, &h.epoch),
557 &mut |row| {
558 if let SnapshotRow::Dedup { origin, .. } = row {
559 seen.push(origin);
560 }
561 Ok(())
562 },
563 )
564 .unwrap();
565 seen
566 }
567
568 #[test]
569 fn a_spooled_stream_round_trips_under_epoch_zero_and_later_epochs() {
570 for number in [0, 1] {
571 let (identity, epoch) = key(number);
572 let dir = tempfile::tempdir().unwrap();
573 let path = dir.path().join("snapshot.spool");
574 let rows = rows(3);
575 let bytes = stream(&identity, epoch, &rows);
576
577 let (header, manifest) = spool(bytes.as_slice(), &path, &|h| {
578 SnapshotKey::derive(&identity, &h.epoch)
579 })
580 .unwrap();
581
582 assert_eq!(header.snapshot_id, "s1");
583 assert_eq!(manifest.state.seq, 9);
584 assert_eq!(
585 origins(&path, &identity),
586 vec!["origin-0", "origin-1", "origin-2"]
587 );
588 }
589 }
590
591 #[test]
592 fn rows_beyond_one_frame_split_into_several() {
593 let (identity, epoch) = key(1);
594 let dir = tempfile::tempdir().unwrap();
595 let path = dir.path().join("snapshot.spool");
596 let long = "x".repeat(FRAME_BYTES / 3);
597 let rows: Vec<SnapshotRow> = (0..5)
598 .map(|i| SnapshotRow::Dedup {
599 origin: format!("{i}{long}"),
600 lo: 0,
601 hi: 0,
602 })
603 .collect();
604 let bytes = stream(&identity, epoch, &rows);
605 let (_, manifest) = spool(bytes.as_slice(), &path, &|h| {
606 SnapshotKey::derive(&identity, &h.epoch)
607 })
608 .unwrap();
609 assert!(manifest.frames >= 2, "{}", manifest.frames);
610 assert_eq!(origins(&path, &identity).len(), 5);
611 }
612
613 #[test]
614 fn a_cut_stream_leaves_no_spool() {
615 let (identity, epoch) = key(1);
616 let dir = tempfile::tempdir().unwrap();
617 let path = dir.path().join("snapshot.spool");
618 let bytes = stream(&identity, epoch, &rows(3));
619 let cut = &bytes[..bytes.len() - 5];
620
621 assert!(spool(cut, &path, &|h| SnapshotKey::derive(&identity, &h.epoch)).is_err());
622 assert!(!path.exists() && !part_path(&path).exists());
623 }
624
625 #[test]
626 fn a_stream_missing_a_row_frame_fails_its_manifest() {
627 let (identity, epoch) = key(1);
628 let dir = tempfile::tempdir().unwrap();
629 let path = dir.path().join("snapshot.spool");
630 let long = "y".repeat(FRAME_BYTES);
631 let rows: Vec<SnapshotRow> = (0..2)
632 .map(|i| SnapshotRow::Dedup {
633 origin: format!("{i}{long}"),
634 lo: 0,
635 hi: 0,
636 })
637 .collect();
638 let bytes = stream(&identity, epoch, &rows);
639 let header_end = bytes.iter().position(|b| *b == b'\n').unwrap() + 1;
640 let mut input = &bytes[header_end..];
641 let (_, first) = read_frame(&mut input).unwrap().unwrap();
642 let mut dropped = bytes[..header_end].to_vec();
643 dropped.extend_from_slice(input);
644 assert!(!first.is_empty());
645
646 let err = spool(dropped.as_slice(), &path, &|h| {
647 SnapshotKey::derive(&identity, &h.epoch)
648 })
649 .unwrap_err();
650 assert!(format!("{err:#}").contains("manifest"), "{err:#}");
651 assert!(!path.exists());
652 }
653
654 #[test]
655 fn another_identitys_key_opens_nothing() {
656 let (identity, epoch) = key(1);
657 let (other, _) = key(1);
658 let dir = tempfile::tempdir().unwrap();
659 let path = dir.path().join("snapshot.spool");
660 let bytes = stream(&identity, epoch, &rows(1));
661 let foreign = |h: &SnapshotHeader| {
662 SnapshotKey::derive(
663 &other,
664 &Epoch {
665 number: h.epoch.number,
666 key_id: other.key_id(),
667 },
668 )
669 };
670 let _ = KeyId([0; 8]);
671 assert!(spool(bytes.as_slice(), &path, &foreign).is_err());
672 }
673}