use std::fmt;
use std::fs::{self, File};
use std::io::{BufRead, BufReader, BufWriter, Read, Write};
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, anyhow, bail, ensure};
use chacha20poly1305::aead::{Aead, KeyInit, OsRng, Payload};
use chacha20poly1305::{XChaCha20Poly1305, XNonce};
use hkdf::Hkdf;
use rand_core::RngCore;
use second_brain_core::keys::{Epoch, IdentityKey, KeyGuard, Subkey, derive_subkey};
use second_brain_core::sqlite_store::{SnapshotRow, SnapshotState};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use crate::protocol::HandshakeKeys;
pub const SNAPSHOT_FORMAT: u32 = 1;
pub const FRAME_BYTES: usize = 4 * 1024 * 1024;
const MAX_ROW_BYTES: usize = 48 * 1024 * 1024;
const MAX_FRAME_BYTES: usize = MAX_ROW_BYTES + 16 * 1024 * 1024;
const HEADER_PREFIX: &str = "second-brain-snapshot";
const AAD_DOMAIN: &[u8] = b"second-brain snapshot frame v1";
const EPOCH_ZERO_INFO: &[u8] = b"second-brain snapshot key over the epoch 0 wire key v1";
const NONCE_LEN: usize = 24;
const KIND_ROWS: u8 = 1;
const KIND_MANIFEST: u8 = 2;
const ZSTD_LEVEL: i32 = 3;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SnapshotHeader {
pub format: u32,
pub snapshot_id: String,
pub exporter: String,
pub epoch: Epoch,
pub sync_protocol: u32,
pub schema_version: i64,
pub created_at: i64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Manifest {
pub state: SnapshotState,
pub frames: u64,
pub hash: String,
pub keys: Option<HandshakeKeys>,
}
pub struct SnapshotKey {
epoch: Epoch,
key: KeyGuard<[u8; 32]>,
}
impl fmt::Debug for SnapshotKey {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SnapshotKey")
.field("epoch", &self.epoch)
.finish_non_exhaustive()
}
}
impl SnapshotKey {
pub fn derive(identity: &IdentityKey, epoch: &Epoch) -> Result<Self> {
if epoch.number != 0 {
return Ok(Self {
epoch: *epoch,
key: derive_subkey(identity, epoch, Subkey::Snapshot)?,
});
}
let wire = derive_subkey(identity, epoch, Subkey::Wire)?;
let mut key = KeyGuard::new([0u8; 32]);
Hkdf::<Sha256>::new(None, wire.expose())
.expand(EPOCH_ZERO_INFO, key.expose_mut())
.map_err(|e| anyhow!("deriving the epoch 0 snapshot key: {e}"))?;
Ok(Self { epoch: *epoch, key })
}
pub fn epoch(&self) -> &Epoch {
&self.epoch
}
fn cipher(&self) -> XChaCha20Poly1305 {
XChaCha20Poly1305::new(self.key.expose().into())
}
}
fn aad(header: &SnapshotHeader, kind: u8, index: u64) -> Vec<u8> {
let mut aad = Vec::with_capacity(AAD_DOMAIN.len() + 13 + header.snapshot_id.len() + 9);
aad.extend_from_slice(AAD_DOMAIN);
aad.extend_from_slice(&header.epoch.number.to_be_bytes());
aad.extend_from_slice(&header.epoch.key_id.0);
aad.extend_from_slice(header.snapshot_id.as_bytes());
aad.push(kind);
aad.extend_from_slice(&index.to_be_bytes());
aad
}
fn seal(
key: &SnapshotKey,
header: &SnapshotHeader,
kind: u8,
index: u64,
plain: &[u8],
) -> Result<Vec<u8>> {
let mut nonce = [0u8; NONCE_LEN];
OsRng.fill_bytes(&mut nonce);
let ct = key
.cipher()
.encrypt(
XNonce::from_slice(&nonce),
Payload {
msg: plain,
aad: &aad(header, kind, index),
},
)
.map_err(|e| anyhow!("sealing a snapshot frame: {e}"))?;
let mut body = Vec::with_capacity(NONCE_LEN + ct.len());
body.extend_from_slice(&nonce);
body.extend_from_slice(&ct);
Ok(body)
}
fn open(
key: &SnapshotKey,
header: &SnapshotHeader,
kind: u8,
index: u64,
body: &[u8],
) -> Result<Vec<u8>> {
ensure!(
body.len() > NONCE_LEN,
"snapshot frame {index} is too short"
);
let (nonce, ct) = body.split_at(NONCE_LEN);
key.cipher()
.decrypt(
XNonce::from_slice(nonce),
Payload {
msg: ct,
aad: &aad(header, kind, index),
},
)
.map_err(|e| anyhow!("snapshot frame {index} does not open (tampered or reordered): {e}"))
}
fn write_frame<W: Write>(out: &mut W, kind: u8, body: &[u8]) -> Result<()> {
out.write_all(&[kind])?;
out.write_all(&u32::try_from(body.len())?.to_be_bytes())?;
out.write_all(body)?;
Ok(())
}
fn read_frame<R: Read>(input: &mut R) -> Result<Option<(u8, Vec<u8>)>> {
let mut kind = [0u8; 1];
match input.read_exact(&mut kind) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None),
Err(e) => return Err(e.into()),
}
let mut len = [0u8; 4];
input
.read_exact(&mut len)
.context("the snapshot stream ended inside a frame")?;
let len = u32::from_be_bytes(len) as usize;
ensure!(
len <= MAX_FRAME_BYTES,
"snapshot frame of {len} bytes is too large"
);
let mut body = vec![0u8; len];
input
.read_exact(&mut body)
.context("the snapshot stream ended inside a frame")?;
Ok(Some((kind[0], body)))
}
pub fn write_header<W: Write>(out: &mut W, header: &SnapshotHeader) -> Result<()> {
writeln!(out, "{HEADER_PREFIX} {}", serde_json::to_string(header)?)?;
Ok(())
}
pub fn read_header<R: BufRead>(input: &mut R) -> Result<SnapshotHeader> {
let mut line = String::new();
if input.read_line(&mut line)? == 0 {
bail!("the snapshot stream is empty");
}
let json = line
.trim()
.strip_prefix(HEADER_PREFIX)
.context("the stream is not a second-brain snapshot")?;
let header: SnapshotHeader =
serde_json::from_str(json.trim()).context("parsing the snapshot header")?;
ensure!(
header.format == SNAPSHOT_FORMAT,
"snapshot format {} is not the {SNAPSHOT_FORMAT} this binary reads",
header.format
);
Ok(header)
}
pub struct SnapshotWriter<W: Write> {
out: W,
key: SnapshotKey,
header: SnapshotHeader,
buf: Vec<u8>,
frames: u64,
hash: Sha256,
}
impl<W: Write> SnapshotWriter<W> {
pub fn new(mut out: W, key: SnapshotKey, header: SnapshotHeader) -> Result<Self> {
ensure!(
key.epoch == header.epoch,
"the snapshot key is not the header's epoch"
);
write_header(&mut out, &header)?;
Ok(Self {
out,
key,
header,
buf: Vec::with_capacity(FRAME_BYTES),
frames: 0,
hash: Sha256::new(),
})
}
pub fn row(&mut self, row: &SnapshotRow) -> Result<()> {
let mut line = serde_json::to_vec(row)?;
line.push(b'\n');
if line.len() > MAX_ROW_BYTES {
tracing::warn!(
section = row.section(),
bytes = line.len(),
"a snapshot row larger than sync carries was left out"
);
return Ok(());
}
if !self.buf.is_empty() && self.buf.len() + line.len() > FRAME_BYTES {
self.flush_rows()?;
}
self.buf.extend_from_slice(&line);
if self.buf.len() >= FRAME_BYTES {
self.flush_rows()?;
}
Ok(())
}
fn flush_rows(&mut self) -> Result<()> {
if self.buf.is_empty() {
return Ok(());
}
let compressed = zstd::encode_all(self.buf.as_slice(), ZSTD_LEVEL)?;
let body = seal(&self.key, &self.header, KIND_ROWS, self.frames, &compressed)?;
self.hash.update(&body);
write_frame(&mut self.out, KIND_ROWS, &body)?;
self.out.flush()?;
self.frames += 1;
self.buf.clear();
Ok(())
}
pub fn finish(mut self, state: SnapshotState, keys: Option<HandshakeKeys>) -> Result<W> {
self.flush_rows()?;
let manifest = Manifest {
state,
frames: self.frames,
hash: hex(&self.hash.clone().finalize()),
keys,
};
let body = seal(
&self.key,
&self.header,
KIND_MANIFEST,
self.frames,
&serde_json::to_vec(&manifest)?,
)?;
write_frame(&mut self.out, KIND_MANIFEST, &body)?;
self.out.flush()?;
Ok(self.out)
}
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
pub fn spool<R: Read>(
input: R,
path: &Path,
key_for: &dyn Fn(&SnapshotHeader) -> Result<SnapshotKey>,
) -> Result<(SnapshotHeader, Manifest)> {
let part = part_path(path);
let checked = copy_to(input, &part).and_then(|()| verify(&part, key_for));
match checked {
Ok(checked) => {
fs::rename(&part, path)
.with_context(|| format!("moving the spool to {}", path.display()))?;
Ok(checked)
}
Err(e) => {
if let Err(removed) = fs::remove_file(&part)
&& removed.kind() != std::io::ErrorKind::NotFound
{
tracing::warn!("removing the snapshot spool {}: {removed}", part.display());
}
Err(e)
}
}
}
fn copy_to<R: Read>(input: R, part: &Path) -> Result<()> {
let mut input = BufReader::new(input);
let header = read_header(&mut input)?;
let mut out =
BufWriter::new(File::create(part).with_context(|| format!("creating {}", part.display()))?);
write_header(&mut out, &header)?;
let mut ended = false;
while let Some((kind, body)) = read_frame(&mut input)? {
ensure!(
!ended,
"the snapshot stream carries frames after its manifest"
);
write_frame(&mut out, kind, &body)?;
ended = kind == KIND_MANIFEST;
}
ensure!(ended, "the snapshot stream ended before its manifest");
out.into_inner()
.map_err(|e| anyhow!("writing the snapshot spool: {e}"))?
.sync_all()?;
Ok(())
}
fn part_path(path: &Path) -> PathBuf {
let mut name = path.file_name().unwrap_or_default().to_os_string();
name.push(".part");
path.with_file_name(name)
}
pub fn verify(
path: &Path,
key_for: &dyn Fn(&SnapshotHeader) -> Result<SnapshotKey>,
) -> Result<(SnapshotHeader, Manifest)> {
let mut input = BufReader::new(File::open(path)?);
let header = read_header(&mut input)?;
let key = key_for(&header)?;
let mut hash = Sha256::new();
let mut frames = 0u64;
while let Some((kind, body)) = read_frame(&mut input)? {
match kind {
KIND_ROWS => {
hash.update(&body);
frames += 1;
}
KIND_MANIFEST => {
let plain = open(&key, &header, KIND_MANIFEST, frames, &body).context(
"the snapshot's manifest does not open at its place: it was cut or altered",
)?;
let manifest: Manifest =
serde_json::from_slice(&plain).context("parsing the snapshot manifest")?;
ensure!(
manifest.frames == frames && manifest.hash == hex(&hash.finalize()),
"the snapshot does not match its manifest: it was cut or altered"
);
return Ok((header, manifest));
}
other => bail!("unknown snapshot frame kind {other}"),
}
}
bail!("the snapshot spool has no manifest")
}
pub fn spool_rows(
path: &Path,
key_for: &dyn Fn(&SnapshotHeader) -> Result<SnapshotKey>,
each: &mut dyn FnMut(SnapshotRow) -> Result<()>,
) -> Result<()> {
let mut input = BufReader::new(File::open(path)?);
let header = read_header(&mut input)?;
let key = key_for(&header)?;
let mut index = 0u64;
while let Some((kind, body)) = read_frame(&mut input)? {
if kind != KIND_ROWS {
break;
}
let compressed = open(&key, &header, KIND_ROWS, index, &body)?;
let jsonl = zstd::decode_all(compressed.as_slice())?;
for line in jsonl.split(|b| *b == b'\n').filter(|l| !l.is_empty()) {
each(serde_json::from_slice(line).context("parsing a snapshot row")?)?;
}
index += 1;
}
Ok(())
}
#[cfg(test)]
pub(crate) fn snapshot_format() -> String {
format!(
"{HEADER_PREFIX} <json header> \\n, then frames of kind u8 (1 rows, 2 manifest) | len u32 be \
| nonce[24] | xchacha20poly1305(zstd(jsonl rows) or json manifest), aad = {:?} | epoch \
u32 be | key id[8] | snapshot id | kind | frame index u64 be",
String::from_utf8_lossy(AAD_DOMAIN)
)
}
#[cfg(test)]
mod tests {
use second_brain_core::keys::KeyId;
use super::*;
fn key(number: u32) -> (IdentityKey, Epoch) {
let identity = IdentityKey::generate().unwrap();
let epoch = Epoch {
number,
key_id: identity.key_id(),
};
(identity, epoch)
}
fn header(epoch: Epoch) -> SnapshotHeader {
SnapshotHeader {
format: SNAPSHOT_FORMAT,
snapshot_id: "s1".to_string(),
exporter: "a".to_string(),
epoch,
sync_protocol: crate::protocol::SYNC_PROTOCOL_VERSION,
schema_version: 12,
created_at: 0,
}
}
fn rows(n: usize) -> Vec<SnapshotRow> {
(0..n)
.map(|i| SnapshotRow::Dedup {
origin: format!("origin-{i}"),
lo: i as i64,
hi: i as i64 + 10,
})
.collect()
}
fn stream(identity: &IdentityKey, epoch: Epoch, rows: &[SnapshotRow]) -> Vec<u8> {
let mut writer = SnapshotWriter::new(
Vec::new(),
SnapshotKey::derive(identity, &epoch).unwrap(),
header(epoch),
)
.unwrap();
for row in rows {
writer.row(row).unwrap();
}
writer
.finish(
SnapshotState {
seq: 9,
..SnapshotState::default()
},
None,
)
.unwrap()
}
fn noise(len: usize) -> String {
let mut state = 0x9e37_79b9_7f4a_7c15u64;
(0..len)
.map(|_| {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
char::from(b'!' + (state % 90) as u8)
})
.collect()
}
fn large(origin: String) -> SnapshotRow {
SnapshotRow::Dedup {
origin,
lo: 0,
hi: 1,
}
}
#[test]
fn a_row_too_large_for_a_shared_frame_travels_in_one_of_its_own() {
let (identity, epoch) = key(1);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("snapshot.spool");
let big = noise(3 * FRAME_BYTES);
let mut sent = rows(2);
sent.insert(1, large(big.clone()));
let bytes = stream(&identity, epoch, &sent);
spool(bytes.as_slice(), &path, &|h| {
SnapshotKey::derive(&identity, &h.epoch)
})
.unwrap();
let got = origins(&path, &identity);
assert_eq!(
got,
vec!["origin-0".to_string(), big, "origin-1".to_string()]
);
}
#[test]
fn a_row_larger_than_sync_carries_is_left_out_and_the_rest_arrive() {
let (identity, epoch) = key(1);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("snapshot.spool");
let mut sent = rows(2);
sent.insert(1, large("x".repeat(MAX_ROW_BYTES)));
let bytes = stream(&identity, epoch, &sent);
spool(bytes.as_slice(), &path, &|h| {
SnapshotKey::derive(&identity, &h.epoch)
})
.unwrap();
assert_eq!(origins(&path, &identity), vec!["origin-0", "origin-1"]);
}
fn origins(path: &Path, identity: &IdentityKey) -> Vec<String> {
let mut seen = Vec::new();
spool_rows(
path,
&|h| SnapshotKey::derive(identity, &h.epoch),
&mut |row| {
if let SnapshotRow::Dedup { origin, .. } = row {
seen.push(origin);
}
Ok(())
},
)
.unwrap();
seen
}
#[test]
fn a_spooled_stream_round_trips_under_epoch_zero_and_later_epochs() {
for number in [0, 1] {
let (identity, epoch) = key(number);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("snapshot.spool");
let rows = rows(3);
let bytes = stream(&identity, epoch, &rows);
let (header, manifest) = spool(bytes.as_slice(), &path, &|h| {
SnapshotKey::derive(&identity, &h.epoch)
})
.unwrap();
assert_eq!(header.snapshot_id, "s1");
assert_eq!(manifest.state.seq, 9);
assert_eq!(
origins(&path, &identity),
vec!["origin-0", "origin-1", "origin-2"]
);
}
}
#[test]
fn rows_beyond_one_frame_split_into_several() {
let (identity, epoch) = key(1);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("snapshot.spool");
let long = "x".repeat(FRAME_BYTES / 3);
let rows: Vec<SnapshotRow> = (0..5)
.map(|i| SnapshotRow::Dedup {
origin: format!("{i}{long}"),
lo: 0,
hi: 0,
})
.collect();
let bytes = stream(&identity, epoch, &rows);
let (_, manifest) = spool(bytes.as_slice(), &path, &|h| {
SnapshotKey::derive(&identity, &h.epoch)
})
.unwrap();
assert!(manifest.frames >= 2, "{}", manifest.frames);
assert_eq!(origins(&path, &identity).len(), 5);
}
#[test]
fn a_cut_stream_leaves_no_spool() {
let (identity, epoch) = key(1);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("snapshot.spool");
let bytes = stream(&identity, epoch, &rows(3));
let cut = &bytes[..bytes.len() - 5];
assert!(spool(cut, &path, &|h| SnapshotKey::derive(&identity, &h.epoch)).is_err());
assert!(!path.exists() && !part_path(&path).exists());
}
#[test]
fn a_stream_missing_a_row_frame_fails_its_manifest() {
let (identity, epoch) = key(1);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("snapshot.spool");
let long = "y".repeat(FRAME_BYTES);
let rows: Vec<SnapshotRow> = (0..2)
.map(|i| SnapshotRow::Dedup {
origin: format!("{i}{long}"),
lo: 0,
hi: 0,
})
.collect();
let bytes = stream(&identity, epoch, &rows);
let header_end = bytes.iter().position(|b| *b == b'\n').unwrap() + 1;
let mut input = &bytes[header_end..];
let (_, first) = read_frame(&mut input).unwrap().unwrap();
let mut dropped = bytes[..header_end].to_vec();
dropped.extend_from_slice(input);
assert!(!first.is_empty());
let err = spool(dropped.as_slice(), &path, &|h| {
SnapshotKey::derive(&identity, &h.epoch)
})
.unwrap_err();
assert!(format!("{err:#}").contains("manifest"), "{err:#}");
assert!(!path.exists());
}
#[test]
fn another_identitys_key_opens_nothing() {
let (identity, epoch) = key(1);
let (other, _) = key(1);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("snapshot.spool");
let bytes = stream(&identity, epoch, &rows(1));
let foreign = |h: &SnapshotHeader| {
SnapshotKey::derive(
&other,
&Epoch {
number: h.epoch.number,
key_id: other.key_id(),
},
)
};
let _ = KeyId([0; 8]);
assert!(spool(bytes.as_slice(), &path, &foreign).is_err());
}
}