use std::sync::{Arc, PoisonError, RwLock};
use std::time::{Duration, Instant};
use anyhow::Result;
use second_brain_core::index::raw::RawIndex;
use second_brain_core::sealed::{SealKeyId, SealKeys};
use second_brain_core::sqlite_store::{AtRestPurpose, SqliteStore};
use second_brain_core::transcript_archive::TranscriptArchive;
pub const CHECK_INTERVAL: Duration = Duration::from_secs(60);
pub trait Reseal: Send + Sync {
fn reseal_under(&self, keys: SealKeys) -> Result<usize>;
fn sealed_under(&self) -> Option<SealKeyId>;
}
impl Reseal for TranscriptArchive {
fn reseal_under(&self, keys: SealKeys) -> Result<usize> {
self.set_seal_keys(keys);
self.reseal()
}
fn sealed_under(&self) -> Option<SealKeyId> {
if self.holds_unsealed() {
return None;
}
self.seal_keys().map(|keys| keys.current_id())
}
}
impl Reseal for RwLock<RawIndex> {
fn reseal_under(&self, keys: SealKeys) -> Result<usize> {
let mut index = self.write().unwrap_or_else(PoisonError::into_inner);
index.set_seal_keys(keys)?;
index.reseal()
}
fn sealed_under(&self) -> Option<SealKeyId> {
self.read()
.unwrap_or_else(PoisonError::into_inner)
.seal_keys()
.map(|keys| keys.current_id())
}
}
pub struct Sealer {
pub store: Arc<SqliteStore>,
pub purpose: AtRestPurpose,
pub target: Arc<dyn Reseal>,
sealed_under: Option<SealKeyId>,
}
impl Sealer {
pub fn new(store: Arc<SqliteStore>, purpose: AtRestPurpose, target: Arc<dyn Reseal>) -> Self {
Self {
store,
purpose,
target,
sealed_under: None,
}
}
pub fn spawn(mut self) {
std::thread::Builder::new()
.name(format!("seal-{:?}", self.purpose).to_lowercase())
.spawn(move || {
loop {
self.check();
std::thread::sleep(CHECK_INTERVAL);
}
})
.expect("spawning an at-rest sealing thread");
}
pub fn check(&mut self) -> bool {
let keys = match self.store.at_rest_keys(self.purpose) {
Ok(Some(keys)) => keys,
Ok(None) => return false,
Err(e) => {
tracing::warn!("reading the {:?} at-rest keys failed: {e:#}", self.purpose);
return false;
}
};
let current = keys.keys.current_id();
if keys.settled
&& (self.sealed_under == Some(current) || self.target.sealed_under() == Some(current))
{
self.sealed_under = Some(current);
return false;
}
let started = Instant::now();
let rewritten = match self.target.reseal_under(keys.keys) {
Ok(n) => n,
Err(e) => {
tracing::warn!(
"sealing the {:?} files failed and will retry: {e:#}",
self.purpose
);
return false;
}
};
if let Err(e) = self.store.settle_at_rest_keys(self.purpose, current) {
tracing::warn!(
"recording the sealed {:?} files failed: {e:#}",
self.purpose
);
return false;
}
self.sealed_under = Some(current);
if rewritten > 0 {
tracing::info!(
"{:?}: {rewritten} files sealed under key {current} in {:.1}s",
self.purpose,
started.elapsed().as_secs_f64()
);
}
true
}
}
#[cfg(test)]
mod tests {
use std::path::Path;
use second_brain_core::keys::StoreKey;
use second_brain_core::sealed::sealed_key_id;
use second_brain_core::sqlite_store::encrypt_migration::RewriteTo;
use second_brain_core::transcript_archive::{MANIFEST_FILE, SEALED_MANIFEST_FILE};
use super::*;
fn keyed_store(dir: &Path) -> Arc<SqliteStore> {
let key = StoreKey::generate().unwrap();
Arc::new(SqliteStore::open_with_key(&dir.join("brain.db"), "m".to_string(), &key).unwrap())
}
fn files_under(dir: &Path) -> Vec<std::path::PathBuf> {
let mut out = Vec::new();
let mut stack = vec![dir.to_path_buf()];
while let Some(d) = stack.pop() {
for entry in std::fs::read_dir(d).unwrap() {
let path = entry.unwrap().path();
if path.is_dir() {
stack.push(path);
} else if !path.to_string_lossy().ends_with(".lock") {
out.push(path);
}
}
}
out
}
fn assert_sealed_under(dir: &Path, key: SealKeyId) {
let files = files_under(dir);
assert!(!files.is_empty());
for path in files {
if path
.file_name()
.is_some_and(|n| n == MANIFEST_FILE || n == SEALED_MANIFEST_FILE)
{
let manifest = std::fs::read_to_string(&path).unwrap();
assert!(!manifest.contains("session.jsonl"), "{manifest}");
continue;
}
assert_eq!(
sealed_key_id(&path).unwrap(),
Some(key),
"{}",
path.display()
);
}
}
#[test]
fn an_archive_written_before_encryption_is_sealed_then_resealed_after_a_rotation() {
let data = tempfile::tempdir().unwrap();
let store = keyed_store(data.path());
let archive_dir = data.path().join("transcripts");
let archive = Arc::new(TranscriptArchive::open(&archive_dir).unwrap());
let source = data.path().join("session.jsonl");
std::fs::write(
&source,
"{\"text\":\"the zanzibar ferry leaves at noon\"}\n",
)
.unwrap();
archive.archive(&source).unwrap();
let mut sealer = Sealer::new(
store.clone(),
AtRestPurpose::TranscriptArchive,
archive.clone(),
);
assert!(sealer.check());
let first = store
.at_rest_keys(AtRestPurpose::TranscriptArchive)
.unwrap()
.unwrap();
assert!(first.settled);
assert_sealed_under(&archive_dir, first.keys.current_id());
assert!(!sealer.check());
let mut after_restart = Sealer::new(
store.clone(),
AtRestPurpose::TranscriptArchive,
archive.clone(),
);
assert!(
!after_restart.check(),
"a settled archive was resealed again"
);
store
.rewrite_under_key(RewriteTo::Key(&StoreKey::generate().unwrap()))
.unwrap();
assert!(sealer.check());
let second = store
.at_rest_keys(AtRestPurpose::TranscriptArchive)
.unwrap()
.unwrap();
assert_ne!(second.keys.current_id(), first.keys.current_id());
assert!(second.keys.retired_ids().is_empty());
assert_sealed_under(&archive_dir, second.keys.current_id());
assert!(archive.read(&source).unwrap().unwrap().contains("zanzibar"));
}
#[test]
fn a_settled_archive_holding_entries_an_older_binary_wrote_is_resealed() {
let data = tempfile::tempdir().unwrap();
let store = keyed_store(data.path());
let archive_dir = data.path().join("transcripts");
let keys = || {
store
.at_rest_keys(AtRestPurpose::TranscriptArchive)
.unwrap()
.unwrap()
};
let first = data.path().join("first.jsonl");
std::fs::write(&first, "{\"text\":\"the zanzibar ferry\"}\n").unwrap();
let archive = Arc::new(TranscriptArchive::open_sealed(&archive_dir, keys().keys).unwrap());
archive.archive(&first).unwrap();
assert!(
Sealer::new(
store.clone(),
AtRestPurpose::TranscriptArchive,
archive.clone()
)
.check()
);
assert!(keys().settled);
drop(archive);
let older = data.path().join("older-binary");
let later = data.path().join("later.jsonl");
std::fs::write(&later, "{\"text\":\"the harbour at dusk\"}\n").unwrap();
let plain = TranscriptArchive::open(&older).unwrap();
plain.archive(&later).unwrap();
let entry = plain.entry(&later).unwrap();
drop(plain);
std::fs::copy(older.join(&entry.file), archive_dir.join(&entry.file)).unwrap();
std::fs::copy(older.join(MANIFEST_FILE), archive_dir.join(MANIFEST_FILE)).unwrap();
let archive = Arc::new(TranscriptArchive::open_sealed(&archive_dir, keys().keys).unwrap());
assert_eq!(archive.len(), 2);
let mut after_restart = Sealer::new(
store.clone(),
AtRestPurpose::TranscriptArchive,
archive.clone(),
);
assert!(after_restart.check());
assert_sealed_under(&archive_dir, keys().keys.current_id());
assert!(archive.read(&later).unwrap().unwrap().contains("harbour"));
}
#[test]
fn a_plaintext_store_seals_nothing() {
let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
let data = tempfile::tempdir().unwrap();
let archive = Arc::new(TranscriptArchive::open(data.path()).unwrap());
let mut sealer = Sealer::new(store, AtRestPurpose::TranscriptArchive, archive);
assert!(!sealer.check());
}
#[test]
fn a_raw_index_is_resealed_under_the_store_keys() {
let data = tempfile::tempdir().unwrap();
let store = keyed_store(data.path());
let raw_dir = data.path().join("brain.raw-index");
let index = Arc::new(RwLock::new(
RawIndex::open_keyed(
&raw_dir,
&store
.at_rest_keys(AtRestPurpose::RawIndex)
.unwrap()
.unwrap()
.keys,
)
.unwrap(),
));
let mut sealer = Sealer::new(store.clone(), AtRestPurpose::RawIndex, index.clone());
assert!(sealer.check());
store
.rewrite_under_key(RewriteTo::Key(&StoreKey::generate().unwrap()))
.unwrap();
assert!(sealer.check());
let keys = store
.at_rest_keys(AtRestPurpose::RawIndex)
.unwrap()
.unwrap();
assert!(keys.settled);
assert_sealed_under(&raw_dir, keys.keys.current_id());
}
#[test]
fn a_raw_index_opened_before_the_store_was_encrypted_is_reopened_sealed() {
let data = tempfile::tempdir().unwrap();
let store = keyed_store(data.path());
let raw_dir = data.path().join("brain.raw-index");
let mut plain = RawIndex::open(&raw_dir).unwrap();
let mut embedding = vec![0.0f32; second_brain_core::index::EMBEDDING_DIM];
embedding[3] = 1.0;
let passage = second_brain_core::index::raw::RawPassage {
text: "the zanzibar ferry leaves at noon".to_string(),
at: None,
project: None,
memory: None,
};
let origin = second_brain_core::index::raw::RawOrigin::Transcript {
sha256: "ab".repeat(32),
};
plain.add_source(&origin, &[(passage, embedding)]).unwrap();
plain.commit().unwrap();
let index = Arc::new(RwLock::new(plain));
let mut sealer = Sealer::new(store.clone(), AtRestPurpose::RawIndex, index.clone());
assert!(sealer.check());
let keys = store
.at_rest_keys(AtRestPurpose::RawIndex)
.unwrap()
.unwrap();
assert!(keys.settled);
assert_sealed_under(&raw_dir, keys.keys.current_id());
let guard = index.read().unwrap();
assert_eq!(guard.len(), 1);
assert_eq!(guard.bm25_search("zanzibar", 5, None).unwrap().len(), 1);
}
}