mod common;
use std::path::{Path, PathBuf};
use std::sync::mpsc::Sender;
use std::sync::{Arc, RwLock};
use std::time::Duration;
use anyhow::Result;
use axum::http::StatusCode;
use chrono::{DateTime, Utc};
use serde_json::{Value, json};
use tower::ServiceExt;
use common::{json_body, post_json, state_in};
use second_brain_api::AppState;
use second_brain_api::clock_gate::ClockGate;
use second_brain_api::ingest::{FileOutcome, capture_transcript, ingest_memory_file};
use second_brain_api::maintenance::{DrainLoop, DrainRequest, WriterSlot};
use second_brain_api::raw_index::{RawCoverage, RawIndexer};
use second_brain_api::routes;
use second_brain_core::cold::{ArchiveKeys, ColdTier, Kek, KekLabel, KeyId, Zeroizing};
use second_brain_core::config::MaintenanceConfig;
use second_brain_core::index::raw::RawIndex;
use second_brain_core::index::{EMBEDDING_DIM, IndexClaim, Indexes};
use second_brain_core::schema::{Memory, MemoryType, SyncEntry, SyncRecord};
use second_brain_core::sqlite_store::{DeleteScope, SqliteStore};
use second_brain_core::store::{Store, StoreAdmin};
use second_brain_core::transcript_archive::TranscriptArchive;
const PROJECT: &str = "-srv-garden";
const GONE: &str = "5e55a000-0000-4000-8000-000000000001";
const KEPT: &str = "5e55a000-0000-4000-8000-000000000002";
fn embed(text: &str) -> Vec<f32> {
let mut v = vec![0.0f32; EMBEDDING_DIM];
for word in text.split_whitespace() {
let slot = word
.bytes()
.fold(7usize, |h, b| h.wrapping_mul(31).wrapping_add(b as usize));
v[slot % EMBEDDING_DIM] += 1.0;
}
let norm = v.iter().map(|x| x * x).sum::<f32>().sqrt().max(1e-6);
v.iter().map(|x| x / norm).collect()
}
struct Keys;
fn label() -> KekLabel {
KekLabel {
epoch: 1,
key_id: KeyId([1; 8]),
}
}
impl ArchiveKeys for Keys {
fn current(&self) -> Result<Option<Kek>> {
Ok(Some(Kek {
label: label(),
key: Zeroizing::new([1; 32]),
}))
}
fn by_label(&self, wanted: &KekLabel) -> Result<Option<Zeroizing<[u8; 32]>>> {
Ok((*wanted == label()).then(|| Zeroizing::new([1; 32])))
}
}
fn line(session: &str, kind: &str, text: &str) -> String {
let message = if kind == "assistant" {
json!({"content": [{"type": "text", "text": text}]})
} else {
json!({"content": text})
};
json!({"type": kind, "sessionId": session, "timestamp": "2026-10-01T09:30:05Z",
"message": message})
.to_string()
}
struct Daemon {
dir: tempfile::TempDir,
machine: String,
state: Arc<AppState>,
app: axum::Router,
archive: Arc<TranscriptArchive>,
raw: Arc<RwLock<RawIndex>>,
reader: Option<Arc<RwLock<Indexes>>>,
drain: Option<Sender<DrainRequest>>,
}
fn daemon(machine: &str, with_indexes: bool) -> Daemon {
let dir = tempfile::tempdir().unwrap();
let store = SqliteStore::open(&dir.path().join("brain.db"), machine.to_string()).unwrap();
let tier = Arc::new(ColdTier::open(dir.path().join("brain.cold"), Arc::new(Keys)).unwrap());
assert!(store.attach_cold_tier(tier));
let base = Arc::into_inner(state_in(store, dir.path(), ClockGate::open_for_tests())).unwrap();
let archive = Arc::new(TranscriptArchive::open(dir.path().join("transcripts")).unwrap());
let raw = Arc::new(RwLock::new(
RawIndex::open(&dir.path().join("brain.raw-index")).unwrap(),
));
let (reader, drain) = if with_indexes {
let index_dir = dir.path().join("brain.indexes");
let mut writer = Indexes::open(&index_dir).unwrap();
writer.persist().unwrap();
let reader = Arc::new(RwLock::new(Indexes::open_readonly(&index_dir).unwrap()));
let claim = IndexClaim::acquire(&index_dir).unwrap();
let (tx, requests) = std::sync::mpsc::channel();
DrainLoop {
store: base.sqlite.clone(),
reader: reader.clone(),
writer: WriterSlot::new(writer, claim),
embedder: None,
summary: Arc::new(RwLock::new(None)),
near_duplicates: Arc::new(RwLock::new(None)),
near_duplicate_threshold: 0.97,
config: MaintenanceConfig {
enabled: false,
interval_secs: 3600,
},
requests,
health: base.health.clone(),
}
.spawn();
(Some(reader), Some(tx))
} else {
(None, None)
};
let state = Arc::new(AppState {
indexes: reader.clone(),
raw_index: Some(raw.clone()),
archive: Some(archive.clone()),
repair_tx: drain.clone(),
..base
});
let app = routes::router().with_state(state.clone());
Daemon {
dir,
machine: machine.to_string(),
state,
app,
archive,
raw,
reader,
drain,
}
}
impl Daemon {
fn sqlite(&self) -> &SqliteStore {
&self.state.sqlite
}
fn transcript(&self, session: &str) -> PathBuf {
self.dir
.path()
.join("projects")
.join(PROJECT)
.join(format!("{session}.jsonl"))
}
fn write_session(&self, session: &str, lines: &[String]) -> PathBuf {
let path = self.transcript(session);
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, lines.join("\n") + "\n").unwrap();
path
}
fn capture(&self, path: &Path) -> FileOutcome {
let store: Arc<dyn StoreAdmin> = self.state.sqlite.clone();
capture_transcript(&store, None, Some(&self.archive), path, &self.machine).unwrap()
}
fn indexer(&self) -> RawIndexer {
RawIndexer {
index: self.raw.clone(),
archive: Some(self.archive.clone()),
store: self.state.sqlite.clone(),
embed: Arc::new(|texts: &[&str]| Ok(texts.iter().map(|t| embed(t)).collect())),
pause: Duration::ZERO,
coverage: Arc::new(RawCoverage::new(36500)),
}
}
fn drain_indexes(&self) {
let (reply, done) = std::sync::mpsc::channel();
self.drain
.as_ref()
.unwrap()
.send(DrainRequest::CompactIndexes(reply))
.unwrap();
done.recv_timeout(Duration::from_secs(60)).unwrap().unwrap();
}
fn bm25(&self, words: &str) -> Vec<uuid::Uuid> {
self.reader
.as_ref()
.unwrap()
.read()
.unwrap()
.bm25_search(words, 10)
.unwrap()
.into_iter()
.map(|(id, _)| id)
.collect()
}
fn raw_texts(&self, words: &str) -> Vec<String> {
let index = self.raw.read().unwrap();
index
.bm25_search(words, 10, None)
.unwrap()
.into_iter()
.filter_map(|(key, _)| index.passage(key).unwrap())
.map(|hit| hit.text)
.collect()
}
fn contents(&self) -> Vec<String> {
self.sqlite()
.all_memories_with_embeddings()
.unwrap()
.into_iter()
.map(|m| m.content)
.collect()
}
async fn delete(&self, body: Value) -> (StatusCode, Value) {
let response = self
.app
.clone()
.oneshot(post_json("/v1/delete", body))
.await
.unwrap();
let status = response.status();
(status, json_body(response).await)
}
}
fn old() -> DateTime<Utc> {
"2026-08-01T12:00:00Z".parse().unwrap()
}
fn seeded(d: &Daemon) -> (PathBuf, Memory) {
let gone = d.write_session(
GONE,
&[
line(GONE, "user", "where does the heron nest these days"),
line(
GONE,
"assistant",
"The heron nests behind the boathouse on the east bank of the lake, and it returns there every single spring to raise its young.",
),
],
);
let kept = d.write_session(
KEPT,
&[
line(KEPT, "user", "what colour is the shed"),
line(
KEPT,
"assistant",
"The shed at the bottom of the garden is painted green, and the paint was renewed last summer after the storm damaged it.",
),
],
);
assert_eq!(d.capture(&gone), FileOutcome::Captured);
assert_eq!(d.capture(&kept), FileOutcome::Captured);
let mut note = Memory::new(
"a note about the heron filed under the session".to_string(),
MemoryType::Semantic,
"agent-note".to_string(),
GONE.to_string(),
)
.with_project_path(Some(PROJECT.to_string()));
note.embedding = embed(¬e.content);
d.sqlite().store_memory(¬e).unwrap();
let mut cold = Memory::new(
"an old heron sighting that went cold".to_string(),
MemoryType::Episodic,
"claude-code".to_string(),
gone.to_string_lossy().into_owned(),
)
.with_project_path(Some(PROJECT.to_string()));
cold.embedding = embed(&cold.content);
cold.session_at = Some(old());
d.sqlite().store_memory(&cold).unwrap();
d.sqlite().archive_memory(cold.id).unwrap();
d.sqlite().run_stored_redaction(None).unwrap();
let pass = d
.sqlite()
.run_cold_pass(Utc::now(), Duration::ZERO)
.unwrap();
assert_eq!(pass.sealed_rows, 1);
(gone, note)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn deleting_a_session_removes_it_from_every_layer_and_it_is_never_captured_again() {
let d = daemon("machine-a", true);
let (gone, note) = seeded(&d);
d.drain_indexes();
d.indexer().pass().unwrap();
assert!(!d.bm25("heron boathouse").is_empty());
assert!(!d.raw_texts("heron nest").is_empty());
assert!(d.archive.entry(&gone).is_some());
let (status, body) = d.delete(json!({"session": GONE})).await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["scope"], "session");
assert_eq!(body["target"], GONE);
assert_eq!(body["dry_run"], false);
assert_eq!(body["drained"], true, "{body}");
assert_eq!(body["warnings"], json!([]), "{body}");
assert!(body["removed"]["memories"].as_u64().unwrap() >= 3, "{body}");
assert_eq!(body["removed"]["conversations"], 1, "{body}");
assert_eq!(body["layers"]["archive"], 1, "{body}");
assert_eq!(body["layers"]["cold_records"], 1, "{body}");
assert!(body["layers"]["raw_index"].as_u64().unwrap() > 0, "{body}");
assert!(body["layers"]["vector"].as_u64().unwrap() >= 1, "{body}");
assert!(body["layers"]["bm25"].as_u64().unwrap() >= 2, "{body}");
assert!(d.contents().iter().all(|c| !c.contains("heron")));
assert!(d.contents().iter().any(|c| c.contains("shed")));
assert!(d.bm25("heron boathouse").is_empty());
let nearest = d
.reader
.as_ref()
.unwrap()
.read()
.unwrap()
.vector_search(d.sqlite(), ¬e.embedding, 5)
.unwrap();
assert!(nearest.iter().all(|(m, _)| *m != note.id));
assert!(d.raw_texts("heron nest boathouse").is_empty());
assert!(!d.raw_texts("shed green").is_empty());
assert!(d.archive.entry(&gone).is_none());
assert_eq!(d.sqlite().cold_stats().unwrap().rows, 0);
let mut grown = std::fs::read_to_string(&gone).unwrap();
grown.push_str(&line(
GONE,
"assistant",
"The heron came back today and stood on the jetty for an hour before it flew off towards the reeds on the far shore.",
));
grown.push('\n');
std::fs::write(&gone, grown).unwrap();
assert_eq!(d.capture(&gone), FileOutcome::Skipped);
d.indexer().pass().unwrap();
assert!(d.archive.entry(&gone).is_none());
assert!(d.contents().iter().all(|c| !c.contains("heron")));
assert!(d.raw_texts("heron").is_empty());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_dry_run_counts_and_leaves_everything_in_place() {
let d = daemon("machine-a", true);
let (gone, _) = seeded(&d);
let before = d.contents().len();
let (status, body) = d.delete(json!({"session": GONE, "dry_run": true})).await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["dry_run"], true);
assert!(body["removed"]["memories"].as_u64().unwrap() >= 3, "{body}");
assert_eq!(body["layers"]["archive"], 1, "{body}");
assert_eq!(d.contents().len(), before);
assert!(d.archive.entry(&gone).is_some());
assert!(!d.sqlite().session_deleted(&gone.to_string_lossy()).unwrap());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_bad_or_empty_scope_is_refused() {
let d = daemon("machine-a", false);
seeded(&d);
for body in [
json!({}),
json!({"memory": "not-a-uuid"}),
json!({"session": GONE, "project": PROJECT}),
json!({"session": ""}),
json!({"session": "MEMORY"}),
json!({"project": "/"}),
] {
let (status, reply) = d.delete(body.clone()).await;
assert_eq!(status, StatusCode::BAD_REQUEST, "{body} -> {reply}");
}
let (status, reply) = d
.delete(json!({"session": "5e55a000-0000-4000-8000-0000000000ff"}))
.await;
assert_eq!(status, StatusCode::NOT_FOUND, "{reply}");
let (status, reply) = d.delete(json!({"project": "-srv-gardne"})).await;
assert_eq!(status, StatusCode::NOT_FOUND, "{reply}");
assert!(
reply["error"].as_str().unwrap().contains(PROJECT),
"{reply}"
);
for body in [
json!({"project": "/srv/garden", "dry_run": true}),
json!({"project": "/srv/garden/", "dry_run": true}),
json!({"session": GONE.to_uppercase(), "dry_run": true}),
] {
let (status, reply) = d.delete(body.clone()).await;
assert_eq!(status, StatusCode::OK, "{body} -> {reply}");
assert!(
reply["removed"]["memories"].as_u64().unwrap() > 0,
"{reply}"
);
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn without_open_search_indexes_the_delete_says_their_removal_is_pending() {
let d = daemon("machine-a", false);
seeded(&d);
let (status, body) = d.delete(json!({"project": PROJECT})).await;
assert_eq!(status, StatusCode::OK, "{body}");
assert_eq!(body["drained"], false);
assert_eq!(body["warnings"].as_array().unwrap().len(), 1, "{body}");
assert!(d.contents().is_empty());
assert_eq!(body["layers"]["archive"], 2, "{body}");
}
fn to_record(entry: &SyncEntry) -> SyncRecord {
SyncRecord {
local_seq: entry.local_seq as u64,
origin_machine_id: entry.origin_machine_id.clone(),
origin_seq: entry.origin_seq as u64,
op: entry.op,
node_type: entry.node_type,
node_id: entry.node_id.clone(),
timestamp: entry.timestamp,
data: entry
.data
.as_deref()
.map(|d| serde_json::from_str(d).unwrap()),
}
}
fn relay(from: &SqliteStore, to: &SqliteStore, after: u64) -> u64 {
for entry in from.sync_log_since(after).unwrap() {
to.apply_record(&to_record(&entry)).unwrap();
}
from.max_sync_seq().unwrap()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_peer_that_holds_a_copy_of_the_transcript_drops_it_at_its_next_raw_pass() {
let a = daemon("machine-a", false);
let b = daemon("machine-b", false);
let (gone, _) = seeded(&a);
let on_b = b.write_session(GONE, &[std::fs::read_to_string(&gone).unwrap()]);
assert_eq!(b.capture(&on_b), FileOutcome::Captured);
let seen = relay(a.sqlite(), b.sqlite(), 0);
b.indexer().pass().unwrap();
assert!(!b.raw_texts("heron nest").is_empty());
a.sqlite()
.delete_scope(&DeleteScope::Session(GONE.to_string()), &[GONE.to_string()])
.unwrap();
relay(a.sqlite(), b.sqlite(), seen);
assert!(b.sqlite().session_deleted(&on_b.to_string_lossy()).unwrap());
let summary = b.indexer().pass().unwrap();
assert!(summary.purged > 0);
assert!(b.raw_texts("heron").is_empty());
assert!(b.archive.entry(&on_b).is_none());
assert!(b.contents().iter().all(|c| !c.contains("heron")));
assert_eq!(b.capture(&on_b), FileOutcome::Skipped);
assert!(b.contents().iter().all(|c| !c.contains("heron")));
}
fn write_note(path: &Path, body: &str) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(
path,
format!("---\nname: heron\ndescription: where the heron nests\ntype: project\n---\n{body}"),
)
.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_deleted_memory_note_is_not_ingested_again_while_its_file_keeps_the_text() {
let d = daemon("machine-a", false);
let store: Arc<dyn StoreAdmin> = d.state.sqlite.clone();
let note = d
.dir
.path()
.join("projects")
.join(PROJECT)
.join("memory")
.join("heron.md");
let body = "The heron nests behind the boathouse every spring.\n";
write_note(¬e, body);
assert_eq!(
ingest_memory_file(&store, None, ¬e, &d.machine).unwrap(),
FileOutcome::Captured
);
let id = d
.sqlite()
.all_memories_with_embeddings()
.unwrap()
.into_iter()
.find(|m| m.content.contains("heron nests"))
.unwrap()
.id;
let (status, body_json) = d.delete(json!({"memory": id.to_string()})).await;
assert_eq!(status, StatusCode::OK, "{body_json}");
assert!(body_json["delete_id"].is_string(), "{body_json}");
assert!(d.contents().iter().all(|c| !c.contains("heron nests")));
let head = d.sqlite().max_sync_seq().unwrap();
write_note(¬e, &format!("{body}\n"));
ingest_memory_file(&store, None, ¬e, &d.machine).unwrap();
d.sqlite().clear_reparseable_ingest_log().unwrap();
ingest_memory_file(&store, None, ¬e, &d.machine).unwrap();
assert!(d.contents().iter().all(|c| !c.contains("heron nests")));
assert_eq!(
d.sqlite().max_sync_seq().unwrap(),
head,
"a blocked note logs nothing, not even its conversation"
);
write_note(¬e, "The heron moved to the reeds.\n");
ingest_memory_file(&store, None, ¬e, &d.machine).unwrap();
assert!(
d.contents()
.iter()
.any(|c| c.contains("moved to the reeds"))
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn a_project_delete_status_is_found_by_its_directory() {
let d = daemon("machine-a", false);
seeded(&d);
let (status, body) = d.delete(json!({"project": PROJECT})).await;
assert_eq!(status, StatusCode::OK, "{body}");
let response = d
.app
.clone()
.oneshot(common::get("/v1/delete/%2Fsrv%2Fgarden%2F"))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let found = json_body(response).await;
assert_eq!(found["id"], body["delete_id"], "{found}");
assert_eq!(found["target"], PROJECT);
let response = d
.app
.clone()
.oneshot(common::get("/v1/delete/nothing-here"))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}