use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use anyhow::Result;
use chrono::{DateTime, Utc};
use sha2::{Digest, Sha256};
use second_brain_adapters::markdown::{self, Note, SOURCE};
use second_brain_core::embedding::{DocumentEmbedder, Embedder};
use second_brain_core::redact::redact_memory;
use second_brain_core::schema::{Conversation, IngestRecord, Memory, Relation, RelationType, Tier};
use second_brain_core::sqlite_store::{RetireOutcome, SqliteStore};
use second_brain_core::store::{Store, StoreAdmin};
pub const SCAN_INTERVAL: Duration = Duration::from_secs(120);
const MAX_NOTE_BYTES: u64 = 1024 * 1024;
const DELETED_HASH: &str = "";
pub fn spawn(
store: Arc<SqliteStore>,
embedder: Option<Arc<Embedder>>,
vaults: Vec<PathBuf>,
machine_id: String,
) {
let vaults = configured_vaults(vaults);
if vaults.is_empty() {
return;
}
let embedder = embedder.map(|e| e as Arc<dyn DocumentEmbedder>);
std::thread::spawn(move || {
let mut sync = VaultSync::new(store, embedder, machine_id);
loop {
let stats = sync.scan(&vaults);
if stats.ingested + stats.superseded + stats.retired + stats.errors > 0 {
tracing::info!(
"vaults: {} notes ingested, {} superseded, {} retired, {} errors",
stats.ingested,
stats.superseded,
stats.retired,
stats.errors
);
}
std::thread::sleep(SCAN_INTERVAL);
}
});
}
pub fn configured_vaults(vaults: Vec<PathBuf>) -> Vec<PathBuf> {
vaults
.into_iter()
.filter_map(|vault| {
let path: PathBuf = expand_home(vault.clone()).components().collect();
if path.is_absolute() {
Some(path)
} else {
tracing::warn!(
"vaults: ignoring {}: a vault path must be absolute or start with ~",
vault.display()
);
None
}
})
.collect()
}
fn expand_home(path: PathBuf) -> PathBuf {
match (path.strip_prefix("~"), std::env::var_os("HOME")) {
(Ok(rest), Some(home)) => PathBuf::from(home).join(rest),
_ => path,
}
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct ScanStats {
pub ingested: usize,
pub superseded: usize,
pub retired: usize,
pub unchanged: usize,
pub errors: usize,
}
pub struct VaultSync {
store: Arc<SqliteStore>,
embedder: Option<Arc<dyn DocumentEmbedder>>,
machine_id: String,
seen: HashMap<PathBuf, (SystemTime, u64)>,
unreadable_notes: HashMap<PathBuf, (SystemTime, u64)>,
reconciled: HashSet<PathBuf>,
warned: HashSet<PathBuf>,
vault_projects: HashMap<PathBuf, Option<String>>,
orphans_checked: bool,
projects: Option<Vec<(String, u64)>>,
}
impl VaultSync {
pub fn new(
store: Arc<SqliteStore>,
embedder: Option<Arc<dyn DocumentEmbedder>>,
machine_id: String,
) -> Self {
Self {
store,
embedder,
machine_id,
seen: HashMap::new(),
unreadable_notes: HashMap::new(),
reconciled: HashSet::new(),
warned: HashSet::new(),
vault_projects: HashMap::new(),
orphans_checked: false,
projects: None,
}
}
pub fn scan(&mut self, vaults: &[PathBuf]) -> ScanStats {
let mut stats = ScanStats::default();
self.projects = None;
for vault in vaults {
self.scan_vault(vault, vaults, &mut stats);
}
if !self.orphans_checked {
self.orphans_checked = true;
self.retire_orphans(vaults, &mut stats);
}
stats
}
fn scan_vault(&mut self, vault: &Path, vaults: &[PathBuf], stats: &mut ScanStats) {
let listing = match markdown::discover_notes(vault) {
Ok(listing) => listing,
Err(e) => {
self.warn_once(vault, &format!("cannot be listed ({e})"));
return;
}
};
if listing.notes.is_empty() {
self.warn_once(vault, "lists no notes");
return;
}
self.warned.retain(|p| {
!p.starts_with(vault)
|| (p != vault && vaults.iter().any(|v| v == p))
|| listing.unreadable.contains(p)
});
for dir in &listing.unreadable {
self.warn_once(dir, "is a folder that cannot be listed");
}
let owns = |path: &Path| {
owner(path, vaults) == Some(vault)
&& !listing.unreadable.iter().any(|dir| path.starts_with(dir))
};
let project = self.project_for(vault);
if self.vault_projects.get(vault) != Some(&project) {
self.seen.retain(|p, _| owner(p, vaults) != Some(vault));
self.vault_projects
.insert(vault.to_path_buf(), project.clone());
}
let notes: Vec<&Note> = listing.notes.iter().filter(|n| owns(&n.path)).collect();
let present: HashSet<&Path> = notes.iter().map(|n| n.path.as_path()).collect();
let changed: Vec<&Note> = notes
.iter()
.copied()
.filter(|n| {
let stamp = Some(&(n.modified, n.len));
self.seen.get(&n.path) != stamp && self.unreadable_notes.get(&n.path) != stamp
})
.collect();
let vanished: Vec<PathBuf> = self
.seen
.keys()
.chain(self.unreadable_notes.keys())
.filter(|p| owns(p) && !present.contains(p.as_path()))
.cloned()
.collect();
if changed.is_empty() && vanished.is_empty() && self.reconciled.contains(vault) {
return;
}
let mut live = match self.live_notes(owns) {
Ok(live) => live,
Err(e) => {
tracing::warn!("vaults: reading stored notes for {}: {e}", vault.display());
stats.errors += 1;
return;
}
};
for note in changed {
let key = note.path.to_string_lossy().to_string();
let bytes = if note.len > MAX_NOTE_BYTES {
None
} else {
match std::fs::read(¬e.path) {
Ok(bytes) => Some(bytes),
Err(e) => {
tracing::warn!("vaults: cannot read {key} ({e}); retrying once it changes");
self.unreadable_notes
.insert(note.path.clone(), (note.modified, note.len));
stats.errors += 1;
continue;
}
}
};
let current = live.remove(&key).unwrap_or_default();
match self.reconcile(project.as_deref(), note, &key, bytes, ¤t, stats) {
Ok(()) => {
self.unreadable_notes.remove(¬e.path);
self.seen
.insert(note.path.clone(), (note.modified, note.len));
}
Err(e) => {
tracing::warn!("vaults: ingesting {key}: {e:#}");
stats.errors += 1;
}
}
}
let mut deleted: Vec<String> = live
.keys()
.filter(|key| !present.contains(Path::new(key.as_str())))
.cloned()
.collect();
deleted.extend(vanished.iter().map(|p| p.to_string_lossy().to_string()));
deleted.sort();
deleted.dedup();
for key in deleted {
let current = live.remove(&key).unwrap_or_default();
self.retire_deleted(&key, ¤t, "note deleted from its vault", stats);
}
self.reconciled.insert(vault.to_path_buf());
}
fn retire_orphans(&mut self, vaults: &[PathBuf], stats: &mut ScanStats) {
let orphans = self.live_notes(|path| {
owner(path, vaults).is_none() && matches!(path.try_exists(), Ok(false))
});
match orphans {
Ok(orphans) => {
for (key, current) in orphans {
self.retire_deleted(&key, ¤t, "note's vault moved or was dropped", stats);
}
}
Err(e) => {
tracing::warn!("vaults: reading stored notes outside the vaults: {e}");
stats.errors += 1;
}
}
}
fn retire_deleted(
&mut self,
key: &str,
current: &[Memory],
reason: &str,
stats: &mut ScanStats,
) {
let result = self
.retire(current, reason, stats)
.and_then(|()| self.record(key, DELETED_HASH, 0));
match result {
Ok(()) => {
self.seen.remove(Path::new(key));
self.unreadable_notes.remove(Path::new(key));
}
Err(e) => {
tracing::warn!("vaults: retiring {key}: {e:#}");
stats.errors += 1;
}
}
}
fn warn_once(&mut self, path: &Path, problem: &str) {
if self.warned.insert(path.to_path_buf()) {
tracing::warn!(
"vaults: {} {problem}; the notes stored from it stay as they are",
path.display()
);
}
}
fn reconcile(
&mut self,
project: Option<&str>,
note: &Note,
key: &str,
bytes: Option<Vec<u8>>,
current: &[Memory],
stats: &mut ScanStats,
) -> Result<()> {
let logged = self
.store
.ingest_log_entry(key)?
.and_then(|e| e.content_hash);
let hash = match &bytes {
Some(bytes) => sha256_hex(bytes),
None => format!("oversize:{}", note.len),
};
if logged.as_deref() == Some(hash.as_str()) {
stats.unchanged += 1;
return self.refile(current, project);
}
let Some(bytes) = bytes else {
tracing::info!("vaults: {key} is over {MAX_NOTE_BYTES} bytes and is not kept");
self.retire(current, "note grew past the size limit", stats)?;
return self.record(key, &hash, 0);
};
let raw = String::from_utf8_lossy(&bytes);
let Some(mut memory) = markdown::note_memory(¬e.path, &raw, project) else {
self.retire(current, "note emptied in its vault", stats)?;
return self.record(key, &hash, 0);
};
redact_memory(&mut memory);
let kept = match current.iter().find(|m| m.content == memory.content) {
Some(same) => {
stats.unchanged += 1;
self.refile(std::slice::from_ref(same), project)?;
same.id
}
None => {
memory.machine_id = self.machine_id.clone();
memory.session_at = Some(DateTime::<Utc>::from(note.modified));
self.persist(&mut memory, ¬e.path, project.map(str::to_string))?;
stats.ingested += 1;
memory.id
}
};
for older in current.iter().filter(|m| m.id != kept) {
self.store.supersede_memory(older.id, kept)?;
stats.superseded += 1;
}
self.record(key, &hash, 1)
}
fn refile(&self, current: &[Memory], project: Option<&str>) -> Result<()> {
let Some(project) = project else {
return Ok(());
};
for memory in current {
if memory.project_path.as_deref() != Some(project) {
self.store.refile_memory(memory.id, project)?;
}
}
Ok(())
}
fn persist(&self, memory: &mut Memory, path: &Path, project: Option<String>) -> Result<()> {
if let Some(embedder) = &self.embedder {
match embedder.embed_document(&memory.content) {
Ok(embedding) => memory.embedding = embedding,
Err(e) => tracing::warn!(
"vaults: embedding {} failed, storing it without a vector: {e}",
memory.source_id
),
}
}
let conversation = Conversation {
id: markdown::note_conversation_id(&self.machine_id, path),
source: SOURCE.to_string(),
machine_id: self.machine_id.clone(),
started_at: Utc::now(),
project_path: project,
};
self.store.store_conversation_if_absent(&conversation)?;
Store::store_memory(self.store.as_ref(), memory)?;
self.store.store_relation(&Relation {
from_id: memory.id,
to_id: conversation.id,
relation_type: RelationType::DerivedFrom,
strength: 1.0,
context: None,
})
}
fn retire(&self, current: &[Memory], reason: &str, stats: &mut ScanStats) -> Result<()> {
for memory in current {
if self.store.retire_memory(memory.id, reason)? == RetireOutcome::Retired {
stats.retired += 1;
}
}
Ok(())
}
fn record(&self, key: &str, hash: &str, memory_count: usize) -> Result<()> {
self.store.record_ingested(&IngestRecord {
file_path: key,
content_hash: hash,
session_key: None,
legacy_ids: true,
memory_count,
source: SOURCE,
})
}
fn live_notes(&self, keep: impl Fn(&Path) -> bool) -> Result<HashMap<String, Vec<Memory>>> {
let mut live: HashMap<String, Vec<Memory>> = HashMap::new();
for memory in self.store.memories_by_source(SOURCE)? {
if memory.machine_id == self.machine_id
&& memory.tier == Tier::Hot
&& memory.superseded_by.is_none()
&& Path::new(&memory.source_id).is_absolute()
&& keep(Path::new(&memory.source_id))
{
live.entry(memory.source_id.clone())
.or_default()
.push(memory);
}
}
Ok(live)
}
fn project_for(&mut self, vault: &Path) -> Option<String> {
if self.projects.is_none() {
self.projects = Some(self.store.distinct_project_paths().unwrap_or_else(|e| {
tracing::warn!("vaults: listing projects failed: {e}");
Vec::new()
}));
}
markdown::vault_project(vault, self.projects.as_deref().unwrap_or_default())
}
}
fn owner<'a>(path: &Path, vaults: &'a [PathBuf]) -> Option<&'a Path> {
vaults
.iter()
.filter(|v| path.starts_with(v))
.max_by_key(|v| v.components().count())
.map(PathBuf::as_path)
}
fn sha256_hex(bytes: &[u8]) -> String {
Sha256::digest(bytes)
.iter()
.map(|b| format!("{b:02x}"))
.collect()
}