1use std::collections::{HashMap, HashSet};
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::{Duration, SystemTime};
5
6use anyhow::Result;
7use chrono::{DateTime, Utc};
8use sha2::{Digest, Sha256};
9
10use second_brain_adapters::markdown::{self, Note, SOURCE};
11use second_brain_core::embedding::{DocumentEmbedder, Embedder};
12use second_brain_core::redact::redact_memory;
13use second_brain_core::schema::{Conversation, IngestRecord, Memory, Relation, RelationType, Tier};
14use second_brain_core::sqlite_store::{RetireOutcome, SqliteStore};
15use second_brain_core::store::{Store, StoreAdmin};
16
17pub const SCAN_INTERVAL: Duration = Duration::from_secs(120);
18const MAX_NOTE_BYTES: u64 = 1024 * 1024;
21const DELETED_HASH: &str = "";
24
25pub fn spawn(
28 store: Arc<SqliteStore>,
29 embedder: Option<Arc<Embedder>>,
30 vaults: Vec<PathBuf>,
31 machine_id: String,
32) {
33 let vaults = configured_vaults(vaults);
34 if vaults.is_empty() {
35 return;
36 }
37 let embedder = embedder.map(|e| e as Arc<dyn DocumentEmbedder>);
38 std::thread::spawn(move || {
39 let mut sync = VaultSync::new(store, embedder, machine_id);
40 loop {
41 let stats = sync.scan(&vaults);
42 if stats.ingested + stats.superseded + stats.retired + stats.errors > 0 {
43 tracing::info!(
44 "vaults: {} notes ingested, {} superseded, {} retired, {} errors",
45 stats.ingested,
46 stats.superseded,
47 stats.retired,
48 stats.errors
49 );
50 }
51 std::thread::sleep(SCAN_INTERVAL);
52 }
53 });
54}
55
56pub fn configured_vaults(vaults: Vec<PathBuf>) -> Vec<PathBuf> {
60 vaults
61 .into_iter()
62 .filter_map(|vault| {
63 let path: PathBuf = expand_home(vault.clone()).components().collect();
64 if path.is_absolute() {
65 Some(path)
66 } else {
67 tracing::warn!(
68 "vaults: ignoring {}: a vault path must be absolute or start with ~",
69 vault.display()
70 );
71 None
72 }
73 })
74 .collect()
75}
76
77fn expand_home(path: PathBuf) -> PathBuf {
78 match (path.strip_prefix("~"), std::env::var_os("HOME")) {
79 (Ok(rest), Some(home)) => PathBuf::from(home).join(rest),
80 _ => path,
81 }
82}
83
84#[derive(Debug, Default, PartialEq, Eq)]
85pub struct ScanStats {
86 pub ingested: usize,
87 pub superseded: usize,
88 pub retired: usize,
89 pub unchanged: usize,
90 pub errors: usize,
91}
92
93pub struct VaultSync {
100 store: Arc<SqliteStore>,
101 embedder: Option<Arc<dyn DocumentEmbedder>>,
102 machine_id: String,
103 seen: HashMap<PathBuf, (SystemTime, u64)>,
104 unreadable_notes: HashMap<PathBuf, (SystemTime, u64)>,
105 reconciled: HashSet<PathBuf>,
106 warned: HashSet<PathBuf>,
107 vault_projects: HashMap<PathBuf, Option<String>>,
108 orphans_checked: bool,
109 projects: Option<Vec<(String, u64)>>,
110}
111
112impl VaultSync {
113 pub fn new(
114 store: Arc<SqliteStore>,
115 embedder: Option<Arc<dyn DocumentEmbedder>>,
116 machine_id: String,
117 ) -> Self {
118 Self {
119 store,
120 embedder,
121 machine_id,
122 seen: HashMap::new(),
123 unreadable_notes: HashMap::new(),
124 reconciled: HashSet::new(),
125 warned: HashSet::new(),
126 vault_projects: HashMap::new(),
127 orphans_checked: false,
128 projects: None,
129 }
130 }
131
132 pub fn scan(&mut self, vaults: &[PathBuf]) -> ScanStats {
133 let mut stats = ScanStats::default();
134 self.projects = None;
135 for vault in vaults {
136 self.scan_vault(vault, vaults, &mut stats);
137 }
138 if !self.orphans_checked {
139 self.orphans_checked = true;
140 self.retire_orphans(vaults, &mut stats);
141 }
142 stats
143 }
144
145 fn scan_vault(&mut self, vault: &Path, vaults: &[PathBuf], stats: &mut ScanStats) {
150 let listing = match markdown::discover_notes(vault) {
151 Ok(listing) => listing,
152 Err(e) => {
153 self.warn_once(vault, &format!("cannot be listed ({e})"));
154 return;
155 }
156 };
157 if listing.notes.is_empty() {
158 self.warn_once(vault, "lists no notes");
159 return;
160 }
161 self.warned.retain(|p| {
162 !p.starts_with(vault)
163 || (p != vault && vaults.iter().any(|v| v == p))
164 || listing.unreadable.contains(p)
165 });
166 for dir in &listing.unreadable {
167 self.warn_once(dir, "is a folder that cannot be listed");
168 }
169
170 let owns = |path: &Path| {
171 owner(path, vaults) == Some(vault)
172 && !listing.unreadable.iter().any(|dir| path.starts_with(dir))
173 };
174 let project = self.project_for(vault);
175 if self.vault_projects.get(vault) != Some(&project) {
176 self.seen.retain(|p, _| owner(p, vaults) != Some(vault));
177 self.vault_projects
178 .insert(vault.to_path_buf(), project.clone());
179 }
180
181 let notes: Vec<&Note> = listing.notes.iter().filter(|n| owns(&n.path)).collect();
182 let present: HashSet<&Path> = notes.iter().map(|n| n.path.as_path()).collect();
183 let changed: Vec<&Note> = notes
184 .iter()
185 .copied()
186 .filter(|n| {
187 let stamp = Some(&(n.modified, n.len));
188 self.seen.get(&n.path) != stamp && self.unreadable_notes.get(&n.path) != stamp
189 })
190 .collect();
191 let vanished: Vec<PathBuf> = self
192 .seen
193 .keys()
194 .chain(self.unreadable_notes.keys())
195 .filter(|p| owns(p) && !present.contains(p.as_path()))
196 .cloned()
197 .collect();
198 if changed.is_empty() && vanished.is_empty() && self.reconciled.contains(vault) {
199 return;
200 }
201
202 let mut live = match self.live_notes(owns) {
203 Ok(live) => live,
204 Err(e) => {
205 tracing::warn!("vaults: reading stored notes for {}: {e}", vault.display());
206 stats.errors += 1;
207 return;
208 }
209 };
210
211 for note in changed {
212 let key = note.path.to_string_lossy().to_string();
213 let bytes = if note.len > MAX_NOTE_BYTES {
214 None
215 } else {
216 match std::fs::read(¬e.path) {
217 Ok(bytes) => Some(bytes),
218 Err(e) => {
219 tracing::warn!("vaults: cannot read {key} ({e}); retrying once it changes");
220 self.unreadable_notes
221 .insert(note.path.clone(), (note.modified, note.len));
222 stats.errors += 1;
223 continue;
224 }
225 }
226 };
227 let current = live.remove(&key).unwrap_or_default();
228 match self.reconcile(project.as_deref(), note, &key, bytes, ¤t, stats) {
229 Ok(()) => {
230 self.unreadable_notes.remove(¬e.path);
231 self.seen
232 .insert(note.path.clone(), (note.modified, note.len));
233 }
234 Err(e) => {
235 tracing::warn!("vaults: ingesting {key}: {e:#}");
236 stats.errors += 1;
237 }
238 }
239 }
240
241 let mut deleted: Vec<String> = live
242 .keys()
243 .filter(|key| !present.contains(Path::new(key.as_str())))
244 .cloned()
245 .collect();
246 deleted.extend(vanished.iter().map(|p| p.to_string_lossy().to_string()));
247 deleted.sort();
248 deleted.dedup();
249 for key in deleted {
250 let current = live.remove(&key).unwrap_or_default();
251 self.retire_deleted(&key, ¤t, "note deleted from its vault", stats);
252 }
253 self.reconciled.insert(vault.to_path_buf());
254 }
255
256 fn retire_orphans(&mut self, vaults: &[PathBuf], stats: &mut ScanStats) {
261 let orphans = self.live_notes(|path| {
262 owner(path, vaults).is_none() && matches!(path.try_exists(), Ok(false))
263 });
264 match orphans {
265 Ok(orphans) => {
266 for (key, current) in orphans {
267 self.retire_deleted(&key, ¤t, "note's vault moved or was dropped", stats);
268 }
269 }
270 Err(e) => {
271 tracing::warn!("vaults: reading stored notes outside the vaults: {e}");
272 stats.errors += 1;
273 }
274 }
275 }
276
277 fn retire_deleted(
278 &mut self,
279 key: &str,
280 current: &[Memory],
281 reason: &str,
282 stats: &mut ScanStats,
283 ) {
284 let result = self
285 .retire(current, reason, stats)
286 .and_then(|()| self.record(key, DELETED_HASH, 0));
287 match result {
288 Ok(()) => {
289 self.seen.remove(Path::new(key));
290 self.unreadable_notes.remove(Path::new(key));
291 }
292 Err(e) => {
293 tracing::warn!("vaults: retiring {key}: {e:#}");
294 stats.errors += 1;
295 }
296 }
297 }
298
299 fn warn_once(&mut self, path: &Path, problem: &str) {
300 if self.warned.insert(path.to_path_buf()) {
301 tracing::warn!(
302 "vaults: {} {problem}; the notes stored from it stay as they are",
303 path.display()
304 );
305 }
306 }
307
308 fn reconcile(
312 &mut self,
313 project: Option<&str>,
314 note: &Note,
315 key: &str,
316 bytes: Option<Vec<u8>>,
317 current: &[Memory],
318 stats: &mut ScanStats,
319 ) -> Result<()> {
320 let logged = self
321 .store
322 .ingest_log_entry(key)?
323 .and_then(|e| e.content_hash);
324 let hash = match &bytes {
325 Some(bytes) => sha256_hex(bytes),
326 None => format!("oversize:{}", note.len),
327 };
328 if logged.as_deref() == Some(hash.as_str()) {
329 stats.unchanged += 1;
330 return self.refile(current, project);
331 }
332 let Some(bytes) = bytes else {
333 tracing::info!("vaults: {key} is over {MAX_NOTE_BYTES} bytes and is not kept");
334 self.retire(current, "note grew past the size limit", stats)?;
335 return self.record(key, &hash, 0);
336 };
337
338 let raw = String::from_utf8_lossy(&bytes);
339 let Some(mut memory) = markdown::note_memory(¬e.path, &raw, project) else {
340 self.retire(current, "note emptied in its vault", stats)?;
341 return self.record(key, &hash, 0);
342 };
343 redact_memory(&mut memory);
344
345 let kept = match current.iter().find(|m| m.content == memory.content) {
346 Some(same) => {
347 stats.unchanged += 1;
348 self.refile(std::slice::from_ref(same), project)?;
349 same.id
350 }
351 None => {
352 memory.machine_id = self.machine_id.clone();
353 memory.session_at = Some(DateTime::<Utc>::from(note.modified));
354 self.persist(&mut memory, ¬e.path, project.map(str::to_string))?;
355 stats.ingested += 1;
356 memory.id
357 }
358 };
359 for older in current.iter().filter(|m| m.id != kept) {
360 self.store.supersede_memory(older.id, kept)?;
361 stats.superseded += 1;
362 }
363 self.record(key, &hash, 1)
364 }
365
366 fn refile(&self, current: &[Memory], project: Option<&str>) -> Result<()> {
367 let Some(project) = project else {
368 return Ok(());
369 };
370 for memory in current {
371 if memory.project_path.as_deref() != Some(project) {
372 self.store.refile_memory(memory.id, project)?;
373 }
374 }
375 Ok(())
376 }
377
378 fn persist(&self, memory: &mut Memory, path: &Path, project: Option<String>) -> Result<()> {
379 if let Some(embedder) = &self.embedder {
380 match embedder.embed_document(&memory.content) {
381 Ok(embedding) => memory.embedding = embedding,
382 Err(e) => tracing::warn!(
383 "vaults: embedding {} failed, storing it without a vector: {e}",
384 memory.source_id
385 ),
386 }
387 }
388 let conversation = Conversation {
389 id: markdown::note_conversation_id(&self.machine_id, path),
390 source: SOURCE.to_string(),
391 machine_id: self.machine_id.clone(),
392 started_at: Utc::now(),
393 project_path: project,
394 };
395 self.store.store_conversation_if_absent(&conversation)?;
396 Store::store_memory(self.store.as_ref(), memory)?;
397 self.store.store_relation(&Relation {
398 from_id: memory.id,
399 to_id: conversation.id,
400 relation_type: RelationType::DerivedFrom,
401 strength: 1.0,
402 context: None,
403 })
404 }
405
406 fn retire(&self, current: &[Memory], reason: &str, stats: &mut ScanStats) -> Result<()> {
407 for memory in current {
408 if self.store.retire_memory(memory.id, reason)? == RetireOutcome::Retired {
409 stats.retired += 1;
410 }
411 }
412 Ok(())
413 }
414
415 fn record(&self, key: &str, hash: &str, memory_count: usize) -> Result<()> {
416 self.store.record_ingested(&IngestRecord {
417 file_path: key,
418 content_hash: hash,
419 session_key: None,
420 legacy_ids: true,
423 memory_count,
424 source: SOURCE,
425 })
426 }
427
428 fn live_notes(&self, keep: impl Fn(&Path) -> bool) -> Result<HashMap<String, Vec<Memory>>> {
433 let mut live: HashMap<String, Vec<Memory>> = HashMap::new();
434 for memory in self.store.memories_by_source(SOURCE)? {
435 if memory.machine_id == self.machine_id
436 && memory.tier == Tier::Hot
437 && memory.superseded_by.is_none()
438 && Path::new(&memory.source_id).is_absolute()
439 && keep(Path::new(&memory.source_id))
440 {
441 live.entry(memory.source_id.clone())
442 .or_default()
443 .push(memory);
444 }
445 }
446 Ok(live)
447 }
448
449 fn project_for(&mut self, vault: &Path) -> Option<String> {
450 if self.projects.is_none() {
451 self.projects = Some(self.store.distinct_project_paths().unwrap_or_else(|e| {
452 tracing::warn!("vaults: listing projects failed: {e}");
453 Vec::new()
454 }));
455 }
456 markdown::vault_project(vault, self.projects.as_deref().unwrap_or_default())
457 }
458}
459
460fn owner<'a>(path: &Path, vaults: &'a [PathBuf]) -> Option<&'a Path> {
463 vaults
464 .iter()
465 .filter(|v| path.starts_with(v))
466 .max_by_key(|v| v.components().count())
467 .map(PathBuf::as_path)
468}
469
470fn sha256_hex(bytes: &[u8]) -> String {
471 Sha256::digest(bytes)
472 .iter()
473 .map(|b| format!("{b:02x}"))
474 .collect()
475}