1use std::collections::{HashMap, HashSet};
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::{Duration, SystemTime};
5
6use anyhow::Result;
7use chrono::{DateTime, Utc};
8use sha2::{Digest, Sha256};
9
10use crate::bootstrap::Bootstrap;
11use second_brain_adapters::markdown::{self, Note, SOURCE};
12use second_brain_core::embedding::{DocumentEmbedder, Embedder};
13use second_brain_core::redact::redact_memory;
14use second_brain_core::schema::{Conversation, IngestRecord, Memory, Relation, RelationType, Tier};
15use second_brain_core::sqlite_store::{RetireOutcome, SqliteStore};
16use second_brain_core::store::{Store, StoreAdmin};
17
18use crate::clock_gate::ClockGate;
19
20pub const SCAN_INTERVAL: Duration = Duration::from_secs(120);
21const CLOCK_POLL: Duration = Duration::from_secs(1);
22const MAX_NOTE_BYTES: u64 = 1024 * 1024;
25const DELETED_HASH: &str = "";
28
29pub fn spawn(
33 store: Arc<SqliteStore>,
34 embedder: Option<Arc<Embedder>>,
35 vaults: Vec<PathBuf>,
36 machine_id: String,
37 clock: Arc<ClockGate>,
38 bootstrap: Arc<Bootstrap>,
39) {
40 let vaults = configured_vaults(vaults);
41 if vaults.is_empty() {
42 return;
43 }
44 let embedder = embedder.map(|e| e as Arc<dyn DocumentEmbedder>);
45 std::thread::spawn(move || {
46 let mut sync = VaultSync::new(store, embedder, machine_id);
47 loop {
48 while !clock.is_open() || bootstrap.is_pending() {
49 std::thread::sleep(CLOCK_POLL);
50 }
51 let stats = sync.scan(&vaults);
52 if stats.ingested + stats.superseded + stats.retired + stats.errors > 0 {
53 tracing::info!(
54 "vaults: {} notes ingested, {} superseded, {} retired, {} errors",
55 stats.ingested,
56 stats.superseded,
57 stats.retired,
58 stats.errors
59 );
60 }
61 std::thread::sleep(SCAN_INTERVAL);
62 }
63 });
64}
65
66pub fn configured_vaults(vaults: Vec<PathBuf>) -> Vec<PathBuf> {
70 vaults
71 .into_iter()
72 .filter_map(|vault| {
73 let path: PathBuf = expand_home(vault.clone()).components().collect();
74 if path.is_absolute() {
75 Some(path)
76 } else {
77 tracing::warn!(
78 "vaults: ignoring {}: a vault path must be absolute or start with ~",
79 vault.display()
80 );
81 None
82 }
83 })
84 .collect()
85}
86
87fn expand_home(path: PathBuf) -> PathBuf {
88 match (path.strip_prefix("~"), std::env::var_os("HOME")) {
89 (Ok(rest), Some(home)) => PathBuf::from(home).join(rest),
90 _ => path,
91 }
92}
93
94#[derive(Debug, Default, PartialEq, Eq)]
95pub struct ScanStats {
96 pub ingested: usize,
97 pub superseded: usize,
98 pub retired: usize,
99 pub unchanged: usize,
100 pub errors: usize,
101}
102
103pub struct VaultSync {
110 store: Arc<SqliteStore>,
111 embedder: Option<Arc<dyn DocumentEmbedder>>,
112 machine_id: String,
113 seen: HashMap<PathBuf, (SystemTime, u64)>,
114 unreadable_notes: HashMap<PathBuf, (SystemTime, u64)>,
115 reconciled: HashSet<PathBuf>,
116 warned: HashSet<PathBuf>,
117 vault_projects: HashMap<PathBuf, Option<String>>,
118 orphans_checked: bool,
119 projects: Option<Vec<(String, u64)>>,
120}
121
122impl VaultSync {
123 pub fn new(
124 store: Arc<SqliteStore>,
125 embedder: Option<Arc<dyn DocumentEmbedder>>,
126 machine_id: String,
127 ) -> Self {
128 Self {
129 store,
130 embedder,
131 machine_id,
132 seen: HashMap::new(),
133 unreadable_notes: HashMap::new(),
134 reconciled: HashSet::new(),
135 warned: HashSet::new(),
136 vault_projects: HashMap::new(),
137 orphans_checked: false,
138 projects: None,
139 }
140 }
141
142 pub fn scan(&mut self, vaults: &[PathBuf]) -> ScanStats {
143 let mut stats = ScanStats::default();
144 self.projects = None;
145 for vault in vaults {
146 self.scan_vault(vault, vaults, &mut stats);
147 }
148 if !self.orphans_checked {
149 let errors = stats.errors;
150 self.retire_orphans(vaults, &mut stats);
151 self.orphans_checked = stats.errors == errors;
152 }
153 stats
154 }
155
156 fn scan_vault(&mut self, vault: &Path, vaults: &[PathBuf], stats: &mut ScanStats) {
161 let errors = stats.errors;
162 let listing = match markdown::discover_notes(vault) {
163 Ok(listing) => listing,
164 Err(e) => {
165 self.warn_once(vault, &format!("cannot be listed ({e})"));
166 return;
167 }
168 };
169 if listing.notes.is_empty() {
170 self.warn_once(vault, "lists no notes");
171 return;
172 }
173 self.warned.retain(|p| {
174 !p.starts_with(vault)
175 || (p != vault && vaults.iter().any(|v| v == p))
176 || listing.unreadable.contains(p)
177 });
178 for dir in &listing.unreadable {
179 self.warn_once(dir, "is a folder that cannot be listed");
180 }
181
182 let owns = |path: &Path| {
183 owner(path, vaults) == Some(vault)
184 && !listing.unreadable.iter().any(|dir| path.starts_with(dir))
185 };
186 let project = self.project_for(vault);
187 if self.vault_projects.get(vault) != Some(&project) {
188 self.seen.retain(|p, _| owner(p, vaults) != Some(vault));
189 self.vault_projects
190 .insert(vault.to_path_buf(), project.clone());
191 }
192
193 let notes: Vec<&Note> = listing.notes.iter().filter(|n| owns(&n.path)).collect();
194 let present: HashSet<&Path> = notes.iter().map(|n| n.path.as_path()).collect();
195 let changed: Vec<&Note> = notes
196 .iter()
197 .copied()
198 .filter(|n| {
199 let stamp = Some(&(n.modified, n.len));
200 self.seen.get(&n.path) != stamp && self.unreadable_notes.get(&n.path) != stamp
201 })
202 .collect();
203 let vanished: Vec<PathBuf> = self
204 .seen
205 .keys()
206 .chain(self.unreadable_notes.keys())
207 .filter(|p| owns(p) && !present.contains(p.as_path()))
208 .cloned()
209 .collect();
210 if changed.is_empty() && vanished.is_empty() && self.reconciled.contains(vault) {
211 return;
212 }
213
214 let mut live = match self.live_notes(owns) {
215 Ok(live) => live,
216 Err(e) => {
217 tracing::warn!("vaults: reading stored notes for {}: {e}", vault.display());
218 stats.errors += 1;
219 return;
220 }
221 };
222
223 for note in changed {
224 let key = note.path.to_string_lossy().to_string();
225 let bytes = if note.len > MAX_NOTE_BYTES {
226 None
227 } else {
228 match std::fs::read(¬e.path) {
229 Ok(bytes) => Some(bytes),
230 Err(e) => {
231 tracing::warn!("vaults: cannot read {key} ({e}); retrying once it changes");
232 self.unreadable_notes
233 .insert(note.path.clone(), (note.modified, note.len));
234 stats.errors += 1;
235 continue;
236 }
237 }
238 };
239 let current = live.remove(&key).unwrap_or_default();
240 match self.reconcile(project.as_deref(), note, &key, bytes, ¤t, stats) {
241 Ok(()) => {
242 self.unreadable_notes.remove(¬e.path);
243 self.seen
244 .insert(note.path.clone(), (note.modified, note.len));
245 }
246 Err(e) => {
247 tracing::warn!("vaults: ingesting {key}: {e:#}");
248 stats.errors += 1;
249 }
250 }
251 }
252
253 let mut deleted: Vec<String> = live
254 .keys()
255 .filter(|key| !present.contains(Path::new(key.as_str())))
256 .cloned()
257 .collect();
258 deleted.extend(vanished.iter().map(|p| p.to_string_lossy().to_string()));
259 deleted.sort();
260 deleted.dedup();
261 for key in deleted {
262 let current = live.remove(&key).unwrap_or_default();
263 self.retire_deleted(&key, ¤t, "note deleted from its vault", stats);
264 }
265 if stats.errors == errors {
266 self.reconciled.insert(vault.to_path_buf());
267 }
268 }
269
270 fn retire_orphans(&mut self, vaults: &[PathBuf], stats: &mut ScanStats) {
275 let orphans = self.live_notes(|path| {
276 owner(path, vaults).is_none() && matches!(path.try_exists(), Ok(false))
277 });
278 match orphans {
279 Ok(orphans) => {
280 for (key, current) in orphans {
281 self.retire_deleted(&key, ¤t, "note's vault moved or was dropped", stats);
282 }
283 }
284 Err(e) => {
285 tracing::warn!("vaults: reading stored notes outside the vaults: {e}");
286 stats.errors += 1;
287 }
288 }
289 }
290
291 fn retire_deleted(
292 &mut self,
293 key: &str,
294 current: &[Memory],
295 reason: &str,
296 stats: &mut ScanStats,
297 ) {
298 let result = self
299 .retire(current, reason, stats)
300 .and_then(|()| self.record(key, DELETED_HASH, 0));
301 match result {
302 Ok(()) => {
303 self.seen.remove(Path::new(key));
304 self.unreadable_notes.remove(Path::new(key));
305 }
306 Err(e) => {
307 tracing::warn!("vaults: retiring {key}: {e:#}");
308 stats.errors += 1;
309 }
310 }
311 }
312
313 fn warn_once(&mut self, path: &Path, problem: &str) {
314 if self.warned.insert(path.to_path_buf()) {
315 tracing::warn!(
316 "vaults: {} {problem}; the notes stored from it stay as they are",
317 path.display()
318 );
319 }
320 }
321
322 fn reconcile(
326 &mut self,
327 project: Option<&str>,
328 note: &Note,
329 key: &str,
330 bytes: Option<Vec<u8>>,
331 current: &[Memory],
332 stats: &mut ScanStats,
333 ) -> Result<()> {
334 let logged = self
335 .store
336 .ingest_log_entry(key)?
337 .and_then(|e| e.content_hash);
338 let hash = match &bytes {
339 Some(bytes) => sha256_hex(bytes),
340 None => format!("oversize:{}", note.len),
341 };
342 if logged.as_deref() == Some(hash.as_str()) {
343 stats.unchanged += 1;
344 return self.refile(current, project);
345 }
346 let Some(bytes) = bytes else {
347 tracing::info!("vaults: {key} is over {MAX_NOTE_BYTES} bytes and is not kept");
348 self.retire(current, "note grew past the size limit", stats)?;
349 return self.record(key, &hash, 0);
350 };
351
352 let raw = String::from_utf8_lossy(&bytes);
353 let Some(mut memory) = markdown::note_memory(¬e.path, &raw, project) else {
354 self.retire(current, "note emptied in its vault", stats)?;
355 return self.record(key, &hash, 0);
356 };
357 redact_memory(&mut memory);
358
359 let kept = match current.iter().find(|m| m.content == memory.content) {
360 Some(same) => {
361 stats.unchanged += 1;
362 self.refile(std::slice::from_ref(same), project)?;
363 same.id
364 }
365 None => {
366 memory.machine_id = self.machine_id.clone();
367 memory.session_at = Some(DateTime::<Utc>::from(note.modified));
368 self.persist(&mut memory, ¬e.path, project.map(str::to_string))?;
369 stats.ingested += 1;
370 memory.id
371 }
372 };
373 for older in current.iter().filter(|m| m.id != kept) {
374 self.store.supersede_memory(older.id, kept)?;
375 stats.superseded += 1;
376 }
377 self.record(key, &hash, 1)
378 }
379
380 fn refile(&self, current: &[Memory], project: Option<&str>) -> Result<()> {
381 let Some(project) = project else {
382 return Ok(());
383 };
384 for memory in current {
385 if memory.project_path.as_deref() != Some(project) {
386 self.store.refile_memory(memory.id, project)?;
387 }
388 }
389 Ok(())
390 }
391
392 fn persist(&self, memory: &mut Memory, path: &Path, project: Option<String>) -> Result<()> {
393 if let Some(embedder) = &self.embedder {
394 match embedder.embed_document(&memory.content) {
395 Ok(embedding) => memory.embedding = embedding,
396 Err(e) => tracing::warn!(
397 "vaults: embedding {} failed, storing it without a vector: {e}",
398 memory.source_id
399 ),
400 }
401 }
402 let conversation = Conversation {
403 id: markdown::note_conversation_id(&self.machine_id, path),
404 source: SOURCE.to_string(),
405 machine_id: self.machine_id.clone(),
406 started_at: Utc::now(),
407 project_path: project,
408 };
409 self.store.store_conversation_if_absent(&conversation)?;
410 Store::store_memory(self.store.as_ref(), memory)?;
411 self.store.store_relation(&Relation {
412 from_id: memory.id,
413 to_id: conversation.id,
414 relation_type: RelationType::DerivedFrom,
415 strength: 1.0,
416 context: None,
417 })
418 }
419
420 fn retire(&self, current: &[Memory], reason: &str, stats: &mut ScanStats) -> Result<()> {
421 for memory in current {
422 if self.store.retire_memory(memory.id, reason)? == RetireOutcome::Retired {
423 stats.retired += 1;
424 }
425 }
426 Ok(())
427 }
428
429 fn record(&self, key: &str, hash: &str, memory_count: usize) -> Result<()> {
430 self.store.record_ingested(&IngestRecord {
431 file_path: key,
432 content_hash: hash,
433 session_key: None,
434 legacy_ids: true,
437 memory_count,
438 source: SOURCE,
439 })
440 }
441
442 fn live_notes(&self, keep: impl Fn(&Path) -> bool) -> Result<HashMap<String, Vec<Memory>>> {
447 let mut live: HashMap<String, Vec<Memory>> = HashMap::new();
448 for memory in self.store.memories_by_source(SOURCE)? {
449 if memory.machine_id == self.machine_id
450 && memory.tier == Tier::Hot
451 && memory.superseded_by.is_none()
452 && Path::new(&memory.source_id).is_absolute()
453 && keep(Path::new(&memory.source_id))
454 {
455 live.entry(memory.source_id.clone())
456 .or_default()
457 .push(memory);
458 }
459 }
460 Ok(live)
461 }
462
463 fn project_for(&mut self, vault: &Path) -> Option<String> {
464 if self.projects.is_none() {
465 self.projects = Some(self.store.distinct_project_paths().unwrap_or_else(|e| {
466 tracing::warn!("vaults: listing projects failed: {e}");
467 Vec::new()
468 }));
469 }
470 markdown::vault_project(vault, self.projects.as_deref().unwrap_or_default())
471 }
472}
473
474fn owner<'a>(path: &Path, vaults: &'a [PathBuf]) -> Option<&'a Path> {
477 vaults
478 .iter()
479 .filter(|v| path.starts_with(v))
480 .max_by_key(|v| v.components().count())
481 .map(PathBuf::as_path)
482}
483
484fn sha256_hex(bytes: &[u8]) -> String {
485 Sha256::digest(bytes)
486 .iter()
487 .map(|b| format!("{b:02x}"))
488 .collect()
489}