Skip to main content

second_brain_api/
vaults.rs

1use std::collections::{HashMap, HashSet};
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::{Duration, SystemTime};
5
6use anyhow::Result;
7use chrono::{DateTime, Utc};
8use sha2::{Digest, Sha256};
9
10use second_brain_adapters::markdown::{self, Note, SOURCE};
11use second_brain_core::embedding::{DocumentEmbedder, Embedder};
12use second_brain_core::redact::redact_memory;
13use second_brain_core::schema::{Conversation, IngestRecord, Memory, Relation, RelationType, Tier};
14use second_brain_core::sqlite_store::{RetireOutcome, SqliteStore};
15use second_brain_core::store::{Store, StoreAdmin};
16
17pub const SCAN_INTERVAL: Duration = Duration::from_secs(120);
18/// Larger files are exports or attachments saved as `.md`, not notes, and
19/// are never read.
20const MAX_NOTE_BYTES: u64 = 1024 * 1024;
21/// The ingest-log hash of a deleted note, so a note recreated with the same
22/// bytes reads as changed and comes back.
23const DELETED_HASH: &str = "";
24
25/// Runs `VaultSync` over the configured vaults every `SCAN_INTERVAL`. No
26/// thread starts when no vault is configured.
27pub fn spawn(
28    store: Arc<SqliteStore>,
29    embedder: Option<Arc<Embedder>>,
30    vaults: Vec<PathBuf>,
31    machine_id: String,
32) {
33    let vaults = configured_vaults(vaults);
34    if vaults.is_empty() {
35        return;
36    }
37    let embedder = embedder.map(|e| e as Arc<dyn DocumentEmbedder>);
38    std::thread::spawn(move || {
39        let mut sync = VaultSync::new(store, embedder, machine_id);
40        loop {
41            let stats = sync.scan(&vaults);
42            if stats.ingested + stats.superseded + stats.retired + stats.errors > 0 {
43                tracing::info!(
44                    "vaults: {} notes ingested, {} superseded, {} retired, {} errors",
45                    stats.ingested,
46                    stats.superseded,
47                    stats.retired,
48                    stats.errors
49                );
50            }
51            std::thread::sleep(SCAN_INTERVAL);
52        }
53    });
54}
55
56/// The configured vaults with `~` expanded and the paths normalized. A
57/// relative entry is dropped, because it would resolve against whatever
58/// directory the daemon happened to start in.
59pub fn configured_vaults(vaults: Vec<PathBuf>) -> Vec<PathBuf> {
60    vaults
61        .into_iter()
62        .filter_map(|vault| {
63            let path: PathBuf = expand_home(vault.clone()).components().collect();
64            if path.is_absolute() {
65                Some(path)
66            } else {
67                tracing::warn!(
68                    "vaults: ignoring {}: a vault path must be absolute or start with ~",
69                    vault.display()
70                );
71                None
72            }
73        })
74        .collect()
75}
76
77fn expand_home(path: PathBuf) -> PathBuf {
78    match (path.strip_prefix("~"), std::env::var_os("HOME")) {
79        (Ok(rest), Some(home)) => PathBuf::from(home).join(rest),
80        _ => path,
81    }
82}
83
84#[derive(Debug, Default, PartialEq, Eq)]
85pub struct ScanStats {
86    pub ingested: usize,
87    pub superseded: usize,
88    pub retired: usize,
89    pub unchanged: usize,
90    pub errors: usize,
91}
92
93/// Keeps the `markdown` memories this machine wrote in step with the notes
94/// in its vaults: a new or edited note is stored and supersedes the version
95/// before it, and a deleted or emptied note is retired. A note is re-read
96/// only when its mtime or size moved since the last scan in this process,
97/// and re-stored only when its SHA-256 differs from the ingest log's, so a
98/// restart re-hashes every note once and writes nothing for unchanged ones.
99pub struct VaultSync {
100    store: Arc<SqliteStore>,
101    embedder: Option<Arc<dyn DocumentEmbedder>>,
102    machine_id: String,
103    seen: HashMap<PathBuf, (SystemTime, u64)>,
104    unreadable_notes: HashMap<PathBuf, (SystemTime, u64)>,
105    reconciled: HashSet<PathBuf>,
106    warned: HashSet<PathBuf>,
107    vault_projects: HashMap<PathBuf, Option<String>>,
108    orphans_checked: bool,
109    projects: Option<Vec<(String, u64)>>,
110}
111
112impl VaultSync {
113    pub fn new(
114        store: Arc<SqliteStore>,
115        embedder: Option<Arc<dyn DocumentEmbedder>>,
116        machine_id: String,
117    ) -> Self {
118        Self {
119            store,
120            embedder,
121            machine_id,
122            seen: HashMap::new(),
123            unreadable_notes: HashMap::new(),
124            reconciled: HashSet::new(),
125            warned: HashSet::new(),
126            vault_projects: HashMap::new(),
127            orphans_checked: false,
128            projects: None,
129        }
130    }
131
132    pub fn scan(&mut self, vaults: &[PathBuf]) -> ScanStats {
133        let mut stats = ScanStats::default();
134        self.projects = None;
135        for vault in vaults {
136            self.scan_vault(vault, vaults, &mut stats);
137        }
138        if !self.orphans_checked {
139            self.orphans_checked = true;
140            self.retire_orphans(vaults, &mut stats);
141        }
142        stats
143    }
144
145    /// A vault that cannot be listed, or lists no notes at all, retires
146    /// nothing: an unmounted disk looks exactly like that, and its notes
147    /// must survive until it comes back. Notes under an unreadable folder, or
148    /// under a nested vault, are likewise never retired by this vault.
149    fn scan_vault(&mut self, vault: &Path, vaults: &[PathBuf], stats: &mut ScanStats) {
150        let listing = match markdown::discover_notes(vault) {
151            Ok(listing) => listing,
152            Err(e) => {
153                self.warn_once(vault, &format!("cannot be listed ({e})"));
154                return;
155            }
156        };
157        if listing.notes.is_empty() {
158            self.warn_once(vault, "lists no notes");
159            return;
160        }
161        self.warned.retain(|p| {
162            !p.starts_with(vault)
163                || (p != vault && vaults.iter().any(|v| v == p))
164                || listing.unreadable.contains(p)
165        });
166        for dir in &listing.unreadable {
167            self.warn_once(dir, "is a folder that cannot be listed");
168        }
169
170        let owns = |path: &Path| {
171            owner(path, vaults) == Some(vault)
172                && !listing.unreadable.iter().any(|dir| path.starts_with(dir))
173        };
174        let project = self.project_for(vault);
175        if self.vault_projects.get(vault) != Some(&project) {
176            self.seen.retain(|p, _| owner(p, vaults) != Some(vault));
177            self.vault_projects
178                .insert(vault.to_path_buf(), project.clone());
179        }
180
181        let notes: Vec<&Note> = listing.notes.iter().filter(|n| owns(&n.path)).collect();
182        let present: HashSet<&Path> = notes.iter().map(|n| n.path.as_path()).collect();
183        let changed: Vec<&Note> = notes
184            .iter()
185            .copied()
186            .filter(|n| {
187                let stamp = Some(&(n.modified, n.len));
188                self.seen.get(&n.path) != stamp && self.unreadable_notes.get(&n.path) != stamp
189            })
190            .collect();
191        let vanished: Vec<PathBuf> = self
192            .seen
193            .keys()
194            .chain(self.unreadable_notes.keys())
195            .filter(|p| owns(p) && !present.contains(p.as_path()))
196            .cloned()
197            .collect();
198        if changed.is_empty() && vanished.is_empty() && self.reconciled.contains(vault) {
199            return;
200        }
201
202        let mut live = match self.live_notes(owns) {
203            Ok(live) => live,
204            Err(e) => {
205                tracing::warn!("vaults: reading stored notes for {}: {e}", vault.display());
206                stats.errors += 1;
207                return;
208            }
209        };
210
211        for note in changed {
212            let key = note.path.to_string_lossy().to_string();
213            let bytes = if note.len > MAX_NOTE_BYTES {
214                None
215            } else {
216                match std::fs::read(&note.path) {
217                    Ok(bytes) => Some(bytes),
218                    Err(e) => {
219                        tracing::warn!("vaults: cannot read {key} ({e}); retrying once it changes");
220                        self.unreadable_notes
221                            .insert(note.path.clone(), (note.modified, note.len));
222                        stats.errors += 1;
223                        continue;
224                    }
225                }
226            };
227            let current = live.remove(&key).unwrap_or_default();
228            match self.reconcile(project.as_deref(), note, &key, bytes, &current, stats) {
229                Ok(()) => {
230                    self.unreadable_notes.remove(&note.path);
231                    self.seen
232                        .insert(note.path.clone(), (note.modified, note.len));
233                }
234                Err(e) => {
235                    tracing::warn!("vaults: ingesting {key}: {e:#}");
236                    stats.errors += 1;
237                }
238            }
239        }
240
241        let mut deleted: Vec<String> = live
242            .keys()
243            .filter(|key| !present.contains(Path::new(key.as_str())))
244            .cloned()
245            .collect();
246        deleted.extend(vanished.iter().map(|p| p.to_string_lossy().to_string()));
247        deleted.sort();
248        deleted.dedup();
249        for key in deleted {
250            let current = live.remove(&key).unwrap_or_default();
251            self.retire_deleted(&key, &current, "note deleted from its vault", stats);
252        }
253        self.reconciled.insert(vault.to_path_buf());
254    }
255
256    /// Retires this machine's notes that sit under no configured vault and
257    /// whose file is gone, which is what a moved or dropped vault leaves
258    /// behind. Runs once per process, since the vault list only changes with
259    /// a restart.
260    fn retire_orphans(&mut self, vaults: &[PathBuf], stats: &mut ScanStats) {
261        let orphans = self.live_notes(|path| {
262            owner(path, vaults).is_none() && matches!(path.try_exists(), Ok(false))
263        });
264        match orphans {
265            Ok(orphans) => {
266                for (key, current) in orphans {
267                    self.retire_deleted(&key, &current, "note's vault moved or was dropped", stats);
268                }
269            }
270            Err(e) => {
271                tracing::warn!("vaults: reading stored notes outside the vaults: {e}");
272                stats.errors += 1;
273            }
274        }
275    }
276
277    fn retire_deleted(
278        &mut self,
279        key: &str,
280        current: &[Memory],
281        reason: &str,
282        stats: &mut ScanStats,
283    ) {
284        let result = self
285            .retire(current, reason, stats)
286            .and_then(|()| self.record(key, DELETED_HASH, 0));
287        match result {
288            Ok(()) => {
289                self.seen.remove(Path::new(key));
290                self.unreadable_notes.remove(Path::new(key));
291            }
292            Err(e) => {
293                tracing::warn!("vaults: retiring {key}: {e:#}");
294                stats.errors += 1;
295            }
296        }
297    }
298
299    fn warn_once(&mut self, path: &Path, problem: &str) {
300        if self.warned.insert(path.to_path_buf()) {
301            tracing::warn!(
302                "vaults: {} {problem}; the notes stored from it stay as they are",
303                path.display()
304            );
305        }
306    }
307
308    /// `project` is the vault's project as of this scan; a stored version
309    /// filed elsewhere is refiled, so a vault that maps to a project only
310    /// later still gets its notes attributed.
311    fn reconcile(
312        &mut self,
313        project: Option<&str>,
314        note: &Note,
315        key: &str,
316        bytes: Option<Vec<u8>>,
317        current: &[Memory],
318        stats: &mut ScanStats,
319    ) -> Result<()> {
320        let logged = self
321            .store
322            .ingest_log_entry(key)?
323            .and_then(|e| e.content_hash);
324        let hash = match &bytes {
325            Some(bytes) => sha256_hex(bytes),
326            None => format!("oversize:{}", note.len),
327        };
328        if logged.as_deref() == Some(hash.as_str()) {
329            stats.unchanged += 1;
330            return self.refile(current, project);
331        }
332        let Some(bytes) = bytes else {
333            tracing::info!("vaults: {key} is over {MAX_NOTE_BYTES} bytes and is not kept");
334            self.retire(current, "note grew past the size limit", stats)?;
335            return self.record(key, &hash, 0);
336        };
337
338        let raw = String::from_utf8_lossy(&bytes);
339        let Some(mut memory) = markdown::note_memory(&note.path, &raw, project) else {
340            self.retire(current, "note emptied in its vault", stats)?;
341            return self.record(key, &hash, 0);
342        };
343        redact_memory(&mut memory);
344
345        let kept = match current.iter().find(|m| m.content == memory.content) {
346            Some(same) => {
347                stats.unchanged += 1;
348                self.refile(std::slice::from_ref(same), project)?;
349                same.id
350            }
351            None => {
352                memory.machine_id = self.machine_id.clone();
353                memory.session_at = Some(DateTime::<Utc>::from(note.modified));
354                self.persist(&mut memory, &note.path, project.map(str::to_string))?;
355                stats.ingested += 1;
356                memory.id
357            }
358        };
359        for older in current.iter().filter(|m| m.id != kept) {
360            self.store.supersede_memory(older.id, kept)?;
361            stats.superseded += 1;
362        }
363        self.record(key, &hash, 1)
364    }
365
366    fn refile(&self, current: &[Memory], project: Option<&str>) -> Result<()> {
367        let Some(project) = project else {
368            return Ok(());
369        };
370        for memory in current {
371            if memory.project_path.as_deref() != Some(project) {
372                self.store.refile_memory(memory.id, project)?;
373            }
374        }
375        Ok(())
376    }
377
378    fn persist(&self, memory: &mut Memory, path: &Path, project: Option<String>) -> Result<()> {
379        if let Some(embedder) = &self.embedder {
380            match embedder.embed_document(&memory.content) {
381                Ok(embedding) => memory.embedding = embedding,
382                Err(e) => tracing::warn!(
383                    "vaults: embedding {} failed, storing it without a vector: {e}",
384                    memory.source_id
385                ),
386            }
387        }
388        let conversation = Conversation {
389            id: markdown::note_conversation_id(&self.machine_id, path),
390            source: SOURCE.to_string(),
391            machine_id: self.machine_id.clone(),
392            started_at: Utc::now(),
393            project_path: project,
394        };
395        self.store.store_conversation_if_absent(&conversation)?;
396        Store::store_memory(self.store.as_ref(), memory)?;
397        self.store.store_relation(&Relation {
398            from_id: memory.id,
399            to_id: conversation.id,
400            relation_type: RelationType::DerivedFrom,
401            strength: 1.0,
402            context: None,
403        })
404    }
405
406    fn retire(&self, current: &[Memory], reason: &str, stats: &mut ScanStats) -> Result<()> {
407        for memory in current {
408            if self.store.retire_memory(memory.id, reason)? == RetireOutcome::Retired {
409                stats.retired += 1;
410            }
411        }
412        Ok(())
413    }
414
415    fn record(&self, key: &str, hash: &str, memory_count: usize) -> Result<()> {
416        self.store.record_ingested(&IngestRecord {
417            file_path: key,
418            content_hash: hash,
419            session_key: None,
420            // Note memories have random ids, so a reingest must not forget
421            // these rows and bring back notes the user forgot.
422            legacy_ids: true,
423            memory_count,
424            source: SOURCE,
425        })
426    }
427
428    /// This machine's live `markdown` memories whose note path passes
429    /// `keep`, by note path. Another machine's are left alone even at the
430    /// same path, because its vault is not this one, and so is any row
431    /// without the absolute note path vault sync always writes.
432    fn live_notes(&self, keep: impl Fn(&Path) -> bool) -> Result<HashMap<String, Vec<Memory>>> {
433        let mut live: HashMap<String, Vec<Memory>> = HashMap::new();
434        for memory in self.store.memories_by_source(SOURCE)? {
435            if memory.machine_id == self.machine_id
436                && memory.tier == Tier::Hot
437                && memory.superseded_by.is_none()
438                && Path::new(&memory.source_id).is_absolute()
439                && keep(Path::new(&memory.source_id))
440            {
441                live.entry(memory.source_id.clone())
442                    .or_default()
443                    .push(memory);
444            }
445        }
446        Ok(live)
447    }
448
449    fn project_for(&mut self, vault: &Path) -> Option<String> {
450        if self.projects.is_none() {
451            self.projects = Some(self.store.distinct_project_paths().unwrap_or_else(|e| {
452                tracing::warn!("vaults: listing projects failed: {e}");
453                Vec::new()
454            }));
455        }
456        markdown::vault_project(vault, self.projects.as_deref().unwrap_or_default())
457    }
458}
459
460/// The most specific configured vault holding `path`, so nested vaults
461/// never act on each other's notes.
462fn owner<'a>(path: &Path, vaults: &'a [PathBuf]) -> Option<&'a Path> {
463    vaults
464        .iter()
465        .filter(|v| path.starts_with(v))
466        .max_by_key(|v| v.components().count())
467        .map(PathBuf::as_path)
468}
469
470fn sha256_hex(bytes: &[u8]) -> String {
471    Sha256::digest(bytes)
472        .iter()
473        .map(|b| format!("{b:02x}"))
474        .collect()
475}