Skip to main content

second_brain_api/
vaults.rs

1use std::collections::{HashMap, HashSet};
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::time::{Duration, SystemTime};
5
6use anyhow::Result;
7use chrono::{DateTime, Utc};
8use sha2::{Digest, Sha256};
9
10use crate::bootstrap::Bootstrap;
11use second_brain_adapters::markdown::{self, Note, SOURCE};
12use second_brain_core::embedding::{DocumentEmbedder, Embedder};
13use second_brain_core::redact::redact_memory;
14use second_brain_core::schema::{Conversation, IngestRecord, Memory, Relation, RelationType, Tier};
15use second_brain_core::sqlite_store::{RetireOutcome, SqliteStore};
16use second_brain_core::store::{Store, StoreAdmin};
17
18use crate::clock_gate::ClockGate;
19
20pub const SCAN_INTERVAL: Duration = Duration::from_secs(120);
21const CLOCK_POLL: Duration = Duration::from_secs(1);
22/// Larger files are exports or attachments saved as `.md`, not notes, and
23/// are never read.
24const MAX_NOTE_BYTES: u64 = 1024 * 1024;
25/// The ingest-log hash of a deleted note, so a note recreated with the same
26/// bytes reads as changed and comes back.
27const DELETED_HASH: &str = "";
28
29/// Runs `VaultSync` over the configured vaults every `SCAN_INTERVAL`, once
30/// the clock gate lets writes through. No thread starts when no vault is
31/// configured.
32pub fn spawn(
33    store: Arc<SqliteStore>,
34    embedder: Option<Arc<Embedder>>,
35    vaults: Vec<PathBuf>,
36    machine_id: String,
37    clock: Arc<ClockGate>,
38    bootstrap: Arc<Bootstrap>,
39) {
40    let vaults = configured_vaults(vaults);
41    if vaults.is_empty() {
42        return;
43    }
44    let embedder = embedder.map(|e| e as Arc<dyn DocumentEmbedder>);
45    std::thread::spawn(move || {
46        let mut sync = VaultSync::new(store, embedder, machine_id);
47        loop {
48            while !clock.is_open() || bootstrap.is_pending() {
49                std::thread::sleep(CLOCK_POLL);
50            }
51            let stats = sync.scan(&vaults);
52            if stats.ingested + stats.superseded + stats.retired + stats.errors > 0 {
53                tracing::info!(
54                    "vaults: {} notes ingested, {} superseded, {} retired, {} errors",
55                    stats.ingested,
56                    stats.superseded,
57                    stats.retired,
58                    stats.errors
59                );
60            }
61            std::thread::sleep(SCAN_INTERVAL);
62        }
63    });
64}
65
66/// The configured vaults with `~` expanded and the paths normalized. A
67/// relative entry is dropped, because it would resolve against whatever
68/// directory the daemon happened to start in.
69pub fn configured_vaults(vaults: Vec<PathBuf>) -> Vec<PathBuf> {
70    vaults
71        .into_iter()
72        .filter_map(|vault| {
73            let path: PathBuf = expand_home(vault.clone()).components().collect();
74            if path.is_absolute() {
75                Some(path)
76            } else {
77                tracing::warn!(
78                    "vaults: ignoring {}: a vault path must be absolute or start with ~",
79                    vault.display()
80                );
81                None
82            }
83        })
84        .collect()
85}
86
87fn expand_home(path: PathBuf) -> PathBuf {
88    match (path.strip_prefix("~"), std::env::var_os("HOME")) {
89        (Ok(rest), Some(home)) => PathBuf::from(home).join(rest),
90        _ => path,
91    }
92}
93
94#[derive(Debug, Default, PartialEq, Eq)]
95pub struct ScanStats {
96    pub ingested: usize,
97    pub superseded: usize,
98    pub retired: usize,
99    pub unchanged: usize,
100    pub errors: usize,
101}
102
103/// Keeps the `markdown` memories this machine wrote in step with the notes
104/// in its vaults: a new or edited note is stored and supersedes the version
105/// before it, and a deleted or emptied note is retired. A note is re-read
106/// only when its mtime or size moved since the last scan in this process,
107/// and re-stored only when its SHA-256 differs from the ingest log's, so a
108/// restart re-hashes every note once and writes nothing for unchanged ones.
109pub struct VaultSync {
110    store: Arc<SqliteStore>,
111    embedder: Option<Arc<dyn DocumentEmbedder>>,
112    machine_id: String,
113    seen: HashMap<PathBuf, (SystemTime, u64)>,
114    unreadable_notes: HashMap<PathBuf, (SystemTime, u64)>,
115    reconciled: HashSet<PathBuf>,
116    warned: HashSet<PathBuf>,
117    vault_projects: HashMap<PathBuf, Option<String>>,
118    orphans_checked: bool,
119    projects: Option<Vec<(String, u64)>>,
120}
121
122impl VaultSync {
123    pub fn new(
124        store: Arc<SqliteStore>,
125        embedder: Option<Arc<dyn DocumentEmbedder>>,
126        machine_id: String,
127    ) -> Self {
128        Self {
129            store,
130            embedder,
131            machine_id,
132            seen: HashMap::new(),
133            unreadable_notes: HashMap::new(),
134            reconciled: HashSet::new(),
135            warned: HashSet::new(),
136            vault_projects: HashMap::new(),
137            orphans_checked: false,
138            projects: None,
139        }
140    }
141
142    pub fn scan(&mut self, vaults: &[PathBuf]) -> ScanStats {
143        let mut stats = ScanStats::default();
144        self.projects = None;
145        for vault in vaults {
146            self.scan_vault(vault, vaults, &mut stats);
147        }
148        if !self.orphans_checked {
149            let errors = stats.errors;
150            self.retire_orphans(vaults, &mut stats);
151            self.orphans_checked = stats.errors == errors;
152        }
153        stats
154    }
155
156    /// A vault that cannot be listed, or lists no notes at all, retires
157    /// nothing: an unmounted disk looks exactly like that, and its notes
158    /// must survive until it comes back. Notes under an unreadable folder, or
159    /// under a nested vault, are likewise never retired by this vault.
160    fn scan_vault(&mut self, vault: &Path, vaults: &[PathBuf], stats: &mut ScanStats) {
161        let errors = stats.errors;
162        let listing = match markdown::discover_notes(vault) {
163            Ok(listing) => listing,
164            Err(e) => {
165                self.warn_once(vault, &format!("cannot be listed ({e})"));
166                return;
167            }
168        };
169        if listing.notes.is_empty() {
170            self.warn_once(vault, "lists no notes");
171            return;
172        }
173        self.warned.retain(|p| {
174            !p.starts_with(vault)
175                || (p != vault && vaults.iter().any(|v| v == p))
176                || listing.unreadable.contains(p)
177        });
178        for dir in &listing.unreadable {
179            self.warn_once(dir, "is a folder that cannot be listed");
180        }
181
182        let owns = |path: &Path| {
183            owner(path, vaults) == Some(vault)
184                && !listing.unreadable.iter().any(|dir| path.starts_with(dir))
185        };
186        let project = self.project_for(vault);
187        if self.vault_projects.get(vault) != Some(&project) {
188            self.seen.retain(|p, _| owner(p, vaults) != Some(vault));
189            self.vault_projects
190                .insert(vault.to_path_buf(), project.clone());
191        }
192
193        let notes: Vec<&Note> = listing.notes.iter().filter(|n| owns(&n.path)).collect();
194        let present: HashSet<&Path> = notes.iter().map(|n| n.path.as_path()).collect();
195        let changed: Vec<&Note> = notes
196            .iter()
197            .copied()
198            .filter(|n| {
199                let stamp = Some(&(n.modified, n.len));
200                self.seen.get(&n.path) != stamp && self.unreadable_notes.get(&n.path) != stamp
201            })
202            .collect();
203        let vanished: Vec<PathBuf> = self
204            .seen
205            .keys()
206            .chain(self.unreadable_notes.keys())
207            .filter(|p| owns(p) && !present.contains(p.as_path()))
208            .cloned()
209            .collect();
210        if changed.is_empty() && vanished.is_empty() && self.reconciled.contains(vault) {
211            return;
212        }
213
214        let mut live = match self.live_notes(owns) {
215            Ok(live) => live,
216            Err(e) => {
217                tracing::warn!("vaults: reading stored notes for {}: {e}", vault.display());
218                stats.errors += 1;
219                return;
220            }
221        };
222
223        for note in changed {
224            let key = note.path.to_string_lossy().to_string();
225            let bytes = if note.len > MAX_NOTE_BYTES {
226                None
227            } else {
228                match std::fs::read(&note.path) {
229                    Ok(bytes) => Some(bytes),
230                    Err(e) => {
231                        tracing::warn!("vaults: cannot read {key} ({e}); retrying once it changes");
232                        self.unreadable_notes
233                            .insert(note.path.clone(), (note.modified, note.len));
234                        stats.errors += 1;
235                        continue;
236                    }
237                }
238            };
239            let current = live.remove(&key).unwrap_or_default();
240            match self.reconcile(project.as_deref(), note, &key, bytes, &current, stats) {
241                Ok(()) => {
242                    self.unreadable_notes.remove(&note.path);
243                    self.seen
244                        .insert(note.path.clone(), (note.modified, note.len));
245                }
246                Err(e) => {
247                    tracing::warn!("vaults: ingesting {key}: {e:#}");
248                    stats.errors += 1;
249                }
250            }
251        }
252
253        let mut deleted: Vec<String> = live
254            .keys()
255            .filter(|key| !present.contains(Path::new(key.as_str())))
256            .cloned()
257            .collect();
258        deleted.extend(vanished.iter().map(|p| p.to_string_lossy().to_string()));
259        deleted.sort();
260        deleted.dedup();
261        for key in deleted {
262            let current = live.remove(&key).unwrap_or_default();
263            self.retire_deleted(&key, &current, "note deleted from its vault", stats);
264        }
265        if stats.errors == errors {
266            self.reconciled.insert(vault.to_path_buf());
267        }
268    }
269
270    /// Retires this machine's notes that sit under no configured vault and
271    /// whose file is gone, which is what a moved or dropped vault leaves
272    /// behind. Runs once per process, since the vault list only changes with
273    /// a restart.
274    fn retire_orphans(&mut self, vaults: &[PathBuf], stats: &mut ScanStats) {
275        let orphans = self.live_notes(|path| {
276            owner(path, vaults).is_none() && matches!(path.try_exists(), Ok(false))
277        });
278        match orphans {
279            Ok(orphans) => {
280                for (key, current) in orphans {
281                    self.retire_deleted(&key, &current, "note's vault moved or was dropped", stats);
282                }
283            }
284            Err(e) => {
285                tracing::warn!("vaults: reading stored notes outside the vaults: {e}");
286                stats.errors += 1;
287            }
288        }
289    }
290
291    fn retire_deleted(
292        &mut self,
293        key: &str,
294        current: &[Memory],
295        reason: &str,
296        stats: &mut ScanStats,
297    ) {
298        let result = self
299            .retire(current, reason, stats)
300            .and_then(|()| self.record(key, DELETED_HASH, 0));
301        match result {
302            Ok(()) => {
303                self.seen.remove(Path::new(key));
304                self.unreadable_notes.remove(Path::new(key));
305            }
306            Err(e) => {
307                tracing::warn!("vaults: retiring {key}: {e:#}");
308                stats.errors += 1;
309            }
310        }
311    }
312
313    fn warn_once(&mut self, path: &Path, problem: &str) {
314        if self.warned.insert(path.to_path_buf()) {
315            tracing::warn!(
316                "vaults: {} {problem}; the notes stored from it stay as they are",
317                path.display()
318            );
319        }
320    }
321
322    /// `project` is the vault's project as of this scan; a stored version
323    /// filed elsewhere is refiled, so a vault that maps to a project only
324    /// later still gets its notes attributed.
325    fn reconcile(
326        &mut self,
327        project: Option<&str>,
328        note: &Note,
329        key: &str,
330        bytes: Option<Vec<u8>>,
331        current: &[Memory],
332        stats: &mut ScanStats,
333    ) -> Result<()> {
334        let logged = self
335            .store
336            .ingest_log_entry(key)?
337            .and_then(|e| e.content_hash);
338        let hash = match &bytes {
339            Some(bytes) => sha256_hex(bytes),
340            None => format!("oversize:{}", note.len),
341        };
342        if logged.as_deref() == Some(hash.as_str()) {
343            stats.unchanged += 1;
344            return self.refile(current, project);
345        }
346        let Some(bytes) = bytes else {
347            tracing::info!("vaults: {key} is over {MAX_NOTE_BYTES} bytes and is not kept");
348            self.retire(current, "note grew past the size limit", stats)?;
349            return self.record(key, &hash, 0);
350        };
351
352        let raw = String::from_utf8_lossy(&bytes);
353        let Some(mut memory) = markdown::note_memory(&note.path, &raw, project) else {
354            self.retire(current, "note emptied in its vault", stats)?;
355            return self.record(key, &hash, 0);
356        };
357        redact_memory(&mut memory);
358
359        let kept = match current.iter().find(|m| m.content == memory.content) {
360            Some(same) => {
361                stats.unchanged += 1;
362                self.refile(std::slice::from_ref(same), project)?;
363                same.id
364            }
365            None => {
366                memory.machine_id = self.machine_id.clone();
367                memory.session_at = Some(DateTime::<Utc>::from(note.modified));
368                self.persist(&mut memory, &note.path, project.map(str::to_string))?;
369                stats.ingested += 1;
370                memory.id
371            }
372        };
373        for older in current.iter().filter(|m| m.id != kept) {
374            self.store.supersede_memory(older.id, kept)?;
375            stats.superseded += 1;
376        }
377        self.record(key, &hash, 1)
378    }
379
380    fn refile(&self, current: &[Memory], project: Option<&str>) -> Result<()> {
381        let Some(project) = project else {
382            return Ok(());
383        };
384        for memory in current {
385            if memory.project_path.as_deref() != Some(project) {
386                self.store.refile_memory(memory.id, project)?;
387            }
388        }
389        Ok(())
390    }
391
392    fn persist(&self, memory: &mut Memory, path: &Path, project: Option<String>) -> Result<()> {
393        if let Some(embedder) = &self.embedder {
394            match embedder.embed_document(&memory.content) {
395                Ok(embedding) => memory.embedding = embedding,
396                Err(e) => tracing::warn!(
397                    "vaults: embedding {} failed, storing it without a vector: {e}",
398                    memory.source_id
399                ),
400            }
401        }
402        let conversation = Conversation {
403            id: markdown::note_conversation_id(&self.machine_id, path),
404            source: SOURCE.to_string(),
405            machine_id: self.machine_id.clone(),
406            started_at: Utc::now(),
407            project_path: project,
408        };
409        self.store.store_conversation_if_absent(&conversation)?;
410        Store::store_memory(self.store.as_ref(), memory)?;
411        self.store.store_relation(&Relation {
412            from_id: memory.id,
413            to_id: conversation.id,
414            relation_type: RelationType::DerivedFrom,
415            strength: 1.0,
416            context: None,
417        })
418    }
419
420    fn retire(&self, current: &[Memory], reason: &str, stats: &mut ScanStats) -> Result<()> {
421        for memory in current {
422            if self.store.retire_memory(memory.id, reason)? == RetireOutcome::Retired {
423                stats.retired += 1;
424            }
425        }
426        Ok(())
427    }
428
429    fn record(&self, key: &str, hash: &str, memory_count: usize) -> Result<()> {
430        self.store.record_ingested(&IngestRecord {
431            file_path: key,
432            content_hash: hash,
433            session_key: None,
434            // Note memories have random ids, so a reingest must not forget
435            // these rows and bring back notes the user forgot.
436            legacy_ids: true,
437            memory_count,
438            source: SOURCE,
439        })
440    }
441
442    /// This machine's live `markdown` memories whose note path passes
443    /// `keep`, by note path. Another machine's are left alone even at the
444    /// same path, because its vault is not this one, and so is any row
445    /// without the absolute note path vault sync always writes.
446    fn live_notes(&self, keep: impl Fn(&Path) -> bool) -> Result<HashMap<String, Vec<Memory>>> {
447        let mut live: HashMap<String, Vec<Memory>> = HashMap::new();
448        for memory in self.store.memories_by_source(SOURCE)? {
449            if memory.machine_id == self.machine_id
450                && memory.tier == Tier::Hot
451                && memory.superseded_by.is_none()
452                && Path::new(&memory.source_id).is_absolute()
453                && keep(Path::new(&memory.source_id))
454            {
455                live.entry(memory.source_id.clone())
456                    .or_default()
457                    .push(memory);
458            }
459        }
460        Ok(live)
461    }
462
463    fn project_for(&mut self, vault: &Path) -> Option<String> {
464        if self.projects.is_none() {
465            self.projects = Some(self.store.distinct_project_paths().unwrap_or_else(|e| {
466                tracing::warn!("vaults: listing projects failed: {e}");
467                Vec::new()
468            }));
469        }
470        markdown::vault_project(vault, self.projects.as_deref().unwrap_or_default())
471    }
472}
473
474/// The most specific configured vault holding `path`, so nested vaults
475/// never act on each other's notes.
476fn owner<'a>(path: &Path, vaults: &'a [PathBuf]) -> Option<&'a Path> {
477    vaults
478        .iter()
479        .filter(|v| path.starts_with(v))
480        .max_by_key(|v| v.components().count())
481        .map(PathBuf::as_path)
482}
483
484fn sha256_hex(bytes: &[u8]) -> String {
485    Sha256::digest(bytes)
486        .iter()
487        .map(|b| format!("{b:02x}"))
488        .collect()
489}