Skip to main content

second_brain_api/
backfill.rs

1use std::path::{Path, PathBuf};
2use std::sync::Arc;
3use std::sync::atomic::{AtomicUsize, Ordering};
4use std::thread::JoinHandle;
5use std::time::{Duration, SystemTime};
6
7use chrono::{DateTime, Utc};
8
9use second_brain_core::embedding::Embedder;
10use second_brain_core::residency;
11use second_brain_core::sqlite_store::SqliteStore;
12use second_brain_core::store::StoreAdmin;
13use second_brain_core::transcript_archive::TranscriptArchive;
14
15use crate::health::{BackfillReport, BackfillState, Health};
16use crate::ingest::{FileOutcome, capture_transcript};
17
18pub const PAUSE_BETWEEN_FILES: Duration = Duration::from_millis(200);
19pub const LIVE_POLL: Duration = Duration::from_secs(1);
20
21#[derive(Debug, Default)]
22pub struct LiveIngest {
23    ingesting: AtomicUsize,
24}
25
26impl LiveIngest {
27    pub fn ingesting(&self, files: usize) -> Ingesting<'_> {
28        self.ingesting.store(files, Ordering::Relaxed);
29        Ingesting(self)
30    }
31
32    pub fn busy(&self) -> bool {
33        self.ingesting.load(Ordering::Relaxed) > 0
34    }
35}
36
37/// Clears the busy flag on drop, so a watcher that panics mid-ingest cannot
38/// leave the backfill waiting forever.
39pub struct Ingesting<'a>(&'a LiveIngest);
40
41impl Drop for Ingesting<'_> {
42    fn drop(&mut self) {
43        self.0.ingesting.store(0, Ordering::Relaxed);
44    }
45}
46
47pub struct Backfill {
48    pub store: Arc<SqliteStore>,
49    pub embedder: Option<Arc<Embedder>>,
50    pub machine_id: String,
51    pub projects_dir: PathBuf,
52    pub archive: Option<Arc<TranscriptArchive>>,
53    pub live: Arc<LiveIngest>,
54    pub health: Arc<Health>,
55    pub pause: Duration,
56}
57
58#[derive(Debug, Default, Clone, PartialEq, Eq)]
59pub struct BackfillStats {
60    pub captured: usize,
61    pub skipped: usize,
62    pub failed: usize,
63}
64
65impl Backfill {
66    pub fn spawn(self) -> JoinHandle<()> {
67        std::thread::spawn(move || {
68            match self.run() {
69                Ok(stats) => tracing::info!(
70                    "transcript backfill done: {} captured, {} skipped, {} failed",
71                    stats.captured,
72                    stats.skipped,
73                    stats.failed
74                ),
75                Err(e) => tracing::error!("transcript backfill failed: {e:#}"),
76            }
77            residency::release_free_heap();
78        })
79    }
80
81    /// The first complete run walks every transcript; later runs only catch
82    /// up on transcripts changed since they were archived, such as sessions
83    /// written while the daemon was down.
84    pub fn run(&self) -> anyhow::Result<BackfillStats> {
85        let catching_up = self.store.transcript_backfill_done()?;
86        let files: Vec<PathBuf> = newest_first(&self.projects_dir)
87            .into_iter()
88            .filter(|(mtime, path)| !catching_up || self.changed_since_archived(path, *mtime))
89            .map(|(_, path)| path)
90            .collect();
91        tracing::info!("transcript backfill: {} transcripts to check", files.len());
92        let mut report = BackfillReport {
93            state: BackfillState::Running,
94            total: files.len(),
95            done: 0,
96            failed: 0,
97            updated_at: Some(Utc::now()),
98        };
99        self.health.backfill_progress(report.clone());
100
101        let store: Arc<dyn StoreAdmin> = self.store.clone();
102        let mut stats = BackfillStats::default();
103        for (i, path) in files.iter().enumerate() {
104            if i > 0 {
105                std::thread::sleep(self.pause);
106            }
107            while self.live.busy() {
108                std::thread::sleep(LIVE_POLL);
109            }
110            match capture_transcript(
111                &store,
112                self.embedder.as_deref(),
113                self.archive.as_deref(),
114                path,
115                &self.machine_id,
116            ) {
117                Ok(FileOutcome::Captured) => stats.captured += 1,
118                Ok(FileOutcome::Skipped) => stats.skipped += 1,
119                Err(e) => {
120                    stats.failed += 1;
121                    tracing::warn!("transcript backfill: {}: {e:#}", path.display());
122                }
123            }
124            report.done = i + 1;
125            report.failed = stats.failed;
126            report.updated_at = Some(Utc::now());
127            self.health.backfill_progress(report.clone());
128        }
129
130        let marked = if catching_up || stats.failed > 0 {
131            Ok(())
132        } else {
133            self.store.mark_transcript_backfill_done()
134        };
135        report.state = BackfillState::Done;
136        report.updated_at = Some(Utc::now());
137        self.health.backfill_progress(report);
138        marked?;
139        Ok(stats)
140    }
141
142    fn changed_since_archived(&self, path: &Path, mtime: Option<SystemTime>) -> bool {
143        let Some(archive) = &self.archive else {
144            return false;
145        };
146        match (archive.entry(path), mtime) {
147            (None, _) | (Some(_), None) => true,
148            (Some(entry), Some(mtime)) => DateTime::<Utc>::from(mtime) > entry.archived_at,
149        }
150    }
151}
152
153pub fn transcripts_newest_first(projects_dir: &Path) -> Vec<PathBuf> {
154    newest_first(projects_dir)
155        .into_iter()
156        .map(|(_, path)| path)
157        .collect()
158}
159
160fn newest_first(projects_dir: &Path) -> Vec<(Option<SystemTime>, PathBuf)> {
161    let mut found = transcript_mtimes(projects_dir);
162    found.sort_by(|(a_time, a_path), (b_time, b_path)| {
163        b_time.cmp(a_time).then_with(|| a_path.cmp(b_path))
164    });
165    found
166}
167
168pub(crate) fn transcript_mtimes(dir: &Path) -> Vec<(Option<SystemTime>, PathBuf)> {
169    let mut found = Vec::new();
170    let mut dirs = vec![dir.to_path_buf()];
171    while let Some(d) = dirs.pop() {
172        let Ok(entries) = std::fs::read_dir(&d) else {
173            continue;
174        };
175        for entry in entries.flatten() {
176            let Ok(kind) = entry.file_type() else {
177                continue;
178            };
179            let path = entry.path();
180            if kind.is_dir() {
181                dirs.push(path);
182            } else if kind.is_file() && path.extension().is_some_and(|e| e == "jsonl") {
183                let mtime = entry.metadata().and_then(|m| m.modified()).ok();
184                found.push((mtime, path));
185            }
186        }
187    }
188    found
189}