second_brain_api/
backfill.rs1use std::path::{Path, PathBuf};
2use std::sync::Arc;
3use std::sync::atomic::{AtomicUsize, Ordering};
4use std::thread::JoinHandle;
5use std::time::{Duration, SystemTime};
6
7use chrono::{DateTime, Utc};
8
9use second_brain_core::embedding::Embedder;
10use second_brain_core::residency;
11use second_brain_core::sqlite_store::SqliteStore;
12use second_brain_core::store::StoreAdmin;
13use second_brain_core::transcript_archive::TranscriptArchive;
14
15use crate::health::{BackfillReport, BackfillState, Health};
16use crate::ingest::{FileOutcome, capture_transcript};
17
18pub const PAUSE_BETWEEN_FILES: Duration = Duration::from_millis(200);
19pub const LIVE_POLL: Duration = Duration::from_secs(1);
20
21#[derive(Debug, Default)]
22pub struct LiveIngest {
23 ingesting: AtomicUsize,
24}
25
26impl LiveIngest {
27 pub fn ingesting(&self, files: usize) -> Ingesting<'_> {
28 self.ingesting.store(files, Ordering::Relaxed);
29 Ingesting(self)
30 }
31
32 pub fn busy(&self) -> bool {
33 self.ingesting.load(Ordering::Relaxed) > 0
34 }
35}
36
37pub struct Ingesting<'a>(&'a LiveIngest);
40
41impl Drop for Ingesting<'_> {
42 fn drop(&mut self) {
43 self.0.ingesting.store(0, Ordering::Relaxed);
44 }
45}
46
47pub struct Backfill {
48 pub store: Arc<SqliteStore>,
49 pub embedder: Option<Arc<Embedder>>,
50 pub machine_id: String,
51 pub projects_dir: PathBuf,
52 pub archive: Option<Arc<TranscriptArchive>>,
53 pub live: Arc<LiveIngest>,
54 pub health: Arc<Health>,
55 pub pause: Duration,
56}
57
58#[derive(Debug, Default, Clone, PartialEq, Eq)]
59pub struct BackfillStats {
60 pub captured: usize,
61 pub skipped: usize,
62 pub failed: usize,
63}
64
65impl Backfill {
66 pub fn spawn(self) -> JoinHandle<()> {
67 std::thread::spawn(move || {
68 match self.run() {
69 Ok(stats) => tracing::info!(
70 "transcript backfill done: {} captured, {} skipped, {} failed",
71 stats.captured,
72 stats.skipped,
73 stats.failed
74 ),
75 Err(e) => tracing::error!("transcript backfill failed: {e:#}"),
76 }
77 residency::release_free_heap();
78 })
79 }
80
81 pub fn run(&self) -> anyhow::Result<BackfillStats> {
85 let catching_up = self.store.transcript_backfill_done()?;
86 let files: Vec<PathBuf> = newest_first(&self.projects_dir)
87 .into_iter()
88 .filter(|(mtime, path)| !catching_up || self.changed_since_archived(path, *mtime))
89 .map(|(_, path)| path)
90 .collect();
91 tracing::info!("transcript backfill: {} transcripts to check", files.len());
92 let mut report = BackfillReport {
93 state: BackfillState::Running,
94 total: files.len(),
95 done: 0,
96 failed: 0,
97 updated_at: Some(Utc::now()),
98 };
99 self.health.backfill_progress(report.clone());
100
101 let store: Arc<dyn StoreAdmin> = self.store.clone();
102 let mut stats = BackfillStats::default();
103 for (i, path) in files.iter().enumerate() {
104 if i > 0 {
105 std::thread::sleep(self.pause);
106 }
107 while self.live.busy() {
108 std::thread::sleep(LIVE_POLL);
109 }
110 match capture_transcript(
111 &store,
112 self.embedder.as_deref(),
113 self.archive.as_deref(),
114 path,
115 &self.machine_id,
116 ) {
117 Ok(FileOutcome::Captured) => stats.captured += 1,
118 Ok(FileOutcome::Skipped) => stats.skipped += 1,
119 Err(e) => {
120 stats.failed += 1;
121 tracing::warn!("transcript backfill: {}: {e:#}", path.display());
122 }
123 }
124 report.done = i + 1;
125 report.failed = stats.failed;
126 report.updated_at = Some(Utc::now());
127 self.health.backfill_progress(report.clone());
128 }
129
130 let marked = if catching_up || stats.failed > 0 {
131 Ok(())
132 } else {
133 self.store.mark_transcript_backfill_done()
134 };
135 report.state = BackfillState::Done;
136 report.updated_at = Some(Utc::now());
137 self.health.backfill_progress(report);
138 marked?;
139 Ok(stats)
140 }
141
142 fn changed_since_archived(&self, path: &Path, mtime: Option<SystemTime>) -> bool {
143 let Some(archive) = &self.archive else {
144 return false;
145 };
146 match (archive.entry(path), mtime) {
147 (None, _) | (Some(_), None) => true,
148 (Some(entry), Some(mtime)) => DateTime::<Utc>::from(mtime) > entry.archived_at,
149 }
150 }
151}
152
153pub fn transcripts_newest_first(projects_dir: &Path) -> Vec<PathBuf> {
154 newest_first(projects_dir)
155 .into_iter()
156 .map(|(_, path)| path)
157 .collect()
158}
159
160fn newest_first(projects_dir: &Path) -> Vec<(Option<SystemTime>, PathBuf)> {
161 let mut found = transcript_mtimes(projects_dir);
162 found.sort_by(|(a_time, a_path), (b_time, b_path)| {
163 b_time.cmp(a_time).then_with(|| a_path.cmp(b_path))
164 });
165 found
166}
167
168pub(crate) fn transcript_mtimes(dir: &Path) -> Vec<(Option<SystemTime>, PathBuf)> {
169 let mut found = Vec::new();
170 let mut dirs = vec![dir.to_path_buf()];
171 while let Some(d) = dirs.pop() {
172 let Ok(entries) = std::fs::read_dir(&d) else {
173 continue;
174 };
175 for entry in entries.flatten() {
176 let Ok(kind) = entry.file_type() else {
177 continue;
178 };
179 let path = entry.path();
180 if kind.is_dir() {
181 dirs.push(path);
182 } else if kind.is_file() && path.extension().is_some_and(|e| e == "jsonl") {
183 let mtime = entry.metadata().and_then(|m| m.modified()).ok();
184 found.push((mtime, path));
185 }
186 }
187 }
188 found
189}