1use std::path::{Path, PathBuf};
2use std::sync::Arc;
3use std::sync::atomic::{AtomicUsize, Ordering};
4use std::thread::JoinHandle;
5use std::time::{Duration, SystemTime};
6
7use chrono::{DateTime, Utc};
8
9use second_brain_core::embedding::Embedder;
10use second_brain_core::residency;
11use second_brain_core::sqlite_store::SqliteStore;
12use second_brain_core::store::StoreAdmin;
13use second_brain_core::transcript_archive::TranscriptArchive;
14
15use crate::bootstrap::Bootstrap;
16use crate::clock_gate::ClockGate;
17use crate::health::{BackfillReport, BackfillState, Health};
18use crate::ingest::{
19 FileOutcome, capture_transcript, changed_since_ingest, ingest_memory_file, ingested_as,
20 is_memory_file,
21};
22
23pub const PAUSE_BETWEEN_FILES: Duration = Duration::from_millis(200);
24pub const LIVE_POLL: Duration = Duration::from_secs(1);
25
26#[derive(Debug, Default)]
27pub struct LiveIngest {
28 ingesting: AtomicUsize,
29}
30
31impl LiveIngest {
32 pub fn ingesting(&self, files: usize) -> Ingesting<'_> {
33 self.ingesting.store(files, Ordering::Relaxed);
34 Ingesting(self)
35 }
36
37 pub fn busy(&self) -> bool {
38 self.ingesting.load(Ordering::Relaxed) > 0
39 }
40}
41
42pub struct Ingesting<'a>(&'a LiveIngest);
45
46impl Drop for Ingesting<'_> {
47 fn drop(&mut self) {
48 self.0.ingesting.store(0, Ordering::Relaxed);
49 }
50}
51
52pub struct Backfill {
53 pub store: Arc<SqliteStore>,
54 pub embedder: Option<Arc<Embedder>>,
55 pub machine_id: String,
56 pub projects_dir: PathBuf,
57 pub archive: Option<Arc<TranscriptArchive>>,
58 pub live: Arc<LiveIngest>,
59 pub health: Arc<Health>,
60 pub pause: Duration,
61 pub clock: Arc<ClockGate>,
62}
63
64#[derive(Debug, Default, Clone, PartialEq, Eq)]
65pub struct BackfillStats {
66 pub captured: usize,
67 pub skipped: usize,
68 pub failed: usize,
69}
70
71impl Backfill {
72 pub fn spawn(self) -> JoinHandle<()> {
73 self.spawn_gated(Arc::default())
74 }
75
76 pub fn spawn_gated(self, bootstrap: Arc<Bootstrap>) -> JoinHandle<()> {
78 std::thread::spawn(move || {
79 match self.run_gated(&bootstrap) {
80 Ok(stats) => tracing::info!(
81 "transcript backfill done: {} captured, {} skipped, {} failed",
82 stats.captured,
83 stats.skipped,
84 stats.failed
85 ),
86 Err(e) => tracing::error!("transcript backfill failed: {e:#}"),
87 }
88 residency::release_free_heap();
89 })
90 }
91
92 pub fn run(&self) -> anyhow::Result<BackfillStats> {
98 self.run_gated(&Bootstrap::default())
99 }
100
101 pub fn run_gated(&self, bootstrap: &Bootstrap) -> anyhow::Result<BackfillStats> {
102 while bootstrap.is_pending() {
103 std::thread::sleep(LIVE_POLL);
104 }
105 let catching_up = self.store.transcript_backfill_done()?;
106 let store: Arc<dyn StoreAdmin> = self.store.clone();
107 let (memory_files, transcripts): (Vec<_>, Vec<_>) = watched_file_mtimes(&self.projects_dir)
108 .into_iter()
109 .partition(|(_, path)| is_memory_file(path));
110 let files: Vec<PathBuf> = newest_first(transcripts)
111 .into_iter()
112 .filter(|(mtime, path)| !catching_up || self.needs_catch_up(path, *mtime))
113 .map(|(_, path)| path)
114 .chain(
115 memory_files
116 .into_iter()
117 .map(|(_, path)| path)
118 .filter(|path| changed_since_ingest(store.as_ref(), path)),
119 )
120 .collect();
121 tracing::info!("transcript backfill: {} files to check", files.len());
122 let mut report = BackfillReport {
123 state: BackfillState::Running,
124 total: files.len(),
125 done: 0,
126 failed: 0,
127 updated_at: Some(Utc::now()),
128 };
129 self.health.backfill_progress(report.clone());
130
131 let mut stats = BackfillStats::default();
132 for (i, path) in files.iter().enumerate() {
133 if i > 0 {
134 std::thread::sleep(self.pause);
135 }
136 match self.ingest_once_open(&store, path, bootstrap) {
137 Ok(FileOutcome::Captured) => stats.captured += 1,
138 Ok(FileOutcome::Skipped) => stats.skipped += 1,
139 Err(e) => {
140 stats.failed += 1;
141 tracing::warn!("transcript backfill: {}: {e:#}", path.display());
142 }
143 }
144 report.done = i + 1;
145 report.failed = stats.failed;
146 report.updated_at = Some(Utc::now());
147 self.health.backfill_progress(report.clone());
148 }
149
150 let marked = if catching_up || stats.failed > 0 {
151 Ok(())
152 } else {
153 self.store.mark_transcript_backfill_done()
154 };
155 report.state = BackfillState::Done;
156 report.updated_at = Some(Utc::now());
157 self.health.backfill_progress(report);
158 marked?;
159 Ok(stats)
160 }
161
162 fn ingest_once_open(
165 &self,
166 store: &Arc<dyn StoreAdmin>,
167 path: &Path,
168 bootstrap: &Bootstrap,
169 ) -> anyhow::Result<FileOutcome> {
170 loop {
171 let closures = self.clock.closures();
172 while self.live.busy() || !self.clock.is_open() || bootstrap.is_pending() {
173 std::thread::sleep(LIVE_POLL);
174 }
175 let result = self.ingest(store, path);
176 if result.is_ok() || (self.clock.closures() == closures && self.clock.is_open()) {
177 return result;
178 }
179 }
180 }
181
182 fn ingest(&self, store: &Arc<dyn StoreAdmin>, path: &Path) -> anyhow::Result<FileOutcome> {
183 if is_memory_file(path) {
184 ingest_memory_file(store, self.embedder.as_deref(), path, &self.machine_id)
185 } else {
186 capture_transcript(
187 store,
188 self.embedder.as_deref(),
189 self.archive.as_deref(),
190 path,
191 &self.machine_id,
192 )
193 }
194 }
195
196 fn needs_catch_up(&self, path: &Path, mtime: Option<SystemTime>) -> bool {
199 let Some(archive) = &self.archive else {
200 return false;
201 };
202 let Some(entry) = archive.entry(path) else {
203 return true;
204 };
205 let changed = mtime.is_none_or(|mtime| DateTime::<Utc>::from(mtime) > entry.archived_at);
206 changed || !ingested_as(self.store.as_ref(), path, &entry.sha256)
207 }
208}
209
210pub fn transcripts_newest_first(projects_dir: &Path) -> Vec<PathBuf> {
211 newest_first(transcript_mtimes(projects_dir))
212 .into_iter()
213 .map(|(_, path)| path)
214 .collect()
215}
216
217fn newest_first(
218 mut found: Vec<(Option<SystemTime>, PathBuf)>,
219) -> Vec<(Option<SystemTime>, PathBuf)> {
220 found.sort_by(|(a_time, a_path), (b_time, b_path)| {
221 b_time.cmp(a_time).then_with(|| a_path.cmp(b_path))
222 });
223 found
224}
225
226fn is_transcript(path: &Path) -> bool {
227 path.extension().is_some_and(|e| e == "jsonl")
228}
229
230pub(crate) fn transcript_mtimes(dir: &Path) -> Vec<(Option<SystemTime>, PathBuf)> {
231 files_under(dir, is_transcript)
232}
233
234pub(crate) fn watched_files(dir: &Path) -> Vec<PathBuf> {
236 watched_file_mtimes(dir)
237 .into_iter()
238 .map(|(_, path)| path)
239 .collect()
240}
241
242fn watched_file_mtimes(dir: &Path) -> Vec<(Option<SystemTime>, PathBuf)> {
243 files_under(dir, |path| is_transcript(path) || is_memory_file(path))
244}
245
246fn files_under(dir: &Path, keep: impl Fn(&Path) -> bool) -> Vec<(Option<SystemTime>, PathBuf)> {
247 let mut found = Vec::new();
248 let mut dirs = vec![dir.to_path_buf()];
249 while let Some(d) = dirs.pop() {
250 let Ok(entries) = std::fs::read_dir(&d) else {
251 continue;
252 };
253 for entry in entries.flatten() {
254 let Ok(kind) = entry.file_type() else {
255 continue;
256 };
257 let path = entry.path();
258 if kind.is_dir() {
259 dirs.push(path);
260 } else if kind.is_file() && keep(&path) {
261 let mtime = entry.metadata().and_then(|m| m.modified()).ok();
262 found.push((mtime, path));
263 }
264 }
265 }
266 found
267}