use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::thread::JoinHandle;
use std::time::{Duration, SystemTime};
use chrono::{DateTime, Utc};
use second_brain_core::embedding::Embedder;
use second_brain_core::residency;
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::StoreAdmin;
use second_brain_core::transcript_archive::TranscriptArchive;
use crate::bootstrap::Bootstrap;
use crate::clock_gate::ClockGate;
use crate::health::{BackfillReport, BackfillState, Health};
use crate::ingest::{
FileOutcome, capture_transcript, changed_since_ingest, ingest_memory_file, ingested_as,
is_memory_file,
};
pub const PAUSE_BETWEEN_FILES: Duration = Duration::from_millis(200);
pub const LIVE_POLL: Duration = Duration::from_secs(1);
#[derive(Debug, Default)]
pub struct LiveIngest {
ingesting: AtomicUsize,
}
impl LiveIngest {
pub fn ingesting(&self, files: usize) -> Ingesting<'_> {
self.ingesting.store(files, Ordering::Relaxed);
Ingesting(self)
}
pub fn busy(&self) -> bool {
self.ingesting.load(Ordering::Relaxed) > 0
}
}
pub struct Ingesting<'a>(&'a LiveIngest);
impl Drop for Ingesting<'_> {
fn drop(&mut self) {
self.0.ingesting.store(0, Ordering::Relaxed);
}
}
pub struct Backfill {
pub store: Arc<SqliteStore>,
pub embedder: Option<Arc<Embedder>>,
pub machine_id: String,
pub projects_dir: PathBuf,
pub archive: Option<Arc<TranscriptArchive>>,
pub live: Arc<LiveIngest>,
pub health: Arc<Health>,
pub pause: Duration,
pub clock: Arc<ClockGate>,
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct BackfillStats {
pub captured: usize,
pub skipped: usize,
pub failed: usize,
}
impl Backfill {
pub fn spawn(self) -> JoinHandle<()> {
self.spawn_gated(Arc::default())
}
pub fn spawn_gated(self, bootstrap: Arc<Bootstrap>) -> JoinHandle<()> {
std::thread::spawn(move || {
match self.run_gated(&bootstrap) {
Ok(stats) => tracing::info!(
"transcript backfill done: {} captured, {} skipped, {} failed",
stats.captured,
stats.skipped,
stats.failed
),
Err(e) => tracing::error!("transcript backfill failed: {e:#}"),
}
residency::release_free_heap();
})
}
pub fn run(&self) -> anyhow::Result<BackfillStats> {
self.run_gated(&Bootstrap::default())
}
pub fn run_gated(&self, bootstrap: &Bootstrap) -> anyhow::Result<BackfillStats> {
while bootstrap.is_pending() {
std::thread::sleep(LIVE_POLL);
}
let catching_up = self.store.transcript_backfill_done()?;
let store: Arc<dyn StoreAdmin> = self.store.clone();
let (memory_files, transcripts): (Vec<_>, Vec<_>) = watched_file_mtimes(&self.projects_dir)
.into_iter()
.partition(|(_, path)| is_memory_file(path));
let files: Vec<PathBuf> = newest_first(transcripts)
.into_iter()
.filter(|(mtime, path)| !catching_up || self.needs_catch_up(path, *mtime))
.map(|(_, path)| path)
.chain(
memory_files
.into_iter()
.map(|(_, path)| path)
.filter(|path| changed_since_ingest(store.as_ref(), path)),
)
.collect();
tracing::info!("transcript backfill: {} files to check", files.len());
let mut report = BackfillReport {
state: BackfillState::Running,
total: files.len(),
done: 0,
failed: 0,
updated_at: Some(Utc::now()),
};
self.health.backfill_progress(report.clone());
let mut stats = BackfillStats::default();
for (i, path) in files.iter().enumerate() {
if i > 0 {
std::thread::sleep(self.pause);
}
match self.ingest_once_open(&store, path, bootstrap) {
Ok(FileOutcome::Captured) => stats.captured += 1,
Ok(FileOutcome::Skipped) => stats.skipped += 1,
Err(e) => {
stats.failed += 1;
tracing::warn!("transcript backfill: {}: {e:#}", path.display());
}
}
report.done = i + 1;
report.failed = stats.failed;
report.updated_at = Some(Utc::now());
self.health.backfill_progress(report.clone());
}
let marked = if catching_up || stats.failed > 0 {
Ok(())
} else {
self.store.mark_transcript_backfill_done()
};
report.state = BackfillState::Done;
report.updated_at = Some(Utc::now());
self.health.backfill_progress(report);
marked?;
Ok(stats)
}
fn ingest_once_open(
&self,
store: &Arc<dyn StoreAdmin>,
path: &Path,
bootstrap: &Bootstrap,
) -> anyhow::Result<FileOutcome> {
loop {
let closures = self.clock.closures();
while self.live.busy() || !self.clock.is_open() || bootstrap.is_pending() {
std::thread::sleep(LIVE_POLL);
}
let result = self.ingest(store, path);
if result.is_ok() || (self.clock.closures() == closures && self.clock.is_open()) {
return result;
}
}
}
fn ingest(&self, store: &Arc<dyn StoreAdmin>, path: &Path) -> anyhow::Result<FileOutcome> {
if is_memory_file(path) {
ingest_memory_file(store, self.embedder.as_deref(), path, &self.machine_id)
} else {
capture_transcript(
store,
self.embedder.as_deref(),
self.archive.as_deref(),
path,
&self.machine_id,
)
}
}
fn needs_catch_up(&self, path: &Path, mtime: Option<SystemTime>) -> bool {
let Some(archive) = &self.archive else {
return false;
};
let Some(entry) = archive.entry(path) else {
return true;
};
let changed = mtime.is_none_or(|mtime| DateTime::<Utc>::from(mtime) > entry.archived_at);
changed || !ingested_as(self.store.as_ref(), path, &entry.sha256)
}
}
pub fn transcripts_newest_first(projects_dir: &Path) -> Vec<PathBuf> {
newest_first(transcript_mtimes(projects_dir))
.into_iter()
.map(|(_, path)| path)
.collect()
}
fn newest_first(
mut found: Vec<(Option<SystemTime>, PathBuf)>,
) -> Vec<(Option<SystemTime>, PathBuf)> {
found.sort_by(|(a_time, a_path), (b_time, b_path)| {
b_time.cmp(a_time).then_with(|| a_path.cmp(b_path))
});
found
}
fn is_transcript(path: &Path) -> bool {
path.extension().is_some_and(|e| e == "jsonl")
}
pub(crate) fn transcript_mtimes(dir: &Path) -> Vec<(Option<SystemTime>, PathBuf)> {
files_under(dir, is_transcript)
}
pub(crate) fn watched_files(dir: &Path) -> Vec<PathBuf> {
watched_file_mtimes(dir)
.into_iter()
.map(|(_, path)| path)
.collect()
}
fn watched_file_mtimes(dir: &Path) -> Vec<(Option<SystemTime>, PathBuf)> {
files_under(dir, |path| is_transcript(path) || is_memory_file(path))
}
fn files_under(dir: &Path, keep: impl Fn(&Path) -> bool) -> Vec<(Option<SystemTime>, PathBuf)> {
let mut found = Vec::new();
let mut dirs = vec![dir.to_path_buf()];
while let Some(d) = dirs.pop() {
let Ok(entries) = std::fs::read_dir(&d) else {
continue;
};
for entry in entries.flatten() {
let Ok(kind) = entry.file_type() else {
continue;
};
let path = entry.path();
if kind.is_dir() {
dirs.push(path);
} else if kind.is_file() && keep(&path) {
let mtime = entry.metadata().and_then(|m| m.modified()).ok();
found.push((mtime, path));
}
}
}
found
}