use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;
use anyhow::Result;
use chrono::{DateTime, Utc};
use second_brain_adapters::raw_text::raw_passages;
use second_brain_core::index::EMBEDDING_DIM;
use second_brain_core::index::raw::{RawIndex, RawOrigin, RawPassage};
use second_brain_core::project_path;
use second_brain_core::residency;
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::Store;
use second_brain_core::transcript_archive::{ArchiveEntry, TranscriptArchive};
use uuid::Uuid;
const EMBED_BATCH: usize = 8;
const COMMIT_EVERY: usize = 20;
pub const PAUSE_AFTER_EMBED: Duration = Duration::from_millis(100);
const PASS_INTERVAL: Duration = Duration::from_secs(600);
pub fn open_or_rebuild(dir: &Path) -> Result<RawIndex> {
match RawIndex::open(dir) {
Ok(index) => Ok(index),
Err(e)
if e.chain()
.any(|c| c.downcast_ref::<std::io::Error>().is_some()) =>
{
Err(e)
}
Err(e) => {
tracing::warn!(
"raw index at {} cannot be opened, rebuilding it: {e:#}",
dir.display()
);
std::fs::remove_dir_all(dir)?;
RawIndex::open(dir)
}
}
}
pub type EmbedBatch = dyn Fn(&[&str]) -> Result<Vec<Vec<f32>>> + Send + Sync;
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct PassSummary {
pub transcripts: usize,
pub memories: usize,
pub passages: usize,
pub restored: usize,
pub failed: usize,
pub skipped_by_age: usize,
}
pub struct RawCoverage {
max_age_days: u32,
skipped_by_age: AtomicUsize,
aged: Mutex<HashMap<String, Dated>>,
}
#[derive(Clone, Copy)]
struct Dated {
newest: DateTime<Utc>,
passages: usize,
}
impl RawCoverage {
pub fn new(max_age_days: u32) -> Self {
Self {
max_age_days,
skipped_by_age: AtomicUsize::new(0),
aged: Mutex::new(HashMap::new()),
}
}
pub fn max_age_days(&self) -> u32 {
self.max_age_days
}
pub fn skipped_by_age(&self) -> usize {
self.skipped_by_age.load(Ordering::Relaxed)
}
fn cutoff(&self) -> DateTime<Utc> {
Utc::now()
.checked_sub_signed(chrono::Duration::days(i64::from(self.max_age_days)))
.unwrap_or(DateTime::<Utc>::MIN_UTC)
}
fn aged(&self) -> std::sync::MutexGuard<'_, HashMap<String, Dated>> {
self.aged.lock().unwrap_or_else(|p| p.into_inner())
}
}
pub struct RawIndexer {
pub index: Arc<RwLock<RawIndex>>,
pub archive: Option<Arc<TranscriptArchive>>,
pub store: Arc<SqliteStore>,
pub embed: Arc<EmbedBatch>,
pub pause: Duration,
pub coverage: Arc<RawCoverage>,
}
impl RawIndexer {
pub fn spawn(self) {
std::thread::Builder::new()
.name("raw-index".to_string())
.spawn(move || {
loop {
match self.pass() {
Ok(s)
if PassSummary {
skipped_by_age: 0,
..s
} != PassSummary::default() =>
{
tracing::info!(
"raw index: {} transcripts, {} archived memories, {} passages, {} vectors restored, {} sources failed, {} passages older than {} days skipped",
s.transcripts,
s.memories,
s.passages,
s.restored,
s.failed,
s.skipped_by_age,
self.coverage.max_age_days()
);
residency::release_free_heap();
}
Ok(_) => {}
Err(e) => {
tracing::warn!("raw index pass failed: {e:#}");
residency::release_free_heap();
}
}
std::thread::sleep(PASS_INTERVAL);
}
})
.expect("spawning the raw index thread");
}
pub fn pass(&self) -> Result<PassSummary> {
let mut summary = PassSummary::default();
let mut first_error = None;
let mut pending = 0;
let cutoff = self.coverage.cutoff();
if let Some(archive) = &self.archive {
let mut queue = Vec::new();
let mut aged = HashMap::new();
for entry in archive.entries() {
let origin = RawOrigin::Transcript {
sha256: entry.sha256.clone(),
};
if self.indexed(&origin) {
continue;
}
match self.date_transcript(archive, &entry) {
Ok(dated) if dated.newest < cutoff => {
summary.skipped_by_age += dated.passages;
aged.insert(entry.sha256.clone(), dated);
}
Ok(dated) => queue.push((dated.newest, entry, origin)),
Err(e) => {
summary.failed += 1;
first_error.get_or_insert(e);
}
}
}
*self.coverage.aged() = aged;
queue.sort_by_key(|q| std::cmp::Reverse(q.0));
for (_, entry, origin) in queue {
match self.add_transcript(archive, &entry, &origin) {
Ok(added) => {
summary.passages += added;
summary.transcripts += 1;
pending = self.commit_every(pending + 1)?;
}
Err(e) => {
summary.failed += 1;
first_error.get_or_insert(e);
}
}
}
}
let raws = self.store.archived_raws_since(cutoff)?;
summary.skipped_by_age += raws
.older
.into_iter()
.filter(|&id| !self.indexed(&RawOrigin::Memory { id }))
.count();
for id in raws.recent {
let origin = RawOrigin::Memory { id };
if self.indexed(&origin) {
continue;
}
match self.add_memory(id, &origin) {
Ok((added, restored)) => {
summary.passages += added;
summary.restored += usize::from(restored);
summary.memories += 1;
pending = self.commit_every(pending + 1)?;
}
Err(e) => {
summary.failed += 1;
first_error.get_or_insert(e);
}
}
}
if pending > 0 {
self.write().commit()?;
}
self.coverage
.skipped_by_age
.store(summary.skipped_by_age, Ordering::Relaxed);
if let Some(e) = first_error {
tracing::warn!(
"raw index: {} sources failed and will be retried, first: {e:#}",
summary.failed
);
}
Ok(summary)
}
fn date_transcript(&self, archive: &TranscriptArchive, entry: &ArchiveEntry) -> Result<Dated> {
if let Some(dated) = self.coverage.aged().get(&entry.sha256) {
return Ok(*dated);
}
let source = Path::new(&entry.source);
let passages = match archive.read(source)? {
Some(content) => raw_passages(source, &content),
None => Vec::new(),
};
Ok(Dated {
newest: passages
.iter()
.filter_map(|p| p.at)
.max()
.unwrap_or(entry.archived_at),
passages: passages.len(),
})
}
fn add_transcript(
&self,
archive: &TranscriptArchive,
entry: &ArchiveEntry,
origin: &RawOrigin,
) -> Result<usize> {
let source = Path::new(&entry.source);
let Some(content) = archive.read(source)? else {
return self.add(origin, Vec::new());
};
let project = project_path::from_transcript(&entry.source);
let passages = raw_passages(source, &content)
.into_iter()
.map(|p| RawPassage {
text: p.text,
at: p.at,
project: project.clone(),
memory: p.memory,
})
.collect();
self.add(origin, passages)
}
fn add_memory(&self, id: Uuid, origin: &RawOrigin) -> Result<(usize, bool)> {
let Some(memory) = self.store.get_memory_with_embedding(id)? else {
return Ok((0, false));
};
let mut restored = false;
let embedding = if memory.embedding.len() == EMBEDDING_DIM {
memory.embedding.clone()
} else {
let embedding = self
.embed_and_yield(&[memory.content.as_str()])?
.into_iter()
.next()
.unwrap_or_default();
if embedding.len() == EMBEDDING_DIM {
restored = self.store.restore_archived_embedding(id, &embedding)?;
}
embedding
};
let passage = RawPassage {
text: memory.content.clone(),
at: memory.session_at,
project: memory.project_path.clone(),
memory: Some(id),
};
let added = self.write().add_source(origin, &[(passage, embedding)])?;
Ok((added, restored))
}
fn add(&self, origin: &RawOrigin, passages: Vec<RawPassage>) -> Result<usize> {
let passages: Vec<RawPassage> = {
let index = self.read();
let mut seen = HashSet::new();
passages
.into_iter()
.filter(|p| seen.insert(p.text.trim().to_string()) && !index.has_text(&p.text))
.collect()
};
let mut embedded = Vec::with_capacity(passages.len());
for batch in passages.chunks(EMBED_BATCH) {
let texts: Vec<&str> = batch.iter().map(|p| p.text.as_str()).collect();
let vectors = self.embed_and_yield(&texts)?;
embedded.extend(batch.iter().cloned().zip(vectors));
}
self.write().add_source(origin, &embedded)
}
fn embed_and_yield(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
let vectors = (self.embed)(texts);
std::thread::sleep(self.pause);
vectors
}
fn commit_every(&self, pending: usize) -> Result<usize> {
if pending < COMMIT_EVERY {
return Ok(pending);
}
self.write().commit()?;
Ok(0)
}
fn indexed(&self, origin: &RawOrigin) -> bool {
self.read().has_source(origin)
}
fn read(&self) -> std::sync::RwLockReadGuard<'_, RawIndex> {
self.index.read().unwrap_or_else(|p| p.into_inner())
}
fn write(&self) -> std::sync::RwLockWriteGuard<'_, RawIndex> {
self.index.write().unwrap_or_else(|p| p.into_inner())
}
}