use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
use std::time::Duration;
use anyhow::Result;
use chrono::{DateTime, Utc};
use second_brain_adapters::raw_text::raw_passages;
use second_brain_core::index::EMBEDDING_DIM;
use second_brain_core::index::raw::{RawHit, RawIndex, RawOrigin, RawPassage, text_hash};
use second_brain_core::project_path;
use second_brain_core::residency;
use second_brain_core::sealed::{SealError, SealKeys, seal_error};
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::transcript_archive::{ArchiveEntry, TranscriptArchive};
use uuid::Uuid;
const EMBED_BATCH: usize = 8;
const COMMIT_EVERY: usize = 20;
pub const PAUSE_AFTER_EMBED: Duration = Duration::from_millis(100);
const PASS_INTERVAL: Duration = Duration::from_secs(600);
pub fn open_or_rebuild(dir: &Path) -> Result<RawIndex> {
open_or_rebuild_keyed(dir, None)
}
pub fn open_or_rebuild_keyed(dir: &Path, keys: Option<&SealKeys>) -> Result<RawIndex> {
let open = || match keys {
Some(keys) => RawIndex::open_keyed(dir, keys),
None => RawIndex::open(dir),
};
match open() {
Ok(index) => Ok(index),
Err(e) if seal_error(&e) == Some(&SealError::KeyRequired) => Err(e),
Err(e)
if seal_error(&e).is_none()
&& e.chain()
.any(|c| c.downcast_ref::<std::io::Error>().is_some()) =>
{
Err(e)
}
Err(e) => {
tracing::warn!(
"raw index at {} cannot be opened, rebuilding it: {e:#}",
dir.display()
);
std::fs::remove_dir_all(dir)?;
open()
}
}
}
pub type EmbedBatch = dyn Fn(&[&str]) -> Result<Vec<Vec<f32>>> + Send + Sync;
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct PassSummary {
pub transcripts: usize,
pub memories: usize,
pub passages: usize,
pub restored: usize,
pub failed: usize,
pub skipped_by_age: usize,
pub purged: usize,
}
pub struct RawCoverage {
max_age_days: u32,
skipped_by_age: AtomicUsize,
aged: Mutex<HashMap<String, Dated>>,
}
#[derive(Clone, Copy)]
struct Dated {
newest: DateTime<Utc>,
passages: usize,
}
impl RawCoverage {
pub fn new(max_age_days: u32) -> Self {
Self {
max_age_days,
skipped_by_age: AtomicUsize::new(0),
aged: Mutex::new(HashMap::new()),
}
}
pub fn max_age_days(&self) -> u32 {
self.max_age_days
}
pub fn skipped_by_age(&self) -> usize {
self.skipped_by_age.load(Ordering::Relaxed)
}
fn cutoff(&self) -> DateTime<Utc> {
Utc::now()
.checked_sub_signed(chrono::Duration::days(i64::from(self.max_age_days)))
.unwrap_or(DateTime::<Utc>::MIN_UTC)
}
fn aged(&self) -> std::sync::MutexGuard<'_, HashMap<String, Dated>> {
self.aged.lock().unwrap_or_else(|p| p.into_inner())
}
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct RawPurge {
pub passages: usize,
pub archive_entries: usize,
}
static PURGES: Mutex<()> = Mutex::new(());
pub fn purge_deleted(
index: Option<&RwLock<RawIndex>>,
archive: Option<&TranscriptArchive>,
store: &SqliteStore,
memories: &[Uuid],
) -> Result<RawPurge> {
let _one_at_a_time = PURGES.lock().unwrap_or_else(|p| p.into_inner());
let doomed = match archive {
Some(archive) => doomed_entries(archive, store)?,
None => Vec::new(),
};
let mut gone = Gone::new(store, memories);
match index {
Some(index) => purge_index(index, archive, &doomed, &mut gone),
None => Ok(RawPurge {
passages: 0,
archive_entries: match archive {
Some(archive) => remove_entries(archive, &doomed)?,
None => 0,
},
}),
}
}
fn doomed_entries(archive: &TranscriptArchive, store: &SqliteStore) -> Result<Vec<ArchiveEntry>> {
let mut doomed = Vec::new();
for entry in archive.entries() {
if store.session_deleted(&entry.source)? {
doomed.push(entry);
}
}
Ok(doomed)
}
fn purge_index(
lock: &RwLock<RawIndex>,
archive: Option<&TranscriptArchive>,
doomed: &[ArchiveEntry],
gone: &mut Gone<'_>,
) -> Result<RawPurge> {
let (memories, recorded_sources) = {
let index = read_lock(lock);
(index.passage_memories()?, index.sources()?)
};
let live = live_entries(archive, doomed);
let live_shas: HashSet<&str> = live.iter().map(|e| e.sha256.as_str()).collect();
let mut stale = Vec::new();
for id in memories {
if gone.check(id)? {
stale.push(id);
}
}
let mut sources: Vec<RawOrigin> = doomed
.iter()
.filter(|e| !live_shas.contains(e.sha256.as_str()))
.map(|e| RawOrigin::Transcript {
sha256: e.sha256.clone(),
})
.collect();
for origin in recorded_sources {
if let RawOrigin::Memory { id } = origin
&& gone.check(id)?
{
sources.push(origin);
}
}
if doomed.is_empty() && sources.is_empty() && stale.is_empty() {
return Ok(RawPurge::default());
}
let texts = transcript_texts(archive, doomed)?;
let texts: Vec<&str> = texts.iter().map(String::as_str).collect();
let doomed_sources: HashSet<String> = sources.iter().map(RawOrigin::source_key).collect();
let (candidates, recorded) = {
let index = read_lock(lock);
(
index.matching(&sources, &stale, &texts)?,
recorded_entries(&index, &live),
)
};
let mut holders = Holders::default();
if let Some(archive) = archive
&& !candidates.is_empty()
{
holders.read(archive, &recorded, &candidates, gone)?;
}
let mut index = write_lock(lock);
index.commit_without_merging()?;
let current = index.matching(&sources, &stale, &texts)?;
if let Some(archive) = archive
&& !current.is_empty()
{
let late: Vec<ArchiveEntry> = recorded_entries(&index, &live)
.into_iter()
.filter(|e| !holders.has_read(e))
.collect();
holders.read(archive, &late, ¤t, gone)?;
}
let mut dropped = Vec::new();
for hit in ¤t {
let memory_gone = match hit.memory {
Some(id) => gone.check(id)?,
None => false,
};
let memory_holder = match hit.memory {
Some(id) => !memory_gone && index.has_source(&RawOrigin::Memory { id }),
None => false,
};
match fate(hit, &doomed_sources, &holders, memory_gone, memory_holder) {
Fate::Keep => {}
Fate::Move(holder) => index.rekey(
hit.key,
&holder.origin,
&RawPassage {
text: holder.text,
at: holder.at,
project: holder.project,
memory: holder.memory,
},
)?,
Fate::Drop => dropped.push(hit.key),
}
}
index.withdraw(&dropped)?;
index.forget_sources(&sources)?;
let rewrite = index.seal_for_rewrite(&dropped)?;
index.commit_without_merging()?;
let archive_entries = match archive {
Some(archive) => remove_entries(archive, doomed)?,
None => 0,
};
let compaction = index.start_compaction()?;
drop(index);
let rewritten = rewrite.run();
let merged = compaction.wait();
let mut index = write_lock(lock);
index.reload_sealed(&rewrite)?;
index.finish_compaction(merged)?;
rewritten?;
Ok(RawPurge {
passages: dropped.len(),
archive_entries,
})
}
fn live_entries(archive: Option<&TranscriptArchive>, doomed: &[ArchiveEntry]) -> Vec<ArchiveEntry> {
let Some(archive) = archive else {
return Vec::new();
};
let doomed: HashSet<&str> = doomed.iter().map(|e| e.source.as_str()).collect();
archive
.entries()
.into_iter()
.filter(|e| !doomed.contains(e.source.as_str()))
.collect()
}
fn transcript_texts(
archive: Option<&TranscriptArchive>,
entries: &[ArchiveEntry],
) -> Result<Vec<String>> {
let Some(archive) = archive else {
return Ok(Vec::new());
};
let mut texts = Vec::new();
for entry in entries {
let source = Path::new(&entry.source);
if let Some(content) = archive.read(source)? {
texts.extend(raw_passages(source, &content).into_iter().map(|p| p.text));
}
}
Ok(texts)
}
fn recorded_entries(index: &RawIndex, entries: &[ArchiveEntry]) -> Vec<ArchiveEntry> {
entries
.iter()
.filter(|e| {
index.has_source(&RawOrigin::Transcript {
sha256: e.sha256.clone(),
})
})
.cloned()
.collect()
}
struct Gone<'a> {
store: &'a SqliteStore,
known: HashMap<Uuid, bool>,
}
impl<'a> Gone<'a> {
fn new(store: &'a SqliteStore, just_deleted: &[Uuid]) -> Self {
Self {
store,
known: just_deleted.iter().map(|id| (*id, true)).collect(),
}
}
fn check(&mut self, id: Uuid) -> Result<bool> {
if let Some(gone) = self.known.get(&id) {
return Ok(*gone);
}
let gone = self.store.tombstone_exists(id)?;
self.known.insert(id, gone);
Ok(gone)
}
}
#[derive(Clone)]
struct Holder {
origin: RawOrigin,
text: String,
project: Option<String>,
at: Option<DateTime<Utc>>,
memory: Option<Uuid>,
}
#[derive(Default)]
struct Holders {
by_hash: HashMap<String, Vec<Holder>>,
read: HashSet<String>,
}
impl Holders {
fn has_read(&self, entry: &ArchiveEntry) -> bool {
self.read.contains(&entry.sha256)
}
fn read(
&mut self,
archive: &TranscriptArchive,
entries: &[ArchiveEntry],
hits: &[RawHit],
gone: &mut Gone<'_>,
) -> Result<()> {
let wanted: HashSet<&str> = hits.iter().map(|h| h.text_hash.as_str()).collect();
for entry in entries {
if !self.read.insert(entry.sha256.clone()) {
continue;
}
let source = Path::new(&entry.source);
let Some(content) = archive.read(source)? else {
continue;
};
let project = project_path::from_transcript(&entry.source);
for passage in raw_passages(source, &content) {
let hash = text_hash(&passage.text);
if !wanted.contains(hash.as_str()) {
continue;
}
if let Some(id) = passage.memory
&& gone.check(id)?
{
continue;
}
self.by_hash.entry(hash).or_default().push(Holder {
origin: RawOrigin::Transcript {
sha256: entry.sha256.clone(),
},
text: passage.text,
project: project.clone(),
at: passage.at,
memory: passage.memory,
});
}
}
Ok(())
}
}
enum Fate {
Keep,
Move(Holder),
Drop,
}
fn fate(
hit: &RawHit,
doomed_sources: &HashSet<String>,
holders: &Holders,
memory_gone: bool,
memory_holder: bool,
) -> Fate {
if matches!(hit.origin, RawOrigin::Memory { .. })
&& !doomed_sources.contains(&hit.origin.source_key())
{
return Fate::Keep;
}
if let Some(held) = holders.by_hash.get(&hit.text_hash) {
if let Some(own) = held.iter().find(|h| h.origin == hit.origin) {
return if memory_gone {
Fate::Move(own.clone())
} else {
Fate::Keep
};
}
if let Some(holder) = held.first() {
return Fate::Move(holder.clone());
}
}
match hit.memory {
Some(id) if memory_holder => Fate::Move(Holder {
origin: RawOrigin::Memory { id },
text: hit.text.clone(),
project: hit.project.clone(),
at: hit.at,
memory: Some(id),
}),
_ => Fate::Drop,
}
}
fn read_lock(index: &RwLock<RawIndex>) -> RwLockReadGuard<'_, RawIndex> {
index.read().unwrap_or_else(|p| p.into_inner())
}
fn write_lock(index: &RwLock<RawIndex>) -> RwLockWriteGuard<'_, RawIndex> {
index.write().unwrap_or_else(|p| p.into_inner())
}
fn remove_entries(archive: &TranscriptArchive, doomed: &[ArchiveEntry]) -> Result<usize> {
if doomed.is_empty() {
return Ok(0);
}
let sources: HashSet<&str> = doomed.iter().map(|e| e.source.as_str()).collect();
Ok(archive
.remove_where(|e| sources.contains(e.source.as_str()))?
.len())
}
pub struct RawIndexer {
pub index: Arc<RwLock<RawIndex>>,
pub archive: Option<Arc<TranscriptArchive>>,
pub store: Arc<SqliteStore>,
pub embed: Arc<EmbedBatch>,
pub pause: Duration,
pub coverage: Arc<RawCoverage>,
}
impl RawIndexer {
pub fn spawn(self) {
std::thread::Builder::new()
.name("raw-index".to_string())
.spawn(move || {
loop {
match self.pass() {
Ok(s)
if PassSummary {
skipped_by_age: 0,
..s
} != PassSummary::default() =>
{
tracing::info!(
"raw index: {} transcripts, {} archived memories, {} passages, {} vectors restored, {} sources failed, {} passages older than {} days skipped, {} deleted passages withdrawn",
s.transcripts,
s.memories,
s.passages,
s.restored,
s.failed,
s.skipped_by_age,
self.coverage.max_age_days(),
s.purged
);
residency::release_free_heap();
}
Ok(_) => {}
Err(e) => {
tracing::warn!("raw index pass failed: {e:#}");
residency::release_free_heap();
}
}
std::thread::sleep(PASS_INTERVAL);
}
})
.expect("spawning the raw index thread");
}
pub fn pass(&self) -> Result<PassSummary> {
let mut summary = PassSummary::default();
match purge_deleted(Some(&self.index), self.archive.as_deref(), &self.store, &[]) {
Ok(purge) => summary.purged = purge.passages,
Err(e) => tracing::warn!("raw index: purging deleted sources failed: {e:#}"),
}
let mut first_error = None;
let mut pending = 0;
let cutoff = self.coverage.cutoff();
if let Some(archive) = &self.archive {
let mut queue = Vec::new();
let mut aged = HashMap::new();
for entry in archive.entries() {
let origin = RawOrigin::Transcript {
sha256: entry.sha256.clone(),
};
if self.indexed(&origin) {
continue;
}
match self.date_transcript(archive, &entry) {
Ok(dated) if dated.newest < cutoff => {
summary.skipped_by_age += dated.passages;
aged.insert(entry.sha256.clone(), dated);
}
Ok(dated) => queue.push((dated.newest, entry, origin)),
Err(e) => {
summary.failed += 1;
first_error.get_or_insert(e);
}
}
}
*self.coverage.aged() = aged;
queue.sort_by_key(|q| std::cmp::Reverse(q.0));
for (_, entry, origin) in queue {
match self.add_transcript(archive, &entry, &origin) {
Ok(added) => {
summary.passages += added;
summary.transcripts += 1;
pending = self.commit_every(pending + 1)?;
}
Err(e) => {
summary.failed += 1;
first_error.get_or_insert(e);
}
}
}
}
let raws = self.store.archived_raws_since(cutoff)?;
summary.skipped_by_age += raws
.older
.into_iter()
.filter(|&id| !self.indexed(&RawOrigin::Memory { id }))
.count();
for id in raws.recent {
let origin = RawOrigin::Memory { id };
if self.indexed(&origin) {
continue;
}
match self.add_memory(id, &origin) {
Ok((added, restored)) => {
summary.passages += added;
summary.restored += usize::from(restored);
summary.memories += 1;
pending = self.commit_every(pending + 1)?;
}
Err(e) => {
summary.failed += 1;
first_error.get_or_insert(e);
}
}
}
if pending > 0 {
self.write().commit()?;
}
self.coverage
.skipped_by_age
.store(summary.skipped_by_age, Ordering::Relaxed);
if let Some(e) = first_error {
tracing::warn!(
"raw index: {} sources failed and will be retried, first: {e:#}",
summary.failed
);
}
Ok(summary)
}
fn date_transcript(&self, archive: &TranscriptArchive, entry: &ArchiveEntry) -> Result<Dated> {
if let Some(dated) = self.coverage.aged().get(&entry.sha256) {
return Ok(*dated);
}
let source = Path::new(&entry.source);
let passages = match archive.read(source)? {
Some(content) => raw_passages(source, &content),
None => Vec::new(),
};
Ok(Dated {
newest: passages
.iter()
.filter_map(|p| p.at)
.max()
.unwrap_or(entry.archived_at),
passages: passages.len(),
})
}
fn add_transcript(
&self,
archive: &TranscriptArchive,
entry: &ArchiveEntry,
origin: &RawOrigin,
) -> Result<usize> {
let source = Path::new(&entry.source);
let Some(content) = archive.read(source)? else {
return self.add(origin, Vec::new(), Some((archive, entry)));
};
let project = project_path::from_transcript(&entry.source);
let passages = raw_passages(source, &content)
.into_iter()
.map(|p| RawPassage {
text: p.text,
at: p.at,
project: project.clone(),
memory: p.memory,
})
.collect();
self.add(origin, passages, Some((archive, entry)))
}
fn add_memory(&self, id: Uuid, origin: &RawOrigin) -> Result<(usize, bool)> {
let Some(memory) = self.store.get_memory_full(id)? else {
return Ok((0, false));
};
let mut restored = false;
let embedding = if memory.embedding.len() == EMBEDDING_DIM {
memory.embedding.clone()
} else {
let embedding = self
.embed_and_yield(&[memory.content.as_str()])?
.into_iter()
.next()
.unwrap_or_default();
if embedding.len() == EMBEDDING_DIM {
restored = self.store.restore_archived_embedding(id, &embedding)?;
}
embedding
};
let passage = RawPassage {
text: memory.content.clone(),
at: memory.session_at,
project: memory.project_path.clone(),
memory: Some(id),
};
let added = self.write().add_source(origin, &[(passage, embedding)])?;
Ok((added, restored))
}
fn add(
&self,
origin: &RawOrigin,
passages: Vec<RawPassage>,
archived: Option<(&TranscriptArchive, &ArchiveEntry)>,
) -> Result<usize> {
let (unindexed, generation) = {
let index = self.read();
let unindexed: Vec<RawPassage> = passages
.into_iter()
.filter(|p| !index.has_text(&p.text))
.collect();
(unindexed, index.purge_generation())
};
let mut seen = HashSet::new();
let mut passages = Vec::with_capacity(unindexed.len());
for passage in unindexed {
if let Some(id) = passage.memory
&& self.store.tombstone_exists(id)?
{
continue;
}
if seen.insert(passage.text.trim().to_string()) {
passages.push(passage);
}
}
let mut embedded = Vec::with_capacity(passages.len());
for batch in passages.chunks(EMBED_BATCH) {
let texts: Vec<&str> = batch.iter().map(|p| p.text.as_str()).collect();
let vectors = self.embed_and_yield(&texts)?;
embedded.extend(batch.iter().cloned().zip(vectors));
}
let mut index = self.write();
if let Some((archive, entry)) = archived
&& archive
.entry(Path::new(&entry.source))
.is_none_or(|held| held.sha256 != entry.sha256)
{
return Ok(0);
}
if index.purge_generation() != generation {
return Ok(0);
}
index.add_source(origin, &embedded)
}
fn embed_and_yield(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
let vectors = (self.embed)(texts);
std::thread::sleep(self.pause);
vectors
}
fn commit_every(&self, pending: usize) -> Result<usize> {
if pending < COMMIT_EVERY {
return Ok(pending);
}
self.write().commit()?;
Ok(0)
}
fn indexed(&self, origin: &RawOrigin) -> bool {
self.read().has_source(origin)
}
fn read(&self) -> RwLockReadGuard<'_, RawIndex> {
read_lock(&self.index)
}
fn write(&self) -> RwLockWriteGuard<'_, RawIndex> {
write_lock(&self.index)
}
}