1use std::collections::{HashMap, HashSet};
2use std::path::Path;
3use std::sync::atomic::{AtomicUsize, Ordering};
4use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
5use std::time::Duration;
6
7use anyhow::Result;
8use chrono::{DateTime, Utc};
9use second_brain_adapters::raw_text::raw_passages;
10use second_brain_core::index::EMBEDDING_DIM;
11use second_brain_core::index::raw::{RawHit, RawIndex, RawOrigin, RawPassage, text_hash};
12use second_brain_core::project_path;
13use second_brain_core::residency;
14use second_brain_core::sealed::{SealError, SealKeys, seal_error};
15use second_brain_core::sqlite_store::SqliteStore;
16use second_brain_core::transcript_archive::{ArchiveEntry, TranscriptArchive};
17use uuid::Uuid;
18
19const EMBED_BATCH: usize = 8;
20const COMMIT_EVERY: usize = 20;
21pub const PAUSE_AFTER_EMBED: Duration = Duration::from_millis(100);
25const PASS_INTERVAL: Duration = Duration::from_secs(600);
26
27pub fn open_or_rebuild(dir: &Path) -> Result<RawIndex> {
34 open_or_rebuild_keyed(dir, None)
35}
36
37pub fn open_or_rebuild_keyed(dir: &Path, keys: Option<&SealKeys>) -> Result<RawIndex> {
41 let open = || match keys {
42 Some(keys) => RawIndex::open_keyed(dir, keys),
43 None => RawIndex::open(dir),
44 };
45 match open() {
46 Ok(index) => Ok(index),
47 Err(e) if seal_error(&e) == Some(&SealError::KeyRequired) => Err(e),
48 Err(e)
49 if seal_error(&e).is_none()
50 && e.chain()
51 .any(|c| c.downcast_ref::<std::io::Error>().is_some()) =>
52 {
53 Err(e)
54 }
55 Err(e) => {
56 tracing::warn!(
57 "raw index at {} cannot be opened, rebuilding it: {e:#}",
58 dir.display()
59 );
60 std::fs::remove_dir_all(dir)?;
61 open()
62 }
63 }
64}
65
66pub type EmbedBatch = dyn Fn(&[&str]) -> Result<Vec<Vec<f32>>> + Send + Sync;
67
68#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
69pub struct PassSummary {
70 pub transcripts: usize,
71 pub memories: usize,
72 pub passages: usize,
73 pub restored: usize,
74 pub failed: usize,
76 pub skipped_by_age: usize,
78 pub purged: usize,
80}
81
82pub struct RawCoverage {
86 max_age_days: u32,
87 skipped_by_age: AtomicUsize,
88 aged: Mutex<HashMap<String, Dated>>,
89}
90
91#[derive(Clone, Copy)]
92struct Dated {
93 newest: DateTime<Utc>,
94 passages: usize,
95}
96
97impl RawCoverage {
98 pub fn new(max_age_days: u32) -> Self {
99 Self {
100 max_age_days,
101 skipped_by_age: AtomicUsize::new(0),
102 aged: Mutex::new(HashMap::new()),
103 }
104 }
105
106 pub fn max_age_days(&self) -> u32 {
107 self.max_age_days
108 }
109
110 pub fn skipped_by_age(&self) -> usize {
112 self.skipped_by_age.load(Ordering::Relaxed)
113 }
114
115 fn cutoff(&self) -> DateTime<Utc> {
116 Utc::now()
117 .checked_sub_signed(chrono::Duration::days(i64::from(self.max_age_days)))
118 .unwrap_or(DateTime::<Utc>::MIN_UTC)
119 }
120
121 fn aged(&self) -> std::sync::MutexGuard<'_, HashMap<String, Dated>> {
122 self.aged.lock().unwrap_or_else(|p| p.into_inner())
123 }
124}
125
126#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
128pub struct RawPurge {
129 pub passages: usize,
130 pub archive_entries: usize,
131}
132
133static PURGES: Mutex<()> = Mutex::new(());
134
135pub fn purge_deleted(
148 index: Option<&RwLock<RawIndex>>,
149 archive: Option<&TranscriptArchive>,
150 store: &SqliteStore,
151 memories: &[Uuid],
152) -> Result<RawPurge> {
153 let _one_at_a_time = PURGES.lock().unwrap_or_else(|p| p.into_inner());
154 let doomed = match archive {
155 Some(archive) => doomed_entries(archive, store)?,
156 None => Vec::new(),
157 };
158 let mut gone = Gone::new(store, memories);
159 match index {
160 Some(index) => purge_index(index, archive, &doomed, &mut gone),
161 None => Ok(RawPurge {
162 passages: 0,
163 archive_entries: match archive {
164 Some(archive) => remove_entries(archive, &doomed)?,
165 None => 0,
166 },
167 }),
168 }
169}
170
171fn doomed_entries(archive: &TranscriptArchive, store: &SqliteStore) -> Result<Vec<ArchiveEntry>> {
172 let mut doomed = Vec::new();
173 for entry in archive.entries() {
174 if store.session_deleted(&entry.source)? {
175 doomed.push(entry);
176 }
177 }
178 Ok(doomed)
179}
180
181fn purge_index(
182 lock: &RwLock<RawIndex>,
183 archive: Option<&TranscriptArchive>,
184 doomed: &[ArchiveEntry],
185 gone: &mut Gone<'_>,
186) -> Result<RawPurge> {
187 let (memories, recorded_sources) = {
188 let index = read_lock(lock);
189 (index.passage_memories()?, index.sources()?)
190 };
191 let live = live_entries(archive, doomed);
192 let live_shas: HashSet<&str> = live.iter().map(|e| e.sha256.as_str()).collect();
193 let mut stale = Vec::new();
194 for id in memories {
195 if gone.check(id)? {
196 stale.push(id);
197 }
198 }
199 let mut sources: Vec<RawOrigin> = doomed
200 .iter()
201 .filter(|e| !live_shas.contains(e.sha256.as_str()))
202 .map(|e| RawOrigin::Transcript {
203 sha256: e.sha256.clone(),
204 })
205 .collect();
206 for origin in recorded_sources {
207 if let RawOrigin::Memory { id } = origin
208 && gone.check(id)?
209 {
210 sources.push(origin);
211 }
212 }
213 if doomed.is_empty() && sources.is_empty() && stale.is_empty() {
214 return Ok(RawPurge::default());
215 }
216 let texts = transcript_texts(archive, doomed)?;
217 let texts: Vec<&str> = texts.iter().map(String::as_str).collect();
218 let doomed_sources: HashSet<String> = sources.iter().map(RawOrigin::source_key).collect();
219
220 let (candidates, recorded) = {
221 let index = read_lock(lock);
222 (
223 index.matching(&sources, &stale, &texts)?,
224 recorded_entries(&index, &live),
225 )
226 };
227 let mut holders = Holders::default();
228 if let Some(archive) = archive
229 && !candidates.is_empty()
230 {
231 holders.read(archive, &recorded, &candidates, gone)?;
232 }
233
234 let mut index = write_lock(lock);
235 index.commit_without_merging()?;
236 let current = index.matching(&sources, &stale, &texts)?;
237 if let Some(archive) = archive
238 && !current.is_empty()
239 {
240 let late: Vec<ArchiveEntry> = recorded_entries(&index, &live)
241 .into_iter()
242 .filter(|e| !holders.has_read(e))
243 .collect();
244 holders.read(archive, &late, ¤t, gone)?;
245 }
246 let mut dropped = Vec::new();
247 for hit in ¤t {
248 let memory_gone = match hit.memory {
249 Some(id) => gone.check(id)?,
250 None => false,
251 };
252 let memory_holder = match hit.memory {
253 Some(id) => !memory_gone && index.has_source(&RawOrigin::Memory { id }),
254 None => false,
255 };
256 match fate(hit, &doomed_sources, &holders, memory_gone, memory_holder) {
257 Fate::Keep => {}
258 Fate::Move(holder) => index.rekey(
259 hit.key,
260 &holder.origin,
261 &RawPassage {
262 text: holder.text,
263 at: holder.at,
264 project: holder.project,
265 memory: holder.memory,
266 },
267 )?,
268 Fate::Drop => dropped.push(hit.key),
269 }
270 }
271 index.withdraw(&dropped)?;
272 index.forget_sources(&sources)?;
273 let rewrite = index.seal_for_rewrite(&dropped)?;
274 index.commit_without_merging()?;
275 let archive_entries = match archive {
276 Some(archive) => remove_entries(archive, doomed)?,
277 None => 0,
278 };
279 let compaction = index.start_compaction()?;
280 drop(index);
281
282 let rewritten = rewrite.run();
283 let merged = compaction.wait();
284 let mut index = write_lock(lock);
285 index.reload_sealed(&rewrite)?;
286 index.finish_compaction(merged)?;
287 rewritten?;
288 Ok(RawPurge {
289 passages: dropped.len(),
290 archive_entries,
291 })
292}
293
294fn live_entries(archive: Option<&TranscriptArchive>, doomed: &[ArchiveEntry]) -> Vec<ArchiveEntry> {
295 let Some(archive) = archive else {
296 return Vec::new();
297 };
298 let doomed: HashSet<&str> = doomed.iter().map(|e| e.source.as_str()).collect();
299 archive
300 .entries()
301 .into_iter()
302 .filter(|e| !doomed.contains(e.source.as_str()))
303 .collect()
304}
305
306fn transcript_texts(
307 archive: Option<&TranscriptArchive>,
308 entries: &[ArchiveEntry],
309) -> Result<Vec<String>> {
310 let Some(archive) = archive else {
311 return Ok(Vec::new());
312 };
313 let mut texts = Vec::new();
314 for entry in entries {
315 let source = Path::new(&entry.source);
316 if let Some(content) = archive.read(source)? {
317 texts.extend(raw_passages(source, &content).into_iter().map(|p| p.text));
318 }
319 }
320 Ok(texts)
321}
322
323fn recorded_entries(index: &RawIndex, entries: &[ArchiveEntry]) -> Vec<ArchiveEntry> {
327 entries
328 .iter()
329 .filter(|e| {
330 index.has_source(&RawOrigin::Transcript {
331 sha256: e.sha256.clone(),
332 })
333 })
334 .cloned()
335 .collect()
336}
337
338struct Gone<'a> {
340 store: &'a SqliteStore,
341 known: HashMap<Uuid, bool>,
342}
343
344impl<'a> Gone<'a> {
345 fn new(store: &'a SqliteStore, just_deleted: &[Uuid]) -> Self {
346 Self {
347 store,
348 known: just_deleted.iter().map(|id| (*id, true)).collect(),
349 }
350 }
351
352 fn check(&mut self, id: Uuid) -> Result<bool> {
353 if let Some(gone) = self.known.get(&id) {
354 return Ok(*gone);
355 }
356 let gone = self.store.tombstone_exists(id)?;
357 self.known.insert(id, gone);
358 Ok(gone)
359 }
360}
361
362#[derive(Clone)]
365struct Holder {
366 origin: RawOrigin,
367 text: String,
368 project: Option<String>,
369 at: Option<DateTime<Utc>>,
370 memory: Option<Uuid>,
371}
372
373#[derive(Default)]
376struct Holders {
377 by_hash: HashMap<String, Vec<Holder>>,
378 read: HashSet<String>,
379}
380
381impl Holders {
382 fn has_read(&self, entry: &ArchiveEntry) -> bool {
383 self.read.contains(&entry.sha256)
384 }
385
386 fn read(
387 &mut self,
388 archive: &TranscriptArchive,
389 entries: &[ArchiveEntry],
390 hits: &[RawHit],
391 gone: &mut Gone<'_>,
392 ) -> Result<()> {
393 let wanted: HashSet<&str> = hits.iter().map(|h| h.text_hash.as_str()).collect();
394 for entry in entries {
395 if !self.read.insert(entry.sha256.clone()) {
396 continue;
397 }
398 let source = Path::new(&entry.source);
399 let Some(content) = archive.read(source)? else {
400 continue;
401 };
402 let project = project_path::from_transcript(&entry.source);
403 for passage in raw_passages(source, &content) {
404 let hash = text_hash(&passage.text);
405 if !wanted.contains(hash.as_str()) {
406 continue;
407 }
408 if let Some(id) = passage.memory
409 && gone.check(id)?
410 {
411 continue;
412 }
413 self.by_hash.entry(hash).or_default().push(Holder {
414 origin: RawOrigin::Transcript {
415 sha256: entry.sha256.clone(),
416 },
417 text: passage.text,
418 project: project.clone(),
419 at: passage.at,
420 memory: passage.memory,
421 });
422 }
423 }
424 Ok(())
425 }
426}
427
428enum Fate {
429 Keep,
430 Move(Holder),
431 Drop,
432}
433
434fn fate(
442 hit: &RawHit,
443 doomed_sources: &HashSet<String>,
444 holders: &Holders,
445 memory_gone: bool,
446 memory_holder: bool,
447) -> Fate {
448 if matches!(hit.origin, RawOrigin::Memory { .. })
449 && !doomed_sources.contains(&hit.origin.source_key())
450 {
451 return Fate::Keep;
452 }
453 if let Some(held) = holders.by_hash.get(&hit.text_hash) {
454 if let Some(own) = held.iter().find(|h| h.origin == hit.origin) {
455 return if memory_gone {
456 Fate::Move(own.clone())
457 } else {
458 Fate::Keep
459 };
460 }
461 if let Some(holder) = held.first() {
462 return Fate::Move(holder.clone());
463 }
464 }
465 match hit.memory {
466 Some(id) if memory_holder => Fate::Move(Holder {
467 origin: RawOrigin::Memory { id },
468 text: hit.text.clone(),
469 project: hit.project.clone(),
470 at: hit.at,
471 memory: Some(id),
472 }),
473 _ => Fate::Drop,
474 }
475}
476
477fn read_lock(index: &RwLock<RawIndex>) -> RwLockReadGuard<'_, RawIndex> {
478 index.read().unwrap_or_else(|p| p.into_inner())
479}
480
481fn write_lock(index: &RwLock<RawIndex>) -> RwLockWriteGuard<'_, RawIndex> {
482 index.write().unwrap_or_else(|p| p.into_inner())
483}
484
485fn remove_entries(archive: &TranscriptArchive, doomed: &[ArchiveEntry]) -> Result<usize> {
486 if doomed.is_empty() {
487 return Ok(0);
488 }
489 let sources: HashSet<&str> = doomed.iter().map(|e| e.source.as_str()).collect();
490 Ok(archive
491 .remove_where(|e| sources.contains(e.source.as_str()))?
492 .len())
493}
494
495pub struct RawIndexer {
499 pub index: Arc<RwLock<RawIndex>>,
500 pub archive: Option<Arc<TranscriptArchive>>,
501 pub store: Arc<SqliteStore>,
502 pub embed: Arc<EmbedBatch>,
503 pub pause: Duration,
504 pub coverage: Arc<RawCoverage>,
505}
506
507impl RawIndexer {
508 pub fn spawn(self) {
509 std::thread::Builder::new()
510 .name("raw-index".to_string())
511 .spawn(move || {
512 loop {
513 match self.pass() {
514 Ok(s)
515 if PassSummary {
516 skipped_by_age: 0,
517 ..s
518 } != PassSummary::default() =>
519 {
520 tracing::info!(
521 "raw index: {} transcripts, {} archived memories, {} passages, {} vectors restored, {} sources failed, {} passages older than {} days skipped, {} deleted passages withdrawn",
522 s.transcripts,
523 s.memories,
524 s.passages,
525 s.restored,
526 s.failed,
527 s.skipped_by_age,
528 self.coverage.max_age_days(),
529 s.purged
530 );
531 residency::release_free_heap();
532 }
533 Ok(_) => {}
534 Err(e) => {
535 tracing::warn!("raw index pass failed: {e:#}");
536 residency::release_free_heap();
537 }
538 }
539 std::thread::sleep(PASS_INTERVAL);
540 }
541 })
542 .expect("spawning the raw index thread");
543 }
544
545 pub fn pass(&self) -> Result<PassSummary> {
550 let mut summary = PassSummary::default();
551 match purge_deleted(Some(&self.index), self.archive.as_deref(), &self.store, &[]) {
552 Ok(purge) => summary.purged = purge.passages,
553 Err(e) => tracing::warn!("raw index: purging deleted sources failed: {e:#}"),
554 }
555 let mut first_error = None;
556 let mut pending = 0;
557 let cutoff = self.coverage.cutoff();
558 if let Some(archive) = &self.archive {
559 let mut queue = Vec::new();
560 let mut aged = HashMap::new();
561 for entry in archive.entries() {
562 let origin = RawOrigin::Transcript {
563 sha256: entry.sha256.clone(),
564 };
565 if self.indexed(&origin) {
566 continue;
567 }
568 match self.date_transcript(archive, &entry) {
569 Ok(dated) if dated.newest < cutoff => {
570 summary.skipped_by_age += dated.passages;
571 aged.insert(entry.sha256.clone(), dated);
572 }
573 Ok(dated) => queue.push((dated.newest, entry, origin)),
574 Err(e) => {
575 summary.failed += 1;
576 first_error.get_or_insert(e);
577 }
578 }
579 }
580 *self.coverage.aged() = aged;
581 queue.sort_by_key(|q| std::cmp::Reverse(q.0));
582 for (_, entry, origin) in queue {
583 match self.add_transcript(archive, &entry, &origin) {
584 Ok(added) => {
585 summary.passages += added;
586 summary.transcripts += 1;
587 pending = self.commit_every(pending + 1)?;
588 }
589 Err(e) => {
590 summary.failed += 1;
591 first_error.get_or_insert(e);
592 }
593 }
594 }
595 }
596
597 let raws = self.store.archived_raws_since(cutoff)?;
598 summary.skipped_by_age += raws
599 .older
600 .into_iter()
601 .filter(|&id| !self.indexed(&RawOrigin::Memory { id }))
602 .count();
603 for id in raws.recent {
604 let origin = RawOrigin::Memory { id };
605 if self.indexed(&origin) {
606 continue;
607 }
608 match self.add_memory(id, &origin) {
609 Ok((added, restored)) => {
610 summary.passages += added;
611 summary.restored += usize::from(restored);
612 summary.memories += 1;
613 pending = self.commit_every(pending + 1)?;
614 }
615 Err(e) => {
616 summary.failed += 1;
617 first_error.get_or_insert(e);
618 }
619 }
620 }
621 if pending > 0 {
622 self.write().commit()?;
623 }
624 self.coverage
625 .skipped_by_age
626 .store(summary.skipped_by_age, Ordering::Relaxed);
627 if let Some(e) = first_error {
628 tracing::warn!(
629 "raw index: {} sources failed and will be retried, first: {e:#}",
630 summary.failed
631 );
632 }
633 Ok(summary)
634 }
635
636 fn date_transcript(&self, archive: &TranscriptArchive, entry: &ArchiveEntry) -> Result<Dated> {
640 if let Some(dated) = self.coverage.aged().get(&entry.sha256) {
641 return Ok(*dated);
642 }
643 let source = Path::new(&entry.source);
644 let passages = match archive.read(source)? {
645 Some(content) => raw_passages(source, &content),
646 None => Vec::new(),
647 };
648 Ok(Dated {
649 newest: passages
650 .iter()
651 .filter_map(|p| p.at)
652 .max()
653 .unwrap_or(entry.archived_at),
654 passages: passages.len(),
655 })
656 }
657
658 fn add_transcript(
659 &self,
660 archive: &TranscriptArchive,
661 entry: &ArchiveEntry,
662 origin: &RawOrigin,
663 ) -> Result<usize> {
664 let source = Path::new(&entry.source);
665 let Some(content) = archive.read(source)? else {
666 return self.add(origin, Vec::new(), Some((archive, entry)));
667 };
668 let project = project_path::from_transcript(&entry.source);
669 let passages = raw_passages(source, &content)
670 .into_iter()
671 .map(|p| RawPassage {
672 text: p.text,
673 at: p.at,
674 project: project.clone(),
675 memory: p.memory,
676 })
677 .collect();
678 self.add(origin, passages, Some((archive, entry)))
679 }
680
681 fn add_memory(&self, id: Uuid, origin: &RawOrigin) -> Result<(usize, bool)> {
682 let Some(memory) = self.store.get_memory_full(id)? else {
683 return Ok((0, false));
684 };
685 let mut restored = false;
686 let embedding = if memory.embedding.len() == EMBEDDING_DIM {
687 memory.embedding.clone()
688 } else {
689 let embedding = self
690 .embed_and_yield(&[memory.content.as_str()])?
691 .into_iter()
692 .next()
693 .unwrap_or_default();
694 if embedding.len() == EMBEDDING_DIM {
695 restored = self.store.restore_archived_embedding(id, &embedding)?;
696 }
697 embedding
698 };
699 let passage = RawPassage {
700 text: memory.content.clone(),
701 at: memory.session_at,
702 project: memory.project_path.clone(),
703 memory: Some(id),
704 };
705 let added = self.write().add_source(origin, &[(passage, embedding)])?;
706 Ok((added, restored))
707 }
708
709 fn add(
718 &self,
719 origin: &RawOrigin,
720 passages: Vec<RawPassage>,
721 archived: Option<(&TranscriptArchive, &ArchiveEntry)>,
722 ) -> Result<usize> {
723 let (unindexed, generation) = {
724 let index = self.read();
725 let unindexed: Vec<RawPassage> = passages
726 .into_iter()
727 .filter(|p| !index.has_text(&p.text))
728 .collect();
729 (unindexed, index.purge_generation())
730 };
731 let mut seen = HashSet::new();
732 let mut passages = Vec::with_capacity(unindexed.len());
733 for passage in unindexed {
734 if let Some(id) = passage.memory
735 && self.store.tombstone_exists(id)?
736 {
737 continue;
738 }
739 if seen.insert(passage.text.trim().to_string()) {
740 passages.push(passage);
741 }
742 }
743 let mut embedded = Vec::with_capacity(passages.len());
744 for batch in passages.chunks(EMBED_BATCH) {
745 let texts: Vec<&str> = batch.iter().map(|p| p.text.as_str()).collect();
746 let vectors = self.embed_and_yield(&texts)?;
747 embedded.extend(batch.iter().cloned().zip(vectors));
748 }
749 let mut index = self.write();
750 if let Some((archive, entry)) = archived
751 && archive
752 .entry(Path::new(&entry.source))
753 .is_none_or(|held| held.sha256 != entry.sha256)
754 {
755 return Ok(0);
756 }
757 if index.purge_generation() != generation {
758 return Ok(0);
759 }
760 index.add_source(origin, &embedded)
761 }
762
763 fn embed_and_yield(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
764 let vectors = (self.embed)(texts);
765 std::thread::sleep(self.pause);
766 vectors
767 }
768
769 fn commit_every(&self, pending: usize) -> Result<usize> {
770 if pending < COMMIT_EVERY {
771 return Ok(pending);
772 }
773 self.write().commit()?;
774 Ok(0)
775 }
776
777 fn indexed(&self, origin: &RawOrigin) -> bool {
778 self.read().has_source(origin)
779 }
780
781 fn read(&self) -> RwLockReadGuard<'_, RawIndex> {
782 read_lock(&self.index)
783 }
784
785 fn write(&self) -> RwLockWriteGuard<'_, RawIndex> {
786 write_lock(&self.index)
787 }
788}