1use std::collections::{HashMap, HashSet};
2use std::path::Path;
3use std::sync::atomic::{AtomicUsize, Ordering};
4use std::sync::{Arc, Mutex, RwLock};
5use std::time::Duration;
6
7use anyhow::Result;
8use chrono::{DateTime, Utc};
9use second_brain_adapters::raw_text::raw_passages;
10use second_brain_core::index::EMBEDDING_DIM;
11use second_brain_core::index::raw::{RawIndex, RawOrigin, RawPassage};
12use second_brain_core::project_path;
13use second_brain_core::residency;
14use second_brain_core::sqlite_store::SqliteStore;
15use second_brain_core::store::Store;
16use second_brain_core::transcript_archive::{ArchiveEntry, TranscriptArchive};
17use uuid::Uuid;
18
19const EMBED_BATCH: usize = 8;
20const COMMIT_EVERY: usize = 20;
21pub const PAUSE_AFTER_EMBED: Duration = Duration::from_millis(100);
25const PASS_INTERVAL: Duration = Duration::from_secs(600);
26
27pub fn open_or_rebuild(dir: &Path) -> Result<RawIndex> {
34 match RawIndex::open(dir) {
35 Ok(index) => Ok(index),
36 Err(e)
37 if e.chain()
38 .any(|c| c.downcast_ref::<std::io::Error>().is_some()) =>
39 {
40 Err(e)
41 }
42 Err(e) => {
43 tracing::warn!(
44 "raw index at {} cannot be opened, rebuilding it: {e:#}",
45 dir.display()
46 );
47 std::fs::remove_dir_all(dir)?;
48 RawIndex::open(dir)
49 }
50 }
51}
52
53pub type EmbedBatch = dyn Fn(&[&str]) -> Result<Vec<Vec<f32>>> + Send + Sync;
54
55#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
56pub struct PassSummary {
57 pub transcripts: usize,
58 pub memories: usize,
59 pub passages: usize,
60 pub restored: usize,
61 pub failed: usize,
63 pub skipped_by_age: usize,
65}
66
67pub struct RawCoverage {
71 max_age_days: u32,
72 skipped_by_age: AtomicUsize,
73 aged: Mutex<HashMap<String, Dated>>,
74}
75
76#[derive(Clone, Copy)]
77struct Dated {
78 newest: DateTime<Utc>,
79 passages: usize,
80}
81
82impl RawCoverage {
83 pub fn new(max_age_days: u32) -> Self {
84 Self {
85 max_age_days,
86 skipped_by_age: AtomicUsize::new(0),
87 aged: Mutex::new(HashMap::new()),
88 }
89 }
90
91 pub fn max_age_days(&self) -> u32 {
92 self.max_age_days
93 }
94
95 pub fn skipped_by_age(&self) -> usize {
97 self.skipped_by_age.load(Ordering::Relaxed)
98 }
99
100 fn cutoff(&self) -> DateTime<Utc> {
101 Utc::now()
102 .checked_sub_signed(chrono::Duration::days(i64::from(self.max_age_days)))
103 .unwrap_or(DateTime::<Utc>::MIN_UTC)
104 }
105
106 fn aged(&self) -> std::sync::MutexGuard<'_, HashMap<String, Dated>> {
107 self.aged.lock().unwrap_or_else(|p| p.into_inner())
108 }
109}
110
111pub struct RawIndexer {
115 pub index: Arc<RwLock<RawIndex>>,
116 pub archive: Option<Arc<TranscriptArchive>>,
117 pub store: Arc<SqliteStore>,
118 pub embed: Arc<EmbedBatch>,
119 pub pause: Duration,
120 pub coverage: Arc<RawCoverage>,
121}
122
123impl RawIndexer {
124 pub fn spawn(self) {
125 std::thread::Builder::new()
126 .name("raw-index".to_string())
127 .spawn(move || {
128 loop {
129 match self.pass() {
130 Ok(s)
131 if PassSummary {
132 skipped_by_age: 0,
133 ..s
134 } != PassSummary::default() =>
135 {
136 tracing::info!(
137 "raw index: {} transcripts, {} archived memories, {} passages, {} vectors restored, {} sources failed, {} passages older than {} days skipped",
138 s.transcripts,
139 s.memories,
140 s.passages,
141 s.restored,
142 s.failed,
143 s.skipped_by_age,
144 self.coverage.max_age_days()
145 );
146 residency::release_free_heap();
147 }
148 Ok(_) => {}
149 Err(e) => {
150 tracing::warn!("raw index pass failed: {e:#}");
151 residency::release_free_heap();
152 }
153 }
154 std::thread::sleep(PASS_INTERVAL);
155 }
156 })
157 .expect("spawning the raw index thread");
158 }
159
160 pub fn pass(&self) -> Result<PassSummary> {
165 let mut summary = PassSummary::default();
166 let mut first_error = None;
167 let mut pending = 0;
168 let cutoff = self.coverage.cutoff();
169 if let Some(archive) = &self.archive {
170 let mut queue = Vec::new();
171 let mut aged = HashMap::new();
172 for entry in archive.entries() {
173 let origin = RawOrigin::Transcript {
174 sha256: entry.sha256.clone(),
175 };
176 if self.indexed(&origin) {
177 continue;
178 }
179 match self.date_transcript(archive, &entry) {
180 Ok(dated) if dated.newest < cutoff => {
181 summary.skipped_by_age += dated.passages;
182 aged.insert(entry.sha256.clone(), dated);
183 }
184 Ok(dated) => queue.push((dated.newest, entry, origin)),
185 Err(e) => {
186 summary.failed += 1;
187 first_error.get_or_insert(e);
188 }
189 }
190 }
191 *self.coverage.aged() = aged;
192 queue.sort_by_key(|q| std::cmp::Reverse(q.0));
193 for (_, entry, origin) in queue {
194 match self.add_transcript(archive, &entry, &origin) {
195 Ok(added) => {
196 summary.passages += added;
197 summary.transcripts += 1;
198 pending = self.commit_every(pending + 1)?;
199 }
200 Err(e) => {
201 summary.failed += 1;
202 first_error.get_or_insert(e);
203 }
204 }
205 }
206 }
207
208 let raws = self.store.archived_raws_since(cutoff)?;
209 summary.skipped_by_age += raws
210 .older
211 .into_iter()
212 .filter(|&id| !self.indexed(&RawOrigin::Memory { id }))
213 .count();
214 for id in raws.recent {
215 let origin = RawOrigin::Memory { id };
216 if self.indexed(&origin) {
217 continue;
218 }
219 match self.add_memory(id, &origin) {
220 Ok((added, restored)) => {
221 summary.passages += added;
222 summary.restored += usize::from(restored);
223 summary.memories += 1;
224 pending = self.commit_every(pending + 1)?;
225 }
226 Err(e) => {
227 summary.failed += 1;
228 first_error.get_or_insert(e);
229 }
230 }
231 }
232 if pending > 0 {
233 self.write().commit()?;
234 }
235 self.coverage
236 .skipped_by_age
237 .store(summary.skipped_by_age, Ordering::Relaxed);
238 if let Some(e) = first_error {
239 tracing::warn!(
240 "raw index: {} sources failed and will be retried, first: {e:#}",
241 summary.failed
242 );
243 }
244 Ok(summary)
245 }
246
247 fn date_transcript(&self, archive: &TranscriptArchive, entry: &ArchiveEntry) -> Result<Dated> {
251 if let Some(dated) = self.coverage.aged().get(&entry.sha256) {
252 return Ok(*dated);
253 }
254 let source = Path::new(&entry.source);
255 let passages = match archive.read(source)? {
256 Some(content) => raw_passages(source, &content),
257 None => Vec::new(),
258 };
259 Ok(Dated {
260 newest: passages
261 .iter()
262 .filter_map(|p| p.at)
263 .max()
264 .unwrap_or(entry.archived_at),
265 passages: passages.len(),
266 })
267 }
268
269 fn add_transcript(
270 &self,
271 archive: &TranscriptArchive,
272 entry: &ArchiveEntry,
273 origin: &RawOrigin,
274 ) -> Result<usize> {
275 let source = Path::new(&entry.source);
276 let Some(content) = archive.read(source)? else {
277 return self.add(origin, Vec::new());
278 };
279 let project = project_path::from_transcript(&entry.source);
280 let passages = raw_passages(source, &content)
281 .into_iter()
282 .map(|p| RawPassage {
283 text: p.text,
284 at: p.at,
285 project: project.clone(),
286 memory: p.memory,
287 })
288 .collect();
289 self.add(origin, passages)
290 }
291
292 fn add_memory(&self, id: Uuid, origin: &RawOrigin) -> Result<(usize, bool)> {
293 let Some(memory) = self.store.get_memory_with_embedding(id)? else {
294 return Ok((0, false));
295 };
296 let mut restored = false;
297 let embedding = if memory.embedding.len() == EMBEDDING_DIM {
298 memory.embedding.clone()
299 } else {
300 let embedding = self
301 .embed_and_yield(&[memory.content.as_str()])?
302 .into_iter()
303 .next()
304 .unwrap_or_default();
305 if embedding.len() == EMBEDDING_DIM {
306 restored = self.store.restore_archived_embedding(id, &embedding)?;
307 }
308 embedding
309 };
310 let passage = RawPassage {
311 text: memory.content.clone(),
312 at: memory.session_at,
313 project: memory.project_path.clone(),
314 memory: Some(id),
315 };
316 let added = self.write().add_source(origin, &[(passage, embedding)])?;
317 Ok((added, restored))
318 }
319
320 fn add(&self, origin: &RawOrigin, passages: Vec<RawPassage>) -> Result<usize> {
323 let passages: Vec<RawPassage> = {
324 let index = self.read();
325 let mut seen = HashSet::new();
326 passages
327 .into_iter()
328 .filter(|p| seen.insert(p.text.trim().to_string()) && !index.has_text(&p.text))
329 .collect()
330 };
331 let mut embedded = Vec::with_capacity(passages.len());
332 for batch in passages.chunks(EMBED_BATCH) {
333 let texts: Vec<&str> = batch.iter().map(|p| p.text.as_str()).collect();
334 let vectors = self.embed_and_yield(&texts)?;
335 embedded.extend(batch.iter().cloned().zip(vectors));
336 }
337 self.write().add_source(origin, &embedded)
338 }
339
340 fn embed_and_yield(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
341 let vectors = (self.embed)(texts);
342 std::thread::sleep(self.pause);
343 vectors
344 }
345
346 fn commit_every(&self, pending: usize) -> Result<usize> {
347 if pending < COMMIT_EVERY {
348 return Ok(pending);
349 }
350 self.write().commit()?;
351 Ok(0)
352 }
353
354 fn indexed(&self, origin: &RawOrigin) -> bool {
355 self.read().has_source(origin)
356 }
357
358 fn read(&self) -> std::sync::RwLockReadGuard<'_, RawIndex> {
359 self.index.read().unwrap_or_else(|p| p.into_inner())
360 }
361
362 fn write(&self) -> std::sync::RwLockWriteGuard<'_, RawIndex> {
363 self.index.write().unwrap_or_else(|p| p.into_inner())
364 }
365}