1use codemem_core::{
14 CodememConfig, CodememError, GraphBackend, ScoringWeights, StorageBackend, VectorBackend,
15};
16pub use codemem_storage::graph::GraphEngine;
17pub use codemem_storage::HnswIndex;
18pub use codemem_storage::Storage;
19use std::path::{Path, PathBuf};
20#[cfg(test)]
21use std::sync::atomic::Ordering;
22use std::sync::atomic::{AtomicBool, AtomicI64};
23use std::sync::{Arc, Mutex, OnceLock, RwLock};
24
25pub mod analysis;
26pub mod bm25;
27pub mod compress;
28pub mod consolidation;
29pub mod enrichment;
30mod enrichment_text;
31mod file_indexing;
32mod graph_linking;
33pub mod graph_ops;
34pub mod hooks;
35pub mod index;
36pub mod insights;
37mod memory_ops;
38pub mod metrics;
39pub mod patterns;
40pub mod pca;
41pub mod persistence;
42pub mod recall;
43pub mod review;
44pub mod scoring;
45pub mod search;
46pub mod watch;
47
48#[cfg(test)]
49#[path = "tests/engine_integration_tests.rs"]
50mod integration_tests;
51
52#[cfg(test)]
53#[path = "tests/enrichment_tests.rs"]
54mod enrichment_tests;
55
56#[cfg(test)]
57#[path = "tests/recall_tests.rs"]
58mod recall_tests;
59
60#[cfg(test)]
61#[path = "tests/search_tests.rs"]
62mod search_tests;
63
64#[cfg(test)]
65#[path = "tests/consolidation_tests.rs"]
66mod consolidation_tests;
67
68#[cfg(test)]
69#[path = "tests/analysis_tests.rs"]
70mod analysis_tests;
71
72#[cfg(test)]
73#[path = "tests/persistence_tests.rs"]
74mod persistence_tests;
75
76#[cfg(test)]
77#[path = "tests/memory_expiry_tests.rs"]
78mod memory_expiry_tests;
79
80#[cfg(test)]
81#[path = "tests/scope_tests.rs"]
82mod scope_tests;
83
84#[cfg(test)]
85#[path = "tests/graph_ops_tests.rs"]
86mod graph_ops_tests;
87
88pub use index::{
90 ChunkConfig, CodeChunk, CodeParser, Dependency, IndexAndResolveResult, IndexProgress,
91 IndexResult, Indexer, ManifestResult, ParseResult, Reference, ReferenceKind, ReferenceResolver,
92 ResolvedEdge, Symbol, SymbolKind, Visibility, Workspace,
93};
94
95pub use bm25::Bm25Index;
97pub use metrics::InMemoryMetrics;
98pub use review::{BlastRadiusReport, DiffSymbolMapping, MissingCoChange};
99
100pub use enrichment::{EnrichResult, EnrichmentPipelineResult};
102
103pub use persistence::{edge_weight_for, CrossRepoPersistResult, IndexPersistResult};
105
106pub use recall::{ExpandedResult, NamespaceStats, RecallQuery};
108
109pub use search::{CodeSearchResult, SummaryTreeNode, SymbolSearchResult};
111
112pub use analysis::{
114 DecisionChain, DecisionConnection, DecisionEntry, ImpactResult, SessionCheckpointReport,
115};
116
117#[derive(Debug, Clone)]
119pub struct SplitPart {
120 pub content: String,
121 pub tags: Option<Vec<String>>,
122 pub importance: Option<f64>,
123}
124
125pub struct IndexCache {
129 pub symbols: Vec<Symbol>,
130 pub chunks: Vec<CodeChunk>,
131 pub root_path: String,
132}
133
134pub struct CodememEngine {
148 pub(crate) storage: Box<dyn StorageBackend>,
149 pub(crate) vector: OnceLock<Mutex<Box<dyn VectorBackend>>>,
151 pub(crate) graph: Mutex<Box<dyn GraphBackend>>,
152 pub(crate) embeddings: OnceLock<Option<Mutex<Box<dyn codemem_embeddings::EmbeddingProvider>>>>,
154 pub(crate) db_path: Option<PathBuf>,
156 pub(crate) index_cache: Mutex<Option<IndexCache>>,
158 pub(crate) scoring_weights: RwLock<ScoringWeights>,
160 pub(crate) bm25_index: OnceLock<Mutex<Bm25Index>>,
162 pub(crate) config: CodememConfig,
164 pub(crate) metrics: Arc<InMemoryMetrics>,
166 dirty: AtomicBool,
169 active_session_id: RwLock<Option<String>>,
171 scope: RwLock<Option<codemem_core::ScopeContext>>,
173 change_detector: Mutex<Option<index::incremental::ChangeDetector>>,
176 last_expiry_sweep: AtomicI64,
179}
180
181impl CodememEngine {
182 pub fn new(
184 storage: Box<dyn StorageBackend>,
185 vector: Box<dyn VectorBackend>,
186 graph: Box<dyn GraphBackend>,
187 embeddings: Option<Box<dyn codemem_embeddings::EmbeddingProvider>>,
188 ) -> Self {
189 let config = CodememConfig::load_or_default();
190 Self::new_with_config(storage, vector, graph, embeddings, config)
191 }
192
193 pub fn new_with_config(
195 storage: Box<dyn StorageBackend>,
196 vector: Box<dyn VectorBackend>,
197 graph: Box<dyn GraphBackend>,
198 embeddings: Option<Box<dyn codemem_embeddings::EmbeddingProvider>>,
199 config: CodememConfig,
200 ) -> Self {
201 let vector_lock = OnceLock::new();
202 let _ = vector_lock.set(Mutex::new(vector));
203 let embeddings_lock = OnceLock::new();
204 let _ = embeddings_lock.set(embeddings.map(Mutex::new));
205 let bm25_lock = OnceLock::new();
206 let _ = bm25_lock.set(Mutex::new(Bm25Index::new()));
207 Self {
208 storage,
209 vector: vector_lock,
210 graph: Mutex::new(graph),
211 embeddings: embeddings_lock,
212 db_path: None,
213 index_cache: Mutex::new(None),
214 scoring_weights: RwLock::new(config.scoring.clone()),
215 bm25_index: bm25_lock,
216 config,
217 metrics: Arc::new(InMemoryMetrics::new()),
218 dirty: AtomicBool::new(false),
219 active_session_id: RwLock::new(None),
220 scope: RwLock::new(None),
221 change_detector: Mutex::new(None),
222 last_expiry_sweep: AtomicI64::new(0),
223 }
224 }
225
226 pub fn from_db_path(db_path: &Path) -> Result<Self, CodememError> {
234 if let Some(parent) = db_path.parent() {
236 if !parent.exists() {
237 std::fs::create_dir_all(parent).map_err(|e| {
238 CodememError::Storage(format!(
239 "Failed to create database directory {}: {e}",
240 parent.display()
241 ))
242 })?;
243 }
244 }
245
246 let config = CodememConfig::load_or_default();
247
248 if !config.storage.backend.eq_ignore_ascii_case("sqlite") {
251 return Err(CodememError::Config(format!(
252 "Unsupported storage backend '{}'. Only 'sqlite' is available in this build.",
253 config.storage.backend
254 )));
255 }
256 if !config.vector.backend.eq_ignore_ascii_case("hnsw") {
257 return Err(CodememError::Config(format!(
258 "Unsupported vector backend '{}'. Only 'hnsw' is available in this build.",
259 config.vector.backend
260 )));
261 }
262 if !config.graph.backend.eq_ignore_ascii_case("petgraph") {
263 return Err(CodememError::Config(format!(
264 "Unsupported graph backend '{}'. Only 'petgraph' is available in this build.",
265 config.graph.backend
266 )));
267 }
268
269 let storage = Storage::open_with_config(
271 db_path,
272 Some(config.storage.cache_size_mb),
273 Some(config.storage.busy_timeout_secs),
274 )?;
275
276 let graph = GraphEngine::from_storage(&storage)?;
278
279 let engine = Self {
280 storage: Box::new(storage),
281 vector: OnceLock::new(),
282 graph: Mutex::new(Box::new(graph)),
283 embeddings: OnceLock::new(),
284 db_path: Some(db_path.to_path_buf()),
285 index_cache: Mutex::new(None),
286 scoring_weights: RwLock::new(config.scoring.clone()),
287 bm25_index: OnceLock::new(),
288 config,
289 metrics: Arc::new(InMemoryMetrics::new()),
290 dirty: AtomicBool::new(false),
291 active_session_id: RwLock::new(None),
292 scope: RwLock::new(None),
293 change_detector: Mutex::new(None),
294 last_expiry_sweep: AtomicI64::new(0),
295 };
296
297 Ok(engine)
302 }
303
304 pub fn for_testing() -> Self {
306 let storage = Storage::open_in_memory().unwrap();
307 let graph = GraphEngine::new();
308 let config = CodememConfig::default();
309 let vector_lock = OnceLock::new();
310 let _ = vector_lock.set(Mutex::new(
311 Box::new(HnswIndex::with_defaults().unwrap()) as Box<dyn VectorBackend>
312 ));
313 let embeddings_lock = OnceLock::new();
314 let _ = embeddings_lock.set(None);
315 let bm25_lock = OnceLock::new();
316 let _ = bm25_lock.set(Mutex::new(Bm25Index::new()));
317 Self {
318 storage: Box::new(storage),
319 vector: vector_lock,
320 graph: Mutex::new(Box::new(graph)),
321 embeddings: embeddings_lock,
322 db_path: None,
323 index_cache: Mutex::new(None),
324 scoring_weights: RwLock::new(config.scoring.clone()),
325 bm25_index: bm25_lock,
326 config,
327 metrics: Arc::new(InMemoryMetrics::new()),
328 dirty: AtomicBool::new(false),
329 active_session_id: RwLock::new(None),
330 scope: RwLock::new(None),
331 change_detector: Mutex::new(None),
332 last_expiry_sweep: AtomicI64::new(0),
333 }
334 }
335
336 pub fn lock_vector(
339 &self,
340 ) -> Result<std::sync::MutexGuard<'_, Box<dyn VectorBackend>>, CodememError> {
341 self.vector
342 .get_or_init(|| self.init_vector())
343 .lock()
344 .map_err(|e| CodememError::LockPoisoned(format!("vector: {e}")))
345 }
346
347 pub fn lock_graph(
348 &self,
349 ) -> Result<std::sync::MutexGuard<'_, Box<dyn GraphBackend>>, CodememError> {
350 self.graph
351 .lock()
352 .map_err(|e| CodememError::LockPoisoned(format!("graph: {e}")))
353 }
354
355 pub fn lock_bm25(&self) -> Result<std::sync::MutexGuard<'_, Bm25Index>, CodememError> {
356 self.bm25_index
357 .get_or_init(|| self.init_bm25())
358 .lock()
359 .map_err(|e| CodememError::LockPoisoned(format!("bm25: {e}")))
360 }
361
362 pub fn lock_embeddings(
366 &self,
367 ) -> Result<
368 Option<std::sync::MutexGuard<'_, Box<dyn codemem_embeddings::EmbeddingProvider>>>,
369 CodememError,
370 > {
371 match self.embeddings.get_or_init(|| self.init_embeddings()) {
372 Some(m) => Ok(Some(m.lock().map_err(|e| {
373 CodememError::LockPoisoned(format!("embeddings: {e}"))
374 })?)),
375 None => Ok(None),
376 }
377 }
378
379 fn embeddings_ready(&self) -> bool {
381 self.embeddings.get().is_some_and(|opt| opt.is_some())
382 }
383
384 fn vector_ready(&self) -> bool {
386 self.vector.get().is_some()
387 }
388
389 fn bm25_ready(&self) -> bool {
391 self.bm25_index.get().is_some()
392 }
393
394 pub fn lock_index_cache(
395 &self,
396 ) -> Result<std::sync::MutexGuard<'_, Option<IndexCache>>, CodememError> {
397 self.index_cache
398 .lock()
399 .map_err(|e| CodememError::LockPoisoned(format!("index_cache: {e}")))
400 }
401
402 pub fn scoring_weights(
403 &self,
404 ) -> Result<std::sync::RwLockReadGuard<'_, ScoringWeights>, CodememError> {
405 self.scoring_weights
406 .read()
407 .map_err(|e| CodememError::LockPoisoned(format!("scoring_weights read: {e}")))
408 }
409
410 pub fn scoring_weights_mut(
411 &self,
412 ) -> Result<std::sync::RwLockWriteGuard<'_, ScoringWeights>, CodememError> {
413 self.scoring_weights
414 .write()
415 .map_err(|e| CodememError::LockPoisoned(format!("scoring_weights write: {e}")))
416 }
417
418 fn init_vector(&self) -> Mutex<Box<dyn VectorBackend>> {
422 let vector_config = self.config.vector.clone();
423 let mut vector = HnswIndex::new(vector_config.clone())
424 .unwrap_or_else(|_| HnswIndex::with_defaults().expect("default vector index"));
425
426 if let Some(ref db_path) = self.db_path {
427 let index_path = db_path.with_extension("idx");
428 if index_path.exists() {
429 if let Err(e) = vector.load(&index_path) {
430 tracing::warn!("Stale or corrupt vector index, will rebuild: {e}");
431 }
432 }
433
434 let vector_count = vector.stats().count;
436 if let Ok(db_stats) = self.storage.stats() {
437 let db_embed_count = db_stats.embedding_count;
438 if vector_count != db_embed_count {
439 tracing::warn!(
440 "Vector index ({vector_count}) out of sync with DB ({db_embed_count}), rebuilding..."
441 );
442 if let Ok(mut fresh) = HnswIndex::new(vector_config) {
443 if let Ok(embeddings) = self.storage.list_all_embeddings() {
444 for (id, emb) in &embeddings {
445 if let Err(e) = fresh.insert(id, emb) {
446 tracing::warn!("Failed to re-insert embedding {id}: {e}");
447 }
448 }
449 }
450 vector = fresh;
451 if let Err(e) = vector.save(&index_path) {
452 tracing::warn!("Failed to save rebuilt vector index: {e}");
453 }
454 }
455 }
456 }
457 }
458
459 Mutex::new(Box::new(vector))
460 }
461
462 fn init_bm25(&self) -> Mutex<Bm25Index> {
464 let mut bm25 = Bm25Index::new();
465
466 if let Some(ref db_path) = self.db_path {
467 let bm25_path = db_path.with_extension("bm25");
468 let mut loaded = false;
469 if bm25_path.exists() {
470 if let Ok(data) = std::fs::read(&bm25_path) {
471 if let Ok(index) = Bm25Index::deserialize(&data) {
472 tracing::info!(
473 "Loaded BM25 index from disk ({} documents)",
474 index.doc_count
475 );
476 bm25 = index;
477 loaded = true;
478 }
479 }
480 }
481 if !loaded {
482 if let Ok(ids) = self.storage.list_memory_ids() {
483 let id_refs: Vec<&str> = ids.iter().map(|s| s.as_str()).collect();
484 if let Ok(memories) = self.storage.get_memories_batch(&id_refs) {
485 for m in &memories {
486 bm25.add_document(&m.id, &m.content);
487 }
488 tracing::info!("Rebuilt BM25 index from {} memories", bm25.doc_count);
489 }
490 }
491 }
492 }
493
494 Mutex::new(bm25)
495 }
496
497 fn init_embeddings(&self) -> Option<Mutex<Box<dyn codemem_embeddings::EmbeddingProvider>>> {
502 let provider = match codemem_embeddings::from_env(Some(&self.config.embedding)) {
503 Ok(p) => p,
504 Err(e) => {
505 tracing::warn!("Failed to initialize embedding provider: {e}");
506 return None;
507 }
508 };
509
510 self.backfill_embeddings(&*provider);
512
513 Some(Mutex::new(provider))
514 }
515
516 fn backfill_embeddings(&self, provider: &dyn codemem_embeddings::EmbeddingProvider) {
521 let ids = match self.storage.list_memory_ids() {
522 Ok(ids) => ids,
523 Err(_) => return,
524 };
525
526 let mut to_embed: Vec<(String, String)> = Vec::new();
527 for id in &ids {
528 if self.storage.get_embedding(id).ok().flatten().is_none() {
529 if let Ok(Some(mem)) = self.storage.get_memory_no_touch(id) {
530 let text = self.enrich_memory_text(
531 &mem.content,
532 mem.memory_type,
533 &mem.tags,
534 mem.namespace.as_deref(),
535 Some(&mem.id),
536 );
537 to_embed.push((id.clone(), text));
538 }
539 }
540 }
541
542 if to_embed.is_empty() {
543 return;
544 }
545
546 tracing::info!("Backfilling {} un-embedded memories", to_embed.len());
547 let text_refs: Vec<&str> = to_embed.iter().map(|(_, t)| t.as_str()).collect();
548 match provider.embed_batch(&text_refs) {
549 Ok(embeddings) => {
550 for ((id, _), emb) in to_embed.iter().zip(embeddings.iter()) {
551 let _ = self.storage.store_embedding(id, emb);
552 if let Some(vi_mutex) = self.vector.get() {
554 if let Ok(mut vi) = vi_mutex.lock().map_err(|e| {
555 tracing::warn!("Vector lock failed during backfill: {e}");
556 e
557 }) {
558 let _ = vi.insert(id, emb);
559 }
560 }
561 }
562 tracing::info!("Backfilled {} embeddings", to_embed.len());
563 }
564 Err(e) => tracing::warn!("Backfill embedding failed: {e}"),
565 }
566 }
567
568 pub fn set_active_session(&self, id: Option<String>) {
572 match self.active_session_id.write() {
573 Ok(mut guard) => *guard = id,
574 Err(e) => *e.into_inner() = id,
575 }
576 }
577
578 pub fn active_session_id(&self) -> Option<String> {
580 match self.active_session_id.read() {
581 Ok(guard) => guard.clone(),
582 Err(e) => e.into_inner().clone(),
583 }
584 }
585
586 pub fn set_scope(&self, scope: Option<codemem_core::ScopeContext>) {
590 match self.scope.write() {
591 Ok(mut guard) => *guard = scope,
592 Err(e) => *e.into_inner() = scope,
593 }
594 }
595
596 pub fn scope(&self) -> Option<codemem_core::ScopeContext> {
598 match self.scope.read() {
599 Ok(guard) => guard.clone(),
600 Err(e) => e.into_inner().clone(),
601 }
602 }
603
604 pub fn scope_namespace(&self) -> Option<String> {
606 self.scope().map(|s| s.namespace().to_string())
607 }
608
609 pub fn storage(&self) -> &dyn StorageBackend {
613 &*self.storage
614 }
615
616 pub fn has_embeddings(&self) -> bool {
621 match self.embeddings.get() {
622 Some(opt) => opt.is_some(),
623 None => !self.config.embedding.provider.is_empty(),
624 }
625 }
626
627 pub fn db_path(&self) -> Option<&Path> {
629 self.db_path.as_deref()
630 }
631
632 pub fn config(&self) -> &CodememConfig {
634 &self.config
635 }
636
637 pub fn metrics(&self) -> &Arc<InMemoryMetrics> {
639 &self.metrics
640 }
641
642 pub fn with_graph<F, R>(&self, f: F) -> Result<R, CodememError>
647 where
648 F: FnOnce(&dyn GraphBackend) -> R,
649 {
650 let guard = self.lock_graph()?;
651 Ok(f(&**guard))
652 }
653
654 pub fn with_vector<F, R>(&self, f: F) -> Result<R, CodememError>
657 where
658 F: FnOnce(&dyn VectorBackend) -> R,
659 {
660 let guard = self.lock_vector()?;
661 Ok(f(&**guard))
662 }
663
664 #[cfg(test)]
666 pub(crate) fn is_dirty(&self) -> bool {
667 self.dirty.load(Ordering::Acquire)
668 }
669
670 pub fn list_repos(&self) -> Result<Vec<codemem_core::Repository>, CodememError> {
674 self.storage.list_repos()
675 }
676
677 pub fn add_repo(&self, repo: &codemem_core::Repository) -> Result<(), CodememError> {
679 self.storage.add_repo(repo)
680 }
681
682 pub fn get_repo(&self, id: &str) -> Result<Option<codemem_core::Repository>, CodememError> {
684 self.storage.get_repo(id)
685 }
686
687 pub fn remove_repo(&self, id: &str) -> Result<bool, CodememError> {
689 self.storage.remove_repo(id)
690 }
691
692 pub fn update_repo_status(
694 &self,
695 id: &str,
696 status: &str,
697 indexed_at: Option<&str>,
698 ) -> Result<(), CodememError> {
699 self.storage.update_repo_status(id, status, indexed_at)
700 }
701}
702
703pub use file_indexing::{AnalyzeOptions, AnalyzeProgress, AnalyzeResult, SessionContext};
705
706pub use codemem_embeddings::from_env as embeddings_from_env;
709pub use codemem_embeddings::{EmbeddingProvider, EmbeddingService};