Skip to main content

semantic_memory/
lib.rs

1#![allow(deprecated)]
2
3//! # semantic-memory
4//!
5//! Local-first semantic memory backed by authoritative SQLite state and an optional recoverable
6//! HNSW sidecar.
7//!
8//! The crate stores facts, chunked documents, conversation messages, and searchable episodes in
9//! SQLite. Search combines BM25 (FTS5) and vector retrieval with Reciprocal Rank Fusion, and
10//! `search_explained()` returns the exact scoring breakdown from the live pipeline.
11//!
12//! Concurrency uses one writer connection plus a pool of WAL-enabled reader connections.
13//! Durable writes are committed to SQLite first; any required HNSW sidecar mutations are journaled
14//! in SQLite and replayed on open, flush, rebuild, or reconcile.
15//!
16//! `search()` targets facts, document chunks, and episodes by default. Message retrieval is
17//! available through `search_conversations()` or by opting into
18//! [`SearchSourceType::Messages`].
19//!
20//! Integrity tooling is strict about malformed stored data: invalid roles, JSON, enums, embedding
21//! blobs, quantized blobs, and sidecar drift are surfaced through `verify_integrity()` instead of
22//! being silently converted into defaults. `reconcile()` can rebuild FTS or fully re-embed and
23//! rebuild derived state from SQLite.
24//!
25//! `store.graph_view()` exposes a deterministic graph traversal layer over namespaces, facts,
26//! documents, chunks, sessions, messages, episodes, and semantic/temporal/causal links derived
27//! from SQLite state.
28//!
29//! ## Quick Start
30//!
31//! ```rust,no_run
32//! use semantic_memory::{MemoryConfig, MemoryStore};
33//!
34//! # async fn example() -> Result<(), semantic_memory::MemoryError> {
35//! let store = MemoryStore::open(MemoryConfig::default())?;
36//!
37//! // Store a fact
38//! store.add_fact("general", "Rust was first released in 2015", None, None).await?;
39//!
40//! // Search
41//! let results = store.search("when was Rust released", None, None, None).await?;
42//! # Ok(())
43//! # }
44//! ```
45//!
46//! ## Operational Notes
47//!
48//! - SQLite is authoritative for all durable records and embeddings.
49//! - HNSW is an acceleration sidecar. Pending sidecar mutations are journaled in SQLite, so a
50//!   sidecar failure does not imply the SQLite write rolled back.
51//! - WAL mode plus pooled reader connections allows concurrent reads while writes serialize through
52//!   the writer connection.
53//! - `search_explained()` reflects the exact ranking math used by the active search pipeline,
54//!   including reranking from exact f32 cosine similarity when configured.
55
56// At least one search backend must be enabled.
57#[cfg(not(any(feature = "hnsw", feature = "brute-force", feature = "usearch-backend")))]
58compile_error!(
59    "At least one search backend feature must be enabled: 'hnsw', 'usearch-backend', or 'brute-force'"
60);
61
62pub mod chunker;
63pub mod config;
64pub(crate) mod conversation;
65pub(crate) mod db;
66pub use db::{bytes_to_embedding, decode_f32_le, embedding_to_bytes};
67pub(crate) mod documents;
68pub mod embedder;
69pub(crate) mod episodes;
70pub mod error;
71/// Discord-structured second-order retrieval (graph-neighbour discovery).
72#[cfg(feature = "discord")]
73pub mod discord;
74/// Phase 6: decoder architecture (syndromes and corrections).
75#[cfg(feature = "decoder")]
76pub mod decoder;
77mod graph;
78/// First-class stored graph edges (durable, typed relationships).
79pub(crate) mod graph_edges;
80#[cfg(feature = "hnsw")]
81pub mod hnsw;
82#[cfg(feature = "hnsw")]
83mod hnsw_backend;
84#[cfg(feature = "hnsw")]
85mod hnsw_ops;
86mod json_compat_import;
87pub(crate) mod knowledge;
88mod pool;
89/// Phase 2: semiring provenance (Boolean/Tropical/Probability/Confidence).
90#[cfg(feature = "provenance")]
91pub mod provenance;
92/// Phase 3: temporal field provenance (computed temporal_weight scores).
93#[cfg(feature = "temporal")]
94pub mod temporal;
95mod projection_batch;
96mod projection_derivation;
97/// Compatibility-only legacy import surface.
98///
99/// This module exists only for migration compatibility with pre-V11 import paths.
100#[deprecated(
101    since = "0.6.0",
102    note = "Legacy V10 import path is migration-only. Use `import_projection_batch()` with `ProjectionImportBatchV3` on the canonical lane."
103)]
104#[doc(hidden)]
105pub mod projection_import;
106mod projection_lane;
107mod projection_legacy_compat;
108pub(crate) mod projection_storage;
109/// Multiscale retrieval scheduling pipeline (staged search with budgets).
110#[cfg(feature = "multiscale")]
111pub mod pipeline;
112pub mod quantize;
113pub mod quantize_governed;
114/// Phase 7: lawful subtraction engine.
115#[cfg(feature = "subtraction")]
116pub mod subtraction;
117/// Phase 8: simplified compression governor (importance scoring only).
118#[cfg(feature = "compression-governor")]
119pub mod compression_governor;
120/// Phase 9: adaptive retrieval routing (query-aware stage selection).
121#[cfg(feature = "routing")]
122pub mod routing;
123/// Phase 9b: benchmark harness for routing quality.
124#[cfg(feature = "benchmark")]
125pub mod benchmark;
126/// Phase 10: cross-feature integration wiring.
127#[cfg(feature = "integration")]
128pub mod integration;
129/// Factor graph unification of heterogeneous graph edges (semantic,
130/// temporal, causal, entity) with belief propagation. The single most
131/// novel combination: unified probabilistic reasoning over all edge types.
132#[cfg(feature = "integration")]
133pub mod factor_graph;
134/// ColBERT-style late interaction multi-vector retrieval.
135#[cfg(feature = "late-interaction")]
136pub mod late_interaction;
137/// Persistent homology and topological void detection for knowledge graphs.
138#[cfg(feature = "topology")]
139pub mod topology;
140/// Matryoshka Representation Learning: multi-resolution embedding truncation.
141#[cfg(feature = "matryoshka")]
142pub mod matryoshka;
143/// Leiden community detection with contradiction tracking.
144#[cfg(feature = "community")]
145pub mod community;
146/// RL-trained retrieval routing on receipt replay data.
147#[cfg(feature = "rl-routing")]
148pub mod rl_routing;
149/// Reasoning subgraph pruning with lawful subtraction.
150#[cfg(feature = "subgraph-pruning")]
151pub mod subgraph_pruning;
152pub mod search;
153pub mod storage;
154mod store_support;
155pub mod tokenizer;
156pub mod types;
157#[cfg(feature = "usearch-backend")]
158mod usearch_backend;
159pub mod vector_backend;
160pub mod vector_codec;
161pub mod vector_snapshot;
162
163// Re-export primary public types.
164pub use config::{
165    ChunkingConfig, DerivedVectorBackendPolicy, EmbeddingConfig, MemoryConfig, MemoryLimits,
166    PoolConfig, SearchConfig,
167};
168pub use db::{IntegrityReport, ReconcileAction, VerifyMode};
169pub use embedder::{Embedder, MockEmbedder, OllamaEmbedder};
170pub use error::MemoryError;
171#[cfg(feature = "hnsw")]
172pub use hnsw::{HnswConfig, HnswHit, HnswIndex};
173// Type aliases for the new VectorBackend trait. The Hnsw* names are kept
174// for source compatibility; new code should prefer the Vector* names.
175pub(crate) use projection_lane::projection_import_failure_id;
176pub use projection_lane::{
177    ProjectionImportFailureReceiptEntry, ProjectionImportLogEntry, ProjectionImportResult,
178};
179pub use quantize::{pack_quantized, unpack_quantized, QuantizedVector, Quantizer};
180pub use storage::StoragePaths;
181pub use tokenizer::{EstimateTokenCounter, TokenCounter};
182pub use types::{
183    ChunkManifestChunkMapping, ChunkManifestEntry, ChunkManifestIngestOptions,
184    ChunkManifestIngestResult, DerivedCandidateReceiptV1, Document, EmbeddingDisplacement,
185    EpisodeAsOfReceiptV1, EpisodeMeta, EpisodeOutcome, ExactnessProfile, ExplainedResult,
186    ExplainedResultAnswerV1, ExplainedSearchResponse, Fact, GraphDirection, GraphEdge,
187    GraphEdgeType, GraphView, MemoryStats, Message, NamespaceDeleteReport, ProjectionClaimVersion,
188    ProjectionEntityAlias, ProjectionEpisode, ProjectionEvidenceRef, ProjectionQuery,
189    ProjectionRelationVersion, ProveKvPoolArtifactBuildReceiptV1, ProveKvPoolArtifactStatusV1,
190    ProveKvPoolGenerationStatus, ProveKvPoolGenerationV1, ProveKvPoolItemMapEntryV1, ReceiptMode,
191    Role, ScoreBreakdown, SearchContext, SearchReceiptAnswersV1, SearchReplayReportV1,
192    SearchResponse, SearchResult, SearchSource, SearchSourceType, Session, TextChunk,
193    VectorArtifactBuildReceiptV1, VectorSearchReceiptV1, VerificationStatus,
194};
195pub use graph_edges::{AddGraphEdgeParams, StoredGraphEdge};
196pub use vector_backend::{VectorBackend, VectorHit, VectorIndex, VectorIndexConfig};
197#[cfg(feature = "turbo-quant-codec")]
198pub use vector_codec::TurboQuantCodec;
199pub use vector_codec::{
200    RawF32Codec, Sq8Codec, VectorArtifactV1, VectorCodec, VectorCodecProfileV1,
201};
202pub use vector_snapshot::{build_embedding_snapshot, EmbeddingSnapshotRow, EmbeddingSnapshotV1};
203
204use std::sync::Arc;
205
206const MAX_TOP_K: usize = 1_000;
207#[cfg(feature = "hnsw")]
208const MAX_HNSW_CANDIDATES: usize = 10_000;
209
210pub(crate) use store_support::{
211    as_str_slice, build_episode_search_text, merge_trace_ctx, to_owned_string_vec,
212    verification_status_for_outcome,
213};
214
215#[cfg(feature = "hnsw")]
216fn verify_hnsw_key_level_integrity(
217    conn: &rusqlite::Connection,
218    dimensions: usize,
219    node_vectors: &std::collections::HashMap<usize, Vec<f32>>,
220    sidecar_files_exist: bool,
221) -> Result<Vec<String>, MemoryError> {
222    let mut issues = Vec::new();
223    let mut live_rows: std::collections::HashMap<String, Vec<f32>> =
224        std::collections::HashMap::new();
225
226    let mut live_stmt = conn.prepare(
227        "SELECT 'fact:' || id, embedding FROM facts WHERE embedding IS NOT NULL
228         UNION ALL
229         SELECT 'chunk:' || id, embedding FROM chunks WHERE embedding IS NOT NULL
230         UNION ALL
231         SELECT 'msg:' || id, embedding FROM messages WHERE embedding IS NOT NULL
232         UNION ALL
233         SELECT 'episode:' || episode_id, embedding FROM episodes WHERE embedding IS NOT NULL",
234    )?;
235    let live_iter = live_stmt.query_map([], |row| {
236        Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?))
237    })?;
238    for row in live_iter {
239        let (key, blob) = row?;
240        match db::decode_f32_le(&blob, dimensions) {
241            Ok(vector) => {
242                live_rows.insert(key, vector);
243            }
244            Err(err) => issues.push(format!(
245                "HNSW live embedding row {key} has invalid vector: {err}"
246            )),
247        }
248    }
249
250    if !live_rows.is_empty() && !sidecar_files_exist {
251        issues.push(format!(
252            "HNSW sidecar files are missing while {} embedded rows exist in SQLite",
253            live_rows.len()
254        ));
255    }
256
257    let keymap_exists: bool = conn
258        .query_row(
259            "SELECT COUNT(*) > 0 FROM sqlite_master WHERE type='table' AND name='hnsw_keymap'",
260            [],
261            |row| row.get(0),
262        )
263        .unwrap_or(false);
264    if !keymap_exists {
265        if !live_rows.is_empty() {
266            issues.push("HNSW keymap table missing while embedded SQLite rows exist".to_string());
267        }
268        return Ok(issues);
269    }
270
271    let mut active_keymap: std::collections::HashMap<String, usize> =
272        std::collections::HashMap::new();
273    let mut keymap_stmt =
274        conn.prepare("SELECT node_id, item_key FROM hnsw_keymap WHERE deleted = 0")?;
275    let keymap_iter = keymap_stmt.query_map([], |row| {
276        Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?))
277    })?;
278    for row in keymap_iter {
279        let (node_id_raw, key) = row?;
280        let Some((domain, raw_id)) = key.split_once(':') else {
281            issues.push(format!("HNSW keymap entry has malformed key: {key}"));
282            continue;
283        };
284        if !matches!(domain, "fact" | "chunk" | "msg" | "episode") || raw_id.is_empty() {
285            issues.push(format!(
286                "HNSW keymap entry has unsupported key domain: {key}"
287            ));
288            continue;
289        }
290        if domain == "msg" && raw_id.parse::<i64>().is_err() {
291            issues.push(format!("HNSW message key has non-integer row id: {key}"));
292            continue;
293        }
294        let node_id = match usize::try_from(node_id_raw) {
295            Ok(node_id) => node_id,
296            Err(err) => {
297                issues.push(format!(
298                    "HNSW keymap node_id {node_id_raw} is invalid: {err}"
299                ));
300                continue;
301            }
302        };
303        active_keymap.insert(key, node_id);
304    }
305
306    for key in live_rows.keys() {
307        if !active_keymap.contains_key(key) {
308            issues.push(format!(
309                "HNSW keymap missing live embedded SQLite row: {key}"
310            ));
311        }
312    }
313
314    for (key, node_id) in &active_keymap {
315        let Some(live_vector) = live_rows.get(key) else {
316            issues.push(format!(
317                "HNSW keymap has stale active entry without live embedded SQLite row: {key}"
318            ));
319            continue;
320        };
321        let Some(index_vector) = node_vectors.get(node_id) else {
322            issues.push(format!(
323                "HNSW keymap entry {key} points to missing in-memory node vector {node_id}"
324            ));
325            continue;
326        };
327        if index_vector.len() != live_vector.len()
328            || index_vector
329                .iter()
330                .zip(live_vector)
331                .any(|(left, right)| left.to_bits() != right.to_bits())
332        {
333            issues.push(format!(
334                "HNSW keymap entry {key} points to node {node_id} whose vector does not match the authoritative SQLite embedding"
335            ));
336        }
337    }
338
339    if active_keymap.len() != live_rows.len() {
340        issues.push(format!(
341            "HNSW keymap drift: {} active keymap rows vs {} embedded SQLite rows",
342            active_keymap.len(),
343            live_rows.len()
344        ));
345    }
346
347    Ok(issues)
348}
349
350/// Compatibility-only public access to retained legacy surfaces.
351#[doc(hidden)]
352pub mod compat {
353    #[deprecated(
354        since = "0.5.0",
355        note = "Legacy ImportEnvelope is migration-only. New integrations should use `ProjectionImportBatchV3` on the canonical lane."
356    )]
357    #[doc(hidden)]
358    #[allow(deprecated)]
359    pub mod legacy_import_envelope {
360        pub use crate::projection_import::{
361            ImportEnvelope, ImportProjectionFreshness, ImportReceipt, ImportRecord, ImportStatus,
362        };
363        pub use stack_ids::EnvelopeId;
364    }
365
366    #[deprecated(
367        since = "0.5.0",
368        note = "Legacy trace_id is migration-only. Use `stack_ids::TraceCtx`."
369    )]
370    #[doc(hidden)]
371    #[allow(deprecated)]
372    pub mod compat_trace_id {
373        pub use crate::types::TraceId;
374    }
375}
376
377/// Thread-safe handle to the memory database.
378///
379/// Clone is cheap (Arc internals). `Send + Sync`.
380#[derive(Clone)]
381pub struct MemoryStore {
382    inner: Arc<MemoryStoreInner>,
383}
384
385struct MemoryStoreInner {
386    pool: pool::SqlitePool,
387    embedder: Box<dyn Embedder>,
388    embedding_permits: Arc<tokio::sync::Semaphore>,
389    config: MemoryConfig,
390    paths: StoragePaths,
391    token_counter: Arc<dyn TokenCounter>,
392    #[cfg(feature = "hnsw")]
393    hnsw_index: std::sync::RwLock<HnswIndex>,
394}
395
396#[cfg(feature = "hnsw")]
397impl Drop for MemoryStoreInner {
398    fn drop(&mut self) {
399        if !self.paths.hnsw_dir.exists() {
400            tracing::debug!(
401                path = %self.paths.hnsw_dir.display(),
402                "Skipping HNSW drop flush because the sidecar directory no longer exists"
403            );
404            return;
405        }
406
407        let pending_ops = match self.pool.with_read_conn(db::pending_index_op_count) {
408            Ok(count) => count,
409            Err(err) => {
410                tracing::warn!("Failed to inspect pending HNSW work on drop: {}", err);
411                0
412            }
413        };
414
415        if pending_ops > 0 {
416            if let Err(err) =
417                hnsw_ops::recover_hnsw_sidecar_sync(&self.pool, &self.paths, &self.config.hnsw)
418            {
419                tracing::error!("Failed to recover and flush HNSW on drop: {}", err);
420            }
421            return;
422        }
423
424        let hnsw_guard = match self.hnsw_index.read() {
425            Ok(g) => g,
426            Err(_) => {
427                tracing::warn!("HNSW RwLock poisoned on drop — skipping save");
428                return;
429            }
430        };
431
432        if let Err(err) = hnsw_ops::save_hnsw_sidecar(
433            &hnsw_guard,
434            &self.paths.hnsw_dir,
435            &self.paths.hnsw_basename,
436        ) {
437            tracing::error!("Failed to save HNSW index on drop: {}", err);
438        }
439
440        // Flush key mappings to SQLite
441        if let Err(e) = self
442            .pool
443            .with_write_conn(|conn| hnsw_guard.flush_keymap(conn))
444        {
445            tracing::error!("Failed to flush HNSW keymap on drop: {}", e);
446        }
447    }
448}
449
450impl MemoryStore {
451    /// Run read-only work on a pooled reader connection on a blocking thread.
452    ///
453    /// This prevents SQLite I/O from stalling the tokio executor while allowing
454    /// multiple concurrent readers under WAL mode.
455    async fn with_read_conn<F, T>(&self, f: F) -> Result<T, MemoryError>
456    where
457        F: FnOnce(&rusqlite::Connection) -> Result<T, MemoryError> + Send + 'static,
458        T: Send + 'static,
459    {
460        let inner = self.inner.clone();
461        tokio::task::spawn_blocking(move || -> Result<T, MemoryError> {
462            inner.pool.with_read_conn(f)
463        })
464        .await
465        .map_err(|e| MemoryError::Other(format!("Blocking task panicked: {}", e)))?
466    }
467
468    /// Run write-capable work on the single writer connection on a blocking thread.
469    async fn with_write_conn<F, T>(&self, f: F) -> Result<T, MemoryError>
470    where
471        F: FnOnce(&rusqlite::Connection) -> Result<T, MemoryError> + Send + 'static,
472        T: Send + 'static,
473    {
474        let inner = self.inner.clone();
475        tokio::task::spawn_blocking(move || -> Result<T, MemoryError> {
476            inner.pool.with_write_conn(f)
477        })
478        .await
479        .map_err(|e| MemoryError::Other(format!("Blocking task panicked: {}", e)))?
480    }
481
482    async fn persist_search_receipt(
483        &self,
484        receipt: &VectorSearchReceiptV1,
485    ) -> Result<(), MemoryError> {
486        let receipt = receipt.clone();
487        self.with_write_conn(move |conn| db::store_search_receipt(conn, &receipt))
488            .await
489    }
490
491    /// Run HNSW search on a blocking thread to avoid holding std::sync::RwLock
492    /// across await points (CONC-001).
493    #[cfg(feature = "hnsw")]
494    async fn hnsw_search_blocking(
495        &self,
496        query_embedding: Vec<f32>,
497        candidates: usize,
498    ) -> Vec<HnswHit> {
499        let inner = self.inner.clone();
500        tokio::task::spawn_blocking(move || {
501            let guard = inner.hnsw_index.read().unwrap_or_else(|e| e.into_inner());
502            match guard.search(&query_embedding, candidates) {
503                Ok(hits) => hits,
504                Err(e) => {
505                    tracing::error!(
506                        "HNSW search failed, falling back to brute-force vector search: {}",
507                        e
508                    );
509                    Vec::new()
510                }
511            }
512        })
513        .await
514        .unwrap_or_else(|e| {
515            tracing::error!("HNSW search blocking task panicked: {}", e);
516            Vec::new()
517        })
518    }
519
520    #[cfg(feature = "hnsw")]
521    fn sync_pending_hnsw_ops_blocking(&self) -> Result<usize, MemoryError> {
522        hnsw_ops::sync_pending_hnsw_sidecar(&self.inner)
523    }
524
525    #[cfg(feature = "hnsw")]
526    async fn sync_pending_hnsw_ops(&self) -> Result<usize, MemoryError> {
527        let inner = self.inner.clone();
528        tokio::task::spawn_blocking(move || hnsw_ops::sync_pending_hnsw_sidecar(&inner))
529            .await
530            .map_err(|e| MemoryError::Other(format!("Blocking task panicked: {}", e)))?
531    }
532
533    #[cfg(feature = "hnsw")]
534    async fn sync_pending_hnsw_ops_best_effort(&self, operation: &'static str) {
535        if let Err(err) = self.sync_pending_hnsw_ops().await {
536            tracing::warn!(
537                operation,
538                error = %err,
539                "SQLite write committed but HNSW sidecar sync is still pending"
540            );
541        } else {
542            self.maybe_flush_hnsw();
543        }
544    }
545
546    /// Open or create a memory store at the configured base directory.
547    ///
548    /// Creates the directory if it doesn't exist, opens/creates SQLite,
549    /// runs migrations, and initializes the HNSW index.
550    pub fn open(config: MemoryConfig) -> Result<Self, MemoryError> {
551        let config = config.normalize_and_validate()?;
552        let embedder = Box::new(OllamaEmbedder::try_new(&config.embedding)?);
553        Self::open_with_embedder(config, embedder)
554    }
555
556    /// Open with a custom embedder (for testing or non-Ollama providers).
557    #[allow(unused_mut)] // `config` is mutated only when the `hnsw` feature is enabled
558    pub fn open_with_embedder(
559        mut config: MemoryConfig,
560        embedder: Box<dyn Embedder>,
561    ) -> Result<Self, MemoryError> {
562        config = config.normalize_and_validate()?;
563        if embedder.dimensions() != config.embedding.dimensions {
564            return Err(MemoryError::DimensionMismatch {
565                expected: config.embedding.dimensions,
566                actual: embedder.dimensions(),
567            });
568        }
569        config.embedding.model = embedder.model_name().to_string();
570
571        let paths = StoragePaths::new(&config.base_dir);
572
573        // Create directory if needed
574        std::fs::create_dir_all(&paths.base_dir).map_err(|e| {
575            MemoryError::StorageError(format!(
576                "Failed to create directory {}: {}",
577                paths.base_dir.display(),
578                e
579            ))
580        })?;
581
582        let pool = pool::SqlitePool::open(&paths.sqlite_path, &config.pool, &config.limits)?;
583        pool.with_write_conn(|conn| db::check_embedding_metadata(conn, &config.embedding))?;
584
585        // Ensure HNSW dimensions match the embedding config
586        #[cfg(feature = "hnsw")]
587        {
588            config.hnsw.dimensions = config.embedding.dimensions;
589        }
590
591        let token_counter = config
592            .token_counter
593            .clone()
594            .unwrap_or_else(tokenizer::default_token_counter);
595
596        #[cfg(feature = "hnsw")]
597        let hnsw_index = {
598            let hnsw_config = config.hnsw.clone();
599
600            let embeddings_dirty = pool.with_read_conn(db::is_embeddings_dirty)?;
601            let pending_index_ops = pool.with_read_conn(db::pending_index_op_count)?;
602
603            if embeddings_dirty {
604                // Embedding model changed — old HNSW index is useless.
605                // Create a fresh index; reembed_all() will rebuild it.
606                tracing::warn!(
607                    "Embedding model changed — creating fresh HNSW index (old index is stale)"
608                );
609                pool.with_write_conn(|conn| {
610                    db::clear_all_pending_index_ops(conn)?;
611                    db::set_sidecar_dirty(conn, false)?;
612                    Ok(())
613                })?;
614                HnswIndex::new(hnsw_config)?
615            } else if pending_index_ops > 0 || pool.with_read_conn(db::is_sidecar_dirty)? {
616                tracing::warn!(
617                    pending_index_ops,
618                    "Recovering HNSW sidecar from SQLite because durable sidecar work exists"
619                );
620                hnsw_ops::recover_hnsw_sidecar_sync(&pool, &paths, &hnsw_config)?
621            } else if paths.hnsw_files_exist() {
622                tracing::info!("Loading HNSW index from {:?}", paths.hnsw_dir);
623                match HnswIndex::load(&paths.hnsw_dir, &paths.hnsw_basename, hnsw_config.clone()) {
624                    Ok(index) => {
625                        // Load key mappings from SQLite
626                        if let Err(e) = pool.with_write_conn(|conn| index.load_keymap(conn)) {
627                            tracing::warn!("Failed to load HNSW key mappings: {}. Mappings will be empty until rebuild.", e);
628                        }
629
630                        // Stale index detection: compare HNSW entry count vs SQLite
631                        // embedding count. A mismatch means the app crashed before
632                        // flushing HNSW, or keys were lost.
633                        let hnsw_count = index.len();
634                        let sqlite_count: i64 = pool.with_read_conn(|conn| {
635                            Ok(conn.query_row(
636                                    "SELECT (SELECT COUNT(*) FROM facts WHERE embedding IS NOT NULL) +
637                                        (SELECT COUNT(*) FROM chunks WHERE embedding IS NOT NULL) +
638                                        (SELECT COUNT(*) FROM messages WHERE embedding IS NOT NULL) +
639                                        (SELECT COUNT(*) FROM episodes WHERE embedding IS NOT NULL)",
640                                    [],
641                                    |row| row.get(0),
642                                )?)
643                        })?;
644
645                        let drift = (sqlite_count - hnsw_count as i64).abs();
646                        if drift > 0 {
647                            tracing::warn!(
648                                hnsw_count,
649                                sqlite_count,
650                                drift,
651                                "HNSW index is stale — {} entries differ from SQLite. \
652                                 Likely caused by unclean shutdown. Triggering inline rebuild.",
653                                drift
654                            );
655                            // Discard the stale index and rebuild from SQLite
656                            let rebuilt =
657                                hnsw_ops::recover_hnsw_sidecar_sync(&pool, &paths, &hnsw_config)?;
658                            tracing::info!(
659                                active = rebuilt.len(),
660                                "HNSW index rebuilt after stale detection"
661                            );
662                            rebuilt
663                        } else {
664                            tracing::info!(
665                                "HNSW index loaded ({} active keys, in sync with SQLite)",
666                                hnsw_count
667                            );
668                            index
669                        }
670                    }
671                    Err(e) => {
672                        tracing::warn!(
673                            "Failed to load HNSW index: {}. Rebuilding sidecar from authoritative SQLite rows.",
674                            e
675                        );
676                        hnsw_ops::recover_hnsw_sidecar_sync(&pool, &paths, &hnsw_config)?
677                    }
678                }
679            } else {
680                // Check if SQLite has embeddings that should be in the index.
681                // This happens when: sidecar files were deleted, data dir was
682                // partially copied, app crashed before first flush, or HNSW was
683                // added after data already existed.
684                let orphan_count: i64 = pool.with_read_conn(|conn| {
685                    Ok(conn.query_row(
686                        "SELECT (SELECT COUNT(*) FROM facts WHERE embedding IS NOT NULL) +
687                                (SELECT COUNT(*) FROM chunks WHERE embedding IS NOT NULL) +
688                                (SELECT COUNT(*) FROM messages WHERE embedding IS NOT NULL) +
689                                (SELECT COUNT(*) FROM episodes WHERE embedding IS NOT NULL)",
690                        [],
691                        |row| row.get(0),
692                    )?)
693                })?;
694
695                if orphan_count > 0 {
696                    tracing::warn!(
697                        orphan_count,
698                        "HNSW sidecar files missing but {} embeddings exist in SQLite — \
699                         rebuilding index inline",
700                        orphan_count
701                    );
702                    let new_index =
703                        hnsw_ops::recover_hnsw_sidecar_sync(&pool, &paths, &hnsw_config)?;
704                    tracing::info!(
705                        active = new_index.len(),
706                        "HNSW index rebuilt from SQLite embeddings"
707                    );
708                    new_index
709                } else {
710                    tracing::info!("Creating new empty HNSW index (no embeddings in SQLite)");
711                    HnswIndex::new(hnsw_config)?
712                }
713            }
714        };
715
716        let store = Self {
717            inner: Arc::new(MemoryStoreInner {
718                pool,
719                embedder,
720                embedding_permits: Arc::new(tokio::sync::Semaphore::new(
721                    config.limits.max_embedding_concurrency,
722                )),
723                config,
724                paths,
725                token_counter,
726                #[cfg(feature = "hnsw")]
727                hnsw_index: std::sync::RwLock::new(hnsw_index),
728            }),
729        };
730
731        #[cfg(feature = "hnsw")]
732        if let Err(err) = store.sync_pending_hnsw_ops_blocking() {
733            tracing::warn!(
734                error = %err,
735                "Failed to reconcile pending HNSW sidecar ops during open; sidecar replay remains pending"
736            );
737        }
738
739        Ok(store)
740    }
741
742    async fn with_embedding_permit(
743        &self,
744    ) -> Result<tokio::sync::OwnedSemaphorePermit, MemoryError> {
745        self.inner
746            .embedding_permits
747            .clone()
748            .acquire_owned()
749            .await
750            .map_err(|e| MemoryError::Other(format!("embedding semaphore closed: {e}")))
751    }
752
753    async fn embed_text_internal(&self, text: &str) -> Result<Vec<f32>, MemoryError> {
754        let _permit = self.with_embedding_permit().await?;
755        let embedding = self.inner.embedder.embed(text).await?;
756        db::validate_embedding(&embedding, self.inner.config.embedding.dimensions)?;
757        Ok(embedding)
758    }
759
760    async fn embed_batch_internal(&self, texts: Vec<String>) -> Result<Vec<Vec<f32>>, MemoryError> {
761        let requested = texts.len();
762        let _permit = self.with_embedding_permit().await?;
763        let embeddings = self.inner.embedder.embed_batch(texts).await?;
764        db::validate_embedding_batch(
765            &embeddings,
766            requested,
767            self.inner.config.embedding.dimensions,
768        )?;
769        Ok(embeddings)
770    }
771
772    fn validate_embedding_dimensions(&self, embedding: &[f32]) -> Result<(), MemoryError> {
773        db::validate_embedding(embedding, self.inner.config.embedding.dimensions)
774    }
775
776    fn validate_content(&self, field: &'static str, content: &str) -> Result<(), MemoryError> {
777        if content.is_empty() {
778            return Err(MemoryError::InvalidConfig {
779                field,
780                reason: "content must not be empty".to_string(),
781            });
782        }
783
784        let limit = self.inner.config.limits.max_content_bytes;
785        if content.len() > limit {
786            return Err(MemoryError::ContentTooLarge {
787                size: content.len(),
788                limit,
789            });
790        }
791
792        Ok(())
793    }
794
795    fn validate_confidence(confidence: f32) -> Result<(), MemoryError> {
796        if !confidence.is_finite() || !(0.0..=1.0).contains(&confidence) {
797            return Err(MemoryError::InvalidConfig {
798                field: "episodes.confidence",
799                reason: "confidence must be finite and within [0.0, 1.0]".to_string(),
800            });
801        }
802        Ok(())
803    }
804
805    // ─── HNSW Management ───────────────────────────────────────
806
807    /// Rebuild feature-gated TurboQuant artifacts from authoritative SQLite f32 embeddings.
808    #[cfg(feature = "turbo-quant-codec")]
809    pub async fn rebuild_vector_artifacts(
810        &self,
811    ) -> Result<VectorArtifactBuildReceiptV1, MemoryError> {
812        let dim = self.inner.config.embedding.dimensions;
813        let search = self.inner.config.search.clone();
814        self.with_write_conn(move |conn| {
815            db::rebuild_turbo_quant_artifacts(
816                conn,
817                dim,
818                search.turbo_quant_bits,
819                search.turbo_quant_projections,
820                search.turbo_quant_seed,
821            )
822        })
823        .await
824    }
825
826    /// Rebuild the HNSW index from SQLite f32 embeddings.
827    ///
828    /// Call this if sidecar files are missing, corrupted, or after `reembed_all()`.
829    #[cfg(feature = "hnsw")]
830    pub async fn rebuild_hnsw_index(
831        &self,
832    ) -> Result<crate::types::VectorArtifactBuildReceiptV1, MemoryError> {
833        tracing::info!("Rebuilding HNSW index from SQLite embeddings...");
834        let hnsw_config = self.inner.config.hnsw.clone();
835        let (new_index, build_receipt) = self
836            .with_read_conn(move |conn| hnsw_ops::rebuild_hnsw_from_sqlite(conn, &hnsw_config))
837            .await?;
838
839        {
840            let mut guard = self
841                .inner
842                .hnsw_index
843                .write()
844                .unwrap_or_else(|e| e.into_inner());
845            *guard = new_index.clone();
846        }
847
848        hnsw_ops::save_hnsw_sidecar(
849            &new_index,
850            &self.inner.paths.hnsw_dir,
851            &self.inner.paths.hnsw_basename,
852        )?;
853        self.inner.pool.with_write_conn(|conn| {
854            new_index.flush_keymap(conn)?;
855            db::clear_all_pending_index_ops(conn)?;
856            db::set_sidecar_dirty(conn, false)?;
857            Ok(())
858        })?;
859
860        tracing::info!(active = new_index.len(), receipt_generation_id = ?build_receipt.generation_id, "HNSW index rebuilt");
861
862        Ok(build_receipt)
863    }
864
865    /// Opportunistically flush HNSW if the configured interval has elapsed.
866    ///
867    /// Cheap no-op when `flush_interval_secs` is None or the interval hasn't
868    /// elapsed yet (just an atomic load + epoch comparison).
869    #[cfg(feature = "hnsw")]
870    fn maybe_flush_hnsw(&self) {
871        if let Some(interval) = self.inner.config.hnsw.flush_interval_secs {
872            let guard = self
873                .inner
874                .hnsw_index
875                .read()
876                .unwrap_or_else(|e| e.into_inner());
877            if guard.should_flush(interval) {
878                drop(guard); // release read lock before flushing
879                if let Err(e) = self.flush_hnsw() {
880                    tracing::warn!("Opportunistic HNSW flush failed: {}", e);
881                } else {
882                    let guard = self
883                        .inner
884                        .hnsw_index
885                        .read()
886                        .unwrap_or_else(|e| e.into_inner());
887                    guard.update_last_flush_epoch();
888                    tracing::info!("Opportunistic HNSW flush completed");
889                }
890            }
891        }
892    }
893
894    /// Persist the HNSW graph, vector data, and key mappings to disk.
895    ///
896    /// Called automatically on drop, but can be called explicitly for durability.
897    #[cfg(feature = "hnsw")]
898    pub fn flush_hnsw(&self) -> Result<(), MemoryError> {
899        let pending_ops = self.inner.pool.with_read_conn(db::pending_index_op_count)?;
900        if pending_ops > 0 {
901            tracing::info!(
902                pending_ops,
903                "Flushing HNSW via authoritative SQLite rebuild because pending durable sidecar work exists"
904            );
905            let rebuilt = hnsw_ops::recover_hnsw_sidecar_sync(
906                &self.inner.pool,
907                &self.inner.paths,
908                &self.inner.config.hnsw,
909            )?;
910            let mut guard = self
911                .inner
912                .hnsw_index
913                .write()
914                .unwrap_or_else(|e| e.into_inner());
915            *guard = rebuilt;
916            return Ok(());
917        }
918
919        let index = self
920            .inner
921            .hnsw_index
922            .write()
923            .unwrap_or_else(|e| e.into_inner());
924        hnsw_ops::save_hnsw_sidecar(
925            &index,
926            &self.inner.paths.hnsw_dir,
927            &self.inner.paths.hnsw_basename,
928        )?;
929
930        // Flush key mappings to SQLite
931        self.inner.pool.with_write_conn(|conn| {
932            index.flush_keymap(conn)?;
933            db::clear_all_pending_index_ops(conn)?;
934            db::set_sidecar_dirty(conn, false)?;
935            Ok(())
936        })?;
937        Ok(())
938    }
939
940    /// Compact the HNSW index by rebuilding without tombstones.
941    ///
942    /// Only rebuilds if the deleted ratio exceeds the compaction threshold.
943    #[cfg(feature = "hnsw")]
944    pub async fn compact_hnsw(&self) -> Result<(), MemoryError> {
945        if !self
946            .inner
947            .hnsw_index
948            .read()
949            .unwrap_or_else(|e| e.into_inner())
950            .needs_compaction()
951        {
952            tracing::info!("HNSW compaction not needed (deleted ratio below threshold)");
953            return Ok(());
954        }
955        let _receipt = self.rebuild_hnsw_index().await?;
956        Ok(())
957    }
958
959    // ─── Integrity & Diagnostics ────────────────────────────────
960
961    /// Verify database integrity.
962    ///
963    /// In `Quick` mode, checks table existence and row counts.
964    /// In `Full` mode, also verifies FTS consistency and runs SQLite integrity_check.
965    pub async fn verify_integrity(
966        &self,
967        mode: db::VerifyMode,
968    ) -> Result<db::IntegrityReport, MemoryError> {
969        let use_writer = mode == db::VerifyMode::Full;
970        let mut report = if use_writer {
971            self.with_write_conn(move |conn| db::verify_integrity_sync(conn, mode))
972                .await?
973        } else {
974            self.with_read_conn(move |conn| db::verify_integrity_sync(conn, mode))
975                .await?
976        };
977
978        #[cfg(feature = "hnsw")]
979        {
980            let hnsw_vectors = self
981                .inner
982                .hnsw_index
983                .read()
984                .unwrap_or_else(|e| e.into_inner())
985                .vector_snapshot();
986            let hnsw_dims = self.inner.config.embedding.dimensions;
987            let hnsw_files_exist = self.inner.paths.hnsw_files_exist();
988
989            let hnsw_issues = if use_writer {
990                let hnsw_vectors = hnsw_vectors.clone();
991                self.with_write_conn(move |conn| {
992                    verify_hnsw_key_level_integrity(
993                        conn,
994                        hnsw_dims,
995                        &hnsw_vectors,
996                        hnsw_files_exist,
997                    )
998                })
999                .await?
1000            } else {
1001                let hnsw_vectors = hnsw_vectors.clone();
1002                self.with_read_conn(move |conn| {
1003                    verify_hnsw_key_level_integrity(
1004                        conn,
1005                        hnsw_dims,
1006                        &hnsw_vectors,
1007                        hnsw_files_exist,
1008                    )
1009                })
1010                .await?
1011            };
1012            report.issues.extend(hnsw_issues);
1013        }
1014
1015        report.ok = report.issues.is_empty();
1016        Ok(report)
1017    }
1018
1019    /// Reconcile detected integrity issues.
1020    ///
1021    /// - `ReportOnly`: no-op, just returns the integrity report.
1022    /// - `RebuildFts`: rebuilds all FTS indexes from source data.
1023    /// - `ReEmbed`: not yet implemented (requires async embedding calls).
1024    pub async fn reconcile(
1025        &self,
1026        action: db::ReconcileAction,
1027    ) -> Result<db::IntegrityReport, MemoryError> {
1028        match action {
1029            db::ReconcileAction::ReportOnly => self.verify_integrity(db::VerifyMode::Full).await,
1030            db::ReconcileAction::RebuildFts => {
1031                self.with_write_conn(db::reconcile_fts).await?;
1032                #[cfg(feature = "hnsw")]
1033                self.sync_pending_hnsw_ops_best_effort("reconcile_rebuild_fts")
1034                    .await;
1035                self.verify_integrity(db::VerifyMode::Full).await
1036            }
1037            db::ReconcileAction::ReEmbed => {
1038                self.reembed_all().await?;
1039                self.verify_integrity(db::VerifyMode::Full).await
1040            }
1041        }
1042    }
1043
1044    /// Get the current configuration.
1045    pub fn config(&self) -> &MemoryConfig {
1046        &self.inner.config
1047    }
1048
1049    /// View the store as a derived graph over documents, chunks, facts, sessions, messages,
1050    /// episodes, namespaces, semantic similarity edges, and first-class stored graph edges.
1051    pub fn graph_view(&self) -> Arc<dyn GraphView> {
1052        graph::graph_view(self.inner.clone())
1053    }
1054
1055    // ─── First-class stored graph edges ──────────────────────────
1056
1057    /// Add a durable, typed graph edge between two nodes.
1058    ///
1059    /// Nodes are identified by prefixed IDs (e.g. `fact:<uuid>`,
1060    /// `namespace:<name>`, `document:<id>`). The edge type must be one of
1061    /// `GraphEdgeType::Semantic`, `Temporal`, `Causal`, or `Entity`.
1062    ///
1063    /// Insertion is idempotent on content digest — inserting the same edge
1064    /// twice returns the existing edge without creating a duplicate.
1065    ///
1066    /// Returns the stored edge including its assigned ID and recorded_at timestamp.
1067    pub async fn add_graph_edge(
1068        &self,
1069        source: &str,
1070        target: &str,
1071        edge_type: GraphEdgeType,
1072        weight: f64,
1073        metadata: Option<serde_json::Value>,
1074    ) -> Result<graph_edges::StoredGraphEdge, MemoryError> {
1075        let params = graph_edges::AddGraphEdgeParams {
1076            source: source.to_string(),
1077            target: target.to_string(),
1078            edge_type,
1079            weight,
1080            metadata,
1081        };
1082        self.with_write_conn(move |conn| graph_edges::insert_graph_edge(conn, &params))
1083            .await
1084    }
1085
1086    /// List all stored graph edges involving a given node (as source or target),
1087    /// excluding invalidated edges.
1088    pub async fn list_graph_edges_for_node(
1089        &self,
1090        node_id: &str,
1091    ) -> Result<Vec<graph_edges::StoredGraphEdge>, MemoryError> {
1092        let node_id = node_id.to_string();
1093        self.with_read_conn(move |conn| graph_edges::list_graph_edges_for_node(conn, &node_id))
1094            .await
1095    }
1096
1097    /// List ALL stored graph edges, excluding invalidated ones.
1098    pub async fn list_all_graph_edges(
1099        &self,
1100    ) -> Result<Vec<graph_edges::StoredGraphEdge>, MemoryError> {
1101        self.with_read_conn(graph_edges::list_all_graph_edges)
1102            .await
1103    }
1104
1105    /// Invalidate a stored graph edge by ID. Append-only — the row is never deleted.
1106    pub async fn invalidate_graph_edge(
1107        &self,
1108        edge_id: &str,
1109        reason: &str,
1110    ) -> Result<(), MemoryError> {
1111        let edge_id = edge_id.to_string();
1112        let reason = reason.to_string();
1113        self.with_write_conn(move |conn| {
1114            graph_edges::invalidate_graph_edge(conn, &edge_id, &reason)
1115        })
1116        .await
1117    }
1118
1119    /// Count non-invalidated stored graph edges.
1120    pub async fn count_graph_edges(&self) -> Result<usize, MemoryError> {
1121        self.with_read_conn(graph_edges::count_graph_edges)
1122            .await
1123    }
1124
1125    // ─── Search ─────────────────────────────────────────────────
1126
1127    /// Hybrid search across facts, document chunks, and searchable episodes.
1128    pub async fn search(
1129        &self,
1130        query: &str,
1131        top_k: Option<usize>,
1132        namespaces: Option<&[&str]>,
1133        source_types: Option<&[SearchSourceType]>,
1134    ) -> Result<Vec<SearchResult>, MemoryError> {
1135        Ok(self
1136            .search_with_context(
1137                query,
1138                top_k,
1139                namespaces,
1140                source_types,
1141                SearchContext::default_now(),
1142            )
1143            .await?
1144            .results)
1145    }
1146
1147    /// Hybrid search with an explicit deterministic context and optional receipt.
1148    pub async fn search_with_context(
1149        &self,
1150        query: &str,
1151        top_k: Option<usize>,
1152        namespaces: Option<&[&str]>,
1153        source_types: Option<&[SearchSourceType]>,
1154        context: SearchContext,
1155    ) -> Result<SearchResponse, MemoryError> {
1156        let k = top_k
1157            .unwrap_or(self.inner.config.search.default_top_k)
1158            .min(MAX_TOP_K);
1159
1160        let query_embedding = self.embed_text_internal(query).await?;
1161
1162        #[cfg(feature = "hnsw")]
1163        let hnsw_hits = if context.exactness_profile == ExactnessProfile::PreferExact
1164            || self.inner.config.search.uses_turbo_quant_backend()
1165        {
1166            Vec::new()
1167        } else {
1168            let candidates = self
1169                .inner
1170                .config
1171                .search
1172                .candidate_pool_size
1173                .max(k.saturating_mul(3))
1174                .min(MAX_HNSW_CANDIDATES);
1175            self.hnsw_search_blocking(query_embedding.clone(), candidates)
1176                .await
1177        };
1178
1179        let q = query.to_string();
1180        let config = self.inner.config.search.clone();
1181        let ns_owned = to_owned_string_vec(namespaces);
1182        let st_owned: Option<Vec<SearchSourceType>> = source_types.map(|s| s.to_vec());
1183        let context_owned = context.clone();
1184
1185        #[cfg(feature = "hnsw")]
1186        let hnsw_hits_owned = hnsw_hits;
1187
1188        let response = self
1189            .with_read_conn(move |conn| {
1190                if db::is_embeddings_dirty(conn)? {
1191                    tracing::warn!(
1192                        "Embeddings are stale after model change — search quality is degraded. \
1193                     Call reembed_all() to regenerate embeddings."
1194                    );
1195                }
1196                let ns_refs = as_str_slice(&ns_owned);
1197                let ns_slice: Option<&[&str]> = ns_refs.as_deref();
1198                let st_slice: Option<&[SearchSourceType]> = st_owned.as_deref();
1199
1200                #[cfg(feature = "hnsw")]
1201                {
1202                    let mut execution = if hnsw_hits_owned.is_empty() {
1203                        search::hybrid_search_detailed_with_context(
1204                            conn,
1205                            &q,
1206                            &query_embedding,
1207                            &config,
1208                            &context_owned,
1209                            k,
1210                            ns_slice,
1211                            st_slice,
1212                            None,
1213                        )
1214                    } else {
1215                        search::hybrid_search_with_hnsw_detailed_with_context(
1216                            conn,
1217                            &q,
1218                            &query_embedding,
1219                            &config,
1220                            &context_owned,
1221                            k,
1222                            ns_slice,
1223                            st_slice,
1224                            None,
1225                            &hnsw_hits_owned,
1226                        )
1227                    }?;
1228                    if context_owned.receipts_enabled()
1229                        && context_owned.exactness_profile == ExactnessProfile::PreferExact
1230                    {
1231                        if let Some(receipt) = execution.receipt.as_mut() {
1232                            receipt.search_profile = "hybrid_prefer_exact".to_string();
1233                        }
1234                    }
1235                    Ok(SearchResponse {
1236                        results: execution
1237                            .results
1238                            .into_iter()
1239                            .map(|result| result.result)
1240                            .collect(),
1241                        receipt: execution.receipt,
1242                    })
1243                }
1244                #[cfg(not(feature = "hnsw"))]
1245                {
1246                    let execution = search::hybrid_search_detailed_with_context(
1247                        conn,
1248                        &q,
1249                        &query_embedding,
1250                        &config,
1251                        &context_owned,
1252                        k,
1253                        ns_slice,
1254                        st_slice,
1255                        None,
1256                    )?;
1257                    Ok(SearchResponse {
1258                        results: execution
1259                            .results
1260                            .into_iter()
1261                            .map(|result| result.result)
1262                            .collect(),
1263                        receipt: execution.receipt,
1264                    })
1265                }
1266            })
1267            .await?;
1268        if let Some(receipt) = &response.receipt {
1269            self.persist_search_receipt(receipt).await?;
1270        }
1271        Ok(response)
1272    }
1273
1274    /// Full-text search only (no embeddings needed).
1275    pub async fn search_fts_only(
1276        &self,
1277        query: &str,
1278        top_k: Option<usize>,
1279        namespaces: Option<&[&str]>,
1280        source_types: Option<&[SearchSourceType]>,
1281    ) -> Result<Vec<SearchResult>, MemoryError> {
1282        let k = top_k
1283            .unwrap_or(self.inner.config.search.default_top_k)
1284            .min(MAX_TOP_K);
1285        let q = query.to_string();
1286        let config = self.inner.config.search.clone();
1287        let ns_owned = to_owned_string_vec(namespaces);
1288        let st_owned: Option<Vec<SearchSourceType>> = source_types.map(|s| s.to_vec());
1289        self.with_read_conn(move |conn| {
1290            let ns_refs = as_str_slice(&ns_owned);
1291            let ns_slice: Option<&[&str]> = ns_refs.as_deref();
1292            let st_slice: Option<&[SearchSourceType]> = st_owned.as_deref();
1293            search::fts_only_search(conn, &q, &config, k, ns_slice, st_slice, None)
1294        })
1295        .await
1296    }
1297
1298    /// Vector similarity search only (no FTS).
1299    pub async fn search_vector_only(
1300        &self,
1301        query: &str,
1302        top_k: Option<usize>,
1303        namespaces: Option<&[&str]>,
1304        source_types: Option<&[SearchSourceType]>,
1305    ) -> Result<Vec<SearchResult>, MemoryError> {
1306        Ok(self
1307            .search_vector_only_with_context(
1308                query,
1309                top_k,
1310                namespaces,
1311                source_types,
1312                SearchContext::default_now(),
1313            )
1314            .await?
1315            .results)
1316    }
1317
1318    /// Vector similarity search with an explicit deterministic context and optional receipt.
1319    pub async fn search_vector_only_with_context(
1320        &self,
1321        query: &str,
1322        top_k: Option<usize>,
1323        namespaces: Option<&[&str]>,
1324        source_types: Option<&[SearchSourceType]>,
1325        context: SearchContext,
1326    ) -> Result<SearchResponse, MemoryError> {
1327        let k = top_k
1328            .unwrap_or(self.inner.config.search.default_top_k)
1329            .min(MAX_TOP_K);
1330        let query_embedding = self.embed_text_internal(query).await?;
1331
1332        #[cfg(feature = "hnsw")]
1333        let hnsw_hits = if context.exactness_profile == ExactnessProfile::PreferExact
1334            || self.inner.config.search.uses_turbo_quant_backend()
1335        {
1336            Vec::new()
1337        } else {
1338            let candidates = self
1339                .inner
1340                .config
1341                .search
1342                .candidate_pool_size
1343                .max(k.saturating_mul(3))
1344                .min(MAX_HNSW_CANDIDATES);
1345            self.hnsw_search_blocking(query_embedding.clone(), candidates)
1346                .await
1347        };
1348
1349        let config = self.inner.config.search.clone();
1350        let ns_owned = to_owned_string_vec(namespaces);
1351        let st_owned: Option<Vec<SearchSourceType>> = source_types.map(|s| s.to_vec());
1352        let context_owned = context.clone();
1353
1354        #[cfg(feature = "hnsw")]
1355        let hnsw_hits_owned = hnsw_hits;
1356
1357        let response = self
1358            .with_read_conn(move |conn| {
1359                if db::is_embeddings_dirty(conn)? {
1360                    tracing::warn!(
1361                        "Embeddings are stale after model change — search quality is degraded. \
1362                     Call reembed_all() to regenerate embeddings."
1363                    );
1364                }
1365                let ns_refs = as_str_slice(&ns_owned);
1366                let ns_slice: Option<&[&str]> = ns_refs.as_deref();
1367                let st_slice: Option<&[SearchSourceType]> = st_owned.as_deref();
1368
1369                #[cfg(feature = "hnsw")]
1370                {
1371                    let mut execution = if hnsw_hits_owned.is_empty() {
1372                        search::vector_only_search_detailed_with_context(
1373                            conn,
1374                            &query_embedding,
1375                            &config,
1376                            &context_owned,
1377                            k,
1378                            ns_slice,
1379                            st_slice,
1380                            None,
1381                        )
1382                    } else {
1383                        search::vector_only_search_with_hnsw_detailed_with_context(
1384                            conn,
1385                            &query_embedding,
1386                            &config,
1387                            &context_owned,
1388                            k,
1389                            ns_slice,
1390                            st_slice,
1391                            None,
1392                            &hnsw_hits_owned,
1393                        )
1394                    }?;
1395                    if context_owned.receipts_enabled()
1396                        && context_owned.exactness_profile == ExactnessProfile::PreferExact
1397                    {
1398                        if let Some(receipt) = execution.receipt.as_mut() {
1399                            receipt.search_profile = "vector_only_prefer_exact".to_string();
1400                        }
1401                    }
1402                    Ok(SearchResponse {
1403                        results: execution
1404                            .results
1405                            .into_iter()
1406                            .map(|result| result.result)
1407                            .collect(),
1408                        receipt: execution.receipt,
1409                    })
1410                }
1411                #[cfg(not(feature = "hnsw"))]
1412                {
1413                    let execution = search::vector_only_search_detailed_with_context(
1414                        conn,
1415                        &query_embedding,
1416                        &config,
1417                        &context_owned,
1418                        k,
1419                        ns_slice,
1420                        st_slice,
1421                        None,
1422                    )?;
1423                    Ok(SearchResponse {
1424                        results: execution
1425                            .results
1426                            .into_iter()
1427                            .map(|result| result.result)
1428                            .collect(),
1429                        receipt: execution.receipt,
1430                    })
1431                }
1432            })
1433            .await?;
1434        if let Some(receipt) = &response.receipt {
1435            self.persist_search_receipt(receipt).await?;
1436        }
1437        Ok(response)
1438    }
1439
1440    // ─── Explainable Search ───────────────────────────────────
1441
1442    /// Search with full score breakdown for each result.
1443    pub async fn search_explained(
1444        &self,
1445        query: &str,
1446        top_k: Option<usize>,
1447        namespaces: Option<&[&str]>,
1448        source_types: Option<&[SearchSourceType]>,
1449    ) -> Result<Vec<types::ExplainedResult>, MemoryError> {
1450        Ok(self
1451            .search_explained_with_context(
1452                query,
1453                top_k,
1454                namespaces,
1455                source_types,
1456                SearchContext::default_now(),
1457            )
1458            .await?
1459            .results)
1460    }
1461
1462    /// Search with full score breakdown under an explicit deterministic context.
1463    pub async fn search_explained_with_context(
1464        &self,
1465        query: &str,
1466        top_k: Option<usize>,
1467        namespaces: Option<&[&str]>,
1468        source_types: Option<&[SearchSourceType]>,
1469        context: SearchContext,
1470    ) -> Result<types::ExplainedSearchResponse, MemoryError> {
1471        let k = top_k
1472            .unwrap_or(self.inner.config.search.default_top_k)
1473            .min(MAX_TOP_K);
1474        let query_embedding = self.embed_text_internal(query).await?;
1475
1476        #[cfg(feature = "hnsw")]
1477        let hnsw_hits = if context.exactness_profile == ExactnessProfile::PreferExact {
1478            Vec::new()
1479        } else {
1480            let candidates = self
1481                .inner
1482                .config
1483                .search
1484                .candidate_pool_size
1485                .max(k.saturating_mul(3))
1486                .min(MAX_HNSW_CANDIDATES);
1487            self.hnsw_search_blocking(query_embedding.clone(), candidates)
1488                .await
1489        };
1490
1491        let q = query.to_string();
1492        let config = self.inner.config.search.clone();
1493        let ns_owned = to_owned_string_vec(namespaces);
1494        let st_owned: Option<Vec<SearchSourceType>> = source_types.map(|value| value.to_vec());
1495        let context_owned = context.clone();
1496
1497        #[cfg(feature = "hnsw")]
1498        let hnsw_hits_owned = hnsw_hits;
1499
1500        let response = self
1501            .with_read_conn(move |conn| {
1502                let ns_refs = as_str_slice(&ns_owned);
1503                let ns_slice: Option<&[&str]> = ns_refs.as_deref();
1504                let st_slice: Option<&[SearchSourceType]> = st_owned.as_deref();
1505
1506                #[cfg(feature = "hnsw")]
1507                {
1508                    let mut execution = if hnsw_hits_owned.is_empty() {
1509                        search::hybrid_search_detailed_with_context(
1510                            conn,
1511                            &q,
1512                            &query_embedding,
1513                            &config,
1514                            &context_owned,
1515                            k,
1516                            ns_slice,
1517                            st_slice,
1518                            None,
1519                        )
1520                    } else {
1521                        search::hybrid_search_with_hnsw_detailed_with_context(
1522                            conn,
1523                            &q,
1524                            &query_embedding,
1525                            &config,
1526                            &context_owned,
1527                            k,
1528                            ns_slice,
1529                            st_slice,
1530                            None,
1531                            &hnsw_hits_owned,
1532                        )
1533                    }?;
1534                    if context_owned.receipts_enabled()
1535                        && context_owned.exactness_profile == ExactnessProfile::PreferExact
1536                    {
1537                        if let Some(receipt) = execution.receipt.as_mut() {
1538                            receipt.search_profile = "hybrid_prefer_exact".to_string();
1539                        }
1540                    }
1541                    Ok(types::ExplainedSearchResponse {
1542                        results: execution.results,
1543                        receipt: execution.receipt,
1544                    })
1545                }
1546                #[cfg(not(feature = "hnsw"))]
1547                {
1548                    let execution = search::hybrid_search_detailed_with_context(
1549                        conn,
1550                        &q,
1551                        &query_embedding,
1552                        &config,
1553                        &context_owned,
1554                        k,
1555                        ns_slice,
1556                        st_slice,
1557                        None,
1558                    )?;
1559                    Ok(types::ExplainedSearchResponse {
1560                        results: execution.results,
1561                        receipt: execution.receipt,
1562                    })
1563                }
1564            })
1565            .await?;
1566        if let Some(receipt) = &response.receipt {
1567            self.persist_search_receipt(receipt).await?;
1568        }
1569        Ok(response)
1570    }
1571
1572    /// Load a durable search receipt by receipt/request ID.
1573    pub async fn get_search_receipt(
1574        &self,
1575        receipt_id: &str,
1576    ) -> Result<Option<VectorSearchReceiptV1>, MemoryError> {
1577        let receipt_id = receipt_id.to_string();
1578        self.with_read_conn(move |conn| db::get_search_receipt(conn, &receipt_id))
1579            .await
1580    }
1581
1582    /// Replay a durable search receipt with caller-supplied query text and filters.
1583    ///
1584    /// Receipts intentionally do not store query text or filter values. The
1585    /// caller supplies those inputs, and the stored receipt supplies the
1586    /// deterministic evaluation time and retrieval family for comparison.
1587    pub async fn replay_search_receipt(
1588        &self,
1589        receipt_id: &str,
1590        query: &str,
1591        top_k: Option<usize>,
1592        namespaces: Option<&[&str]>,
1593        source_types: Option<&[SearchSourceType]>,
1594    ) -> Result<SearchReplayReportV1, MemoryError> {
1595        let original_receipt = self.get_search_receipt(receipt_id).await?.ok_or_else(|| {
1596            MemoryError::SearchReceiptNotFound {
1597                receipt_id: receipt_id.to_string(),
1598            }
1599        })?;
1600
1601        let vector_only = original_receipt.search_profile.starts_with("vector_only");
1602        let replay_top_k = top_k.or_else(|| Some(original_receipt.result_ids.len().max(1)));
1603        let replay_receipt_id = format!("{receipt_id}:replay:{}", uuid::Uuid::new_v4());
1604        let mut context = SearchContext::at(original_receipt.evaluation_time);
1605        context.receipt_mode = ReceiptMode::ReturnReceipt;
1606        context.request_id = Some(replay_receipt_id.clone());
1607        context.trace_id = original_receipt.trace_id.clone();
1608        context.attempt_family_id = original_receipt
1609            .attempt_family_id
1610            .clone()
1611            .or_else(|| Some(original_receipt.receipt_id.clone()));
1612        context.attempt_id = Some(replay_receipt_id.clone());
1613        context.replay_of = Some(original_receipt.receipt_id.clone());
1614        context.query_text_digest = original_receipt.query_text_digest.clone();
1615        context.query_input_digest = original_receipt.query_input_digest.clone();
1616        context.filter_digest = original_receipt.filter_digest.clone();
1617        context.redaction_state = original_receipt.redaction_state.clone();
1618        context.budget_id = original_receipt.budget_id.clone();
1619        context.exactness_profile = if original_receipt.approximate {
1620            ExactnessProfile::AllowApproximate
1621        } else {
1622            ExactnessProfile::PreferExact
1623        };
1624
1625        let replay_response = if vector_only {
1626            self.search_vector_only_with_context(
1627                query,
1628                replay_top_k,
1629                namespaces,
1630                source_types,
1631                context,
1632            )
1633            .await?
1634        } else {
1635            self.search_with_context(query, replay_top_k, namespaces, source_types, context)
1636                .await?
1637        };
1638        let replay_receipt = replay_response
1639            .receipt
1640            .ok_or_else(|| MemoryError::Other("replay did not produce a receipt".to_string()))?;
1641
1642        let query_embedding_digest_matches =
1643            original_receipt.query_embedding_digest == replay_receipt.query_embedding_digest;
1644        let result_ids_match = original_receipt.result_ids == replay_receipt.result_ids;
1645        let missing_result_ids = original_receipt
1646            .result_ids
1647            .iter()
1648            .filter(|id| !replay_receipt.result_ids.contains(*id))
1649            .cloned()
1650            .collect();
1651        let added_result_ids = replay_receipt
1652            .result_ids
1653            .iter()
1654            .filter(|id| !original_receipt.result_ids.contains(*id))
1655            .cloned()
1656            .collect();
1657
1658        Ok(SearchReplayReportV1 {
1659            receipt_id: original_receipt.receipt_id.clone(),
1660            replay_receipt_id,
1661            original_receipt,
1662            replay_receipt,
1663            query_embedding_digest_matches,
1664            result_ids_match,
1665            missing_result_ids,
1666            added_result_ids,
1667            vector_only,
1668        })
1669    }
1670
1671    // ─── Embedding Displacement ───────────────────────────────
1672
1673    /// Compute embedding displacement between two texts.
1674    pub async fn embedding_displacement(
1675        &self,
1676        text_a: &str,
1677        text_b: &str,
1678    ) -> Result<types::EmbeddingDisplacement, MemoryError> {
1679        let emb_a = self.embed_text_internal(text_a).await?;
1680        let emb_b = self.embed_text_internal(text_b).await?;
1681        Self::embedding_displacement_from_vecs(&emb_a, &emb_b)
1682    }
1683
1684    /// Compute embedding displacement from pre-computed vectors.
1685    pub fn embedding_displacement_from_vecs(
1686        a: &[f32],
1687        b: &[f32],
1688    ) -> Result<types::EmbeddingDisplacement, MemoryError> {
1689        if a.len() != b.len() {
1690            return Err(MemoryError::DimensionMismatch {
1691                expected: a.len(),
1692                actual: b.len(),
1693            });
1694        }
1695        let cosine_sim = search::cosine_similarity(a, b)?;
1696
1697        let euclidean_dist: f32 = a
1698            .iter()
1699            .zip(b.iter())
1700            .map(|(x, y)| (x - y) * (x - y))
1701            .sum::<f32>()
1702            .sqrt();
1703
1704        let mag_a: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt();
1705        let mag_b: f32 = b.iter().map(|x| x * x).sum::<f32>().sqrt();
1706
1707        Ok(types::EmbeddingDisplacement {
1708            cosine_similarity: cosine_sim,
1709            euclidean_distance: euclidean_dist,
1710            magnitude_a: mag_a,
1711            magnitude_b: mag_b,
1712        })
1713    }
1714
1715    // ─── Utility ────────────────────────────────────────────────
1716
1717    /// Chunk text using the configured strategy and token counter.
1718    pub fn chunk_text(&self, text: &str) -> Vec<TextChunk> {
1719        chunker::chunk_text(
1720            text,
1721            &self.inner.config.chunking,
1722            self.inner.token_counter.as_ref(),
1723        )
1724    }
1725
1726    /// Embed a single text via the configured provider.
1727    pub async fn embed(&self, text: &str) -> Result<Vec<f32>, MemoryError> {
1728        self.embed_text_internal(text).await
1729    }
1730
1731    /// Embed multiple texts in a batch.
1732    pub async fn embed_batch(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>, MemoryError> {
1733        let owned: Vec<String> = texts.iter().map(|s| s.to_string()).collect();
1734        self.embed_batch_internal(owned).await
1735    }
1736
1737    /// Get database statistics.
1738    pub async fn stats(&self) -> Result<MemoryStats, MemoryError> {
1739        let db_path = self.inner.paths.sqlite_path.clone();
1740        self.with_read_conn(move |conn| {
1741            let total_facts: u64 =
1742                conn.query_row("SELECT COUNT(*) FROM facts", [], |r| r.get(0))?;
1743            let total_documents: u64 =
1744                conn.query_row("SELECT COUNT(*) FROM documents", [], |r| r.get(0))?;
1745            let total_chunks: u64 =
1746                conn.query_row("SELECT COUNT(*) FROM chunks", [], |r| r.get(0))?;
1747            let total_sessions: u64 =
1748                conn.query_row("SELECT COUNT(*) FROM sessions", [], |r| r.get(0))?;
1749            let total_messages: u64 =
1750                conn.query_row("SELECT COUNT(*) FROM messages", [], |r| r.get(0))?;
1751
1752            let db_size = std::fs::metadata(&db_path).map(|m| m.len()).unwrap_or(0);
1753
1754            let (model, dims): (Option<String>, Option<usize>) = conn
1755                .query_row(
1756                    "SELECT model_name, dimensions FROM embedding_metadata WHERE id = 1",
1757                    [],
1758                    |r| Ok((Some(r.get(0)?), Some(r.get(1)?))),
1759                )
1760                .unwrap_or((None, None));
1761
1762            Ok(MemoryStats {
1763                total_facts,
1764                total_documents,
1765                total_chunks,
1766                total_sessions,
1767                total_messages,
1768                database_size_bytes: db_size,
1769                embedding_model: model,
1770                embedding_dimensions: dims,
1771            })
1772        })
1773        .await
1774    }
1775
1776    /// Return distinct scope_domain values stored in document metadata.
1777    ///
1778    /// Queries `json_extract(metadata, '$.scope_domain')` across all documents
1779    /// and returns the unique non-null values. Used by the Recall app to populate
1780    /// the scope picker dynamically instead of relying on a hardcoded list.
1781    pub async fn list_scope_domains(&self) -> Result<Vec<String>, MemoryError> {
1782        self.with_read_conn(|conn| {
1783            let mut stmt = conn.prepare(
1784                "SELECT DISTINCT json_extract(metadata, '$.scope_domain') \
1785                 FROM documents \
1786                 WHERE json_extract(metadata, '$.scope_domain') IS NOT NULL",
1787            )?;
1788            let domains: Vec<String> = stmt
1789                .query_map([], |row| row.get::<_, String>(0))?
1790                .filter_map(|r| r.ok())
1791                .collect();
1792            Ok(domains)
1793        })
1794        .await
1795    }
1796
1797    /// Check if embeddings need re-generation after a model change.
1798    pub async fn embeddings_are_dirty(&self) -> Result<bool, MemoryError> {
1799        self.with_read_conn(db::is_embeddings_dirty).await
1800    }
1801
1802    /// Re-embed all facts, chunks, messages, and episodes. Call after changing embedding models.
1803    pub async fn reembed_all(&self) -> Result<usize, MemoryError> {
1804        let mut count = 0usize;
1805        let batch_size = self.inner.config.embedding.batch_size;
1806        let dims = self.inner.config.embedding.dimensions;
1807
1808        // ─── Facts ──────────────────────────────────────────────────
1809        let fact_contents: Vec<(String, String)> = self
1810            .with_read_conn(|conn| {
1811                let mut stmt = conn.prepare("SELECT id, content FROM facts")?;
1812                let result = stmt
1813                    .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1814                    .collect::<Result<Vec<_>, _>>()?;
1815                Ok(result)
1816            })
1817            .await?;
1818
1819        let mut fact_count = 0usize;
1820        for batch in fact_contents.chunks(batch_size) {
1821            let texts: Vec<String> = batch.iter().map(|(_, c)| c.clone()).collect();
1822            let embeddings = self.embed_batch_internal(texts).await?;
1823            for embedding in &embeddings {
1824                self.validate_embedding_dimensions(embedding)?;
1825            }
1826
1827            let quantizer = Quantizer::new(dims);
1828            let updates: Vec<(String, Vec<u8>, Option<Vec<u8>>)> = batch
1829                .iter()
1830                .zip(embeddings.iter())
1831                .map(|((id, _), emb)| {
1832                    // INTENTIONAL: q8 quantization is an optional search optimization; missing q8 is non-fatal
1833                    let q8 = quantizer
1834                        .quantize(emb)
1835                        .map(|qv| quantize::pack_quantized(&qv))
1836                        .ok();
1837                    (id.clone(), db::embedding_to_bytes(emb), q8)
1838                })
1839                .collect();
1840
1841            self.with_write_conn(move |conn| {
1842                db::with_transaction(conn, |tx| {
1843                    for (fid, bytes, q8) in &updates {
1844                        tx.execute(
1845                            "UPDATE facts SET embedding = ?1, embedding_q8 = ?2, updated_at = datetime('now') WHERE id = ?3",
1846                            rusqlite::params![bytes, q8.as_deref(), fid],
1847                        )?;
1848                        #[cfg(feature = "hnsw")]
1849                        db::queue_pending_index_op(
1850                            tx,
1851                            &format!("fact:{fid}"),
1852                            "fact",
1853                            db::IndexOpKind::Upsert,
1854                        )?;
1855                        db::invalidate_derived_vector_artifact(tx, &format!("fact:{fid}"))?;
1856                    }
1857                    Ok(())
1858                })
1859            })
1860            .await?;
1861
1862            fact_count += batch.len();
1863            count += batch.len();
1864            if fact_count % 100 == 0 || fact_count == count {
1865                tracing::info!(fact_count, "Re-embedded {} facts so far", fact_count);
1866            }
1867        }
1868
1869        // ─── Chunks ─────────────────────────────────────────────────
1870        let chunk_data: Vec<(String, String)> = self
1871            .with_read_conn(|conn| {
1872                let mut stmt = conn.prepare("SELECT id, content FROM chunks")?;
1873                let result = stmt
1874                    .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1875                    .collect::<Result<Vec<_>, _>>()?;
1876                Ok(result)
1877            })
1878            .await?;
1879
1880        let mut chunk_count = 0usize;
1881        for batch in chunk_data.chunks(batch_size) {
1882            let texts: Vec<String> = batch.iter().map(|(_, c)| c.clone()).collect();
1883            let embeddings = self.embed_batch_internal(texts).await?;
1884            for embedding in &embeddings {
1885                self.validate_embedding_dimensions(embedding)?;
1886            }
1887
1888            let quantizer = Quantizer::new(dims);
1889            let updates: Vec<(String, Vec<u8>, Option<Vec<u8>>)> = batch
1890                .iter()
1891                .zip(embeddings.iter())
1892                .map(|((id, _), emb)| {
1893                    // INTENTIONAL: q8 quantization is an optional search optimization; missing q8 is non-fatal
1894                    let q8 = quantizer
1895                        .quantize(emb)
1896                        .map(|qv| quantize::pack_quantized(&qv))
1897                        .ok();
1898                    (id.clone(), db::embedding_to_bytes(emb), q8)
1899                })
1900                .collect();
1901
1902            self.with_write_conn(move |conn| {
1903                db::with_transaction(conn, |tx| {
1904                    for (cid, bytes, q8) in &updates {
1905                        tx.execute(
1906                            "UPDATE chunks SET embedding = ?1, embedding_q8 = ?2 WHERE id = ?3",
1907                            rusqlite::params![bytes, q8.as_deref(), cid],
1908                        )?;
1909                        #[cfg(feature = "hnsw")]
1910                        db::queue_pending_index_op(
1911                            tx,
1912                            &format!("chunk:{cid}"),
1913                            "chunk",
1914                            db::IndexOpKind::Upsert,
1915                        )?;
1916                        db::invalidate_derived_vector_artifact(tx, &format!("chunk:{cid}"))?;
1917                    }
1918                    Ok(())
1919                })
1920            })
1921            .await?;
1922
1923            chunk_count += batch.len();
1924            count += batch.len();
1925            if chunk_count % 100 == 0 {
1926                tracing::info!(chunk_count, "Re-embedded {} chunks so far", chunk_count);
1927            }
1928        }
1929
1930        // ─── Messages ───────────────────────────────────────────────
1931        let message_data: Vec<(i64, String)> = self
1932            .with_read_conn(|conn| {
1933                let mut stmt = conn.prepare("SELECT id, content FROM messages")?;
1934                let result = stmt
1935                    .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1936                    .collect::<Result<Vec<_>, _>>()?;
1937                Ok(result)
1938            })
1939            .await?;
1940
1941        let mut msg_count = 0usize;
1942        for batch in message_data.chunks(batch_size) {
1943            let texts: Vec<String> = batch.iter().map(|(_, c)| c.clone()).collect();
1944            let embeddings = self.embed_batch_internal(texts).await?;
1945            for embedding in &embeddings {
1946                self.validate_embedding_dimensions(embedding)?;
1947            }
1948
1949            let quantizer = Quantizer::new(dims);
1950            let updates: Vec<(i64, Vec<u8>, Option<Vec<u8>>)> = batch
1951                .iter()
1952                .zip(embeddings.iter())
1953                .map(|((id, _), emb)| {
1954                    // INTENTIONAL: q8 quantization is an optional search optimization; missing q8 is non-fatal
1955                    let q8 = quantizer
1956                        .quantize(emb)
1957                        .map(|qv| quantize::pack_quantized(&qv))
1958                        .ok();
1959                    (*id, db::embedding_to_bytes(emb), q8)
1960                })
1961                .collect();
1962
1963            self.with_write_conn(move |conn| {
1964                db::with_transaction(conn, |tx| {
1965                    for (mid, bytes, q8) in &updates {
1966                        tx.execute(
1967                            "UPDATE messages SET embedding = ?1, embedding_q8 = ?2 WHERE id = ?3",
1968                            rusqlite::params![bytes, q8.as_deref(), mid],
1969                        )?;
1970                        #[cfg(feature = "hnsw")]
1971                        db::queue_pending_index_op(
1972                            tx,
1973                            &format!("msg:{mid}"),
1974                            "message",
1975                            db::IndexOpKind::Upsert,
1976                        )?;
1977                        db::invalidate_derived_vector_artifact(tx, &format!("msg:{mid}"))?;
1978                    }
1979                    Ok(())
1980                })
1981            })
1982            .await?;
1983
1984            msg_count += batch.len();
1985            count += batch.len();
1986            if msg_count % 100 == 0 {
1987                tracing::info!(msg_count, "Re-embedded {} messages so far", msg_count);
1988            }
1989        }
1990
1991        // ─── Episodes ───────────────────────────────────────────────
1992        let episode_data: Vec<(String, String)> = self
1993            .with_read_conn(|conn| {
1994                let mut stmt = conn.prepare("SELECT episode_id, search_text FROM episodes")?;
1995                let result = stmt
1996                    .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
1997                    .collect::<Result<Vec<_>, _>>()?;
1998                Ok(result)
1999            })
2000            .await?;
2001
2002        let mut episode_count = 0usize;
2003        for batch in episode_data.chunks(batch_size) {
2004            let texts: Vec<String> = batch.iter().map(|(_, text)| text.clone()).collect();
2005            let embeddings = self.embed_batch_internal(texts).await?;
2006            for embedding in &embeddings {
2007                self.validate_embedding_dimensions(embedding)?;
2008            }
2009
2010            let quantizer = Quantizer::new(dims);
2011            let updates: Vec<(String, Vec<u8>, Option<Vec<u8>>)> = batch
2012                .iter()
2013                .zip(embeddings.iter())
2014                .map(|((episode_id, _), embedding)| {
2015                    // INTENTIONAL: q8 quantization is an optional search optimization; missing q8 is non-fatal
2016                    let q8 = quantizer
2017                        .quantize(embedding)
2018                        .map(|vector| quantize::pack_quantized(&vector))
2019                        .ok();
2020                    (episode_id.clone(), db::embedding_to_bytes(embedding), q8)
2021                })
2022                .collect();
2023
2024            self.with_write_conn(move |conn| {
2025                db::with_transaction(conn, |tx| {
2026                    for (episode_id, bytes, q8) in &updates {
2027                        tx.execute(
2028                            "UPDATE episodes
2029                             SET embedding = ?1,
2030                                 embedding_q8 = ?2,
2031                                 updated_at = datetime('now')
2032                             WHERE episode_id = ?3",
2033                            rusqlite::params![bytes, q8.as_deref(), episode_id],
2034                        )?;
2035                        #[cfg(feature = "hnsw")]
2036                        db::queue_pending_index_op(
2037                            tx,
2038                            &episodes::episode_item_key(episode_id),
2039                            "episode",
2040                            db::IndexOpKind::Upsert,
2041                        )?;
2042                        db::invalidate_derived_vector_artifact(
2043                            tx,
2044                            &episodes::episode_item_key(episode_id),
2045                        )?;
2046                    }
2047                    Ok(())
2048                })
2049            })
2050            .await?;
2051
2052            episode_count += batch.len();
2053            count += batch.len();
2054            if episode_count % 100 == 0 {
2055                tracing::info!(
2056                    episode_count,
2057                    "Re-embedded {} episodes so far",
2058                    episode_count
2059                );
2060            }
2061        }
2062
2063        // Clear the dirty flag
2064        self.with_write_conn(db::clear_embeddings_dirty).await?;
2065
2066        tracing::info!(
2067            facts = fact_count,
2068            chunks = chunk_count,
2069            messages = msg_count,
2070            episodes = episode_count,
2071            total = count,
2072            "Re-embedding complete"
2073        );
2074
2075        // Rebuild HNSW after re-embedding
2076        #[cfg(feature = "hnsw")]
2077        {
2078            tracing::info!("Rebuilding HNSW index after re-embedding...");
2079            let _receipt = self.rebuild_hnsw_index().await?;
2080        }
2081
2082        Ok(count)
2083    }
2084
2085    /// Vacuum the database (reclaim space after deletions).
2086    pub async fn vacuum(&self) -> Result<(), MemoryError> {
2087        self.with_write_conn(|conn| {
2088            conn.execute_batch("VACUUM")?;
2089            Ok(())
2090        })
2091        .await
2092    }
2093
2094    // ─── Projection Import ─────────────────────────────────────
2095
2096    /// Import a projection envelope atomically (V10 legacy path).
2097    ///
2098    /// ## Phase status: compatibility / migration-only
2099    ///
2100    /// This method is the V10 legacy import path. New integrations should use
2101    /// [`import_projection_batch()`](Self::import_projection_batch) instead,
2102    /// which accepts the canonical `ProjectionImportBatchV3` format from
2103    /// `forge-memory-bridge`.
2104    ///
2105    /// **Removal condition**: removed when all callers migrate to the bridge pipeline.
2106    ///
2107    /// **Idempotent**: re-importing the same envelope (same `envelope_id` +
2108    /// `schema_version` + `content_digest`) returns a receipt with
2109    /// `was_duplicate = true` and does not modify data.
2110    ///
2111    /// **Atomic**: all records are committed in a single transaction. On any
2112    /// failure the entire import is rolled back — no partial visibility.
2113    ///
2114    /// **Provenance**: each imported record's metadata is tagged with the
2115    /// envelope_id and source_authority for traceability.
2116    #[deprecated(
2117        since = "0.5.0",
2118        note = "Legacy V10 import envelope path is compatibility-only. Use `import_projection_batch()` and `ProjectionImportBatchV3` on the canonical lane."
2119    )]
2120    #[doc(hidden)]
2121    #[allow(deprecated)]
2122    pub async fn import_envelope(
2123        &self,
2124        envelope: &projection_import::ImportEnvelope,
2125    ) -> Result<projection_import::ImportReceipt, MemoryError> {
2126        projection_legacy_compat::import_envelope(self, envelope).await
2127    }
2128
2129    /// Check whether an envelope has already been imported.
2130    #[deprecated(
2131        since = "0.5.0",
2132        note = "Legacy V10 import envelope status reads are compatibility-only. Prefer the projection import log."
2133    )]
2134    #[doc(hidden)]
2135    #[allow(deprecated)]
2136    pub async fn import_status(
2137        &self,
2138        envelope_id: &projection_import::EnvelopeId,
2139    ) -> Result<Vec<projection_import::ImportReceipt>, MemoryError> {
2140        projection_legacy_compat::import_status(self, envelope_id).await
2141    }
2142
2143    /// List recent imports, optionally filtered by namespace.
2144    #[deprecated(
2145        since = "0.5.0",
2146        note = "Legacy V10 import log access is compatibility-only. Prefer new projection-import metadata."
2147    )]
2148    #[doc(hidden)]
2149    #[allow(deprecated)]
2150    pub async fn list_imports(
2151        &self,
2152        namespace: Option<&str>,
2153        limit: usize,
2154    ) -> Result<Vec<projection_import::ImportReceipt>, MemoryError> {
2155        projection_legacy_compat::list_imports(self, namespace, limit).await
2156    }
2157
2158    /// Get the most recent successful import timestamp for a namespace.
2159    #[allow(deprecated)]
2160    pub async fn last_import_at(&self, namespace: &str) -> Result<Option<String>, MemoryError> {
2161        projection_legacy_compat::last_import_at(self, namespace).await
2162    }
2163
2164    /// Query imported claim projection rows through the supported public read surface.
2165    pub async fn query_claim_versions(
2166        &self,
2167        query: ProjectionQuery,
2168    ) -> Result<Vec<ProjectionClaimVersion>, MemoryError> {
2169        self.with_read_conn(move |conn| projection_storage::query_claim_versions(conn, &query))
2170            .await
2171    }
2172
2173    /// Query imported relation projection rows through the supported public read surface.
2174    pub async fn query_relation_versions(
2175        &self,
2176        query: ProjectionQuery,
2177    ) -> Result<Vec<ProjectionRelationVersion>, MemoryError> {
2178        self.with_read_conn(move |conn| projection_storage::query_relation_versions(conn, &query))
2179            .await
2180    }
2181
2182    /// Query imported episode projection rows through the supported public read surface.
2183    pub async fn query_episodes(
2184        &self,
2185        query: ProjectionQuery,
2186    ) -> Result<Vec<ProjectionEpisode>, MemoryError> {
2187        self.with_read_conn(move |conn| projection_storage::query_episode_rows(conn, &query))
2188            .await
2189    }
2190
2191    /// Query imported entity-alias rows through the supported public read surface.
2192    pub async fn query_entity_aliases(
2193        &self,
2194        query: ProjectionQuery,
2195    ) -> Result<Vec<ProjectionEntityAlias>, MemoryError> {
2196        self.with_read_conn(move |conn| projection_storage::query_entity_aliases(conn, &query))
2197            .await
2198    }
2199
2200    /// Query imported evidence-reference rows through the supported public read surface.
2201    pub async fn query_evidence_refs(
2202        &self,
2203        query: ProjectionQuery,
2204    ) -> Result<Vec<ProjectionEvidenceRef>, MemoryError> {
2205        self.with_read_conn(move |conn| projection_storage::query_evidence_refs(conn, &query))
2206            .await
2207    }
2208
2209    /// Execute raw SQL. For testing only — not part of the stable public API.
2210    #[cfg(any(test, feature = "testing"))]
2211    pub async fn raw_execute(&self, sql: &str, params: Vec<String>) -> Result<usize, MemoryError> {
2212        let sql = sql.to_string();
2213        self.with_write_conn(move |conn| {
2214            let param_refs: Vec<&dyn rusqlite::types::ToSql> = params
2215                .iter()
2216                .map(|s| s as &dyn rusqlite::types::ToSql)
2217                .collect();
2218            Ok(conn.execute(&sql, &*param_refs)?)
2219        })
2220        .await
2221    }
2222}