leindex 1.7.1

// Core search engine implementation
//
// # Thread Safety
//
// `SearchEngine` is NOT thread-safe for concurrent writes. However:
// - `&SearchEngine` (shared reference) can be safely used for concurrent reads
// - `&mut SearchEngine` requires exclusive access for writes
// - VectorIndex uses internal HashMap which is not thread-safe
//
// For concurrent access, wrap in `Arc<RwLock<SearchEngine>>`.

use crate::search::hnsw::{HNSWIndex, HNSWParams};
use crate::search::quantization::int8_hnsw::{Int8HnswIndex, Int8HnswParams};
use crate::search::query::{MAX_EMBEDDING_DIMENSION, MIN_EMBEDDING_DIMENSION};
use crate::search::ranking::{HybridScorer, Score};
use crate::search::vector::VectorIndex;
use lru::LruCache;
use serde::{Deserialize, Serialize};
use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
use std::num::NonZeroUsize;

// ============================================================================
// CONSTANTS & VALIDATION
// ============================================================================

/// Default embedding dimension (CodeRank-compatible)
pub const DEFAULT_EMBEDDING_DIMENSION: usize = 768;

/// Maximum number of nodes that can be indexed (prevents memory exhaustion)
pub const MAX_NODES: usize = 1_000_000;

// ============================================================================
// A+ BOUND-GATED INDEXING, SELECTIVE PRUNING, AND WORK HOISTING
// ============================================================================

/// Conservative bound for the maximum number of nodes admitted in a single
/// indexing batch. When a batch exceeds this, the gate sheds or defers
/// additional nodes instead of admitting unbounded resident state growth.
pub const INDEXING_BATCH_NODE_CAP: usize = 50_000;

/// Conservative bound for the maximum total content bytes admitted in a single
/// indexing batch. Nodes whose cumulative content exceeds this are shed.
pub const INDEXING_BATCH_BYTE_CAP: usize = 512 * 1024 * 1024; // 512 MiB

/// Maximum number of entries in the work-hoister cache.
pub const WORK_HOISTER_MAX_ENTRIES: usize = 4_096;

/// Maximum byte budget for the work-hoister cache.
pub const WORK_HOISTER_MAX_BYTES: usize = 8 * 1024 * 1024; // 8 MiB

/// Decision returned by the content pruner for each node.
///
/// This enum is externally observable in tests so that pruning behavior
/// remains transparent and reversible at the contract surface.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PruningDecision {
    /// Node is kept for full indexing.
    Keep,
    /// Node is pruned because it appears to be generated code.
    GeneratedCode(String),
    /// Node is pruned because it has very low information content.
    LowInformation(String),
}

/// Selective content pruner for A+ memory discipline.
///
/// Identifies low-information nodes and recognized generated-code patterns
/// so they can be excluded from heavyweight indexing/search execution paths.
/// The pruner is conservative: user-authored high-signal files remain eligible.
///
/// Pruning is externally observable through the `PruningDecision` return value
/// and is reversible — the decision can be overridden by the caller.
#[derive(Debug, Clone)]
pub struct ContentPruner {
    /// Recognized generated-code file name suffixes.
    generated_suffixes: Vec<String>,
    /// Recognized generated-code path substrings.
    generated_path_patterns: Vec<String>,
    /// Minimum content length (bytes) below which a node is considered
    /// low-information. Nodes with content shorter than this are candidates
    /// for pruning unless they have a non-trivial symbol name.
    min_content_bytes: usize,
}

impl Default for ContentPruner {
    fn default() -> Self {
        Self::new()
    }
}

impl ContentPruner {
    /// Create a new pruner with default generated-code patterns.
    pub fn new() -> Self {
        Self {
            generated_suffixes: vec![
                ".min.js".into(),
                ".min.css".into(),
                ".pb.go".into(),
                ".generated.rs".into(),
                ".bundle.js".into(),
                ".chunk.js".into(),
                "_pb2.py".into(),
                ".d.ts".into(),
            ],
            generated_path_patterns: vec![
                "node_modules".into(),
                "/vendor/".into(),
                "/generated/".into(),
                "/autogenerated/".into(),
                "\\vendor\\".into(),
            ],
            min_content_bytes: 16,
        }
    }

    /// Evaluate whether a node should be pruned from heavyweight paths.
    ///
    /// Returns a `PruningDecision` explaining the outcome. The caller can
    /// inspect the decision and choose to override it.
    pub fn evaluate(&self, file_path: &str, content: &str, symbol_name: &str) -> PruningDecision {
        // Check generated-code file patterns first (highest confidence).
        let fp_lower = file_path.to_ascii_lowercase();
        for suffix in &self.generated_suffixes {
            if fp_lower.ends_with(suffix) {
                return PruningDecision::GeneratedCode(format!(
                    "file ends with generated suffix '{}'",
                    suffix
                ));
            }
        }
        for pattern in &self.generated_path_patterns {
            if fp_lower.contains(pattern) {
                return PruningDecision::GeneratedCode(format!(
                    "file path contains generated pattern '{}'",
                    pattern
                ));
            }
        }

        // Check low-information content. Very short content with a trivial
        // symbol name is unlikely to carry meaningful search signal.
        if content.len() < self.min_content_bytes && symbol_name.len() < 3 {
            return PruningDecision::LowInformation(format!(
                "content {} bytes < min {}, symbol '{}' < 3 chars",
                content.len(),
                self.min_content_bytes,
                symbol_name
            ));
        }

        PruningDecision::Keep
    }

    /// Check if a file path matches generated-code patterns without
    /// considering content. Useful for pre-filtering during file scanning.
    pub fn is_generated_path(&self, file_path: &str) -> bool {
        let fp_lower = file_path.to_ascii_lowercase();
        for suffix in &self.generated_suffixes {
            if fp_lower.ends_with(suffix) {
                return true;
            }
        }
        for pattern in &self.generated_path_patterns {
            if fp_lower.contains(pattern) {
                return true;
            }
        }
        false
    }
}

/// Bound-gated indexing admission control.
///
/// When indexing pressure exceeds configured conservative bounds, the gate
/// sheds, defers, or otherwise gates additional parse/index work instead of
/// admitting unbounded resident state growth.
///
/// The gate tracks cumulative node count and content bytes within a single
/// indexing batch. When either cap is exceeded, subsequent nodes are shed
/// (skipped) rather than admitted.
#[derive(Debug)]
pub struct IndexingAdmissionGate {
    /// Maximum nodes to admit in this batch.
    node_cap: usize,
    /// Maximum total content bytes to admit in this batch.
    byte_cap: usize,
    /// Nodes admitted so far.
    nodes_admitted: usize,
    /// Content bytes admitted so far.
    bytes_admitted: usize,
    /// Nodes shed (rejected) so far.
    nodes_shed: usize,
}

impl IndexingAdmissionGate {
    /// Create a new gate with default A+ caps.
    pub fn new() -> Self {
        Self {
            node_cap: INDEXING_BATCH_NODE_CAP,
            byte_cap: INDEXING_BATCH_BYTE_CAP,
            nodes_admitted: 0,
            bytes_admitted: 0,
            nodes_shed: 0,
        }
    }

    /// Create a gate with custom caps (for testing).
    pub fn with_caps(node_cap: usize, byte_cap: usize) -> Self {
        Self {
            node_cap,
            byte_cap,
            nodes_admitted: 0,
            bytes_admitted: 0,
            nodes_shed: 0,
        }
    }

    /// Try to admit a node. Returns `true` if the node is admitted, `false`
    /// if it is shed due to exceeding bounds.
    pub fn try_admit(&mut self, content_bytes: usize) -> bool {
        if self.nodes_admitted >= self.node_cap {
            self.nodes_shed += 1;
            return false;
        }
        if self.bytes_admitted + content_bytes > self.byte_cap {
            self.nodes_shed += 1;
            return false;
        }
        self.nodes_admitted += 1;
        self.bytes_admitted += content_bytes;
        true
    }

    /// Number of nodes admitted so far.
    pub fn nodes_admitted(&self) -> usize {
        self.nodes_admitted
    }

    /// Number of nodes shed (rejected) so far.
    pub fn nodes_shed(&self) -> usize {
        self.nodes_shed
    }

    /// Total content bytes admitted so far.
    pub fn bytes_admitted(&self) -> usize {
        self.bytes_admitted
    }

    /// Reset the gate for a new batch.
    pub fn reset(&mut self) {
        self.nodes_admitted = 0;
        self.bytes_admitted = 0;
        self.nodes_shed = 0;
    }
}

impl Default for IndexingAdmissionGate {
    fn default() -> Self {
        Self::new()
    }
}

/// Cached intermediate work entry for the work hoister.
struct HoistedWork {
    /// The TF-IDF embedding computed for this content.
    embedding: Vec<f32>,
    /// The neural embedding computed for this content (if available).
    /// Cached alongside TF-IDF to avoid redundant ONNX inference on cache hits.
    neural_embedding: Option<Vec<f32>>,
    /// Approximate byte size of this cache entry.
    byte_size: usize,
}

/// Bounded repeated-work hoister.
///
/// Caches intermediate TF-IDF and neural embedding results keyed by a BLAKE3
/// content hash within a bounded window. When the same content is encountered again
/// (e.g., during incremental reindexing of unchanged files), the hoisted
/// result is reused instead of recomputing the embedding.
///
/// The cache is bounded by both entry count and total bytes. When either
/// limit is exceeded, the least-recently-used entry is evicted.
pub struct WorkHoister {
    cache: LruCache<blake3::Hash, HoistedWork>,
    tracked_bytes: usize,
    max_bytes: usize,
}

impl WorkHoister {
    /// Create a new hoister with default A+ bounds.
    pub fn new() -> Self {
        Self {
            cache: LruCache::new(NonZeroUsize::new(WORK_HOISTER_MAX_ENTRIES).unwrap()),
            tracked_bytes: 0,
            max_bytes: WORK_HOISTER_MAX_BYTES,
        }
    }

    /// Create a hoister with custom bounds (for testing).
    pub fn with_bounds(max_entries: usize, max_bytes: usize) -> Self {
        Self {
            cache: LruCache::new(NonZeroUsize::new(max_entries.max(1)).unwrap()),
            tracked_bytes: 0,
            max_bytes,
        }
    }

    /// Look up a previously hoisted embedding for the given content.
    /// Returns `Some((tfidf_embedding, neural_embedding))` if found, `None` otherwise.
    pub fn lookup(&mut self, content: &str) -> Option<(Vec<f32>, Option<Vec<f32>>)> {
        let hash = blake3::hash(content.as_bytes());
        self.cache.get(&hash).map(|entry| {
            (entry.embedding.clone(), entry.neural_embedding.clone())
        })
    }

    /// Store a computed embedding for the given content.
    /// If the cache is full, evicts the least-recently-used entry first.
    pub fn store(&mut self, content: &str, embedding: Vec<f32>, neural_embedding: Option<Vec<f32>>) {
        let hash = blake3::hash(content.as_bytes());
        let neural_byte_size = neural_embedding.as_ref().map_or(0, |v| v.len() * std::mem::size_of::<f32>());
        let byte_size = 32 + embedding.len() * std::mem::size_of::<f32>() + neural_byte_size;
        let entry = HoistedWork {
            embedding,
            neural_embedding,
            byte_size,
        };

        if let Some(existing) = self.cache.put(hash, entry) {
            self.tracked_bytes = self.tracked_bytes.saturating_sub(existing.byte_size);
        }

        // Evict until there is room.
        while self.tracked_bytes + byte_size > self.max_bytes && !self.cache.is_empty() {
            if let Some((evicted_hash, evicted)) = self.cache.pop_lru() {
                self.tracked_bytes = self.tracked_bytes.saturating_sub(evicted.byte_size);
                // If the just-inserted entry was evicted, stop — don't count it.
                if evicted_hash == hash {
                    return;
                }
            }
        }

        // Only increment tracked_bytes if the entry is still in the cache after eviction.
        if self.cache.contains(&hash) {
            self.tracked_bytes += byte_size;
        }
    }

    /// Number of entries currently in the hoister.
    pub fn len(&self) -> usize {
        self.cache.len()
    }

    /// Whether the hoister is empty.
    pub fn is_empty(&self) -> bool {
        self.cache.is_empty()
    }

    /// Tracked byte usage.
    pub fn bytes_used(&self) -> usize {
        self.tracked_bytes
    }

    /// Clear the hoister.
    pub fn clear(&mut self) {
        self.cache.clear();
        self.tracked_bytes = 0;
    }
}

impl Default for WorkHoister {
    fn default() -> Self {
        Self::new()
    }
}

// ============================================================================
// VECTOR INDEX IMPLEMENTATION
// ============================================================================

/// Vector index implementation
///
/// Enum that wraps either the brute-force VectorIndex or the HNSW-based HNSWIndex.
/// This allows switching between implementations at runtime.
pub enum VectorIndexImpl {
    /// Brute-force vector index (exact search)
    BruteForce(VectorIndex),

    /// HNSW-based approximate nearest neighbor index
    HNSW(Box<HNSWIndex>),
    /// INT8 quantized HNSW-based approximate nearest neighbor index
    HNSWQuantized(Box<Int8HnswIndex>),
}

impl VectorIndexImpl {
    /// Get the number of vectors in the index
    #[must_use]
    pub fn len(&self) -> usize {
        match self {
            Self::BruteForce(idx) => idx.len(),
            Self::HNSW(idx) => idx.len(),
            Self::HNSWQuantized(idx) => idx.len(),
        }
    }

    /// Check if the index is empty
    #[must_use]
    pub fn is_empty(&self) -> bool {
        match self {
            Self::BruteForce(idx) => idx.is_empty(),
            Self::HNSW(idx) => idx.is_empty(),
            Self::HNSWQuantized(idx) => idx.is_empty(),
        }
    }

    /// Get the embedding dimension
    #[must_use]
    pub fn dimension(&self) -> usize {
        match self {
            Self::BruteForce(idx) => idx.dimension(),
            Self::HNSW(idx) => idx.dimension(),
            Self::HNSWQuantized(idx) => idx.dimension(),
        }
    }

    /// Search for similar vectors
    pub fn search(&self, query: &[f32], top_k: usize) -> Vec<(String, f32)> {
        match self {
            Self::BruteForce(idx) => idx.search(query, top_k),
            Self::HNSW(idx) => idx.search(query, top_k),
            Self::HNSWQuantized(idx) => idx.search(query, top_k),
        }
    }

    /// Insert a vector into the index
    pub fn insert(&mut self, node_id: String, vector: Vec<f32>) -> Result<(), VectorIndexError> {
        match self {
            Self::BruteForce(idx) => idx
                .insert(node_id, vector)
                .map_err(|e| VectorIndexError::InsertionFailed(e.to_string())),
            Self::HNSW(idx) => idx
                .insert(node_id, vector)
                .map_err(|e| VectorIndexError::InsertionFailed(e.to_string())),
            Self::HNSWQuantized(idx) => idx
                .insert(node_id, vector)
                .map_err(|e| VectorIndexError::InsertionFailed(e.to_string())),
        }
    }

    /// Clear all vectors from the index
    pub fn clear(&mut self) {
        match self {
            Self::BruteForce(idx) => idx.clear(),
            Self::HNSW(idx) => idx.clear(),
            Self::HNSWQuantized(idx) => idx.clear(),
        }
    }

    /// Remove a vector from the index by node ID.
    ///
    /// Returns `true` if the node was found and removed, `false` otherwise.
    /// For HNSW indexes, removal is lazy (marks as deleted); use `rebuild()` to reclaim memory.
    pub fn remove(&mut self, node_id: &str) -> bool {
        match self {
            Self::BruteForce(idx) => idx.remove(node_id),
            Self::HNSW(idx) => idx.remove(node_id),
            Self::HNSWQuantized(idx) => idx.remove(node_id),
        }
    }

    /// Check if HNSW is enabled
    #[must_use]
    pub fn is_hnsw_enabled(&self) -> bool {
        matches!(self, Self::HNSW(_) | Self::HNSWQuantized(_))
    }

    /// Get estimated memory usage in bytes
    #[must_use]
    pub fn estimated_memory_bytes(&self) -> usize {
        match self {
            Self::BruteForce(idx) => (*idx).estimated_memory_bytes(),
            Self::HNSW(idx) => (*idx).estimated_memory_bytes(),
            Self::HNSWQuantized(idx) => (*idx).estimated_memory_bytes(),
        }
    }
}

/// Vector index errors
#[derive(Debug, thiserror::Error)]
pub enum VectorIndexError {
    /// Failed to insert a vector into the index
    #[error("Insertion failed: {0}")]
    InsertionFailed(String),

    /// General index operation failure
    #[error("Index operation failed: {0}")]
    IndexOperationFailed(String),
}

// ============================================================================
// NODE INFORMATION
// ============================================================================

/// Node information for indexing
///
/// This represents a single code node (function, class, module) that can be
/// indexed and searched.
///
/// ## Serialization compatibility
///
/// The legacy `embedding: Option<Vec<f32>>` field is no longer stored on the
/// struct. During deserialization, payloads that contain the old `embedding`
/// field (and may lack `tfidf_embedding`) are accepted through a compatibility
/// bridge: the bridge prefers `tfidf_embedding` when present and non-empty,
/// otherwise promotes the legacy `embedding` value, otherwise defaults to empty.
/// Serialization always emits only the new layout (no `embedding` field).
#[derive(Debug, Clone)]
pub struct NodeInfo {
    /// Unique node ID
    pub node_id: String,

    /// File path
    pub file_path: String,

    /// Symbol name
    pub symbol_name: String,

    /// Programming language
    pub language: String,

    /// Source content
    pub content: String,

    /// Byte range in source
    pub byte_range: (usize, usize),

    /// TF-IDF embedding (always present, 768-dim, for hybrid search)
    pub tfidf_embedding: Vec<f32>,

    /// Neural/remote embedding (optional enhancement, for hybrid search)
    pub neural_embedding: Option<Vec<f32>>,

    /// Complexity score (0-100+, higher = more complex)
    pub complexity: u32,

    /// Cached signature extracted from content (for search results)
    /// This is extracted before content is cleared during T13 optimization
    pub signature: Option<String>,

    /// Pre-tokenized search tokens (lowercased, filtered by length >= 2).
    ///
    /// When `Some`, these tokens are used directly for the inverted index
    /// instead of re-tokenizing from `content`. This enables callers that
    /// already have tokenized content (e.g., `index_builder`) to skip the
    /// redundant split+lowercase pass.
    ///
    /// Backward-compatible: `None` falls back to `content.split()` tokenization.
    pub pre_tokenized: Option<Vec<String>>,
}

// ---------------------------------------------------------------------------
// Compatibility bridge: deserialize both old and new payload shapes
// ---------------------------------------------------------------------------

/// Intermediate representation used during deserialization to accept both the
/// legacy shape (`embedding: Option<Vec<f32>>`) and the new shape
/// (`tfidf_embedding: Vec<f32>`).
#[derive(Deserialize)]
struct NodeInfoRepr {
    node_id: String,
    file_path: String,
    symbol_name: String,
    language: String,
    content: String,
    byte_range: (usize, usize),

    #[serde(default)]
    tfidf_embedding: Vec<f32>,

    #[serde(default)]
    neural_embedding: Option<Vec<f32>>,

    /// Legacy field — accepted from old payloads but never written back out.
    #[serde(default, alias = "embedding")]
    legacy_embedding: Option<Vec<f32>>,

    #[serde(default)]
    complexity: u32,

    #[serde(default)]
    signature: Option<String>,

    #[serde(default)]
    pre_tokenized: Option<Vec<String>>,
}

impl<'de> Deserialize<'de> for NodeInfo {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: serde::Deserializer<'de>,
    {
        let repr = NodeInfoRepr::deserialize(deserializer)?;

        // Resolution rule (per spec §5.5):
        //   1. Prefer tfidf_embedding if present and non-empty.
        //   2. Otherwise promote legacy embedding value if present and non-empty.
        //   3. Otherwise default to empty.
        let tfidf_embedding = if !repr.tfidf_embedding.is_empty() {
            repr.tfidf_embedding
        } else if let Some(legacy) = repr.legacy_embedding {
            if !legacy.is_empty() {
                legacy
            } else {
                Vec::new()
            }
        } else {
            Vec::new()
        };

        Ok(Self {
            node_id: repr.node_id,
            file_path: repr.file_path,
            symbol_name: repr.symbol_name,
            language: repr.language,
            content: repr.content,
            byte_range: repr.byte_range,
            tfidf_embedding,
            neural_embedding: repr.neural_embedding,
            complexity: repr.complexity,
            signature: repr.signature,
            pre_tokenized: repr.pre_tokenized,
        })
    }
}

impl Serialize for NodeInfo {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: serde::Serializer,
    {
        // Always serialize the new layout — never write the legacy `embedding` field.
        #[derive(Serialize)]
        struct NodeInfoNew<'a> {
            node_id: &'a str,
            file_path: &'a str,
            symbol_name: &'a str,
            language: &'a str,
            content: &'a str,
            byte_range: (usize, usize),
            tfidf_embedding: &'a [f32],
            neural_embedding: &'a Option<Vec<f32>>,
            complexity: u32,
            signature: &'a Option<String>,
            pre_tokenized: &'a Option<Vec<String>>,
        }

        NodeInfoNew {
            node_id: &self.node_id,
            file_path: &self.file_path,
            symbol_name: &self.symbol_name,
            language: &self.language,
            content: &self.content,
            byte_range: self.byte_range,
            tfidf_embedding: &self.tfidf_embedding,
            neural_embedding: &self.neural_embedding,
            complexity: self.complexity,
            signature: &self.signature,
            pre_tokenized: &self.pre_tokenized,
        }
        .serialize(serializer)
    }
}

/// Pre-computed query data for optimized text scoring
///
/// This struct holds data that is pre-computed once per search to avoid
/// repeated allocations in the hot path. When searching N nodes, this reduces
/// allocations from O(N) to O(1).
struct TextQueryPreprocessed {
    /// Lowercase query for case-insensitive matching
    query_lower: String,
    /// Query tokens for overlap calculation
    query_tokens: HashSet<String>,
}

impl TextQueryPreprocessed {
    /// Create pre-computed query data
    fn from_query(query: &str) -> Self {
        let query_lower = query.to_ascii_lowercase();
        // Tokenize using the same logic as the content indexing
        let query_tokens: HashSet<_> = query
            .split(|c: char| !c.is_alphanumeric())
            .map(|s| s.to_ascii_lowercase())
            .filter(|s| s.len() >= 2)
            .collect();

        Self {
            query_lower,
            query_tokens,
        }
    }
}

// ============================================================================
// SEARCH QUERY
// ============================================================================

/// Search query
///
/// This represents a search request with all parameters needed to execute
/// a search operation.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SearchQuery {
    /// Query text
    pub query: String,

    /// Maximum results to return (validated by QueryParser)
    pub top_k: usize,

    /// Token budget for context expansion (validated by QueryParser)
    pub token_budget: Option<usize>,

    /// Whether to use semantic search
    pub semantic: bool,

    /// Whether to expand context using graph traversal
    pub expand_context: bool,

    /// Optional query embedding for semantic search
    pub query_embedding: Option<Vec<f32>>,

    /// Minimum relevance threshold (0.0-1.0)
    pub threshold: Option<f32>,

    /// Query type for adaptive ranking
    pub query_type: Option<crate::search::ranking::QueryType>,
}

// ============================================================================
// SEARCH RESULT
// ============================================================================

/// Search result
///
/// This represents a single search result with all relevant metadata.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SearchResult {
    /// Result rank (1-based)
    pub rank: usize,

    /// Node ID
    pub node_id: String,

    /// File path
    pub file_path: String,

    /// Symbol name
    pub symbol_name: String,

    /// Symbol type: function | method | class | variable | module
    ///
    /// Populated by `LeIndex::search()` from PDG node type.
    /// `None` when the node is not in the PDG (e.g., external refs).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub symbol_type: Option<String>,

    /// First line of the symbol's source (declaration / signature).
    ///
    /// Extracted from `node.content` — the second line after the
    /// `// name in path` header comment, trimmed of leading whitespace.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub signature: Option<String>,

    /// Cyclomatic complexity score of the symbol.
    pub complexity: u32,

    /// Number of call-sites that invoke this symbol (direct callers in PDG).
    ///
    /// Populated by `LeIndex::search()`. `None` if PDG is unavailable.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub caller_count: Option<usize>,

    /// Number of symbols this symbol depends on (outgoing PDG edges).
    ///
    /// Populated by `LeIndex::search()`. `None` if PDG is unavailable.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub dependency_count: Option<usize>,

    /// Programming language
    pub language: String,

    /// Relevance score
    pub score: Score,

    /// Expanded context (if requested)
    pub context: Option<String>,

    /// Byte range in source
    pub byte_range: (usize, usize),
}

// ============================================================================
// SEARCH ENGINE
// ============================================================================

// ---------------------------------------------------------------------------
// B-phase compact resident metadata (Plan 2)
// ---------------------------------------------------------------------------

/// Compact token-to-rows index using integer-backed addressing.
///
/// Maps each token to a set of row indices (`u32`) instead of string node IDs.
/// This is the compressed form of the inverted index for resident search state.
#[derive(Debug, Clone)]
pub struct CompactTokenIndex {
    /// Token → set of row indices.
    token_rows: HashMap<String, HashSet<u32>>,
}

impl CompactTokenIndex {
    /// Return the set of row indices that contain the given token.
    ///
    /// Returns an empty set if the token is not in the index.
    pub fn nodes_for_token(&self, token: &str) -> &HashSet<u32> {
        static EMPTY: std::sync::OnceLock<HashSet<u32>> = std::sync::OnceLock::new();
        self.token_rows
            .get(token)
            .unwrap_or_else(|| EMPTY.get_or_init(HashSet::new))
    }

    /// Return the number of distinct tokens in the index.
    pub fn token_count(&self) -> usize {
        self.token_rows.len()
    }
}

/// Compact, row-oriented snapshot of resident search metadata.
///
/// Uses `u32` row indices instead of string-heavy node-ID maps. This is the
/// B-phase compressed resident state that reduces memory overhead while
/// preserving stable lookup semantics (VAL-BPHASE-041).
#[derive(Debug, Clone)]
pub struct CompactNodeMetadata {
    /// Node ID → row index mapping (compact u32 addressing).
    row_map: Vec<(String, u32)>,
    /// Complexity values indexed by row (compact u32 array).
    complexity_by_row: Vec<u32>,
    /// Token index using row-based addressing.
    token_index: CompactTokenIndex,
}

impl CompactNodeMetadata {
    /// Look up the row index for a given node ID.
    ///
    /// Returns `None` if the node is not in the compact metadata.
    pub fn row_index(&self, node_id: &str) -> Option<u32> {
        self.row_map
            .iter()
            .find(|(id, _)| id == node_id)
            .map(|(_, row)| *row)
    }

    /// Look up the complexity for a given row index.
    ///
    /// Returns `None` if the row is out of range.
    pub fn complexity_by_row(&self, row: u32) -> Option<u32> {
        self.complexity_by_row.get(row as usize).copied()
    }

    /// Return the compact token index.
    pub fn token_index(&self) -> &CompactTokenIndex {
        &self.token_index
    }

    /// Return the number of nodes in the compact metadata.
    pub fn node_count(&self) -> usize {
        self.row_map.len()
    }
}
/// Search engine combining vector and graph search
///
/// This is the main entry point for search operations. It combines:
/// - Text-based search for keyword matching
/// - Vector-based semantic search for similarity
/// - Hybrid scoring combining multiple signals
///
/// Supports both brute-force and HNSW vector search backends.
///
/// # Thread Safety
///
/// - Reads (`&SearchEngine`) are thread-safe for concurrent access
/// - Writes (`&mut SearchEngine`) require exclusive access
/// - The internal VectorIndexImpl is NOT thread-safe for concurrent writes
///
/// For concurrent read-write access, wrap in `Arc<RwLock<SearchEngine>>`.
///
/// # Example
///
/// ```ignore
/// let mut engine = SearchEngine::new();
/// engine.index_nodes(nodes);
/// let results = engine.search(query)?;
/// ```
pub struct SearchEngine {
    nodes: Vec<NodeInfo>,
    scorer: HybridScorer,
    vector_index: VectorIndexImpl,
    /// Complexity cache for O(1) lookups (fixes O(n²) bug)
    complexity_cache: HashMap<String, u32>,
    /// Inverted index for O(1) text lookups: token -> set of node IDs
    /// This allows sub-linear text search instead of O(N) scan
    text_index: HashMap<String, HashSet<String>>,
    /// Node ID to index mapping for O(1) node lookups (fixes A1)
    /// Populated during index_nodes() and maintained on updates
    node_id_to_idx: HashMap<String, usize>,
    /// Per-node token cache: node_id -> set of normalized tokens
    /// Populated during index_nodes() to avoid re-tokenization in scoring
    node_tokens: HashMap<String, HashSet<String>>,
    /// Result cache for repeated queries (A+ Section 8.1: bounded by entries and bytes)
    search_cache: LruCache<String, Vec<SearchResult>>,
    /// Tracked byte estimate for the search cache
    search_cache_bytes: usize,
}

// A+ Search cache budget constants (Section 8.1)
/// Maximum entries in the search cache.
pub const SEARCH_CACHE_MAX_ENTRIES: usize = 256;
/// Maximum total bytes for the search cache.
pub const SEARCH_CACHE_MAX_BYTES: usize = 16 * 1024 * 1024; // 16 MiB

// ============================================================================
// STAGED RETRIEVAL (Plan 2 — VAL-BPHASE-044, VAL-BPHASE-045)
// ============================================================================

/// Configuration for staged retrieval: coarse candidate generation followed
/// by exact rerank.
///
/// Staged retrieval reduces exact-stage work by first narrowing the candidate
/// set with a cheap coarse pass (TF-IDF vector similarity only), then applying
/// the full hybrid scoring (text + TF-IDF + structural) only to the reduced
/// candidate set.
///
/// **Important**: This is a coarse-prefilter-plus-exact-rerank design. It does
/// **not** replace the approved INT8/default quality-gated path with
/// binary-quantization-first search. The existing `search()` method remains
/// the authoritative default; staged retrieval is an opt-in optimization.
#[derive(Debug, Clone)]
pub struct StagedRetrievalConfig {
    /// Whether staged retrieval is enabled.
    ///
    /// When `false`, `search_staged` falls back to the standard `search` path.
    /// When `true`, the coarse-then-exact pipeline is used.
    pub enabled: bool,

    /// Multiplier applied to `top_k` to determine the coarse candidate set
    /// size. For example, with `top_k = 10` and `coarse_multiplier = 5`,
    /// the coarse phase retrieves `50` candidates, then the exact rerank
    /// narrows to the best `10`.
    ///
    /// Must be >= 1. Higher values improve recall at the cost of more exact
    /// scoring work.
    pub coarse_multiplier: usize,
}

impl Default for StagedRetrievalConfig {
    fn default() -> Self {
        Self {
            enabled: false,
            coarse_multiplier: 5,
        }
    }
}

impl StagedRetrievalConfig {
    /// Create a new config with staged retrieval enabled and the given
    /// coarse multiplier.
    pub fn enabled_with_multiplier(coarse_multiplier: usize) -> Self {
        Self {
            enabled: true,
            coarse_multiplier: coarse_multiplier.max(1),
        }
    }

    /// Create a disabled config (staged retrieval off).
    pub fn disabled() -> Self {
        Self {
            enabled: false,
            ..Default::default()
        }
    }
}

/// Metrics reported by a staged retrieval pass.
///
/// Allows tests and callers to observe that the staged path actually reduced
/// exact-stage work (VAL-BPHASE-045).
#[derive(Debug, Clone, Default)]
pub struct StagedRetrievalMetrics {
    /// Number of candidates produced by the coarse phase.
    pub coarse_candidates: usize,
    /// Number of candidates scored by the exact rerank phase.
    pub exact_scored: usize,
    /// Final number of results returned after rerank.
    pub results_returned: usize,
    /// Whether staged retrieval was actually used (vs. fallback to standard).
    pub staged_used: bool,
}

// ============================================================================
// INT8 QUALITY GATE (Plan 2 — VAL-BPHASE-030 through VAL-BPHASE-034)
// ============================================================================

/// Thresholds for the INT8 quality gate.
///
/// INT8 promotion is only allowed when all three thresholds are satisfied:
/// - NDCG@10 degradation vs FP32 baseline ≤ 1%
/// - p50 latency ≤ FP32 baseline + 5%
/// - p99 latency ≤ FP32 baseline + 10%
///
/// These thresholds come from the binding spec §6.7.
#[derive(Debug, Clone)]
pub struct Int8QualityThresholds {
    /// Maximum allowed NDCG@10 drop (fraction, e.g. 0.01 = 1%).
    pub ndcg10_max_drop: f64,
    /// Maximum allowed p50 latency increase (fraction, e.g. 0.05 = 5%).
    pub p50_max_increase: f64,
    /// Maximum allowed p99 latency increase (fraction, e.g. 0.10 = 10%).
    pub p99_max_increase: f64,
}

impl Default for Int8QualityThresholds {
    fn default() -> Self {
        Self {
            ndcg10_max_drop: 0.01,
            p50_max_increase: 0.05,
            p99_max_increase: 0.10,
        }
    }
}

/// Result of comparing INT8 search quality against an FP32 baseline.
///
/// Each field reports the measured metric and whether it passed the
/// corresponding threshold.
#[derive(Debug, Clone)]
pub struct Int8QualityReport {
    /// NDCG@10 measured on the FP32 baseline.
    pub baseline_ndcg10: f64,
    /// NDCG@10 measured on the INT8 path.
    pub int8_ndcg10: f64,
    /// NDCG@10 drop (baseline − INT8). Positive means INT8 is worse.
    pub ndcg10_drop: f64,
    /// Whether the NDCG@10 drop is within the allowed threshold.
    pub ndcg10_passed: bool,

    /// p50 latency on the FP32 baseline (nanoseconds).
    pub baseline_p50_ns: u64,
    /// p50 latency on the INT8 path (nanoseconds).
    pub int8_p50_ns: u64,
    /// p50 latency increase fraction.
    pub p50_increase: f64,
    /// Whether p50 latency is within the allowed threshold.
    pub p50_passed: bool,

    /// p99 latency on the FP32 baseline (nanoseconds).
    pub baseline_p99_ns: u64,
    /// p99 latency on the INT8 path (nanoseconds).
    pub int8_p99_ns: u64,
    /// p99 latency increase fraction.
    pub p99_increase: f64,
    /// Whether p99 latency is within the allowed threshold.
    pub p99_passed: bool,

    /// Overall pass: `true` only when all three thresholds pass.
    pub overall_passed: bool,
}

/// INT8 quality gate.
///
/// Evaluates whether INT8 quantized search meets the quality and latency
/// thresholds required for promotion. The gate is explicit: INT8 is **not**
/// promoted unless all thresholds pass (VAL-BPHASE-030).
///
/// The FP32 comparison path remains available regardless of the gate outcome
/// (VAL-BPHASE-034).
///
/// # Usage
///
/// ```ignore
/// let gate = Int8QualityGate::new(Int8QualityThresholds::default());
/// let report = gate.evaluate(&baseline_results, &int8_results, &baseline_latencies, &int8_latencies);
/// if report.overall_passed {
///     // Safe to promote INT8
/// } else {
///     // Keep FP32 as default
/// }
/// ```
#[derive(Debug, Clone)]
pub struct Int8QualityGate {
    thresholds: Int8QualityThresholds,
}

impl Int8QualityGate {
    /// Create a new quality gate with the given thresholds.
    pub fn new(thresholds: Int8QualityThresholds) -> Self {
        Self { thresholds }
    }

    /// Create a gate with default thresholds from the spec.
    pub fn with_default_thresholds() -> Self {
        Self::new(Int8QualityThresholds::default())
    }

    /// Return the configured thresholds.
    pub fn thresholds(&self) -> &Int8QualityThresholds {
        &self.thresholds
    }

    /// Compute NDCG@10 given a ranked list of returned IDs and a set of
    /// relevant (ground-truth) IDs.
    ///
    /// `returned_ids` is ordered by rank (position 0 = rank 1).
    /// `relevant_ids` is the set of IDs that are relevant for the query.
    pub fn ndcg_at_10(returned_ids: &[String], relevant_ids: &HashSet<String>) -> f64 {
        // DCG@10
        let k = 10.min(returned_ids.len());
        let mut dcg: f64 = 0.0;
        for (i, id) in returned_ids.iter().take(k).enumerate() {
            if relevant_ids.contains(id) {
                let rank = (i + 1) as f64;
                dcg += 1.0 / (rank + 1.0).log2();
            }
        }

        // Ideal DCG@10
        let ideal_k = 10.min(relevant_ids.len());
        let mut idcg: f64 = 0.0;
        for i in 0..ideal_k {
            let rank = (i + 1) as f64;
            idcg += 1.0 / (rank + 1.0).log2();
        }

        if idcg == 0.0 {
            0.0
        } else {
            dcg / idcg
        }
    }

    /// Compute a latency percentile from a sorted list of nanosecond samples.
    ///
    /// `samples` must be sorted in ascending order.
    /// `percentile` is in [0.0, 1.0] (e.g. 0.5 for p50, 0.99 for p99).
    pub fn latency_percentile(sorted_samples: &[u64], percentile: f64) -> u64 {
        if sorted_samples.is_empty() {
            return 0;
        }
        let idx = ((sorted_samples.len() as f64) * percentile) as usize;
        let idx = idx.min(sorted_samples.len() - 1);
        sorted_samples[idx]
    }

    /// Evaluate the INT8 quality gate against measured metrics.
    ///
    /// Returns an [`Int8QualityReport`] indicating whether each threshold
    /// passed and whether the overall gate passed.
    pub fn evaluate(
        &self,
        baseline_ndcg10: f64,
        int8_ndcg10: f64,
        baseline_latencies_ns: &[u64],
        int8_latencies_ns: &[u64],
    ) -> Int8QualityReport {
        let ndcg10_drop = baseline_ndcg10 - int8_ndcg10;
        let ndcg10_passed = ndcg10_drop <= self.thresholds.ndcg10_max_drop;

        let mut baseline_sorted = baseline_latencies_ns.to_vec();
        baseline_sorted.sort();
        let mut int8_sorted = int8_latencies_ns.to_vec();
        int8_sorted.sort();

        let baseline_p50 = Self::latency_percentile(&baseline_sorted, 0.50);
        let int8_p50 = Self::latency_percentile(&int8_sorted, 0.50);
        let p50_increase = if baseline_p50 == 0 {
            0.0
        } else {
            (int8_p50 as f64 - baseline_p50 as f64) / baseline_p50 as f64
        };
        let p50_passed = p50_increase <= self.thresholds.p50_max_increase;

        let baseline_p99 = Self::latency_percentile(&baseline_sorted, 0.99);
        let int8_p99 = Self::latency_percentile(&int8_sorted, 0.99);
        let p99_increase = if baseline_p99 == 0 {
            0.0
        } else {
            (int8_p99 as f64 - baseline_p99 as f64) / baseline_p99 as f64
        };
        let p99_passed = p99_increase <= self.thresholds.p99_max_increase;

        let overall_passed = ndcg10_passed && p50_passed && p99_passed;

        Int8QualityReport {
            baseline_ndcg10,
            int8_ndcg10,
            ndcg10_drop,
            ndcg10_passed,
            baseline_p50_ns: baseline_p50,
            int8_p50_ns: int8_p50,
            p50_increase,
            p50_passed,
            baseline_p99_ns: baseline_p99,
            int8_p99_ns: int8_p99,
            p99_increase,
            p99_passed,
            overall_passed,
        }
    }
}

impl Default for Int8QualityGate {
    fn default() -> Self {
        Self::with_default_thresholds()
    }
}

/// Promotion decision for INT8 as the default search path.
///
/// The system does not silently promote INT8 — the decision is explicit and
/// observable (VAL-BPHASE-030).
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Int8PromotionDecision {
    /// INT8 may be promoted as the default path.
    Promote,
    /// INT8 must not be promoted; FP32 remains the default.
    /// Contains a human-readable reason.
    Block(String),
}

impl Int8QualityReport {
    /// Convert the report into a promotion decision.
    ///
    /// Returns [`Int8PromotionDecision::Promote`] only when all thresholds
    /// pass. Otherwise returns [`Int8PromotionDecision::Block`] with a
    /// description of which thresholds failed.
    pub fn promotion_decision(&self) -> Int8PromotionDecision {
        if self.overall_passed {
            Int8PromotionDecision::Promote
        } else {
            let mut reasons = Vec::new();
            if !self.ndcg10_passed {
                reasons.push(format!(
                    "NDCG@10 drop {:.4} > {:.4}",
                    self.ndcg10_drop, 0.01
                ));
            }
            if !self.p50_passed {
                reasons.push(format!(
                    "p50 latency increase {:.2}% > {:.2}%",
                    self.p50_increase * 100.0,
                    5.0
                ));
            }
            if !self.p99_passed {
                reasons.push(format!(
                    "p99 latency increase {:.2}% > {:.2}%",
                    self.p99_increase * 100.0,
                    10.0
                ));
            }
            Int8PromotionDecision::Block(reasons.join("; "))
        }
    }
}

impl SearchEngine {
    /// Create a new search engine with default 768-dim embeddings
    ///
    /// Uses brute-force vector search by default.
    ///
    /// # Panics
    ///
    /// This never panics - all initialization is infallible.
    #[must_use]
    pub fn new() -> Self {
        Self {
            nodes: Vec::new(),
            scorer: HybridScorer::new(),
            vector_index: VectorIndexImpl::BruteForce(VectorIndex::new(
                DEFAULT_EMBEDDING_DIMENSION,
            )),
            complexity_cache: HashMap::new(),
            text_index: HashMap::new(),
            node_id_to_idx: HashMap::new(),
            node_tokens: HashMap::new(),
            search_cache: LruCache::new(NonZeroUsize::new(SEARCH_CACHE_MAX_ENTRIES).unwrap()),
            search_cache_bytes: 0,
        }
    }

    /// Create a new search engine with custom embedding dimension
    ///
    /// Uses brute-force vector search by default.
    ///
    /// # Arguments
    ///
    /// * `dimension` - Embedding vector dimension (1-10000)
    ///
    /// # Panics
    ///
    /// Panics if dimension is 0 or exceeds MAX_EMBEDDING_DIMENSION.
    ///
    /// # Example
    ///
    /// ```ignore
    /// let engine = SearchEngine::with_dimension(128);
    /// ```
    #[must_use]
    pub fn with_dimension(dimension: usize) -> Self {
        // Validate dimension at construction time
        if !(MIN_EMBEDDING_DIMENSION..=MAX_EMBEDDING_DIMENSION).contains(&dimension) {
            panic!(
                "Invalid embedding dimension: {} (must be between {} and {})",
                dimension, MIN_EMBEDDING_DIMENSION, MAX_EMBEDDING_DIMENSION
            );
        }

        Self {
            nodes: Vec::new(),
            scorer: HybridScorer::new(),
            vector_index: VectorIndexImpl::BruteForce(VectorIndex::new(dimension)),
            complexity_cache: HashMap::new(),
            text_index: HashMap::new(),
            node_id_to_idx: HashMap::new(),
            node_tokens: HashMap::new(),
            search_cache: LruCache::new(NonZeroUsize::new(SEARCH_CACHE_MAX_ENTRIES).unwrap()),
            search_cache_bytes: 0,
        }
    }

    /// Index nodes for searching
    ///
    /// This builds the internal indexes needed for search:
    /// - Text search index (stored in self.nodes)
    /// - Vector index (built from embeddings)
    /// - Complexity cache (for O(1) complexity lookups)
    ///
    /// # Arguments
    ///
    /// * `nodes` - Vector of nodes to index
    ///
    /// # Performance
    ///
    /// - Time complexity: O(n) where n is number of nodes
    /// - Space complexity: O(n) for storage + O(n) for embeddings
    ///
    /// # Panics
    ///
    /// Panics if node count exceeds MAX_NODES (prevents memory exhaustion).
    pub fn index_nodes(&mut self, mut nodes: Vec<NodeInfo>) {
        if nodes.len() > MAX_NODES {
            panic!(
                "Cannot index more than {} nodes (provided: {})",
                MAX_NODES,
                nodes.len()
            );
        }

        // Clear cache when re-indexing
        self.complexity_cache.clear();
        self.text_index.clear();
        self.search_cache.clear();
        self.search_cache_bytes = 0;
        self.node_id_to_idx.clear();
        self.node_tokens.clear();
        self.vector_index.clear();

        // Build node_id_to_idx for O(1) node lookups (A1 optimization)
        // Build complexity cache, inverted index, and token cache before taking ownership
        for (idx, node) in nodes.iter().enumerate() {
            self.node_id_to_idx.insert(node.node_id.clone(), idx);
            self.complexity_cache
                .insert(node.node_id.clone(), node.complexity);

            // Build inverted index for O(1) text lookups
            // This maps each token to the set of node IDs containing it
            // Also build per-node token cache for scoring (T14 optimization)
            //
            // R8: Use pre-tokenized tokens when available to skip re-tokenization.
            // Falls back to content-based tokenization for backward compatibility.
            let mut tokens = HashSet::new();
            if let Some(pre_tok) = &node.pre_tokenized {
                // Use pre-computed tokens directly (already lowercased, filtered >= 2 chars)
                for token in pre_tok {
                    self.text_index
                        .entry(token.clone())
                        .or_default()
                        .insert(node.node_id.clone());
                    tokens.insert(token.clone());
                }
            } else {
                for token in node.content.split(|c: char| !c.is_alphanumeric()) {
                    let normalized_token: String = token.to_ascii_lowercase();
                    // Skip empty tokens and very short ones (< 2 chars) to reduce noise
                    if normalized_token.len() >= 2 {
                        self.text_index
                            .entry(normalized_token.clone())
                            .or_default()
                            .insert(node.node_id.clone());
                        tokens.insert(normalized_token);
                    }
                }
            }
            self.node_tokens.insert(node.node_id.clone(), tokens);
        }

        // Build vector index from TF-IDF embeddings — clone only embeddings (A4 optimization)
        // All other node content is moved via ownership, avoiding a full Vec clone
        for node in nodes.iter_mut() {
            // Use tfidf_embedding (always present) instead of optional embedding
            if !node.tfidf_embedding.is_empty() {
                if let Err(e) = self
                    .vector_index
                    .insert(node.node_id.clone(), node.tfidf_embedding.clone())
                {
                    tracing::warn!(
                        "Failed to insert TF-IDF embedding for node {}: {:?}",
                        node.node_id,
                        e
                    );
                }
            }
        }

        // Move nodes into storage — no clone needed since indexes are already built
        self.nodes = nodes;

        // Extract signatures before clearing content (for search results)
        // This must happen before T13 optimization clears the content
        for node in &mut self.nodes {
            node.signature = Self::extract_signature_from_content(&node.content);
        }

        // Free content memory after all indexes are built (T13 optimization)
        // The inverted index (text_index) already captures all tokens,
        // and the Storage layer retains original source files on disk.
        // This reduces memory by ~15MB at 5K nodes.
        for node in &mut self.nodes {
            node.content.clear();
        }
    }

    /// Extract signature from node content.
    ///
    /// Returns the first non-empty, non-comment line after the header.
    pub fn extract_signature_from_content(content: &str) -> Option<String> {
        content
            .lines()
            .skip(1) // skip "// name in path" header
            .map(|l| l.trim())
            .find(|l| !l.is_empty() && !l.starts_with("// [No source") && !l.starts_with("// ["))
            .map(|l| l.to_string())
    }

    /// Apply an incremental delta update to the text index.
    ///
    /// This removes and adds/updates nodes without rebuilding the entire index,
    /// making it significantly faster than `index_nodes()` for small changes.
    ///
    /// # Arguments
    ///
    /// * `delta` - The delta describing nodes to remove and add/update.
    ///
    /// # Performance
    ///
    /// - Time complexity: O(K) where K is the number of changed nodes
    /// - Full rebuild is O(N) — incremental is faster when K << N
    ///
    /// # Panics
    ///
    /// Panics if the total node count after the update exceeds `MAX_NODES`.
    ///
    /// # Example
    ///
    /// ```ignore
    /// let delta = TextIndexDelta {
    ///     removed_node_ids: vec!["old_func".to_string()],
    ///     updated_nodes: vec![new_node],
    /// };
    /// engine.incremental_reindex(delta);
    /// ```
    pub fn incremental_reindex(&mut self, delta: TextIndexDelta) {
        // Invalidate search cache — results may change
        self.search_cache.clear();
        self.search_cache_bytes = 0;

        // Phase 1: Remove nodes
        for node_id in &delta.removed_node_ids {
            self.remove_node_from_index(node_id);
        }

        // Phase 2: Add/update nodes
        for node in delta.updated_nodes {
            self.add_node_to_index(node);
        }

        // Verify we don't exceed limits
        if self.nodes.len() > MAX_NODES {
            panic!(
                "Cannot index more than {} nodes (current: {})",
                MAX_NODES,
                self.nodes.len()
            );
        }
    }

    /// Remove a single node from all index structures.
    ///
    /// This is O(T) where T is the number of unique tokens in the removed node.
    fn remove_node_from_index(&mut self, node_id: &str) {
        // Remove from node_id_to_idx
        let Some(removed_idx) = self.node_id_to_idx.remove(node_id) else {
            return; // Node not in index, nothing to do
        };

        // Remove from text_index: for each token the node contributed to,
        // remove the node_id from the token's set. Clean up empty sets.
        if let Some(tokens) = self.node_tokens.remove(node_id) {
            for token in tokens {
                if let Entry::Occupied(mut entry) = self.text_index.entry(token) {
                    entry.get_mut().remove(node_id);
                    if entry.get().is_empty() {
                        entry.remove();
                    }
                }
            }
        }

        // Remove from complexity_cache
        self.complexity_cache.remove(node_id);

        // Remove from vector_index
        self.vector_index.remove(node_id);

        // Remove from nodes Vec and fix indices
        // Swap-remove for O(1) removal, then fix the swapped node's index
        if removed_idx < self.nodes.len() {
            self.nodes.swap_remove(removed_idx);
            // If we didn't remove the last element, the swapped element needs
            // its index updated in node_id_to_idx
            if removed_idx < self.nodes.len() {
                let swapped_id = self.nodes[removed_idx].node_id.clone();
                self.node_id_to_idx.insert(swapped_id, removed_idx);
            }
        }
    }

    /// Add or update a single node in all index structures.
    ///
    /// If the node already exists (same `node_id`), it is removed first, then
    /// re-added with the new data.
    fn add_node_to_index(&mut self, mut node: NodeInfo) {
        // If node already exists, remove the old version first
        if self.node_id_to_idx.contains_key(&node.node_id) {
            self.remove_node_from_index(&node.node_id);
        }

        let node_id = node.node_id.clone();
        let new_idx = self.nodes.len();

        // Build inverted index entries and token cache for this node
        //
        // R8: Use pre-tokenized tokens when available to skip re-tokenization.
        // Falls back to content-based tokenization for backward compatibility.
        let mut tokens = HashSet::new();
        if let Some(pre_tok) = &node.pre_tokenized {
            for token in pre_tok {
                self.text_index
                    .entry(token.clone())
                    .or_default()
                    .insert(node_id.clone());
                tokens.insert(token.clone());
            }
        } else {
            for token in node.content.split(|c: char| !c.is_alphanumeric()) {
                let normalized_token: String = token.to_ascii_lowercase();
                if normalized_token.len() >= 2 {
                    self.text_index
                        .entry(normalized_token.clone())
                        .or_default()
                        .insert(node_id.clone());
                    tokens.insert(normalized_token);
                }
            }
        }
        self.node_tokens.insert(node_id.clone(), tokens);

        // Update node_id_to_idx
        self.node_id_to_idx.insert(node_id.clone(), new_idx);

        // Update complexity_cache
        self.complexity_cache
            .insert(node_id.clone(), node.complexity);

        // Insert TF-IDF embedding into vector index (always present)
        if !node.tfidf_embedding.is_empty() {
            if let Err(e) = self
                .vector_index
                .insert(node_id.clone(), node.tfidf_embedding.clone())
            {
                tracing::warn!(
                    "Failed to insert TF-IDF embedding for node {}: {:?}",
                    node_id,
                    e
                );
            }
        }

        // Extract signature before clearing content (same as index_nodes does)
        node.signature = Self::extract_signature_from_content(&node.content);

        // Clear content to save memory (same as index_nodes does)
        node.content.clear();

        // Add to nodes Vec
        self.nodes.push(node);
    }

    /// Get the number of indexed nodes
    ///
    /// # Returns
    ///
    /// The number of nodes currently indexed.
    #[must_use]
    pub fn node_count(&self) -> usize {
        self.nodes.len()
    }

    /// Collect all (node_id, embedding) pairs from the indexed nodes.
    ///
    /// Returns only nodes that have a TF-IDF embedding. Used by the mmap
    /// persistence layer to write embeddings to disk.
    pub fn collect_embeddings(&self) -> Vec<(String, Vec<f32>)> {
        self.nodes
            .iter()
            .filter(|n| !n.tfidf_embedding.is_empty())
            .map(|n| (n.node_id.clone(), n.tfidf_embedding.clone()))
            .collect()
    }

    /// Check if the index is empty
    ///
    /// # Returns
    ///
    /// `true` if no nodes are indexed, `false` otherwise.
    pub fn is_empty(&self) -> bool {
        self.nodes.is_empty()
    }

    // ----------------------------------------------------------------
    // B-phase residency accessors (Plan 2)
    // ----------------------------------------------------------------

    /// Return the internal index position for a given node ID.
    ///
    /// Returns `None` if the node is not in the index. This is the
    /// row-oriented position used by the residency layer.
    pub fn node_index(&self, node_id: &str) -> Option<usize> {
        self.node_id_to_idx.get(node_id).copied()
    }

    /// Return the number of live (non-tombstoned) nodes in the index.
    ///
    /// Equivalent to `node_count()` but named for clarity in residency
    /// contexts where the distinction between live and tombstoned matters.
    pub fn live_node_count(&self) -> usize {
        self.node_id_to_idx.len()
    }

    /// Check whether a node ID is currently in the live index.
    pub fn contains_node(&self, node_id: &str) -> bool {
        self.node_id_to_idx.contains_key(node_id)
    }

    /// Return the node IDs currently in the live index.
    pub fn live_node_ids(&self) -> Vec<String> {
        self.node_id_to_idx.keys().cloned().collect()
    }

    /// Return the complexity score for a given node.
    pub fn node_complexity(&self, node_id: &str) -> Option<u32> {
        self.complexity_cache.get(node_id).copied()
    }

    /// Return the tokens associated with a given node.
    pub fn node_tokens(&self, node_id: &str) -> Option<&HashSet<String>> {
        self.node_tokens.get(node_id)
    }

    /// Check whether a token exists in the text index and, if so, which
    /// node IDs contain it.
    pub fn token_lookup(&self, token: &str) -> Option<&HashSet<String>> {
        self.text_index.get(token)
    }

    /// Return the number of entries currently in the search cache.
    ///
    /// Part of the B-phase memory accounting surface (VAL-BPHASE-024).
    pub fn search_cache_len(&self) -> usize {
        self.search_cache.len()
    }

    /// Return the tracked byte estimate for the search cache.
    ///
    /// Part of the B-phase memory accounting surface (VAL-BPHASE-024).
    pub fn search_cache_bytes(&self) -> usize {
        self.search_cache_bytes
    }

    /// Produce a compact, row-oriented snapshot of the resident search
    /// metadata.
    ///
    /// The returned [`CompactNodeMetadata`] uses compact integer-backed
    /// addressing (row indices as `u32`) instead of string-heavy
    /// identifier-based maps. This is the B-phase compressed resident
    /// state (VAL-BPHASE-041).
    pub fn compact_metadata(&self) -> CompactNodeMetadata {
        // Build compact node-id → row index map (u32 keys)
        let mut row_map: Vec<(String, u32)> = Vec::with_capacity(self.nodes.len());
        let mut complexity_by_row: Vec<u32> = Vec::with_capacity(self.nodes.len());

        for (idx, node) in self.nodes.iter().enumerate() {
            row_map.push((node.node_id.clone(), idx as u32));
            complexity_by_row.push(node.complexity);
        }

        // Build compact token → set-of-rows index
        let mut token_rows: HashMap<String, HashSet<u32>> = HashMap::new();
        for (token, node_ids) in &self.text_index {
            let mut rows = HashSet::new();
            for node_id in node_ids {
                if let Some(&idx) = self.node_id_to_idx.get(node_id) {
                    rows.insert(idx as u32);
                }
            }
            token_rows.insert(token.clone(), rows);
        }

        CompactNodeMetadata {
            row_map,
            complexity_by_row,
            token_index: CompactTokenIndex { token_rows },
        }
    }

    /// Validate internal coherence of all index structures.
    ///
    /// Returns `Ok(())` if all structures agree, or a description of the
    /// first incoherence found. Used by residency tests to verify that
    /// compaction and delta updates maintain structural coherence.
    pub fn validate_coherence(&self) -> Result<(), String> {
        // node_id_to_idx and nodes must agree on count
        if self.node_id_to_idx.len() != self.nodes.len() {
            return Err(format!(
                "node_id_to_idx len ({}) != nodes len ({})",
                self.node_id_to_idx.len(),
                self.nodes.len()
            ));
        }

        // Every entry in node_id_to_idx must point to the correct node
        for (id, &idx) in &self.node_id_to_idx {
            if idx >= self.nodes.len() {
                return Err(format!(
                    "node_id_to_idx[{}] = {} >= nodes.len() = {}",
                    id,
                    idx,
                    self.nodes.len()
                ));
            }
            if self.nodes[idx].node_id != *id {
                return Err(format!(
                    "nodes[{}].node_id = '{}' != node_id_to_idx key '{}'",
                    idx, self.nodes[idx].node_id, id
                ));
            }
        }

        // complexity_cache must have an entry for every live node
        for node in &self.nodes {
            match self.complexity_cache.get(&node.node_id) {
                Some(c) if *c == node.complexity => {}
                Some(c) => {
                    return Err(format!(
                        "complexity_cache[{}] = {} != node.complexity = {}",
                        node.node_id, c, node.complexity
                    ))
                }
                None => {
                    return Err(format!(
                        "complexity_cache missing entry for {}",
                        node.node_id
                    ))
                }
            }
        }

        // node_tokens must have an entry for every live node
        for node in &self.nodes {
            if !self.node_tokens.contains_key(&node.node_id) {
                return Err(format!("node_tokens missing entry for {}", node.node_id));
            }
        }

        // text_index must not reference removed nodes
        for (token, node_ids) in &self.text_index {
            for id in node_ids {
                if !self.node_id_to_idx.contains_key(id) {
                    return Err(format!(
                        "text_index token '{}' references non-live node '{}'",
                        token, id
                    ));
                }
            }
        }

        Ok(())
    }

    /// Execute a search query
    ///
    /// This performs a hybrid search combining:
    /// - Text matching (substring + token overlap)
    /// - Semantic similarity (if embeddings available)
    /// - Structural relevance (complexity-based)
    ///
    /// # Arguments
    ///
    /// * `query` - Search query with all parameters
    ///
    /// # Returns
    ///
    /// Vector of search results sorted by relevance (highest first).
    ///
    /// # Performance
    ///
    /// - Time complexity: O(n) where n is number of nodes
    /// - Space complexity: O(k) where k is top_k (results)
    ///
    /// # Errors
    ///
    /// Returns `Error::QueryFailed` if the search operation fails.
    pub fn search(&mut self, query: SearchQuery) -> Result<Vec<SearchResult>, Error> {
        if self.nodes.is_empty() {
            return Ok(Vec::new());
        }

        // Check cache first
        let cache_key = format!(
            "{}:{}:{:?}:{}",
            query.query, query.top_k, query.threshold, query.semantic
        );
        if let Some(cached) = self.search_cache.get(&cache_key) {
            return Ok(cached.clone());
        }

        let mut results = Vec::new();

        // Pre-compute vector search if semantic search is requested
        let vector_results: std::collections::HashMap<String, f32> = if query.semantic {
            // Use provided query embedding if available
            let embedding = if let Some(emb) = query.query_embedding {
                Some(emb)
            } else {
                // Fallback: find if there's a node with a TF-IDF embedding we can use
                // This is legacy behavior, should be avoided
                self.nodes
                    .iter()
                    .find_map(|n| {
                        if n.tfidf_embedding.is_empty() {
                            None
                        } else {
                            Some(&n.tfidf_embedding)
                        }
                    })
                    .cloned()
            };

            if let Some(emb) = embedding {
                self.vector_index
                    .search(&emb, query.top_k)
                    .into_iter()
                    .collect()
            } else {
                std::collections::HashMap::new()
            }
        } else {
            std::collections::HashMap::new()
        };

        // Pre-compute query data for optimized text scoring
        // This reduces allocations from O(N) to O(1) per search
        let text_query = TextQueryPreprocessed::from_query(&query.query);

        // Use inverted index to filter candidates - only check nodes that contain query terms
        // This reduces search complexity from O(N) to O(M) where M is number of matching nodes
        let candidates = if text_query.query_tokens.is_empty() {
            // No query tokens, check all nodes
            self.nodes.iter().collect::<Vec<_>>()
        } else {
            // Build candidate set using inverted index - O(1) per token lookup
            let mut candidate_ids: HashSet<&str> = HashSet::new();

            for token in &text_query.query_tokens {
                if let Some(node_ids) = self.text_index.get(token) {
                    for node_id in node_ids {
                        candidate_ids.insert(node_id.as_str());
                    }
                }
            }

            // If no matches in inverted index, return empty results early
            if candidate_ids.is_empty() && !query.semantic {
                return Ok(Vec::new());
            }

            // Convert candidate IDs to node references
            if candidate_ids.is_empty() {
                // If no text matches, but we have semantic search, check all nodes
                // (Optimization: we could limit to just the vector search hits)
                self.nodes.iter().collect()
            } else {
                // We have text matches. If we also have semantic results, we must include them
                // even if they don't match keywords.
                if vector_results.is_empty() {
                    self.nodes
                        .iter()
                        .filter(|node| candidate_ids.contains(node.node_id.as_str()))
                        .collect()
                } else {
                    // Union of text matches and semantic matches
                    self.nodes
                        .iter()
                        .filter(|node| {
                            candidate_ids.contains(node.node_id.as_str())
                                || vector_results.contains_key(&node.node_id)
                        })
                        .collect()
                }
            }
        };

        for node in candidates {
            let text_score = self.calculate_text_score_optimized(
                &text_query,
                &node.node_id,
                &node.symbol_name,
                &node.file_path,
            );

            // Get TF-IDF score from vector search results
            let tfidf_score = if query.semantic {
                *vector_results.get(&node.node_id).unwrap_or(&0.0)
            } else {
                0.0
            };

            // For now, if no text match and not semantic, skip
            if text_score == 0.0 && !query.semantic && tfidf_score == 0.0 {
                continue;
            }

            // Normalize complexity to 0-1 range (divide by 100, not 10)
            let structural_score = (node.complexity as f32 / 100.0).min(1.0);

            // Neural score is 0.0 in this context (vector index only has TF-IDF embeddings)
            let neural_score = 0.0;

            // Use custom weights based on query type if provided
            let score = if let Some(qt) = query.query_type {
                match qt {
                    crate::search::ranking::QueryType::Text => {
                        // Prose/Text mode: heavily favor keyword overlap
                        self.scorer
                            .with_weights_hybrid(0.2, 0.05, 0.05, 0.7)
                            .score_hybrid(tfidf_score, neural_score, structural_score, text_score)
                    }
                    crate::search::ranking::QueryType::Semantic => {
                        // Semantic-heavy mode
                        self.scorer
                            .with_weights_hybrid(0.7, 0.1, 0.1, 0.1)
                            .score_hybrid(tfidf_score, neural_score, structural_score, text_score)
                    }
                    crate::search::ranking::QueryType::Structural => {
                        // Structural-heavy mode
                        self.scorer
                            .with_weights_hybrid(0.3, 0.0, 0.5, 0.2)
                            .score_hybrid(tfidf_score, neural_score, structural_score, text_score)
                    }
                }
            } else {
                // Default hybrid scoring
                self.scorer
                    .score_hybrid(tfidf_score, neural_score, structural_score, text_score)
            };

            if score.overall > 0.0 {
                // Apply relevance threshold if specified
                if let Some(threshold) = query.threshold {
                    if score.overall < threshold {
                        continue;
                    }
                }

                // Use cached signature (extracted before content was cleared)
                let signature = node.signature.clone();

                results.push(SearchResult {
                    rank: 0, // Will be set after sorting
                    node_id: node.node_id.clone(),
                    file_path: node.file_path.clone(),
                    symbol_name: node.symbol_name.clone(),
                    symbol_type: None, // enriched by LeIndex::search()
                    signature,
                    complexity: node.complexity,
                    caller_count: None,     // enriched by LeIndex::search()
                    dependency_count: None, // enriched by LeIndex::search()
                    language: node.language.clone(),
                    score,
                    context: None,
                    byte_range: node.byte_range,
                });
            }
        }

        // Sort by score (descending)
        results.sort_by(|a, b| {
            b.score
                .overall
                .partial_cmp(&a.score.overall)
                .unwrap_or(std::cmp::Ordering::Equal)
        });

        // Take top_k
        let top_k = results.into_iter().take(query.top_k).collect::<Vec<_>>();

        let mut final_results = top_k;
        for (i, result) in final_results.iter_mut().enumerate() {
            result.rank = i + 1;
        }

        // Cache results with byte-budget enforcement (A+ Section 8.1)
        {
            let results_bytes = Self::estimate_search_results_bytes(&final_results);
            // Guard: skip insertion if a single entry exceeds the cache budget.
            if results_bytes < SEARCH_CACHE_MAX_BYTES {
                // If replacing an existing entry, subtract its bytes first
                if let Some(existing) = self.search_cache.get(&cache_key) {
                    self.search_cache_bytes = self
                        .search_cache_bytes
                        .saturating_sub(Self::estimate_search_results_bytes(existing));
                }
                // Evict until there is room
                while self.search_cache_bytes + results_bytes > SEARCH_CACHE_MAX_BYTES
                    && !self.search_cache.is_empty()
                {
                    if let Some((_, evicted)) = self.search_cache.pop_lru() {
                        self.search_cache_bytes = self
                            .search_cache_bytes
                            .saturating_sub(Self::estimate_search_results_bytes(&evicted));
                    }
                }
                self.search_cache_bytes += results_bytes;
                self.search_cache.put(cache_key, final_results.clone());
            }
        }

        Ok(final_results)
    }

    /// Execute a staged retrieval search: coarse candidate generation followed
    /// by exact rerank (Plan 2 — VAL-BPHASE-044, VAL-BPHASE-045).
    ///
    /// When `config.enabled` is `false`, this falls back to the standard
    /// [`search`](Self::search) path and reports `staged_used: false`.
    ///
    /// When enabled, the pipeline is:
    /// 1. **Coarse phase**: Retrieve `top_k * coarse_multiplier` candidates
    ///    using TF-IDF vector similarity only (cheap).
    /// 2. **Exact rerank phase**: Apply the full hybrid scoring (text +
    ///    TF-IDF + structural) only to the coarse candidate set, then return
    ///    the top-K results.
    ///
    /// This reduces exact-stage work without replacing the approved
    /// INT8/default quality-gated path with binary-quantization-first search.
    /// The staged path is opt-in; the existing `search()` method remains the
    /// authoritative default.
    ///
    /// # Returns
    ///
    /// A tuple of `(results, metrics)` where `metrics` carries observability
    /// data about the staged pipeline (candidate counts, whether staged was
    /// used, etc.).
    ///
    /// # Errors
    ///
    /// Same error conditions as [`search`](Self::search).
    pub fn search_staged(
        &mut self,
        query: SearchQuery,
        config: &StagedRetrievalConfig,
    ) -> Result<(Vec<SearchResult>, StagedRetrievalMetrics), Error> {
        // If staged retrieval is disabled, fall back to the standard path.
        if !config.enabled {
            let results = self.search(query)?;
            let count = results.len();
            return Ok((
                results,
                StagedRetrievalMetrics {
                    coarse_candidates: 0,
                    exact_scored: count,
                    results_returned: count,
                    staged_used: false,
                },
            ));
        }

        if self.nodes.is_empty() {
            return Ok((
                Vec::new(),
                StagedRetrievalMetrics {
                    staged_used: true,
                    ..Default::default()
                },
            ));
        }

        // Check staged-search cache (key includes query, top_k, threshold, semantic, coarse_multiplier, query_type)
        let cache_key = format!(
            "staged:{}:{}:{:?}:{}:{:?}:{:?}",
            query.query, query.top_k, query.threshold, query.semantic, config.coarse_multiplier, query.query_type
        );
        if let Some(cached) = self.search_cache.get(&cache_key) {
            let count = cached.len();
            return Ok((
                cached.clone(),
                StagedRetrievalMetrics {
                    coarse_candidates: 0,
                    exact_scored: count,
                    results_returned: count,
                    staged_used: true,
                },
            ));
        }

        let mut metrics = StagedRetrievalMetrics {
            staged_used: true,
            ..Default::default()
        };

        // ====================================================================
        // Phase 1: Coarse candidate generation
        //
        // Use TF-IDF vector similarity AND text-index lookup to retrieve a
        // larger candidate set. This is cheap because vector search computes
        // cosine similarity against the index without text/structural scoring,
        // and text lookup is O(1) per token via the inverted index.
        //
        // The coarse candidate set is the UNION of vector-similarity hits and
        // text-index hits, ensuring that nodes relevant by either signal are
        // included for the exact rerank phase.
        // ====================================================================
        let coarse_top_k = query.top_k.saturating_mul(config.coarse_multiplier);

        // Start with text-index candidates (always included)
        let text_query = TextQueryPreprocessed::from_query(&query.query);
        let mut coarse_candidate_ids: HashSet<String> = HashSet::new();
        for token in &text_query.query_tokens {
            if let Some(node_ids) = self.text_index.get(token) {
                for id in node_ids {
                    coarse_candidate_ids.insert(id.clone());
                }
            }
        }

        // Add vector-similarity candidates if semantic search is requested
        let vector_results: HashMap<String, f32> = if query.semantic {
            if let Some(ref emb) = query.query_embedding {
                let vec_hits = self.vector_index.search(emb, coarse_top_k);
                for (id, _) in &vec_hits {
                    coarse_candidate_ids.insert(id.clone());
                }
                vec_hits.into_iter().collect()
            } else {
                HashMap::new()
            }
        } else {
            HashMap::new()
        };

        metrics.coarse_candidates = coarse_candidate_ids.len();

        // If no coarse candidates, return early
        if coarse_candidate_ids.is_empty() {
            return Ok((Vec::new(), metrics));
        }

        // ====================================================================
        // Phase 2: Exact rerank
        //
        // Apply the full hybrid scoring (text + TF-IDF + structural) only to
        // the reduced candidate set from the coarse phase.
        // ====================================================================

        let mut results = Vec::new();

        for node in &self.nodes {
            // Only score nodes that passed the coarse filter
            if !coarse_candidate_ids.contains(&node.node_id) {
                continue;
            }

            let text_score = self.calculate_text_score_optimized(
                &text_query,
                &node.node_id,
                &node.symbol_name,
                &node.file_path,
            );

            let tfidf_score = if query.semantic {
                *vector_results.get(&node.node_id).unwrap_or(&0.0)
            } else {
                0.0
            };

            if text_score == 0.0 && !query.semantic && tfidf_score == 0.0 {
                continue;
            }

            let structural_score = (node.complexity as f32 / 100.0).min(1.0);
            let neural_score = 0.0;

            let score = if let Some(qt) = query.query_type {
                match qt {
                    crate::search::ranking::QueryType::Text => self
                        .scorer
                        .with_weights_hybrid(0.2, 0.05, 0.05, 0.7)
                        .score_hybrid(tfidf_score, neural_score, structural_score, text_score),
                    crate::search::ranking::QueryType::Semantic => self
                        .scorer
                        .with_weights_hybrid(0.7, 0.1, 0.1, 0.1)
                        .score_hybrid(tfidf_score, neural_score, structural_score, text_score),
                    crate::search::ranking::QueryType::Structural => self
                        .scorer
                        .with_weights_hybrid(0.3, 0.0, 0.5, 0.2)
                        .score_hybrid(tfidf_score, neural_score, structural_score, text_score),
                }
            } else {
                self.scorer
                    .score_hybrid(tfidf_score, neural_score, structural_score, text_score)
            };

            if score.overall > 0.0 {
                if let Some(threshold) = query.threshold {
                    if score.overall < threshold {
                        continue;
                    }
                }

                let signature = node.signature.clone();
                results.push(SearchResult {
                    rank: 0,
                    node_id: node.node_id.clone(),
                    file_path: node.file_path.clone(),
                    symbol_name: node.symbol_name.clone(),
                    symbol_type: None,
                    signature,
                    complexity: node.complexity,
                    caller_count: None,
                    dependency_count: None,
                    language: node.language.clone(),
                    score,
                    context: None,
                    byte_range: node.byte_range,
                });
            }
        }

        metrics.exact_scored = results.len();

        // Sort by score (descending)
        results.sort_by(|a, b| {
            b.score
                .overall
                .partial_cmp(&a.score.overall)
                .unwrap_or(std::cmp::Ordering::Equal)
        });

        // Take top_k
        let mut final_results: Vec<SearchResult> = results.into_iter().take(query.top_k).collect();
        for (i, result) in final_results.iter_mut().enumerate() {
            result.rank = i + 1;
        }

        metrics.results_returned = final_results.len();

        // Cache results with byte-budget enforcement
        {
            let results_bytes = Self::estimate_search_results_bytes(&final_results);
            // Guard: skip insertion if a single entry exceeds the cache budget.
            if results_bytes < SEARCH_CACHE_MAX_BYTES {
                if let Some(existing) = self.search_cache.get(&cache_key) {
                    self.search_cache_bytes = self
                        .search_cache_bytes
                        .saturating_sub(Self::estimate_search_results_bytes(existing));
                }
                while self.search_cache_bytes + results_bytes > SEARCH_CACHE_MAX_BYTES
                    && !self.search_cache.is_empty()
                {
                    if let Some((_, evicted)) = self.search_cache.pop_lru() {
                        self.search_cache_bytes = self
                            .search_cache_bytes
                            .saturating_sub(Self::estimate_search_results_bytes(&evicted));
                    }
                }
                self.search_cache_bytes += results_bytes;
                self.search_cache.put(cache_key, final_results.clone());
            }
        }

        Ok((final_results, metrics))
    }

    /// Optimized text score calculation using cached node tokens and pre-computed query data
    ///
    /// Uses the node_tokens HashMap for O(1) token overlap calculation instead of
    /// iterating over the inverted index per query token. Tokens are cached during
    /// index_nodes() — no re-tokenization in the scoring hot path.
    ///
    /// # Performance
    ///
    /// - Time complexity: O(min(q, t)) where q = query tokens, t = node tokens (set intersection)
    /// - Space complexity: O(1) — no allocations per call
    fn calculate_text_score_optimized(
        &self,
        precomputed: &TextQueryPreprocessed,
        node_id: &str,
        symbol_name: &str,
        file_path: &str,
    ) -> f32 {
        // Boost matches in symbol name
        let symbol_boost = if symbol_name
            .to_ascii_lowercase()
            .contains(&precomputed.query_lower)
        {
            0.5
        } else {
            0.0
        };

        // Penalty for test-related files to address Limitation 4
        let test_penalty = if file_path.to_ascii_lowercase().contains("test")
            || symbol_name.to_ascii_lowercase().contains("test")
        {
            0.3
        } else {
            0.0
        };

        // Use cached node tokens for overlap calculation (T14 optimization)
        // Tokens were cached during index_nodes() — no re-tokenization needed.
        // This avoids iterating over each query token and checking the inverted index,
        // replacing it with a single set intersection on pre-cached per-node tokens.
        let base_score = if precomputed.query_tokens.is_empty() {
            // No meaningful tokens in query
            0.0
        } else if let Some(node_tokens) = self.node_tokens.get(node_id) {
            // Count overlap between query tokens and cached node tokens
            let matching = precomputed.query_tokens.intersection(node_tokens).count();
            matching as f32 / precomputed.query_tokens.len() as f32
        } else {
            0.0
        };

        ((base_score + symbol_boost) - test_penalty).clamp(0.0, 1.0)
    }

    /// Semantic search for entry points
    ///
    /// This method performs vector similarity search using cosine similarity.
    /// For now, it requires pre-computed embeddings in the indexed nodes.
    ///
    /// # Arguments
    ///
    /// * `query_embedding` - Query embedding vector (must match index dimension)
    /// * `top_k` - Maximum number of results to return
    ///
    /// # Returns
    ///
    /// Vector of semantic entries sorted by similarity score
    ///
    /// # Example
    ///
    /// ```ignore
    /// let query_embedding = vec![0.1, 0.2, 0.3, ...]; // 768-dim vector
    /// let results = engine.semantic_search(&query_embedding, 10).await?;
    /// ```
    ///
    /// # Errors
    ///
    /// Returns `Error::QueryFailed` if dimension mismatch or search fails.
    pub fn semantic_search(
        &self,
        query_embedding: &[f32],
        top_k: usize,
    ) -> Result<Vec<SemanticEntry>, Error> {
        // Return early if index is empty (no need to validate dimensions in this case)
        if self.vector_index.is_empty() {
            return Ok(Vec::new());
        }

        // Validate embedding dimension (only needed when we actually have embeddings)
        if query_embedding.len() != self.vector_index.dimension() {
            return Err(Error::QueryFailed(format!(
                "Embedding dimension mismatch: expected {}, got {}",
                self.vector_index.dimension(),
                query_embedding.len()
            )));
        }

        // Perform vector similarity search
        let results = self.vector_index.search(query_embedding, top_k);

        // Convert to SemanticEntry format using O(1) HashMap lookup
        let entries = results
            .into_iter()
            .map(|(node_id, score)| {
                // O(1) lookup via node_id_to_idx instead of O(N) linear scan
                let entry_type = self
                    .node_id_to_idx
                    .get(&node_id)
                    .and_then(|&idx| self.nodes.get(idx))
                    .map(|_| EntryType::Function)
                    .unwrap_or(EntryType::Function);

                SemanticEntry {
                    node_id,
                    relevance: score,
                    entry_type,
                }
            })
            .collect();

        Ok(entries)
    }

    /// Get the vector index for direct access
    ///
    /// This provides access to the underlying vector index for advanced use cases.
    ///
    /// # Example
    ///
    /// ```ignore
    /// let dimension = engine.vector_index().dimension();
    /// let count = engine.vector_index().len();
    /// ```
    #[must_use]
    pub fn vector_index(&self) -> &VectorIndexImpl {
        &self.vector_index
    }

    /// Get mutable access to the vector index
    ///
    /// This allows direct manipulation of the vector index.
    ///
    /// # Thread Safety
    ///
    /// **WARNING:** This method requires `&mut self` which ensures exclusive access.
    /// Never call this concurrently with any other method on the same instance.
    ///
    /// # Example
    ///
    /// ```ignore
    /// let index = engine.vector_index_mut();
    /// index.insert("new_node", embedding)?;
    /// ```
    pub fn vector_index_mut(&mut self) -> &mut VectorIndexImpl {
        &mut self.vector_index
    }

    /// Enable HNSW for faster approximate search
    ///
    /// This converts the vector index from brute-force to HNSW-based.
    /// Existing indexed vectors are **NOT** automatically migrated - you must
    /// re-index your data after enabling HNSW.
    ///
    /// # Arguments
    ///
    /// * `params` - Optional HNSW parameters (uses defaults if None)
    ///
    /// # Example
    ///
    /// ```ignore
    /// engine.enable_hnsw(None);
    /// engine.index_nodes(nodes); // Re-index with HNSW
    /// ```
    pub fn enable_hnsw(&mut self, params: Option<HNSWParams>) {
        let dimension = self.vector_index.dimension();
        let params = params.unwrap_or_default();
        self.vector_index =
            VectorIndexImpl::HNSW(Box::new(HNSWIndex::with_params(dimension, params)));
    }

    /// Check if HNSW is currently enabled
    #[must_use]
    pub fn is_hnsw_enabled(&self) -> bool {
        matches!(
            self.vector_index,
            VectorIndexImpl::HNSW(_) | VectorIndexImpl::HNSWQuantized(_)
        )
    }

    /// Disable HNSW and switch back to brute-force search
    ///
    /// This clears the current vector index and creates a new brute-force index.
    /// You'll need to re-index your data after disabling HNSW.
    pub fn disable_hnsw(&mut self) {
        let dimension = self.vector_index.dimension();
        self.vector_index = VectorIndexImpl::BruteForce(VectorIndex::new(dimension));
    }

    /// Create a new search engine with HNSW enabled
    ///
    /// # Arguments
    ///
    /// * `dimension` - Embedding vector dimension
    /// * `params` - HNSW parameters
    ///
    /// # Example
    ///
    /// ```ignore
    /// let engine = SearchEngine::with_hnsw(128, HNSWParams::default());
    /// ```
    #[must_use]
    pub fn with_hnsw(dimension: usize, params: HNSWParams) -> Self {
        let mut engine = Self::with_dimension(dimension);
        engine.enable_hnsw(Some(params));
        engine
    }

    /// Enable INT8 quantized HNSW for memory-efficient search
    ///
    /// This provides ~74% memory reduction compared to f32 HNSW while
    /// maintaining search accuracy through asymmetric distance computation.
    ///
    /// # Arguments
    ///
    /// * `params` - Optional INT8 HNSW parameters (uses defaults if None)
    ///
    /// # Example
    ///
    /// ```ignore
    /// engine.enable_int8_hnsw(None);
    /// engine.index_nodes(nodes); // Re-index with INT8 quantization
    /// ```
    pub fn enable_int8_hnsw(&mut self, params: Option<Int8HnswParams>) {
        let dimension = self.vector_index.dimension();
        let params = params.unwrap_or_default();
        self.vector_index =
            VectorIndexImpl::HNSWQuantized(Box::new(Int8HnswIndex::with_params(dimension, params)));
    }

    /// Check if the current index is quantized
    #[must_use]
    pub fn is_quantized(&self) -> bool {
        matches!(self.vector_index, VectorIndexImpl::HNSWQuantized(_))
    }

    /// Estimate memory usage in bytes
    #[must_use]
    pub fn estimated_memory_bytes(&self) -> usize {
        // Rough estimate based on implementation
        // Content is cleared after indexing (T13), so no +256 content estimate needed
        let nodes_size = self.nodes.len() * std::mem::size_of::<NodeInfo>();
        let cache_size = self.complexity_cache.len()
            * (std::mem::size_of::<String>() + std::mem::size_of::<u32>());
        let text_index_size = self
            .text_index
            .values()
            .map(|set| set.len() * std::mem::size_of::<String>())
            .sum::<usize>();

        nodes_size + cache_size + text_index_size + self.vector_index.estimated_memory_bytes()
    }

    /// Estimate byte size of a slice of search results for cache accounting.
    fn estimate_search_results_bytes(results: &[SearchResult]) -> usize {
        results
            .iter()
            .map(|r| {
                r.node_id.len()
                    + r.file_path.len()
                    + r.symbol_name.len()
                    + r.symbol_type.as_ref().map_or(0, |s| s.len())
                    + r.signature.as_ref().map_or(0, |s| s.len())
                    + r.language.len()
                    + r.context.as_ref().map_or(0, |c| c.len())
                    + 128 // overhead estimate for rank, score, complexity, byte_range, etc.
            })
            .sum()
    }
}

/// Delta update for the text index.
///
/// Describes which nodes to remove and which to add/update, enabling
/// incremental reindexing without rebuilding the entire index from scratch.
///
/// # Performance
///
/// Incremental updates are O(K) where K is the number of changed nodes,
/// compared to O(N) for a full `index_nodes()` rebuild.
///
/// # Example
///
/// ```ignore
/// let delta = TextIndexDelta {
///     removed_node_ids: vec!["old_func".to_string()],
///     updated_nodes: vec![updated_node_info],
/// };
/// engine.incremental_reindex(delta);
/// ```
#[derive(Debug, Default)]
pub struct TextIndexDelta {
    /// Node IDs to remove from the index.
    pub removed_node_ids: Vec<String>,
    /// New or updated nodes to add to the index.
    /// Nodes whose `node_id` already exists will be replaced in-place.
    pub updated_nodes: Vec<NodeInfo>,
}

impl Default for SearchEngine {
    fn default() -> Self {
        Self::new()
    }
}

/// Entry type for semantic search results
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum EntryType {
    /// Function entry point
    Function,
    /// Method entry point
    Method,
    /// Class/struct entry point
    Class,
    /// Module-level entry point
    Module,
}

/// Semantic entry for entry point detection
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SemanticEntry {
    /// Node ID
    pub node_id: String,
    /// Relevance score
    pub relevance: f32,
    /// Entry type
    pub entry_type: EntryType,
}

/// Search errors
#[derive(Debug, thiserror::Error)]
pub enum Error {
    /// Query execution failed
    #[error("Query failed: {0}")]
    QueryFailed(String),

    /// Index is empty
    #[error("Index is empty")]
    EmptyIndex,

    /// Dimension mismatch
    #[error("Dimension mismatch: expected {expected}, got {got}")]
    DimensionMismatch {
        /// Expected dimension
        expected: usize,
        /// Actual dimension received
        got: usize,
    },
}

#[cfg(test)]
mod tests {
    use super::*;

    fn create_test_nodes() -> Vec<NodeInfo> {
        vec![
            NodeInfo {
                node_id: "func1".to_string(),
                file_path: "test.rs".to_string(),
                symbol_name: "func1".to_string(),
                language: "rust".to_string(),
                content: "fn func1() { println!(\"hello\"); }".to_string(),
                byte_range: (0, 40),
                tfidf_embedding: vec![1.0, 0.0, 0.0],
                neural_embedding: None,
                complexity: 2,
                signature: None,
                pre_tokenized: None,
            },
            NodeInfo {
                node_id: "func2".to_string(),
                file_path: "test.rs".to_string(),
                symbol_name: "func2".to_string(),
                language: "rust".to_string(),
                content: "fn func2() { println!(\"world\"); }".to_string(),
                byte_range: (42, 82),
                tfidf_embedding: vec![0.0, 1.0, 0.0],
                neural_embedding: None,
                complexity: 2,
                signature: None,
                pre_tokenized: None,
            },
        ]
    }

    #[test]
    fn test_search_engine_creation() {
        let engine = SearchEngine::new();
        assert_eq!(engine.node_count(), 0);
        assert!(engine.is_empty());
    }

    #[test]
    fn test_index_nodes() {
        let mut engine = SearchEngine::new();
        let nodes = create_test_nodes();
        engine.index_nodes(nodes);
        assert_eq!(engine.node_count(), 2);
        assert!(!engine.is_empty());
    }

    #[test]
    fn test_search_empty_index() {
        let mut engine = SearchEngine::new();
        let query = SearchQuery {
            query: "test".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert!(results.is_empty());
    }

    #[test]
    fn test_semantic_search_empty_index() {
        let engine = SearchEngine::new();
        let results = engine.semantic_search(&[0.1, 0.2, 0.3], 10).unwrap();
        assert!(results.is_empty());
    }

    #[test]
    fn test_search_with_results() {
        let mut engine = SearchEngine::new();
        let nodes = create_test_nodes();
        engine.index_nodes(nodes);

        let query = SearchQuery {
            query: "func1".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert!(!results.is_empty());
        assert_eq!(results[0].node_id, "func1");
    }

    #[test]
    fn test_semantic_search() {
        let mut engine = SearchEngine::with_dimension(3);
        let nodes = create_test_nodes();
        engine.index_nodes(nodes);

        // Search with query vector similar to func1
        let results = engine.semantic_search(&[1.0, 0.0, 0.0], 1).unwrap();
        assert!(!results.is_empty());
        assert_eq!(results[0].node_id, "func1");
    }

    #[test]
    fn test_dimension_validation() {
        let engine = SearchEngine::with_dimension(128);
        assert_eq!(engine.vector_index().dimension(), 128);
    }

    #[test]
    fn test_dimension_mismatch_error() {
        let mut engine = SearchEngine::with_dimension(3);
        let nodes = create_test_nodes();
        engine.index_nodes(nodes);

        // Try searching with wrong dimension
        let result = engine.semantic_search(&[0.1, 0.2], 10);
        assert!(result.is_err());
    }

    #[test]
    fn test_hnsw_enable() {
        let mut engine = SearchEngine::with_dimension(128);
        engine.enable_hnsw(None);
        assert!(engine.vector_index().is_hnsw_enabled());
    }

    #[test]
    fn test_top_k_limit() {
        let mut engine = SearchEngine::new();
        let nodes = create_test_nodes();
        engine.index_nodes(nodes);

        let query = SearchQuery {
            query: "fn".to_string(),
            top_k: 1,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert_eq!(results.len(), 1);
    }

    #[test]
    fn test_relevance_threshold() {
        let mut engine = SearchEngine::new();
        let nodes = create_test_nodes();
        engine.index_nodes(nodes);

        let query = SearchQuery {
            query: "nonexistent".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: Some(0.5),
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert!(results.is_empty());
    }

    #[test]
    fn test_node_id_to_idx_populated() {
        let mut engine = SearchEngine::new();
        let nodes = create_test_nodes();
        engine.index_nodes(nodes);

        // Verify node_id_to_idx is populated with correct indices
        assert_eq!(engine.node_id_to_idx.len(), 2);
        assert_eq!(engine.node_id_to_idx.get("func1"), Some(&0));
        assert_eq!(engine.node_id_to_idx.get("func2"), Some(&1));
    }

    #[test]
    fn test_node_id_to_idx_o1_lookup_in_semantic_search() {
        let mut engine = SearchEngine::with_dimension(3);
        let nodes = create_test_nodes();
        engine.index_nodes(nodes);

        // Verify semantic_search uses node_id_to_idx for O(1) lookup
        // by checking that results are still correct after optimization
        let results = engine.semantic_search(&[1.0, 0.0, 0.0], 10).unwrap();
        assert!(!results.is_empty());

        // The top result should be func1 (closest to query vector)
        assert_eq!(results[0].node_id, "func1");
        assert_eq!(results[0].entry_type, EntryType::Function);

        // Verify all results have correct entry type
        for entry in &results {
            assert_eq!(entry.entry_type, EntryType::Function);
        }
    }

    #[test]
    fn test_node_id_to_idx_cleared_on_reindex() {
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());
        assert_eq!(engine.node_id_to_idx.len(), 2);

        // Re-index with different nodes - should clear and repopulate
        engine.index_nodes(vec![NodeInfo {
            node_id: "new_func".to_string(),
            file_path: "new.rs".to_string(),
            symbol_name: "new_func".to_string(),
            language: "rust".to_string(),
            content: "fn new_func() {}".to_string(),
            byte_range: (0, 18),
            tfidf_embedding: vec![],
            neural_embedding: None,
            complexity: 1,
            signature: None,
            pre_tokenized: None,
        }]);
        assert_eq!(engine.node_id_to_idx.len(), 1);
        assert_eq!(engine.node_id_to_idx.get("new_func"), Some(&0));
        assert_eq!(engine.node_id_to_idx.get("func1"), None);
    }

    #[test]
    fn test_content_cleared_after_indexing() {
        // T13: Verify that NodeInfo.content is cleared after index_nodes()
        // to reduce memory footprint. The inverted index (text_index) preserves
        // all token information for search.
        let mut engine = SearchEngine::new();
        let nodes = create_test_nodes();
        engine.index_nodes(nodes);

        // Content should be empty (cleared) for all nodes
        for node in &engine.nodes {
            assert!(
                node.content.is_empty(),
                "Node {} content should be cleared after indexing, but got: {:?}",
                node.node_id,
                node.content
            );
        }

        // But text search should still work via inverted index
        let query = SearchQuery {
            query: "func1".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert!(
            !results.is_empty(),
            "Search should still find results via inverted index after content cleared"
        );
        assert_eq!(results[0].node_id, "func1");

        // Also verify text_index is populated
        assert!(
            !engine.text_index.is_empty(),
            "text_index should be populated"
        );
        assert!(
            engine.text_index.contains_key("func1"),
            "text_index should contain 'func1' token"
        );
        assert!(
            engine.text_index.contains_key("func2"),
            "text_index should contain 'func2' token"
        );
    }

    #[test]
    fn test_node_tokens_populated() {
        // T14: Verify that node_tokens cache is populated during index_nodes()
        let mut engine = SearchEngine::new();
        let nodes = create_test_nodes();
        engine.index_nodes(nodes);

        // node_tokens should have an entry for each node
        assert_eq!(engine.node_tokens.len(), 2);
        assert!(engine.node_tokens.contains_key("func1"));
        assert!(engine.node_tokens.contains_key("func2"));

        // Verify tokens contain expected normalized content
        let func1_tokens = engine.node_tokens.get("func1").unwrap();
        assert!(
            func1_tokens.contains("func1"),
            "func1 tokens should contain 'func1', got: {:?}",
            func1_tokens
        );

        let func2_tokens = engine.node_tokens.get("func2").unwrap();
        assert!(
            func2_tokens.contains("func2"),
            "func2 tokens should contain 'func2', got: {:?}",
            func2_tokens
        );
    }

    #[test]
    fn test_node_tokens_cleared_on_reindex() {
        // T14: Verify node_tokens is cleared when re-indexing
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());
        assert_eq!(engine.node_tokens.len(), 2);

        // Re-index with different nodes
        engine.index_nodes(vec![NodeInfo {
            node_id: "new_func".to_string(),
            file_path: "test.rs".to_string(),
            symbol_name: "new_func".to_string(),
            language: "rust".to_string(),
            content: "fn new_func() {}".to_string(),
            byte_range: (0, 18),
            tfidf_embedding: vec![],
            neural_embedding: None,
            complexity: 1,
            signature: None,
            pre_tokenized: None,
        }]);
        assert_eq!(engine.node_tokens.len(), 1);
        assert!(engine.node_tokens.contains_key("new_func"));
        assert!(!engine.node_tokens.contains_key("func1"));
    }

    #[test]
    fn test_node_tokens_used_in_scoring() {
        // T14: Verify that scoring uses cached tokens (no re-tokenization)
        // by checking that search results are correct after content is cleared.
        // This implicitly tests that calculate_text_score_optimized uses node_tokens.
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());

        // Content is cleared (T13), but tokens are cached (T14)
        for node in &engine.nodes {
            assert!(node.content.is_empty());
        }

        // Search for a term that appears in content — should still find it via cached tokens
        let query = SearchQuery {
            query: "println hello".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();

        // Should find results since "println" and "hello" appear in node content and tokens are cached
        assert!(
            !results.is_empty(),
            "Search should find results using cached node_tokens even after content is cleared"
        );
        // func1 contains both "println" and "hello", should be top result
        assert_eq!(results[0].node_id, "func1");
    }

    // ----------------------------------------------------------------
    // T28: Incremental reindex tests
    // ----------------------------------------------------------------

    #[test]
    fn test_incremental_reindex_add_nodes() {
        // T28: Adding nodes via incremental_reindex should update all indexes
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());
        assert_eq!(engine.node_count(), 2);

        let delta = TextIndexDelta {
            removed_node_ids: vec![],
            updated_nodes: vec![NodeInfo {
                node_id: "func3".to_string(),
                file_path: "test.rs".to_string(),
                symbol_name: "func3".to_string(),
                language: "rust".to_string(),
                content: "fn func3() { db_query(); }".to_string(),
                byte_range: (100, 130),
                tfidf_embedding: vec![0.0, 0.0, 1.0],
                neural_embedding: None,
                complexity: 3,
                signature: None,
                pre_tokenized: None,
            }],
        };
        engine.incremental_reindex(delta);

        // Should now have 3 nodes
        assert_eq!(engine.node_count(), 3);
        assert_eq!(engine.node_id_to_idx.len(), 3);
        assert_eq!(engine.node_tokens.len(), 3);
        assert_eq!(engine.complexity_cache.len(), 3);

        // Search should find the new node
        let query = SearchQuery {
            query: "func3".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert!(!results.is_empty());
        assert_eq!(results[0].node_id, "func3");

        // text_index should contain "func3" token
        assert!(engine.text_index.contains_key("func3"));
        // "db" and "query" tokens should also be indexed
        assert!(engine.text_index.contains_key("query"));
    }

    #[test]
    fn test_incremental_reindex_remove_nodes() {
        // T28: Removing nodes via incremental_reindex should clean up all indexes
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());
        assert_eq!(engine.node_count(), 2);

        let delta = TextIndexDelta {
            removed_node_ids: vec!["func1".to_string()],
            updated_nodes: vec![],
        };
        engine.incremental_reindex(delta);

        // Should now have 1 node
        assert_eq!(engine.node_count(), 1);
        assert_eq!(engine.node_id_to_idx.len(), 1);
        assert!(!engine.node_id_to_idx.contains_key("func1"));
        assert!(engine.node_id_to_idx.contains_key("func2"));

        // func1's tokens should be removed from text_index
        // "func1" token should no longer map to func1
        if let Some(ids) = engine.text_index.get("func1") {
            assert!(
                !ids.contains("func1"),
                "func1 should be removed from text_index"
            );
        }

        // node_tokens should not contain func1
        assert!(!engine.node_tokens.contains_key("func1"));
        assert!(engine.node_tokens.contains_key("func2"));

        // Search for func1 should not find it
        let query = SearchQuery {
            query: "func1".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert!(
            results.is_empty(),
            "func1 should not be found after removal"
        );
    }

    #[test]
    fn test_incremental_reindex_update_existing_node() {
        // T28: Updating an existing node should replace it correctly
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());

        // Update func1 with new content
        let delta = TextIndexDelta {
            removed_node_ids: vec![],
            updated_nodes: vec![NodeInfo {
                node_id: "func1".to_string(),
                file_path: "updated.rs".to_string(),
                symbol_name: "func1_renamed".to_string(),
                language: "rust".to_string(),
                content: "fn func1_renamed() { new_logic(); }".to_string(),
                byte_range: (0, 35),
                tfidf_embedding: vec![0.5, 0.5, 0.0],
                neural_embedding: None,
                complexity: 5,
                signature: None,
                pre_tokenized: None,
            }],
        };
        engine.incremental_reindex(delta);

        // Should still have 2 nodes
        assert_eq!(engine.node_count(), 2);

        // Complexity cache should reflect the update
        assert_eq!(engine.complexity_cache.get("func1"), Some(&5));

        // New tokens should be indexed
        assert!(engine.node_tokens.get("func1").unwrap().contains("logic"));
        assert!(engine.text_index.contains_key("logic"));

        // Search for new content should work
        let query = SearchQuery {
            query: "new_logic".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert!(!results.is_empty());
        assert_eq!(results[0].node_id, "func1");
    }

    #[test]
    fn test_incremental_reindex_combined_add_remove() {
        // T28: Combined add and remove in one delta
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());

        let delta = TextIndexDelta {
            removed_node_ids: vec!["func1".to_string()],
            updated_nodes: vec![
                NodeInfo {
                    node_id: "func3".to_string(),
                    file_path: "new.rs".to_string(),
                    symbol_name: "func3".to_string(),
                    language: "rust".to_string(),
                    content: "fn func3() {}".to_string(),
                    byte_range: (0, 14),
                    tfidf_embedding: vec![],
                    neural_embedding: None,
                    complexity: 1,
                    signature: None,
                    pre_tokenized: None,
                },
                NodeInfo {
                    node_id: "func4".to_string(),
                    file_path: "new.rs".to_string(),
                    symbol_name: "func4".to_string(),
                    language: "rust".to_string(),
                    content: "fn func4() { helper(); }".to_string(),
                    byte_range: (15, 40),
                    tfidf_embedding: vec![],
                    neural_embedding: None,
                    complexity: 2,
                    signature: None,
                    pre_tokenized: None,
                },
            ],
        };
        engine.incremental_reindex(delta);

        // Should have func2 (original) + func3 + func4 = 3 nodes
        assert_eq!(engine.node_count(), 3);
        assert_eq!(engine.node_id_to_idx.len(), 3);

        // func1 should be gone
        assert!(!engine.node_id_to_idx.contains_key("func1"));
        // func2, func3, func4 should exist
        assert!(engine.node_id_to_idx.contains_key("func2"));
        assert!(engine.node_id_to_idx.contains_key("func3"));
        assert!(engine.node_id_to_idx.contains_key("func4"));

        // Search for func2 should still work
        let query = SearchQuery {
            query: "func2".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert!(!results.is_empty());
        assert_eq!(results[0].node_id, "func2");
    }

    #[test]
    fn test_incremental_reindex_empty_delta() {
        // T28: Empty delta should not change anything
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());

        let delta = TextIndexDelta {
            removed_node_ids: vec![],
            updated_nodes: vec![],
        };
        engine.incremental_reindex(delta);

        assert_eq!(engine.node_count(), 2);
        assert_eq!(engine.node_id_to_idx.len(), 2);
    }

    #[test]
    fn test_incremental_reindex_removes_empty_token_sets() {
        // T28: When removing the last node for a token, the token entry should be removed
        let mut engine = SearchEngine::new();
        engine.index_nodes(vec![
            NodeInfo {
                node_id: "unique1".to_string(),
                file_path: "test.rs".to_string(),
                symbol_name: "unique1".to_string(),
                language: "rust".to_string(),
                content: "fn unique1() { zebra(); }".to_string(),
                byte_range: (0, 25),
                tfidf_embedding: vec![],
                neural_embedding: None,
                complexity: 1,
                signature: None,
                pre_tokenized: None,
            },
            NodeInfo {
                node_id: "unique2".to_string(),
                file_path: "test.rs".to_string(),
                symbol_name: "unique2".to_string(),
                language: "rust".to_string(),
                content: "fn unique2() { apple(); }".to_string(),
                byte_range: (26, 52),
                tfidf_embedding: vec![],
                neural_embedding: None,
                complexity: 1,
                signature: None,
                pre_tokenized: None,
            },
        ]);

        // "zebra" token should exist and map to unique1 only
        assert!(engine.text_index.contains_key("zebra"));

        // Remove unique1 — "zebra" token set should be cleaned up entirely
        let delta = TextIndexDelta {
            removed_node_ids: vec!["unique1".to_string()],
            updated_nodes: vec![],
        };
        engine.incremental_reindex(delta);

        // "zebra" token should no longer exist in text_index (no remaining nodes have it)
        assert!(
            !engine.text_index.contains_key("zebra"),
            "Token with no remaining nodes should be removed from text_index"
        );

        // "apple" should still exist
        assert!(engine.text_index.contains_key("apple"));
    }

    #[test]
    fn test_incremental_reindex_correctness_vs_full_rebuild() {
        // T28: Incremental reindex should produce identical results to a full rebuild
        let mut engine_inc = SearchEngine::new();
        let mut engine_full = SearchEngine::new();

        // Start with same initial nodes
        let initial = create_test_nodes();
        engine_inc.index_nodes(initial.clone());
        engine_full.index_nodes(initial);

        // Apply delta incrementally
        let delta = TextIndexDelta {
            removed_node_ids: vec!["func1".to_string()],
            updated_nodes: vec![NodeInfo {
                node_id: "func3".to_string(),
                file_path: "new.rs".to_string(),
                symbol_name: "func3".to_string(),
                language: "rust".to_string(),
                content: "fn func3() { compute(); }".to_string(),
                byte_range: (0, 25),
                tfidf_embedding: vec![1.0, 1.0, 0.0],
                neural_embedding: None,
                complexity: 4,
                signature: None,
                pre_tokenized: None,
            }],
        };
        engine_inc.incremental_reindex(delta);

        // Apply same changes via full rebuild
        engine_full.index_nodes(vec![
            NodeInfo {
                node_id: "func2".to_string(),
                file_path: "test.rs".to_string(),
                symbol_name: "func2".to_string(),
                language: "rust".to_string(),
                content: "fn func2() { println!(\"world\"); }".to_string(),
                byte_range: (42, 82),
                tfidf_embedding: vec![0.0, 1.0, 0.0],
                neural_embedding: None,
                complexity: 2,
                signature: None,
                pre_tokenized: None,
            },
            NodeInfo {
                node_id: "func3".to_string(),
                file_path: "new.rs".to_string(),
                symbol_name: "func3".to_string(),
                language: "rust".to_string(),
                content: "fn func3() { compute(); }".to_string(),
                byte_range: (0, 25),
                tfidf_embedding: vec![1.0, 1.0, 0.0],
                neural_embedding: None,
                complexity: 4,
                signature: None,
                pre_tokenized: None,
            },
        ]);

        // Both engines should have same node count
        assert_eq!(engine_inc.node_count(), engine_full.node_count());

        // Both should have same node_ids
        let inc_ids: std::collections::BTreeSet<_> =
            engine_inc.nodes.iter().map(|n| n.node_id.clone()).collect();
        let full_ids: std::collections::BTreeSet<_> = engine_full
            .nodes
            .iter()
            .map(|n| n.node_id.clone())
            .collect();
        assert_eq!(inc_ids, full_ids);

        // Search should produce same results
        let query = SearchQuery {
            query: "func2".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let inc_results = engine_inc.search(query.clone()).unwrap();
        let full_results = engine_full.search(query).unwrap();
        assert_eq!(inc_results.len(), full_results.len());
        if !inc_results.is_empty() {
            assert_eq!(inc_results[0].node_id, full_results[0].node_id);
        }

        // Semantic search should also produce same results
        let inc_sem = engine_inc.semantic_search(&[1.0, 1.0, 0.0], 10).unwrap();
        let full_sem = engine_full.semantic_search(&[1.0, 1.0, 0.0], 10).unwrap();
        assert_eq!(inc_sem.len(), full_sem.len());
        if !inc_sem.is_empty() {
            assert_eq!(inc_sem[0].node_id, full_sem[0].node_id);
        }
    }

    #[test]
    fn test_incremental_reindex_semantic_search_after_update() {
        // T28: Semantic search should work correctly after incremental update
        let mut engine = SearchEngine::with_dimension(3);
        engine.index_nodes(create_test_nodes());

        // Add a new node with a distinct embedding
        let delta = TextIndexDelta {
            removed_node_ids: vec![],
            updated_nodes: vec![NodeInfo {
                node_id: "func3".to_string(),
                file_path: "test.rs".to_string(),
                symbol_name: "func3".to_string(),
                language: "rust".to_string(),
                content: "fn func3() {}".to_string(),
                byte_range: (0, 14),
                tfidf_embedding: vec![0.1, 0.1, 0.9],
                neural_embedding: None,
                complexity: 1,
                signature: None,
                pre_tokenized: None,
            }],
        };
        engine.incremental_reindex(delta);

        // Search for vec close to func3's embedding
        let results = engine.semantic_search(&[0.1, 0.1, 0.9], 1).unwrap();
        assert!(!results.is_empty());
        assert_eq!(results[0].node_id, "func3");
    }

    #[test]
    fn test_incremental_reindex_node_id_to_idx_consistency() {
        // T28: node_id_to_idx should be consistent after multiple incremental updates
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());

        // Add func3
        engine.incremental_reindex(TextIndexDelta {
            removed_node_ids: vec![],
            updated_nodes: vec![NodeInfo {
                node_id: "func3".to_string(),
                file_path: "test.rs".to_string(),
                symbol_name: "func3".to_string(),
                language: "rust".to_string(),
                content: "fn func3() {}".to_string(),
                byte_range: (0, 14),
                tfidf_embedding: vec![],
                neural_embedding: None,
                complexity: 1,
                signature: None,
                pre_tokenized: None,
            }],
        });

        // Remove func1 (swap-remove may swap func3 into func1's slot)
        engine.incremental_reindex(TextIndexDelta {
            removed_node_ids: vec!["func1".to_string()],
            updated_nodes: vec![],
        });

        // Verify all indices are consistent
        assert_eq!(engine.node_id_to_idx.len(), engine.nodes.len());
        for (idx, node) in engine.nodes.iter().enumerate() {
            assert_eq!(
                engine.node_id_to_idx.get(&node.node_id),
                Some(&idx),
                "node_id_to_idx mismatch for node {}",
                node.node_id
            );
        }
    }

    #[test]
    fn test_incremental_reindex_removes_nonexistent_node() {
        // T28: Removing a node that doesn't exist should be a no-op
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());

        let delta = TextIndexDelta {
            removed_node_ids: vec!["nonexistent".to_string()],
            updated_nodes: vec![],
        };
        engine.incremental_reindex(delta);

        assert_eq!(engine.node_count(), 2);
        assert_eq!(engine.node_id_to_idx.len(), 2);
    }

    #[test]
    fn test_incremental_reindex_content_cleared() {
        // T28: Content should be cleared on newly added nodes (same as index_nodes)
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());

        engine.incremental_reindex(TextIndexDelta {
            removed_node_ids: vec![],
            updated_nodes: vec![NodeInfo {
                node_id: "func3".to_string(),
                file_path: "test.rs".to_string(),
                symbol_name: "func3".to_string(),
                language: "rust".to_string(),
                content: "fn func3() { important_content(); }".to_string(),
                byte_range: (0, 40),
                tfidf_embedding: vec![],
                neural_embedding: None,
                complexity: 3,
                signature: None,
                pre_tokenized: None,
            }],
        });

        // Content should be cleared for all nodes
        for node in &engine.nodes {
            assert!(
                node.content.is_empty(),
                "Node {} content should be cleared, got: {:?}",
                node.node_id,
                node.content
            );
        }

        // But func3 tokens should still be searchable
        assert!(engine
            .node_tokens
            .get("func3")
            .unwrap()
            .contains("important"));
    }

    // ----------------------------------------------------------------
    // R8: Pre-tokenized search engine tests
    // ----------------------------------------------------------------

    #[test]
    fn test_pre_tokenized_produces_identical_search_results() {
        // R8: NodeInfo with pre_tokenized = Some(...) should produce identical
        // search results to the re-tokenization path.
        let content = "fn calculate_total(price: f64, tax: f64) -> f64 { price + tax }";

        // Compute search tokens the same way index_builder does
        let search_tokens: Vec<String> = content
            .split(|c: char| !c.is_alphanumeric())
            .map(|s| s.to_ascii_lowercase())
            .filter(|s| s.len() >= 2)
            .collect();

        // Engine with pre-tokenized tokens
        let mut engine_pre = SearchEngine::new();
        engine_pre.index_nodes(vec![NodeInfo {
            node_id: "calc_total".to_string(),
            file_path: "math.rs".to_string(),
            symbol_name: "calculate_total".to_string(),
            language: "rust".to_string(),
            content: content.to_string(),
            byte_range: (0, content.len()),
            tfidf_embedding: vec![],
            neural_embedding: None,
            complexity: 3,
            signature: None,
            pre_tokenized: Some(search_tokens),
        }]);

        // Engine with re-tokenization (pre_tokenized = None)
        let mut engine_fallback = SearchEngine::new();
        engine_fallback.index_nodes(vec![NodeInfo {
            node_id: "calc_total".to_string(),
            file_path: "math.rs".to_string(),
            symbol_name: "calculate_total".to_string(),
            language: "rust".to_string(),
            content: content.to_string(),
            byte_range: (0, content.len()),
            tfidf_embedding: vec![],
            neural_embedding: None,
            complexity: 3,
            signature: None,
            pre_tokenized: None,
        }]);

        // Both inverted indexes should be identical
        assert_eq!(
            engine_pre.text_index, engine_fallback.text_index,
            "Pre-tokenized and fallback should produce identical text_index"
        );
        assert_eq!(
            engine_pre.node_tokens, engine_fallback.node_tokens,
            "Pre-tokenized and fallback should produce identical node_tokens"
        );

        // Search for "calculate" should find the node in both
        let query = SearchQuery {
            query: "calculate".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results_pre = engine_pre.search(query.clone()).unwrap();
        let results_fallback = engine_fallback.search(query).unwrap();
        assert_eq!(results_pre.len(), results_fallback.len());
        assert!(!results_pre.is_empty());
        assert_eq!(results_pre[0].node_id, results_fallback[0].node_id);
    }

    #[test]
    fn test_pre_tokenized_none_falls_back_to_content() {
        // R8: NodeInfo with pre_tokenized = None should use content-based
        // tokenization (backward compatibility).
        let mut engine = SearchEngine::new();
        engine.index_nodes(vec![NodeInfo {
            node_id: "backward_compat".to_string(),
            file_path: "compat.rs".to_string(),
            symbol_name: "legacy_func".to_string(),
            language: "rust".to_string(),
            content: "fn legacy_func() { return 42; }".to_string(),
            byte_range: (0, 30),
            tfidf_embedding: vec![],
            neural_embedding: None,
            complexity: 1,
            signature: None,
            pre_tokenized: None,
        }]);

        // Should still find via content-based tokenization
        assert!(engine.text_index.contains_key("legacy"));
        assert!(engine.text_index.contains_key("func"));
        assert!(engine.node_tokens.contains_key("backward_compat"));

        let query = SearchQuery {
            query: "legacy".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert!(!results.is_empty());
        assert_eq!(results[0].node_id, "backward_compat");
    }

    #[test]
    fn test_pre_tokenized_and_content_produce_same_inverted_index() {
        // R8: Both paths produce the same inverted index for the same content.
        let content = "pub async fn handle_http_request(req: Request) -> Response { ... }";

        let tokens: Vec<String> = content
            .split(|c: char| !c.is_alphanumeric())
            .map(|s| s.to_ascii_lowercase())
            .filter(|s| s.len() >= 2)
            .collect();

        // Verify our manual tokenization produces expected tokens
        assert!(tokens.contains(&"handle".to_string()));
        assert!(tokens.contains(&"http".to_string()));
        assert!(tokens.contains(&"request".to_string()));
        assert!(tokens.contains(&"response".to_string()));

        // Engine A: pre-tokenized
        let mut engine_a = SearchEngine::new();
        engine_a.index_nodes(vec![NodeInfo {
            node_id: "handler".to_string(),
            file_path: "server.rs".to_string(),
            symbol_name: "handle_http_request".to_string(),
            language: "rust".to_string(),
            content: content.to_string(),
            byte_range: (0, content.len()),
            tfidf_embedding: vec![],
            neural_embedding: None,
            complexity: 5,
            signature: None,
            pre_tokenized: Some(tokens),
        }]);

        // Engine B: content-based
        let mut engine_b = SearchEngine::new();
        engine_b.index_nodes(vec![NodeInfo {
            node_id: "handler".to_string(),
            file_path: "server.rs".to_string(),
            symbol_name: "handle_http_request".to_string(),
            language: "rust".to_string(),
            content: content.to_string(),
            byte_range: (0, content.len()),
            tfidf_embedding: vec![],
            neural_embedding: None,
            complexity: 5,
            signature: None,
            pre_tokenized: None,
        }]);

        // Both should have identical text_index entries
        for token in &["handle", "http", "request", "response", "pub", "async"] {
            assert_eq!(
                engine_a.text_index.get(*token),
                engine_b.text_index.get(*token),
                "Mismatch for token '{}': pre_tokenized={:?}, content={:?}",
                token,
                engine_a.text_index.get(*token),
                engine_b.text_index.get(*token)
            );
        }
    }

    #[test]
    fn test_pre_tokenized_incremental_reindex() {
        // R8: Pre-tokenized tokens should work correctly with incremental reindex.
        let mut engine = SearchEngine::new();
        engine.index_nodes(create_test_nodes());

        let new_content = "fn compute_metrics(data: &[f64]) -> Metrics { ... }";
        let tokens: Vec<String> = new_content
            .split(|c: char| !c.is_alphanumeric())
            .map(|s| s.to_ascii_lowercase())
            .filter(|s| s.len() >= 2)
            .collect();

        let delta = TextIndexDelta {
            removed_node_ids: vec![],
            updated_nodes: vec![NodeInfo {
                node_id: "metrics".to_string(),
                file_path: "metrics.rs".to_string(),
                symbol_name: "compute_metrics".to_string(),
                language: "rust".to_string(),
                content: new_content.to_string(),
                byte_range: (0, new_content.len()),
                tfidf_embedding: vec![],
                neural_embedding: None,
                complexity: 4,
                signature: None,
                pre_tokenized: Some(tokens),
            }],
        };
        engine.incremental_reindex(delta);

        // Should have 3 nodes now
        assert_eq!(engine.node_count(), 3);

        // Pre-tokenized tokens should be in the inverted index
        assert!(engine.text_index.contains_key("compute"));
        assert!(engine.text_index.contains_key("metrics"));
        assert!(engine.text_index.contains_key("data"));

        // Search should find the new node
        let query = SearchQuery {
            query: "compute metrics".to_string(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert!(!results.is_empty());
        assert_eq!(results[0].node_id, "metrics");
    }

    // A+ VAL-APLUS-014: Search cache is hard-capped without semantic regression
    #[test]
    fn test_search_cache_hard_capped() {
        let mut engine = SearchEngine::new();

        // Index some nodes
        let nodes: Vec<NodeInfo> = (0..50)
            .map(|i| NodeInfo {
                node_id: format!("node_{}", i),
                file_path: format!("file_{}.rs", i),
                symbol_name: format!("symbol_{}", i),
                language: "rust".to_string(),
                content: format!("fn symbol_{}() {{}}", i),
                byte_range: (0, 16),
                tfidf_embedding: vec![0.0; 768],
                neural_embedding: None,
                complexity: 1,
                signature: None,
                pre_tokenized: None,
            })
            .collect();
        engine.index_nodes(nodes);

        // Run many searches to fill the cache
        for i in 0..300 {
            let query = SearchQuery {
                query: format!("query_{}", i),
                top_k: 10,
                token_budget: None,
                semantic: false,
                expand_context: false,
                query_embedding: None,
                threshold: None,
                query_type: None,
            };
            let _ = engine.search(query);
        }

        // Cache should not exceed entry limit
        assert!(
            engine.search_cache.len() <= SEARCH_CACHE_MAX_ENTRIES,
            "search cache entries ({}) should not exceed max ({})",
            engine.search_cache.len(),
            SEARCH_CACHE_MAX_ENTRIES
        );

        // Cache bytes should not exceed byte limit
        assert!(
            engine.search_cache_bytes <= SEARCH_CACHE_MAX_BYTES,
            "search cache bytes ({}) should not exceed max ({})",
            engine.search_cache_bytes,
            SEARCH_CACHE_MAX_BYTES
        );

        // Verify search still returns correct results (no semantic regression)
        let query = SearchQuery {
            query: "symbol_0".to_string(),
            top_k: 5,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };
        let results = engine.search(query).unwrap();
        assert!(!results.is_empty(), "search should still return results");
    }

    // ================================================================
    // A+ VAL-APLUS-037: Bound-gated indexing admission tests
    // ================================================================

    #[test]
    fn test_admission_gate_admits_within_bounds() {
        let mut gate = IndexingAdmissionGate::with_caps(100, 1024);
        assert!(gate.try_admit(10));
        assert!(gate.try_admit(10));
        assert_eq!(gate.nodes_admitted(), 2);
        assert_eq!(gate.nodes_shed(), 0);
    }

    #[test]
    fn test_admission_gate_sheds_over_node_cap() {
        let mut gate = IndexingAdmissionGate::with_caps(3, 1_000_000);
        assert!(gate.try_admit(10));
        assert!(gate.try_admit(10));
        assert!(gate.try_admit(10));
        // 4th node should be shed
        assert!(!gate.try_admit(10));
        assert!(!gate.try_admit(10));
        assert_eq!(gate.nodes_admitted(), 3);
        assert_eq!(gate.nodes_shed(), 2);
    }

    #[test]
    fn test_admission_gate_sheds_over_byte_cap() {
        let mut gate = IndexingAdmissionGate::with_caps(100, 50);
        assert!(gate.try_admit(20));
        assert!(gate.try_admit(20));
        // 3rd node would exceed byte cap (20+20+20=60 > 50)
        assert!(!gate.try_admit(20));
        assert_eq!(gate.nodes_admitted(), 2);
        assert_eq!(gate.bytes_admitted(), 40);
        assert_eq!(gate.nodes_shed(), 1);
    }

    #[test]
    fn test_admission_gate_resets() {
        let mut gate = IndexingAdmissionGate::with_caps(2, 100);
        assert!(gate.try_admit(10));
        assert!(gate.try_admit(10));
        assert!(!gate.try_admit(10));
        gate.reset();
        // After reset, should admit again
        assert!(gate.try_admit(10));
        assert_eq!(gate.nodes_admitted(), 1);
        assert_eq!(gate.nodes_shed(), 0);
    }

    #[test]
    fn test_admission_gate_default_caps() {
        let gate = IndexingAdmissionGate::new();
        assert_eq!(gate.nodes_admitted(), 0);
        assert_eq!(gate.nodes_shed(), 0);
        assert_eq!(gate.bytes_admitted(), 0);
    }

    #[test]
    fn test_admission_gate_oversized_single_node() {
        // A single node whose content exceeds the byte cap should be shed.
        let mut gate = IndexingAdmissionGate::with_caps(100, 50);
        assert!(!gate.try_admit(100));
        assert_eq!(gate.nodes_shed(), 1);
        assert_eq!(gate.nodes_admitted(), 0);
    }

    #[test]
    fn test_admission_gate_bursty_workload() {
        // Simulate bursty indexing: many nodes arriving at once.
        let mut gate = IndexingAdmissionGate::with_caps(10, 10_000);
        let mut admitted = 0;
        let mut shed = 0;
        for _ in 0..50 {
            if gate.try_admit(100) {
                admitted += 1;
            } else {
                shed += 1;
            }
        }
        assert_eq!(admitted, 10);
        assert_eq!(shed, 40);
        assert_eq!(gate.nodes_admitted(), 10);
        assert_eq!(gate.nodes_shed(), 40);
    }

    // ================================================================
    // A+ VAL-APLUS-038: Selective pruning tests
    // ================================================================

    #[test]
    fn test_pruner_keeps_user_authored_code() {
        let pruner = ContentPruner::new();
        let decision = pruner.evaluate("src/main.rs", "fn main() { println!(\"hello\"); }", "main");
        assert_eq!(decision, PruningDecision::Keep);
    }

    #[test]
    fn test_pruner_prunes_minified_js() {
        let pruner = ContentPruner::new();
        let decision = pruner.evaluate(
            "static/app.min.js",
            "var a=1,b=2;function c(){return a+b}",
            "c",
        );
        assert!(matches!(decision, PruningDecision::GeneratedCode(_)));
    }

    #[test]
    fn test_pruner_prunes_generated_protobuf() {
        let pruner = ContentPruner::new();
        let decision = pruner.evaluate(
            "proto/user.pb.go",
            "func (m *User) GetName() string { return m.Name }",
            "GetName",
        );
        assert!(matches!(decision, PruningDecision::GeneratedCode(_)));
    }

    #[test]
    fn test_pruner_prunes_generated_rust() {
        let pruner = ContentPruner::new();
        let decision = pruner.evaluate(
            "src/types.generated.rs",
            "pub fn generated_fn() -> i32 { 42 }",
            "generated_fn",
        );
        assert!(matches!(decision, PruningDecision::GeneratedCode(_)));
    }

    #[test]
    fn test_pruner_prunes_bundle_js() {
        let pruner = ContentPruner::new();
        let decision = pruner.evaluate(
            "dist/app.bundle.js",
            "module.exports=function(n){return n+1}",
            "anonymous",
        );
        assert!(matches!(decision, PruningDecision::GeneratedCode(_)));
    }

    #[test]
    fn test_pruner_prunes_node_modules() {
        let pruner = ContentPruner::new();
        let decision = pruner.evaluate(
            "node_modules/lodash/index.js",
            "function debounce(fn, ms) { /* ... */ }",
            "debounce",
        );
        assert!(matches!(decision, PruningDecision::GeneratedCode(_)));
    }

    #[test]
    fn test_pruner_prunes_low_information() {
        let pruner = ContentPruner::new();
        // Very short content with trivial symbol name
        let decision = pruner.evaluate("src/x.rs", "fn x() {}", "x");
        assert!(matches!(decision, PruningDecision::LowInformation(_)));
    }

    #[test]
    fn test_pruner_keeps_short_content_with_meaningful_name() {
        let pruner = ContentPruner::new();
        // Short content but meaningful symbol name — should be kept
        let decision = pruner.evaluate("src/lib.rs", "fn compute() {}", "compute");
        assert_eq!(decision, PruningDecision::Keep);
    }

    #[test]
    fn test_pruner_is_generated_path() {
        let pruner = ContentPruner::new();
        assert!(pruner.is_generated_path("static/app.min.js"));
        assert!(pruner.is_generated_path("proto/user.pb.go"));
        assert!(pruner.is_generated_path("src/types.generated.rs"));
        assert!(pruner.is_generated_path("node_modules/react/index.js"));
        assert!(!pruner.is_generated_path("src/main.rs"));
        assert!(!pruner.is_generated_path("lib/parser.py"));
    }

    #[test]
    fn test_pruner_decision_is_observable() {
        // VAL-APLUS-038: The pruned-vs-kept decision is externally observable.
        let pruner = ContentPruner::new();

        let kept = pruner.evaluate("src/main.rs", "fn main() { /* ... */ }", "main");
        let generated = pruner.evaluate("src/types.generated.rs", "fn gen() {}", "gen");
        let low_info = pruner.evaluate("src/x.rs", "fn x() {}", "x");

        // Each decision variant is distinguishable
        assert_eq!(kept, PruningDecision::Keep);
        match generated {
            PruningDecision::GeneratedCode(reason) => {
                assert!(
                    reason.contains("generated"),
                    "reason should mention generated: {}",
                    reason
                );
            }
            other => panic!("expected GeneratedCode, got {:?}", other),
        }
        match low_info {
            PruningDecision::LowInformation(reason) => {
                assert!(
                    reason.contains("bytes"),
                    "reason should mention bytes: {}",
                    reason
                );
            }
            other => panic!("expected LowInformation, got {:?}", other),
        }
    }

    #[test]
    fn test_pruner_does_not_remove_high_signal_files() {
        let pruner = ContentPruner::new();
        // User-authored high-signal files should always be kept
        let cases = vec![
            (
                "src/lib.rs",
                "pub fn connect(db: &Database) -> Result<Connection> { /* ... */ }",
                "connect",
            ),
            (
                "src/api/handlers.rs",
                "async fn handle_request(req: Request) -> Response { /* ... */ }",
                "handle_request",
            ),
            (
                "src/models/user.rs",
                "struct User { name: String, email: String, created_at: DateTime }",
                "User",
            ),
            (
                "app/controllers/application_controller.rb",
                "def index; @items = Item.all; end",
                "index",
            ),
        ];
        for (path, content, symbol) in cases {
            let decision = pruner.evaluate(path, content, symbol);
            assert_eq!(
                decision,
                PruningDecision::Keep,
                "high-signal file {} should be kept, got {:?}",
                path,
                decision
            );
        }
    }

    // ================================================================
    // A+ VAL-APLUS-039: Repeated-work hoisting tests
    // ================================================================

    #[test]
    fn test_work_hoister_stores_and_retrieves() {
        let mut hoister = WorkHoister::with_bounds(100, 1_000_000);
        let content = "fn compute(x: i32) -> i32 { x + 1 }";
        let embedding = vec![0.1, 0.2, 0.3];

        hoister.store(content, embedding.clone(), None);
        let retrieved = hoister.lookup(content);

        assert!(retrieved.is_some());
        assert_eq!(retrieved.unwrap().0, embedding);
    }

    #[test]
    fn test_work_hoister_miss_for_unseen_content() {
        let mut hoister = WorkHoister::with_bounds(100, 1_000_000);
        assert!(hoister.lookup("unseen content").is_none());
    }

    #[test]
    fn test_work_hoister_reuses_identical_content() {
        let mut hoister = WorkHoister::with_bounds(100, 1_000_000);
        let content = "fn identical() {}";
        let embedding = vec![0.5, 0.5, 0.5];

        hoister.store(content, embedding.clone(), None);

        // Second lookup for identical content should return the same embedding
        let retrieved = hoister.lookup(content);
        assert_eq!(retrieved.unwrap().0, embedding);
    }

    #[test]
    fn test_work_hoister_distinguishes_different_content() {
        let mut hoister = WorkHoister::with_bounds(100, 1_000_000);
        let content_a = "fn alpha() {}";
        let content_b = "fn beta() {}";
        let embedding_a = vec![1.0, 0.0, 0.0];
        let embedding_b = vec![0.0, 1.0, 0.0];

        hoister.store(content_a, embedding_a.clone(), None);
        hoister.store(content_b, embedding_b.clone(), None);

        assert_eq!(hoister.lookup(content_a).unwrap().0, embedding_a);
        assert_eq!(hoister.lookup(content_b).unwrap().0, embedding_b);
    }

    #[test]
    fn test_work_hoister_evicts_on_entry_cap() {
        let mut hoister = WorkHoister::with_bounds(3, 1_000_000);

        hoister.store("content_1", vec![1.0], None);
        hoister.store("content_2", vec![2.0], None);
        hoister.store("content_3", vec![3.0], None);
        assert_eq!(hoister.len(), 3);

        // Adding a 4th should evict the LRU entry
        hoister.store("content_4", vec![4.0], None);
        assert_eq!(hoister.len(), 3);

        // content_1 should have been evicted (LRU)
        assert!(hoister.lookup("content_1").is_none());
        // content_4 should be present
        assert!(hoister.lookup("content_4").is_some());
    }

    #[test]
    fn test_work_hoister_evicts_on_byte_cap() {
        // Very small byte cap: only room for ~1 embedding
        // Each entry: 32-byte BLAKE3 hash key + 3 * 4 bytes = 44 bytes
        let mut hoister = WorkHoister::with_bounds(100, 50);

        hoister.store("short", vec![1.0, 2.0, 3.0], None);
        assert_eq!(hoister.len(), 1);

        // Adding another should evict the first (44 + 44 = 88 > 50)
        hoister.store("another", vec![4.0, 5.0, 6.0], None);
        assert!(hoister.lookup("short").is_none());
        assert!(hoister.lookup("another").is_some());
    }

    #[test]
    fn test_work_hoister_clear() {
        let mut hoister = WorkHoister::with_bounds(100, 1_000_000);
        hoister.store("content", vec![1.0], None);
        assert!(!hoister.is_empty());

        hoister.clear();
        assert!(hoister.is_empty());
        assert_eq!(hoister.bytes_used(), 0);
    }

    #[test]
    fn test_work_hoister_preserves_search_results() {
        // VAL-APLUS-039: Reusing hoisted work should produce identical results.
        let mut hoister = WorkHoister::with_bounds(100, 1_000_000);
        let content = "fn compute(data: &[f64]) -> f64 { data.iter().sum() }";
        let embedding = vec![0.1, 0.2, 0.3, 0.4, 0.5];

        // Store the embedding
        hoister.store(content, embedding.clone(), None);

        // Retrieve and verify it matches
        let retrieved = hoister.lookup(content).unwrap().0;
        assert_eq!(retrieved, embedding);

        // Using the retrieved embedding in a search engine should produce
        // the same results as using the original.
        let mut engine_a = SearchEngine::with_dimension(5);
        let mut engine_b = SearchEngine::with_dimension(5);

        engine_a.index_nodes(vec![NodeInfo {
            node_id: "compute".to_string(),
            file_path: "math.rs".to_string(),
            symbol_name: "compute".to_string(),
            language: "rust".to_string(),
            content: content.to_string(),
            byte_range: (0, content.len()),
            tfidf_embedding: embedding,
            neural_embedding: None,
            complexity: 3,
            signature: None,
            pre_tokenized: None,
        }]);

        engine_b.index_nodes(vec![NodeInfo {
            node_id: "compute".to_string(),
            file_path: "math.rs".to_string(),
            symbol_name: "compute".to_string(),
            language: "rust".to_string(),
            content: content.to_string(),
            byte_range: (0, content.len()),
            tfidf_embedding: retrieved,
            neural_embedding: None,
            complexity: 3,
            signature: None,
            pre_tokenized: None,
        }]);

        // Both engines should produce identical semantic search results
        let results_a = engine_a
            .semantic_search(&[0.1, 0.2, 0.3, 0.4, 0.5], 5)
            .unwrap();
        let results_b = engine_b
            .semantic_search(&[0.1, 0.2, 0.3, 0.4, 0.5], 5)
            .unwrap();
        assert_eq!(results_a.len(), results_b.len());
        if !results_a.is_empty() {
            assert_eq!(results_a[0].node_id, results_b[0].node_id);
            assert!((results_a[0].relevance - results_b[0].relevance).abs() < 1e-6);
        }
    }

    #[test]
    fn test_work_hoister_duplicate_work_suppressed() {
        // VAL-APLUS-039: Duplicate expensive work is suppressed.
        let mut hoister = WorkHoister::with_bounds(100, 1_000_000);
        let content = "fn expensive() { /* lots of work */ }";
        let embedding = vec![0.42; 768];

        // First time: store
        hoister.store(content, embedding.clone(), None);

        // Second time: lookup should return the same result without recomputation
        let lookup_result = hoister.lookup(content);
        assert!(lookup_result.is_some());
        assert_eq!(lookup_result.unwrap().0, embedding);

        // Verify the hoister has exactly 1 entry (no duplicate)
        assert_eq!(hoister.len(), 1);
    }

    #[test]
    fn test_work_hoister_updates_existing_embedding() {
        let mut hoister = WorkHoister::with_bounds(100, 1_000_000);
        let content = "fn update_me() {}";
        let first_embedding = vec![1.0, 2.0, 3.0];
        let second_embedding = vec![4.0, 5.0];

        hoister.store(content, first_embedding.clone(), None);
        let initial_bytes = hoister.bytes_used();

        hoister.store(content, second_embedding.clone(), None);

        assert_eq!(hoister.lookup(content).unwrap().0, second_embedding);
        assert_eq!(hoister.len(), 1);
        assert_eq!(
            hoister.bytes_used(),
            initial_bytes - first_embedding.len() * std::mem::size_of::<f32>()
                + second_embedding.len() * std::mem::size_of::<f32>()
        );
    }

    // ========================================================================
    // VAL-APLUS-015 through VAL-APLUS-021: NodeInfo compatibility bridge
    // and duplicate TF-IDF ownership removal tests
    // ========================================================================

    /// Helper: create a minimal NodeInfo for serialization tests.
    fn make_node_info(tfidf: Vec<f32>) -> NodeInfo {
        NodeInfo {
            node_id: "test_node".into(),
            file_path: "test.rs".into(),
            symbol_name: "test_fn".into(),
            language: "rust".into(),
            content: "fn test_fn() {}".into(),
            byte_range: (0, 16),
            tfidf_embedding: tfidf,
            neural_embedding: None,
            complexity: 1,
            signature: None,
            pre_tokenized: None,
        }
    }

    /// VAL-APLUS-015: Legacy NodeInfo payloads remain readable during the
    /// one-minor compatibility window.
    ///
    /// A payload serialized with the old `embedding` field (and no
    /// `tfidf_embedding`) must deserialize successfully.
    #[test]
    fn test_legacy_payload_remains_readable() {
        let legacy_json = r#"{
            "node_id": "legacy_node",
            "file_path": "legacy.rs",
            "symbol_name": "legacy_fn",
            "language": "rust",
            "content": "fn legacy_fn() {}",
            "byte_range": [0, 16],
            "embedding": [0.1, 0.2, 0.3, 0.4],
            "neural_embedding": null,
            "complexity": 5,
            "signature": null,
            "pre_tokenized": null
        }"#;

        let node: NodeInfo = serde_json::from_str(legacy_json)
            .expect("Legacy payload must deserialize during compatibility window");

        assert_eq!(node.node_id, "legacy_node");
        // The legacy embedding should have been promoted to tfidf_embedding
        assert_eq!(node.tfidf_embedding, vec![0.1, 0.2, 0.3, 0.4]);
    }

    /// VAL-APLUS-016: New NodeInfo payloads serialize only the new shape.
    ///
    /// Fresh serialization must emit `tfidf_embedding` and must NOT emit
    /// the legacy `embedding` field.
    #[test]
    fn test_new_payload_serializes_only_new_shape() {
        let node = make_node_info(vec![1.0, 2.0, 3.0]);
        let json = serde_json::to_string(&node).expect("Serialization must succeed");

        // Must contain tfidf_embedding
        assert!(
            json.contains("\"tfidf_embedding\""),
            "Serialized output must contain tfidf_embedding field"
        );

        // Must NOT contain the legacy "embedding" field (not "tfidf_embedding")
        // We check for the exact key by looking for the pattern that would indicate
        // a standalone "embedding" key (not preceded by "tfidf_")
        let has_legacy_embedding =
            json.contains("\"embedding\":") && !json.contains("\"tfidf_embedding\":");
        assert!(
            !has_legacy_embedding,
            "Serialized output must not contain legacy 'embedding' field. JSON: {}",
            json
        );
    }

    /// VAL-APLUS-017: Compatibility resolution prefers non-empty tfidf_embedding.
    ///
    /// When both old and new shapes are present, deserialization uses the
    /// non-empty new TF-IDF field first.
    #[test]
    fn test_compat_prefers_tfidf_embedding_over_legacy() {
        let dual_json = r#"{
            "node_id": "dual_node",
            "file_path": "dual.rs",
            "symbol_name": "dual_fn",
            "language": "rust",
            "content": "fn dual_fn() {}",
            "byte_range": [0, 14],
            "tfidf_embedding": [0.5, 0.6, 0.7],
            "embedding": [0.1, 0.2, 0.3],
            "neural_embedding": null,
            "complexity": 3,
            "signature": null,
            "pre_tokenized": null
        }"#;

        let node: NodeInfo =
            serde_json::from_str(dual_json).expect("Dual-shape payload must deserialize");

        // Should prefer tfidf_embedding (the new field)
        assert_eq!(
            node.tfidf_embedding,
            vec![0.5, 0.6, 0.7],
            "Must prefer tfidf_embedding when both fields are present"
        );
    }

    /// VAL-APLUS-018: Compatibility fallback promotes legacy embedding only
    /// when needed.
    ///
    /// If the new TF-IDF field is absent or empty and the legacy field is
    /// populated, deserialization promotes the legacy value.
    #[test]
    fn test_compat_fallback_promotes_legacy_when_needed() {
        // Case 1: tfidf_embedding absent, legacy present
        let legacy_only_json = r#"{
            "node_id": "fallback_node",
            "file_path": "fallback.rs",
            "symbol_name": "fallback_fn",
            "language": "rust",
            "content": "fn fallback_fn() {}",
            "byte_range": [0, 18],
            "embedding": [0.9, 0.8, 0.7],
            "neural_embedding": null,
            "complexity": 2,
            "signature": null,
            "pre_tokenized": null
        }"#;

        let node: NodeInfo =
            serde_json::from_str(legacy_only_json).expect("Legacy-only payload must deserialize");
        assert_eq!(
            node.tfidf_embedding,
            vec![0.9, 0.8, 0.7],
            "Must promote legacy embedding when tfidf_embedding is absent"
        );

        // Case 2: tfidf_embedding present but empty, legacy present
        let empty_new_json = r#"{
            "node_id": "empty_new_node",
            "file_path": "empty.rs",
            "symbol_name": "empty_fn",
            "language": "rust",
            "content": "fn empty_fn() {}",
            "byte_range": [0, 14],
            "tfidf_embedding": [],
            "embedding": [0.4, 0.5, 0.6],
            "neural_embedding": null,
            "complexity": 1,
            "signature": null,
            "pre_tokenized": null
        }"#;

        let node2: NodeInfo = serde_json::from_str(empty_new_json)
            .expect("Empty-new + legacy payload must deserialize");
        assert_eq!(
            node2.tfidf_embedding,
            vec![0.4, 0.5, 0.6],
            "Must promote legacy embedding when tfidf_embedding is empty"
        );
    }

    /// VAL-APLUS-019: Empty legacy and new embeddings degrade safely.
    ///
    /// If neither shape provides a usable vector, deserialization succeeds
    /// with an empty TF-IDF vector rather than crashing or inventing state.
    #[test]
    fn test_empty_embeddings_degrade_safely() {
        let empty_json = r#"{
            "node_id": "empty_node",
            "file_path": "empty.rs",
            "symbol_name": "empty_fn",
            "language": "rust",
            "content": "fn empty_fn() {}",
            "byte_range": [0, 14],
            "neural_embedding": null,
            "complexity": 0,
            "signature": null,
            "pre_tokenized": null
        }"#;

        let node: NodeInfo = serde_json::from_str(empty_json)
            .expect("Payload with no embeddings must deserialize successfully");

        assert!(
            node.tfidf_embedding.is_empty(),
            "Must degrade to empty tfidf_embedding, got {:?}",
            node.tfidf_embedding
        );
        assert_eq!(node.node_id, "empty_node");
    }

    /// VAL-APLUS-020: Search semantics are unchanged after duplicate embedding
    /// removal.
    ///
    /// Removing duplicate TF-IDF ownership does not alter observable search
    /// results or ranking behavior for existing scenarios.
    #[test]
    fn test_search_semantics_unchanged_after_dedup() {
        // Create a node with tfidf_embedding and index it
        let node = NodeInfo {
            node_id: "dedup_node".into(),
            file_path: "dedup.rs".into(),
            symbol_name: "dedup_fn".into(),
            language: "rust".into(),
            content: "fn dedup_fn() { compute_value(); }".into(),
            byte_range: (0, 32),
            tfidf_embedding: vec![1.0, 0.0, 0.0],
            neural_embedding: None,
            complexity: 3,
            signature: None,
            pre_tokenized: None,
        };

        let mut engine = SearchEngine::with_dimension(3);
        engine.index_nodes(vec![node]);

        // Search with a vector close to the indexed embedding
        let query = SearchQuery {
            query: "dedup_fn".into(),
            top_k: 10,
            token_budget: None,
            semantic: true,
            expand_context: false,
            query_embedding: Some(vec![0.9, 0.1, 0.0]),
            threshold: None,
            query_type: None,
        };

        let results = engine.search(query).unwrap();
        assert!(
            !results.is_empty(),
            "Search must return results for indexed node"
        );
        assert_eq!(
            results[0].node_id, "dedup_node",
            "Search must find the correct node"
        );

        // Verify round-trip: serialize then deserialize and search again
        let node_v2 = NodeInfo {
            node_id: "dedup_node_v2".into(),
            file_path: "dedup.rs".into(),
            symbol_name: "dedup_fn_v2".into(),
            language: "rust".into(),
            content: "fn dedup_fn_v2() { compute_other(); }".into(),
            byte_range: (0, 36),
            tfidf_embedding: vec![0.0, 1.0, 0.0],
            neural_embedding: None,
            complexity: 4,
            signature: None,
            pre_tokenized: None,
        };

        // Serialize and deserialize the node to verify the round-trip
        let serialized = serde_json::to_string(&node_v2).unwrap();
        let deserialized: NodeInfo = serde_json::from_str(&serialized).unwrap();
        assert_eq!(deserialized.tfidf_embedding, node_v2.tfidf_embedding);

        let mut engine2 = SearchEngine::with_dimension(3);
        engine2.index_nodes(vec![deserialized]);

        let query2 = SearchQuery {
            query: "dedup_fn_v2".into(),
            top_k: 10,
            token_budget: None,
            semantic: true,
            expand_context: false,
            query_embedding: Some(vec![0.0, 0.9, 0.1]),
            threshold: None,
            query_type: None,
        };

        let results2 = engine2.search(query2).unwrap();
        assert!(
            !results2.is_empty(),
            "Search must return results after round-trip serialization"
        );
        assert_eq!(
            results2[0].node_id, "dedup_node_v2",
            "Search must find the correct node after round-trip"
        );
    }

    /// VAL-APLUS-021: Post-index content clearing behavior is preserved.
    ///
    /// A+ dedup work does not regress the existing behavior that clears
    /// bulky source content after indexing.
    #[test]
    fn test_post_index_content_clearing_preserved() {
        let nodes = vec![
            NodeInfo {
                node_id: "clear_node_1".into(),
                file_path: "clear.rs".into(),
                symbol_name: "clear_fn_1".into(),
                language: "rust".into(),
                content: "fn clear_fn_1() { /* some content that should be cleared */ }".into(),
                byte_range: (0, 60),
                tfidf_embedding: vec![1.0, 0.0, 0.0],
                neural_embedding: None,
                complexity: 2,
                signature: Some("fn clear_fn_1()".into()),
                pre_tokenized: None,
            },
            NodeInfo {
                node_id: "clear_node_2".into(),
                file_path: "clear.rs".into(),
                symbol_name: "clear_fn_2".into(),
                language: "rust".into(),
                content: "fn clear_fn_2() { /* more content to be cleared */ }".into(),
                byte_range: (60, 110),
                tfidf_embedding: vec![0.0, 1.0, 0.0],
                neural_embedding: None,
                complexity: 3,
                signature: Some("fn clear_fn_2()".into()),
                pre_tokenized: None,
            },
        ];

        let mut engine = SearchEngine::with_dimension(3);
        engine.index_nodes(nodes);

        // Verify content was cleared on the stored nodes
        for node in &engine.nodes {
            assert!(
                node.content.is_empty(),
                "Node {} content should be cleared after indexing, but got: {:?}",
                node.node_id,
                node.content
            );
        }

        // Verify search still works after content clearing (uses inverted index)
        let query = SearchQuery {
            query: "clear_fn".into(),
            top_k: 10,
            token_budget: None,
            semantic: false,
            expand_context: false,
            query_embedding: None,
            threshold: None,
            query_type: None,
        };

        let results = engine.search(query).unwrap();
        assert!(
            !results.is_empty(),
            "Search should still find results via inverted index after content cleared"
        );
    }
}