velesdb-core 7.0.0

//! Native HNSW index - standalone implementation without `hnsw_rs` dependency.
//!
//! This module provides `NativeHnswIndex`, a complete HNSW index using our native
//! implementation. It can be used as a drop-in replacement for `HnswIndex` when
//! the `native-hnsw` feature is enabled.
//!
//! # Feature Flag
//!
//! Enable with `native-hnsw` feature in `Cargo.toml`:
//! ```toml
//! [dependencies]
//! velesdb-core = { version = "0.8", features = ["native-hnsw"] }
//! ```

use super::native_inner::NativeHnswInner;
use super::params::{HnswParams, SearchQuality};
use super::sharded_mappings::ShardedMappings;
use super::upsert;
use crate::distance::DistanceMetric;
use crate::index::VectorIndex;
use crate::scored_result::ScoredResult;
use crate::validation::validate_dimension_match;
use parking_lot::{Mutex, RwLock};

/// Native HNSW index for efficient approximate nearest neighbor search.
///
/// This is a standalone implementation that doesn't depend on `hnsw_rs`.
/// It provides the same API as `HnswIndex` for easy migration.
///
/// # Performance Characteristics
///
/// - **Recall**: matched `hnsw_rs` until 1.0.0 removed it (`docs/reference/NATIVE_HNSW.md`)
/// - **Insert**: Comparable performance with SIMD distance calculations
/// - **Search**: Optimized with `CachedSimdDistance` engine
/// - **Persistence**: Native binary format (not compatible with `hnsw_rs` format)
pub struct NativeHnswIndex {
    pub(crate) dimension: usize,
    pub(crate) metric: DistanceMetric,
    pub(crate) inner: RwLock<NativeHnswInner>,
    pub(crate) mappings: ShardedMappings,
    pub(crate) enable_vector_storage: bool,
    #[allow(dead_code)] // Retained for future vacuum/rebuild operations
    pub(crate) params: HnswParams,
    /// Serializes saves of this index (#2262), as `HnswIndex::saving` does
    /// for the other one. A save rewrites its vectors and graph files in
    /// place and stamps every file with the generation after the one it read
    /// from the directory, so two saves into one directory at once read that
    /// generation together and stamp it twice: the load then took one save's
    /// mappings beside the other's graph, and a crash between two renames
    /// stopped being detectable. Taken before `inner`.
    pub(crate) saving: Mutex<()>,
}

impl NativeHnswIndex {
    /// Creates a new native HNSW index with auto-tuned parameters.
    ///
    /// # Errors
    ///
    /// Returns an error if vector storage pre-allocation fails.
    pub fn new(dimension: usize, metric: DistanceMetric) -> crate::error::Result<Self> {
        Self::with_params(dimension, metric, HnswParams::auto(dimension))
    }

    /// Creates a new native HNSW index with custom parameters.
    ///
    /// # Errors
    ///
    /// Returns an error if vector storage pre-allocation fails.
    pub fn with_params(
        dimension: usize,
        metric: DistanceMetric,
        params: HnswParams,
    ) -> crate::error::Result<Self> {
        let inner = NativeHnswInner::new_with_options(
            metric,
            params.max_connections,
            params.max_elements,
            params.ef_construction,
            dimension,
            params.storage_mode,
            params.alpha,
        )?;

        Ok(Self {
            dimension,
            metric,
            inner: RwLock::new(inner),
            mappings: ShardedMappings::new(),
            enable_vector_storage: true,
            params,
            saving: Mutex::new(()),
        })
    }

    /// Creates a turbo mode index for maximum insert throughput.
    ///
    /// # Errors
    ///
    /// Returns an error if vector storage pre-allocation fails.
    pub fn new_turbo(dimension: usize, metric: DistanceMetric) -> crate::error::Result<Self> {
        Self::with_params(dimension, metric, HnswParams::turbo())
    }

    /// Creates an index optimized for fast inserts, with its exact-distance
    /// features off: [`Self::brute_force_search_parallel`] returns nothing. The
    /// graph still stores every vector.
    ///
    /// # Errors
    ///
    /// Returns an error if vector storage pre-allocation fails.
    pub fn new_fast_insert(dimension: usize, metric: DistanceMetric) -> crate::error::Result<Self> {
        let mut index = Self::new(dimension, metric)?;
        index.enable_vector_storage = false;
        Ok(index)
    }

    /// Returns the dimension of vectors in this index.
    #[inline]
    #[must_use]
    pub fn dimension(&self) -> usize {
        self.dimension
    }

    /// Returns the distance metric used by this index.
    #[inline]
    #[must_use]
    pub fn metric(&self) -> DistanceMetric {
        self.metric
    }

    /// Returns the number of live vectors in the index.
    ///
    /// This reflects the mapping count (excluding tombstones), consistent
    /// with `HnswIndex::len()`.
    #[inline]
    #[must_use]
    pub fn len(&self) -> usize {
        self.mappings.len()
    }

    /// Returns true if the index contains no live vectors.
    #[inline]
    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.mappings.is_empty()
    }

    /// Returns whether exact-distance features are on — `false` for an index
    /// built with [`Self::new_fast_insert`], or loaded from one. The graph stores
    /// its vectors either way.
    #[inline]
    #[must_use]
    pub fn has_vector_storage(&self) -> bool {
        self.enable_vector_storage
    }

    /// Searches for the k nearest neighbors.
    #[must_use]
    pub fn search(&self, query: &[f32], k: usize) -> Vec<ScoredResult> {
        self.search_with_quality(query, k, SearchQuality::Balanced)
    }

    /// Searches with a specific quality profile.
    ///
    /// Uses `NativeHnswInner::search_auto` (private) so that when the `gpu`
    /// feature is enabled and the index is large enough (> 500K vectors,
    /// Standard backend), layer-0 traversal is offloaded to the GPU. Falls
    /// back to CPU otherwise.
    #[must_use]
    pub fn search_with_quality(
        &self,
        query: &[f32],
        k: usize,
        quality: SearchQuality,
    ) -> Vec<ScoredResult> {
        let ef_search = quality.ef_search_for_scale(k, self.len());
        let inner = self.inner.read();
        let neighbors = inner.search_auto(query, k, ef_search);

        let results: Vec<ScoredResult> = neighbors
            .into_iter()
            .filter_map(|(node_id, raw_dist)| {
                self.mappings.get_id(node_id).map(|id| {
                    let score = inner.transform_score(raw_dist);
                    ScoredResult::new(id, score)
                })
            })
            .collect();
        drop(inner);
        results
    }

    /// Inserts or updates a single vector (upsert semantics).
    ///
    /// The graph places the vector first; `id` is then mapped to the slot it
    /// was given (#2246). If `id` already existed, its old node becomes a
    /// tombstone, filtered out during search via the reverse mapping.
    ///
    /// # Errors
    ///
    /// Returns an error if the dimension does not match or the graph refuses
    /// the vector; the mappings are untouched in both cases.
    pub fn insert(&self, id: u64, vector: &[f32]) -> crate::error::Result<()> {
        validate_dimension_match(self.dimension, vector.len())?;
        // Held until the id is mapped, as in `HnswIndex`: a slot must not be
        // renumbered between the push that returned it and the mapping.
        let inner = self.inner.read();
        let placed = inner.place(vector)?;
        self.mappings.assign(id, placed);
        drop(inner);
        Ok(())
    }

    /// Batch insert or update multiple vectors (upsert semantics).
    ///
    /// Every dimension is checked before the graph sees any vector; each id
    /// is then mapped to the slot the graph gave its vector (#2246).
    ///
    /// # Errors
    ///
    /// Returns an error if any dimension does not match or the graph refuses
    /// the batch; the mappings are untouched in both cases.
    pub fn insert_batch(&self, items: &[(u64, Vec<f32>)]) -> crate::error::Result<()> {
        for (_, vector) in items {
            validate_dimension_match(self.dimension, vector.len())?;
        }
        let vectors: Vec<&[f32]> = items.iter().map(|(_, vector)| vector.as_slice()).collect();
        // Held until every id is mapped, as in `HnswIndex`.
        let inner = self.inner.read();
        let placements = inner.place_parallel(&vectors)?;
        for ((id, _), placed) in items.iter().zip(placements) {
            self.mappings.assign(*id, placed);
        }
        drop(inner);
        Ok(())
    }

    /// Removes a vector by ID (soft delete).
    ///
    /// Removes the ID from the mappings. The HNSW graph node becomes a
    /// tombstone, filtered out during search via the reverse mapping.
    ///
    /// Delegates to `upsert::soft_delete` (private), shared with
    /// `HnswIndex::remove` (#448 Group F).
    pub fn remove(&self, id: u64) -> bool {
        upsert::soft_delete(&self.mappings, id, &self.inner.read())
    }

    /// Sets searching mode (no-op for native implementation).
    ///
    /// This method exists for API compatibility with `HnswIndex`.
    /// The native implementation doesn't require mode switching.
    #[allow(clippy::unused_self)]
    pub fn set_searching_mode(&self) {}

    /// Parallel batch insert - API compatible with `HnswIndex`.
    ///
    /// # Returns
    ///
    /// Number of vectors inserted.
    #[allow(clippy::needless_pass_by_value)]
    pub fn insert_batch_parallel<I>(&self, items: I) -> usize
    where
        I: IntoIterator<Item = (u64, Vec<f32>)>,
    {
        let items: Vec<_> = items.into_iter().collect();
        let count = items.len();
        if let Err(e) = self.insert_batch(items.as_slice()) {
            tracing::error!("insert_batch_parallel failed: {e}");
            return 0;
        }
        count
    }

    /// Batch search with parallel execution.
    ///
    /// # Arguments
    ///
    /// * `queries` - Slice of query vector slices
    /// * `k` - Number of nearest neighbors per query
    /// * `quality` - Search quality profile
    ///
    /// # Returns
    ///
    /// Vector of results for each query.
    #[must_use]
    pub fn search_batch_parallel(
        &self,
        queries: &[&[f32]],
        k: usize,
        quality: SearchQuality,
    ) -> Vec<Vec<ScoredResult>> {
        use rayon::prelude::*;

        queries
            .par_iter()
            .map(|q| self.search_with_quality(q, k, quality))
            .collect()
    }

    /// Brute-force exact nearest neighbor search with parallel execution.
    ///
    /// Computes distances to all vectors in the index and returns the k nearest.
    /// It returns the exact top-k under the index's own distance, ties aside,
    /// at O(n) cost.
    ///
    /// # Arguments
    ///
    /// * `query` - Query vector
    /// * `k` - Number of nearest neighbors to return
    ///
    /// # Returns
    ///
    /// Vector of (id, distance) tuples sorted by distance.
    ///
    /// # Use Cases
    ///
    /// - **Recall validation**: Compare HNSW results against brute-force
    /// - **Small datasets**: When n < 10k, brute-force may be faster
    /// - **Critical accuracy**: When the exact top-k is required
    // Held across the rayon scan deliberately; the comment below explains
    // why this cannot deadlock a worker (inserts mutate under read guards
    // through interior mutability, so no exclusive write() exists to queue
    // behind).
    #[expect(clippy::significant_drop_tightening)]
    #[must_use]
    pub fn brute_force_search_parallel(&self, query: &[f32], k: usize) -> Vec<ScoredResult> {
        use rayon::prelude::*;

        // Preserve fast-insert semantics: exact-distance features are gated
        // on `enable_vector_storage` even though the graph stores vectors.
        if !self.enable_vector_storage {
            return Vec::new();
        }

        let inner = self.inner.read();

        // Snapshot the contiguous slab into an owned buffer (one memcpy) so the
        // rayon scan below iterates a private copy. The `inner` read lock is
        // held for the whole scan (the closure calls `inner.compute_distance`),
        // which is safe here: native inserts also run under `inner.read()` and
        // mutate the graph through interior mutability, so there is no
        // exclusive `inner.write()` this shared read could deadlock a rayon
        // worker against. Tombstoned slots (deleted/upserted vectors) have no
        // reverse mapping and are skipped by `mappings.get_id`.
        let (flat, dimension) = inner.with_contiguous_vectors(|vectors| {
            (vectors.as_flat_slice().to_vec(), vectors.dimension())
        });
        if flat.is_empty() || dimension == 0 {
            return Vec::new();
        }

        let mut results: Vec<ScoredResult> = flat
            .par_chunks(dimension)
            .enumerate()
            .filter_map(|(idx, vec)| {
                let id = self.mappings.get_id(idx)?;
                let raw_distance = inner.compute_distance(query, vec);
                // Reason: compute_distance returns squared L2 for Euclidean
                // (CachedSimdDistance optimization). Apply transform_score to
                // restore actual Euclidean distance for user-visible scores.
                let score = inner.transform_score(raw_distance);
                Some(ScoredResult::new(id, score))
            })
            .collect();

        self.metric.sort_scored_results(&mut results);

        results.truncate(k);
        results
    }
}

impl VectorIndex for NativeHnswIndex {
    fn insert(&self, id: u64, vector: &[f32]) {
        if let Err(e) = NativeHnswIndex::insert(self, id, vector) {
            tracing::error!("NativeHnswIndex::insert failed for id={id}: {e}");
        }
    }

    fn remove(&self, id: u64) -> bool {
        NativeHnswIndex::remove(self, id)
    }

    fn search(&self, query: &[f32], k: usize) -> Vec<ScoredResult> {
        NativeHnswIndex::search(self, query, k)
    }

    fn len(&self) -> usize {
        NativeHnswIndex::len(self)
    }

    fn dimension(&self) -> usize {
        self.dimension
    }

    fn metric(&self) -> DistanceMetric {
        self.metric
    }
}

// ============================================================================
// Tests moved to native_index_tests.rs per project rules