use std::collections::HashSet;
use std::time::{Duration, Instant};
use anndists::dist::distances::DistCosine;
use hnsw_rs::hnsw::Hnsw;
use crate::error::Result;
use crate::index::hnsw::{HnswConfig, HnswIndex};
#[derive(Debug, Clone)]
pub struct RebuildResult {
pub embeddings_processed: usize,
pub embeddings_included: usize,
pub embeddings_excluded: usize,
pub duration: Duration,
}
pub struct HnswRebuilder;
impl HnswRebuilder {
pub fn new() -> Self {
Self
}
pub fn rebuild_clean(
&self,
source_index: &HnswIndex,
deleted_internal_ids: &HashSet<usize>,
config: &HnswConfig,
) -> Result<(Hnsw<'static, f32, DistCosine>, RebuildResult)> {
let start = Instant::now();
let new_hnsw = Hnsw::new(
config.max_connections,
config.max_elements,
16, config.ef_construction,
DistCosine {},
);
let cache = source_index.get_embedding_cache().read();
let mut embeddings_processed = 0;
let mut embeddings_included = 0;
let mut embeddings_excluded = 0;
for (internal_id, embedding) in cache.iter_valid() {
embeddings_processed += 1;
if deleted_internal_ids.contains(&internal_id) {
embeddings_excluded += 1;
} else {
new_hnsw.insert_slice((embedding, internal_id));
embeddings_included += 1;
}
}
let duration = start.elapsed();
let result = RebuildResult {
embeddings_processed,
embeddings_included,
embeddings_excluded,
duration,
};
tracing::info!(
"HNSW rebuild complete: {} processed, {} included, {} excluded in {:?}",
embeddings_processed,
embeddings_included,
embeddings_excluded,
duration
);
Ok((new_hnsw, result))
}
}
impl Default for HnswRebuilder {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::ChunkId;
fn normalize(v: &mut [f32]) {
let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm > 0.0 {
for x in v.iter_mut() {
*x /= norm;
}
}
}
#[test]
fn test_rebuild_clean_empty() {
let config = HnswConfig {
max_elements: 100,
dimension: 4,
..Default::default()
};
let source = HnswIndex::new(config.clone());
let deleted = HashSet::new();
let rebuilder = HnswRebuilder::new();
let (_, result) = rebuilder.rebuild_clean(&source, &deleted, &config).unwrap();
assert_eq!(result.embeddings_processed, 0);
assert_eq!(result.embeddings_included, 0);
assert_eq!(result.embeddings_excluded, 0);
}
#[test]
fn test_rebuild_clean_no_deletions() {
let config = HnswConfig {
max_elements: 100,
dimension: 4,
..Default::default()
};
let source = HnswIndex::new(config.clone());
for i in 0..5 {
let chunk_id = ChunkId::new();
let mut emb = vec![i as f32, (i + 1) as f32, (i + 2) as f32, (i + 3) as f32];
normalize(&mut emb);
source.insert(&chunk_id, &emb).unwrap();
}
let deleted = HashSet::new();
let rebuilder = HnswRebuilder::new();
let (_, result) = rebuilder.rebuild_clean(&source, &deleted, &config).unwrap();
assert_eq!(result.embeddings_processed, 5);
assert_eq!(result.embeddings_included, 5);
assert_eq!(result.embeddings_excluded, 0);
}
#[test]
fn test_rebuild_clean_with_deletions() {
let config = HnswConfig {
max_elements: 100,
dimension: 4,
..Default::default()
};
let source = HnswIndex::new(config.clone());
for i in 0..5 {
let chunk_id = ChunkId::new();
let mut emb = vec![i as f32, (i + 1) as f32, (i + 2) as f32, (i + 3) as f32];
normalize(&mut emb);
source.insert(&chunk_id, &emb).unwrap();
}
let mut deleted = HashSet::new();
deleted.insert(1);
deleted.insert(3);
let rebuilder = HnswRebuilder::new();
let (_, result) = rebuilder.rebuild_clean(&source, &deleted, &config).unwrap();
assert_eq!(result.embeddings_processed, 5);
assert_eq!(result.embeddings_included, 3); assert_eq!(result.embeddings_excluded, 2); }
#[test]
fn swap_graph_replaces_live_hnsw() {
let config = HnswConfig {
max_elements: 100,
dimension: 4,
..Default::default()
};
let source = HnswIndex::new(config.clone());
let mut chunk_ids = Vec::new();
for i in 0..5 {
let chunk_id = ChunkId::new();
let mut emb = vec![i as f32, (i + 1) as f32, (i + 2) as f32, (i + 3) as f32];
normalize(&mut emb);
source.insert(&chunk_id, &emb).unwrap();
chunk_ids.push(chunk_id);
}
let mut query = vec![0.0, 1.0, 2.0, 3.0];
normalize(&mut query);
let before = source.search(&query, 5).unwrap();
assert!(
before.iter().any(|r| r.chunk_id == chunk_ids[0]),
"sanity: source graph must return chunk 0 before the swap"
);
let internal_id_0 = source
.get_mapping()
.read()
.get_internal_id(&chunk_ids[0])
.unwrap();
let mut deleted = HashSet::new();
deleted.insert(internal_id_0);
let rebuilder = HnswRebuilder::new();
let (new_hnsw, result) = rebuilder.rebuild_clean(&source, &deleted, &config).unwrap();
assert_eq!(result.embeddings_excluded, 1);
assert_eq!(result.embeddings_included, 4);
source.swap_graph(new_hnsw, &deleted);
let after = source.search(&query, 5).unwrap();
assert!(
!after.iter().any(|r| r.chunk_id == chunk_ids[0]),
"after swap_graph, the rebuilt graph must not surface the excluded chunk; \
got hits: {:?}",
after.iter().map(|r| r.chunk_id.clone()).collect::<Vec<_>>()
);
assert!(after.len() <= 4, "only four points remain in the graph");
}
#[test]
fn test_rebuild_result_duration() {
let config = HnswConfig {
max_elements: 100,
dimension: 4,
..Default::default()
};
let source = HnswIndex::new(config.clone());
let deleted = HashSet::new();
let rebuilder = HnswRebuilder::new();
let (_, result) = rebuilder.rebuild_clean(&source, &deleted, &config).unwrap();
let _duration = result.duration;
}
}