use std::path::{Path, PathBuf};
use anyhow::{Context, Result, anyhow};
use usearch::{Index, IndexOptions, MetricKind, ScalarKind};
pub const EMBEDDING_DIM: usize = 384;
const INITIAL_CAPACITY: usize = 1024;
#[derive(Clone, Copy, PartialEq, Eq)]
enum VectorMode {
ReadWrite,
View,
}
pub struct VectorIndex {
index: Index,
path: PathBuf,
mode: VectorMode,
scalar_kind: ScalarKind,
}
fn new_index(scalar_kind: ScalarKind) -> Result<Index> {
let options = IndexOptions {
dimensions: EMBEDDING_DIM,
metric: MetricKind::Cos,
quantization: scalar_kind,
..IndexOptions::default()
};
Index::new(&options).map_err(|e| anyhow!("creating usearch index: {e}"))
}
fn check_on_disk_kind(path: &Path, path_str: &str, expected: ScalarKind) -> Result<()> {
let actual = read_on_disk_kind(path, path_str)?;
if actual != expected {
return Err(anyhow!(
"vector index at {} was written as {:?} but is being opened as {:?}",
path.display(),
actual,
expected
));
}
Ok(())
}
fn read_on_disk_kind(path: &Path, path_str: &str) -> Result<ScalarKind> {
Ok(Index::metadata(path_str)
.map_err(|e| anyhow!("reading usearch header at {}: {e}", path.display()))?
.quantization)
}
impl VectorIndex {
pub fn on_disk_kind(path: &Path) -> Result<Option<ScalarKind>> {
if !path.exists() {
return Ok(None);
}
let path_str = path.to_str().context("vector index path is not utf-8")?;
read_on_disk_kind(path, path_str).map(Some)
}
pub fn open_with_kind(path: &Path, scalar_kind: ScalarKind) -> Result<Self> {
let path_str = path.to_str().context("vector index path is not utf-8")?;
let index = new_index(scalar_kind)?;
if path.exists() {
check_on_disk_kind(path, path_str, scalar_kind)?;
index
.load(path_str)
.map_err(|e| anyhow!("loading usearch index from {path_str}: {e}"))?;
} else {
index
.reserve(INITIAL_CAPACITY)
.map_err(|e| anyhow!("reserving usearch capacity: {e}"))?;
}
Ok(Self {
index,
path: path.to_path_buf(),
mode: VectorMode::ReadWrite,
scalar_kind,
})
}
pub fn open_view_with_kind(path: &Path, scalar_kind: ScalarKind) -> Result<Self> {
if !path.exists() {
return Err(anyhow!(
"vector index view: {} does not exist",
path.display()
));
}
let path_str = path.to_str().context("vector index path is not utf-8")?;
check_on_disk_kind(path, path_str, scalar_kind)?;
let index = new_index(scalar_kind)?;
index
.view(path_str)
.map_err(|e| anyhow!("viewing usearch index at {path_str}: {e}"))?;
Ok(Self {
index,
path: path.to_path_buf(),
mode: VectorMode::View,
scalar_kind,
})
}
pub fn empty_view_with_kind(path: &Path, scalar_kind: ScalarKind) -> Result<Self> {
Ok(Self {
index: new_index(scalar_kind)?,
path: path.to_path_buf(),
mode: VectorMode::View,
scalar_kind,
})
}
pub fn scalar_kind(&self) -> ScalarKind {
self.scalar_kind
}
fn require_writable(&self) -> Result<()> {
match self.mode {
VectorMode::ReadWrite => Ok(()),
VectorMode::View => Err(anyhow!(
"vector index at {} is a read-only view; mutation is not permitted",
self.path.display()
)),
}
}
pub fn upsert(&mut self, key: u64, embedding: &[f32]) -> Result<()> {
self.require_writable()?;
self.ensure_capacity(1)?;
self.index
.remove(key)
.map_err(|e| anyhow!("removing key {key}: {e}"))?;
self.index
.add(key, embedding)
.map_err(|e| anyhow!("adding key {key}: {e}"))?;
Ok(())
}
pub fn remove(&mut self, key: u64) -> Result<()> {
self.require_writable()?;
self.index
.remove(key)
.map_err(|e| anyhow!("removing key {key}: {e}"))?;
Ok(())
}
pub fn search(&self, query: &[f32], k: usize) -> Result<Vec<(u64, f32)>> {
let matches = self
.index
.search(query, k)
.map_err(|e| anyhow!("usearch search: {e}"))?;
Ok(matches
.keys
.iter()
.zip(matches.distances.iter())
.map(|(key, distance)| (*key, 1.0 - *distance))
.collect())
}
pub fn len(&self) -> usize {
self.index.size()
}
pub fn memory_usage(&self) -> usize {
self.index.memory_usage()
}
pub fn save(&self) -> Result<()> {
self.require_writable()?;
let tmp_path = self.path.with_extension("usearch.tmp");
let tmp_str = tmp_path
.to_str()
.context("vector index tmp path is not utf-8")?;
self.index
.save(tmp_str)
.map_err(|e| anyhow!("saving usearch index to {tmp_str}: {e}"))?;
let tmp_file = std::fs::File::open(&tmp_path)
.with_context(|| format!("reopening {} to fsync", tmp_path.display()))?;
tmp_file
.sync_all()
.with_context(|| format!("fsyncing {}", tmp_path.display()))?;
drop(tmp_file);
std::fs::rename(&tmp_path, &self.path).with_context(|| {
format!("renaming {} to {}", tmp_path.display(), self.path.display())
})?;
Ok(())
}
pub fn reload_view(&mut self) -> Result<()> {
if self.mode != VectorMode::View {
return Ok(());
}
let path_str = self
.path
.to_str()
.context("vector index path is not utf-8")?;
self.index
.view(path_str)
.map_err(|e| anyhow!("reloading usearch view at {path_str}: {e}"))?;
Ok(())
}
pub fn reset(&mut self) -> Result<()> {
self.require_writable()?;
self.index = new_index(self.scalar_kind)?;
self.index
.reserve(INITIAL_CAPACITY)
.map_err(|e| anyhow!("reserving usearch capacity: {e}"))?;
Ok(())
}
fn ensure_capacity(&self, extra: usize) -> Result<()> {
let needed = self.index.size() + extra;
if needed > self.index.capacity() {
self.index
.reserve((needed * 2).max(INITIAL_CAPACITY))
.map_err(|e| anyhow!("growing usearch capacity: {e}"))?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
fn unit_vector(hot: usize) -> Vec<f32> {
let mut v = vec![0.0f32; EMBEDDING_DIM];
v[hot] = 1.0;
v
}
#[test]
fn f32_index_writes_and_reads_back() {
let dir = tempdir().unwrap();
let path = dir.path().join("vectors.usearch");
let mut idx = VectorIndex::open_with_kind(&path, ScalarKind::F32).unwrap();
assert_eq!(idx.scalar_kind(), ScalarKind::F32);
for i in 0..5u64 {
idx.upsert(i, &unit_vector(i as usize)).unwrap();
}
idx.save().unwrap();
let reopened = VectorIndex::open_with_kind(&path, ScalarKind::F32).unwrap();
assert_eq!(reopened.scalar_kind(), ScalarKind::F32);
let hits = reopened.search(&unit_vector(2), 1).unwrap();
assert_eq!(hits[0].0, 2);
let viewed = VectorIndex::open_view_with_kind(&path, ScalarKind::F32).unwrap();
let hits = viewed.search(&unit_vector(3), 1).unwrap();
assert_eq!(hits[0].0, 3);
}
#[test]
fn opening_f32_file_as_a_different_kind_errors_instead_of_corrupting() {
let dir = tempdir().unwrap();
let path = dir.path().join("vectors.usearch");
let mut idx = VectorIndex::open_with_kind(&path, ScalarKind::F32).unwrap();
idx.upsert(0, &unit_vector(0)).unwrap();
idx.save().unwrap();
let err = VectorIndex::open_with_kind(&path, ScalarKind::I8)
.err()
.expect("opening an F32 file as I8 must error");
assert!(err.to_string().contains("F32"), "error was: {err}");
let err = VectorIndex::open_view_with_kind(&path, ScalarKind::F16)
.err()
.expect("viewing an F32 file as F16 must error");
assert!(err.to_string().contains("F32"), "error was: {err}");
}
#[test]
fn i8_and_f16_kinds_round_trip() {
for kind in [ScalarKind::I8, ScalarKind::F16] {
let dir = tempdir().unwrap();
let path = dir.path().join("vectors.usearch");
let mut idx = VectorIndex::open_with_kind(&path, kind).unwrap();
for i in 0..5u64 {
idx.upsert(i, &unit_vector(i as usize)).unwrap();
}
idx.save().unwrap();
let viewed = VectorIndex::open_view_with_kind(&path, kind).unwrap();
let hits = viewed.search(&unit_vector(4), 1).unwrap();
assert_eq!(hits[0].0, 4, "kind {kind:?} lost the nearest match");
}
}
}