use rustc_hash::FxHashMap;
use crate::db::record::Key;
use crate::error::Result;
use crate::file::ValiseFile;
use crate::format::{CollectionId, EmbeddingSpaceId, FrameId, VectorId};
#[derive(Clone, Debug)]
pub(crate) struct Entry {
pub(crate) frame_id: FrameId,
pub(crate) vectors: Vec<(EmbeddingSpaceId, VectorId)>,
pub(crate) committed: bool,
}
#[derive(Default)]
pub(crate) struct IdentityIndex {
map: FxHashMap<CollectionId, FxHashMap<Vec<u8>, Entry>>,
uncommitted: Vec<(CollectionId, Vec<u8>)>,
}
impl IdentityIndex {
pub fn get(&self, collection: CollectionId, key_bytes: &[u8]) -> Option<&Entry> {
self.map.get(&collection)?.get(key_bytes)
}
pub fn committed_keys(&self, collection: CollectionId) -> Vec<Vec<u8>> {
self.map
.get(&collection)
.map(|inner| {
inner
.iter()
.filter(|(_, e)| e.committed)
.map(|(k, _)| k.clone())
.collect()
})
.unwrap_or_default()
}
pub fn insert(&mut self, collection: CollectionId, key_bytes: Vec<u8>, entry: Entry) {
if !entry.committed {
self.uncommitted.push((collection, key_bytes.clone()));
}
self.map
.entry(collection)
.or_default()
.insert(key_bytes, entry);
}
pub fn remove(&mut self, collection: CollectionId, key_bytes: &[u8]) -> Option<Entry> {
self.map.get_mut(&collection)?.remove(key_bytes)
}
pub fn remove_collection(&mut self, collection: CollectionId) {
self.map.remove(&collection);
self.uncommitted.retain(|(c, _)| *c != collection);
}
pub fn push_vector(
&mut self,
collection: CollectionId,
key_bytes: &[u8],
space_id: EmbeddingSpaceId,
vector_id: VectorId,
) {
if let Some(inner) = self.map.get_mut(&collection)
&& let Some(entry) = inner.get_mut(key_bytes)
{
entry.vectors.push((space_id, vector_id));
}
}
pub fn mark_all_committed(&mut self) {
for (collection, key_bytes) in self.uncommitted.drain(..) {
if let Some(inner) = self.map.get_mut(&collection)
&& let Some(entry) = inner.get_mut(&key_bytes)
{
entry.committed = true;
}
}
}
pub fn rebuild(&mut self, valise: &ValiseFile) -> Result<()> {
self.map.clear();
self.uncommitted.clear();
let mut by_frame: FxHashMap<FrameId, (CollectionId, Vec<u8>)> = FxHashMap::default();
let frame_ids: Vec<(FrameId, CollectionId)> = valise
.active_frames()
.map(|f| (f.frame_id, f.collection_id))
.collect();
for (frame_id, collection_id) in frame_ids {
let Some(uri) = valise.read_frame_uri(frame_id)? else {
continue;
};
if Key::decode(&uri).is_none() {
continue;
}
by_frame.insert(frame_id, (collection_id, uri.clone()));
self.map.entry(collection_id).or_default().insert(
uri,
Entry {
frame_id,
vectors: Vec::new(),
committed: true,
},
);
}
let space_ids: Vec<EmbeddingSpaceId> = valise
.embedding_spaces()
.iter()
.map(|e| e.embedding_space_id)
.collect();
for space_id in space_ids {
let owned: Vec<(FrameId, VectorId)> = valise
.active_vectors(space_id)
.map(|v| (v.owner_frame_id, v.vector_id))
.collect();
for (owner_frame, vector_id) in owned {
if let Some((coll, key)) = by_frame.get(&owner_frame)
&& let Some(inner) = self.map.get_mut(coll)
&& let Some(entry) = inner.get_mut(key)
{
entry.vectors.push((space_id, vector_id));
}
}
}
Ok(())
}
}