uqa-storage-sqlite 0.5.0

SQLite catalog, indexes, compressed storage, graph and key/value providers
Documentation
//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! IVF catalog metadata and candidate-list loading.

use rusqlite::{params, OptionalExtension};
use uqa_core::DocId;

use super::metadata::{usize_to_i64, SQLiteIVFMeta};
use super::SQLiteIVFIndex;
use crate::vector_index::codec::{blob_to_vector, decode_doc_id};
use crate::Result as SQLiteResult;

impl SQLiteIVFIndex {
    pub(super) fn load_meta(&self) -> SQLiteResult<Option<SQLiteIVFMeta>> {
        if let Some(meta) = self.persistent.read_native(super::native::load_metadata)? {
            return Ok(meta);
        }
        let row = self.persistent.conn.with(|conn| {
            Ok(conn
                .query_row(
                    "SELECT dimensions, nlist, nprobe, train_threshold, state,
                        trained_size, deletes_since_train, vector_count
                   FROM _ivf_indexes
                  WHERE table_name = ?1 AND field = ?2",
                    params![self.persistent.table, self.persistent.field],
                    |row| {
                        Ok((
                            row.get::<_, i64>(0)?,
                            row.get::<_, i64>(1)?,
                            row.get::<_, i64>(2)?,
                            row.get::<_, i64>(3)?,
                            row.get::<_, String>(4)?,
                            row.get::<_, i64>(5)?,
                            row.get::<_, i64>(6)?,
                            row.get::<_, i64>(7)?,
                        ))
                    },
                )
                .optional()?)
        })?;
        row.map(super::metadata::decode_metadata).transpose()
    }

    pub(super) fn load_centroids(&self) -> SQLiteResult<Vec<Vec<f32>>> {
        self.persistent.conn.with(|conn| {
            let mut statement = conn.prepare(
                "SELECT vector FROM _ivf_centroids
                  WHERE table_name = ?1 AND field = ?2
                  ORDER BY centroid_id",
            )?;
            let rows = statement.query_map(
                params![self.persistent.table, self.persistent.field],
                |row| row.get::<_, Vec<u8>>(0),
            )?;
            let mut centroids = Vec::new();
            for row in rows {
                let centroid = blob_to_vector(&row?)?;
                self.persistent.validate_dimensions_sqlite(&centroid)?;
                centroids.push(centroid);
            }
            Ok(centroids)
        })
    }

    pub(super) fn load_candidates(
        &self,
        centroids: &[usize],
    ) -> SQLiteResult<Vec<(DocId, Vec<f32>)>> {
        self.persistent.conn.with(|conn| {
            let mut output = Vec::new();
            let mut statement = conn.prepare(
                "SELECT v.doc_id, v.vector
                   FROM _ivf_assignments a
                   JOIN _vectors v
                     ON v.table_name = a.table_name AND v.field = a.field
                    AND v.doc_id = a.doc_id AND v.vector_ordinal = a.vector_ordinal
                  WHERE a.table_name = ?1 AND a.field = ?2 AND a.centroid_id = ?3
                  ORDER BY v.doc_id, v.vector_ordinal",
            )?;
            for centroid in centroids {
                let rows = statement.query_map(
                    params![
                        self.persistent.table,
                        self.persistent.field,
                        usize_to_i64("centroid_id", *centroid)?
                    ],
                    |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Vec<u8>>(1)?)),
                )?;
                for row in rows {
                    let (doc_id, blob) = row?;
                    let vector = blob_to_vector(&blob)?;
                    self.persistent.validate_dimensions_sqlite(&vector)?;
                    output.push((decode_doc_id(doc_id)?, vector));
                }
            }
            Ok(output)
        })
    }
}