uqa-storage-sqlite 0.5.1

SQLite catalog, indexes, compressed storage, graph and key/value providers
Documentation
//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! Canonical vectors on a single committed/private native record boundary.

use std::sync::Arc;

use rusqlite::types::ValueRef;
use uqa_core::{
    memory::{Budgeted, BudgetedVec, MemoryReservation},
    DocId,
};
use uqa_storage::{mvcc::VersionError, read_control::StorageReadControl, KeyValueBatch};

use super::{blob_to_vector, decode_doc_id, SQLiteVectorIndex};
use crate::mvcc::native::{
    NativeRecordFamily as Family, NativeRecordIdentity, NativeRecordOwner, NativeSnapshot,
};
use crate::{Result, SQLiteError};

pub(super) mod canonical;
mod guards;
pub(super) mod identity;
pub(in crate::vector_index) mod publication;
pub(in crate::vector_index) mod records;

// Payload allocations are destroyed before either of their reservations, including on failed reads.
pub(super) struct VectorBuffer<T> {
    pub(super) rows: BudgetedVec<T>,
    pub(super) payload: MemoryReservation,
}

impl<T> VectorBuffer<T> {
    pub(super) fn new(read: &NativeVectorRead<'_>) -> Result<Self> {
        read.snapshot.control.check()?;
        Ok(Self::with_memory(read.snapshot.control.memory()))
    }

    pub(super) fn with_memory(memory: &uqa_core::memory::MemoryBudget) -> Self {
        Self {
            rows: BudgetedVec::new(memory),
            payload: memory.empty_reservation(),
        }
    }

    pub(super) fn finish(self) -> Budgeted<Vec<T>> {
        let (rows, mut memory) = self.rows.into_parts();
        memory.absorb(self.payload);
        Budgeted::new(rows, memory)
    }
}

pub(super) fn integer(value: ValueRef<'_>) -> Result<i64> {
    value
        .as_i64()
        .map_err(|_| SQLiteError::StorageBackend("invalid native vector integer".into()))
}

pub(super) fn blob(value: ValueRef<'_>) -> Result<&[u8]> {
    value
        .as_blob()
        .map_err(|_| SQLiteError::StorageBackend("invalid native vector blob".into()))
}

pub(super) fn text(value: ValueRef<'_>) -> Result<&str> {
    value
        .as_str()
        .map_err(|_| SQLiteError::StorageBackend("invalid native vector text".into()))
}

pub(super) struct NativeVectorRead<'a> {
    pub(super) snapshot: &'a NativeSnapshot,
    pub(super) index: &'a SQLiteVectorIndex,
    pub(super) owner: Option<NativeRecordOwner>,
}

impl SQLiteVectorIndex {
    pub(super) fn native_snapshot(&self) -> Result<Option<Arc<NativeSnapshot>>> {
        match &self.retained {
            Some(snapshot) => Ok(Some(Arc::clone(snapshot))),
            None => self.conn.native_snapshot(),
        }
    }

    pub(super) fn retained_snapshot(&self) -> Result<Self> {
        let mut snapshot = self.clone();
        snapshot.retained = self.native_snapshot()?;
        Ok(snapshot)
    }

    pub(super) fn read_native<R>(
        &self,
        read: impl FnOnce(&NativeVectorRead<'_>) -> Result<R>,
    ) -> Result<Option<R>> {
        self.native_snapshot()?
            .map(|snapshot| read(&NativeVectorRead::new(&snapshot, self)?))
            .transpose()
    }

    pub(super) fn write_native<R>(
        &self,
        write: impl FnOnce(&NativeVectorRead<'_>, &mut dyn KeyValueBatch) -> Result<R>,
    ) -> Result<Option<R>> {
        if self.retained.is_some() {
            return Err(SQLiteError::StorageBackend(
                "a retained vector snapshot is read-only".into(),
            ));
        }
        self.conn.with_native_write(|snapshot, batch| {
            write(&NativeVectorRead::new(snapshot, self)?, batch)
        })
    }
}

impl<'a> NativeVectorRead<'a> {
    pub(super) fn new(snapshot: &'a NativeSnapshot, index: &'a SQLiteVectorIndex) -> Result<Self> {
        Ok(Self {
            snapshot,
            index,
            owner: snapshot.table_owner(&index.table)?,
        })
    }

    pub(super) fn field(&self) -> ValueRef<'_> {
        ValueRef::Text(self.index.field.as_bytes())
    }

    pub(super) fn owned(&self, batch: &mut dyn KeyValueBatch) -> Result<Self> {
        let owner = match self.owner {
            Some(owner) => owner,
            None => self.snapshot.ensure_table_owner(&self.index.table, batch)?,
        };
        Ok(Self {
            snapshot: self.snapshot,
            index: self.index,
            owner: Some(owner),
        })
    }

    pub(super) fn count(&self) -> Result<usize> {
        let Some(owner) = self.owner else {
            return Ok(0);
        };
        let prefix = NativeRecordIdentity::new(Family::Vectors, owner)?
            .encode_prefix(&[self.field()], &self.snapshot.control)?;
        let mut count = 0_usize;
        self.snapshot.view.visit_keys(
            &prefix,
            None,
            usize::MAX,
            &self.snapshot.control,
            &mut |_, record| {
                if record.live {
                    count = count.checked_add(1).ok_or(VersionError::InvalidEncoding(
                        "native vector count overflow",
                    ))?;
                }
                Ok(true)
            },
        )?;
        Ok(count)
    }

    /// Score one canonical vector at a time without retaining the corpus. The internal consumer cannot reenter persistence.
    pub(super) fn visit_vectors(
        &self,
        mut visit: impl FnMut(DocId, &[f32], &StorageReadControl) -> Result<()>,
    ) -> Result<()> {
        self.visit_ordered_vectors(|document, _, vector| {
            visit(document, vector, &self.snapshot.control)
        })
    }

    /// Native vector keys order the field's nonnegative document and ordinal identities lexicographically.
    pub(super) fn visit_ordered_vectors(
        &self,
        mut visit: impl FnMut(DocId, u32, &[f32]) -> Result<()>,
    ) -> Result<()> {
        let mut ordinals = super::codec::PersistedOrdinalSequence::default();
        self.snapshot.control.check()?;
        if let Some(owner) = self.owner {
            self.snapshot
                .visit_rows(Family::Vectors, Some(owner), &[self.field()], |row| {
                    let blob = blob(row[4])?;
                    let mut payload = self.snapshot.control.memory().reserve(blob.len())?;
                    let vector = blob_to_vector(blob)?;
                    let capacity = vector
                        .capacity()
                        .checked_mul(size_of::<f32>())
                        .ok_or(uqa_core::memory::MemoryError::SizeOverflow)?;
                    payload.grow(capacity.saturating_sub(blob.len()))?;
                    self.index.validate_dimensions_sqlite(&vector)?;
                    let ordinal = u32::try_from(integer(row[3])?).map_err(|_| {
                        SQLiteError::StorageBackend("invalid native vector ordinal".into())
                    })?;
                    let id = decode_doc_id(integer(row[2])?)?;
                    ordinals.push(id, ordinal)?;
                    visit(id, ordinal, &vector)?;
                    Ok(())
                })?;
        }
        Ok(())
    }

    pub(super) fn contains_document(&self, doc_id: i64) -> Result<bool> {
        let Some(owner) = self.owner else {
            return Ok(false);
        };
        self.snapshot.contains_row(
            Family::Vectors,
            owner,
            &[
                self.field(),
                ValueRef::Integer(doc_id),
                ValueRef::Integer(0),
            ],
        )
    }

    pub(super) fn replace(
        &self,
        batch: &mut dyn KeyValueBatch,
        doc_id: i64,
        vectors: &[(i64, Vec<u8>)],
    ) -> Result<()> {
        if vectors.is_empty() {
            return self.delete(batch, doc_id);
        }
        let owner = match self.owner {
            Some(owner) => owner,
            None => self.snapshot.ensure_table_owner(&self.index.table, batch)?,
        };
        self.snapshot
            .coordinate_vector_field(batch, owner, self.field(), false)?;
        self.invalidate_origin(batch, owner, doc_id)?;
        self.snapshot.delete_prefix(
            batch,
            Family::Vectors,
            owner,
            &[self.field(), ValueRef::Integer(doc_id)],
        )?;
        for (ordinal, vector) in vectors {
            self.snapshot.put_row(
                batch,
                Family::Vectors,
                owner,
                &[
                    ValueRef::Text(self.index.table.as_bytes()),
                    self.field(),
                    ValueRef::Integer(doc_id),
                    ValueRef::Integer(*ordinal),
                    ValueRef::Blob(vector),
                ],
            )?;
        }
        Ok(())
    }

    pub(super) fn delete(&self, batch: &mut dyn KeyValueBatch, doc_id: i64) -> Result<()> {
        if let Some(owner) = self.owner {
            self.snapshot
                .coordinate_vector_field(batch, owner, self.field(), false)?;
            self.invalidate_origin(batch, owner, doc_id)?;
            self.snapshot.delete_prefix(
                batch,
                Family::Vectors,
                owner,
                &[self.field(), ValueRef::Integer(doc_id)],
            )?;
        }
        Ok(())
    }

    fn invalidate_origin(
        &self,
        batch: &mut dyn KeyValueBatch,
        owner: NativeRecordOwner,
        document: i64,
    ) -> Result<()> {
        let key = NativeRecordIdentity::new(Family::VectorOrigins, owner)?.encode_key(
            &[self.field(), ValueRef::Integer(document)],
            &self.snapshot.control,
        )?;
        batch.invalidate_diskann_origin(&key)?;
        Ok(())
    }

    pub(super) fn clear_family(&self, batch: &mut dyn KeyValueBatch, family: Family) -> Result<()> {
        if let Some(owner) = self.owner {
            if family == Family::Vectors {
                self.snapshot
                    .coordinate_vector_field(batch, owner, self.field(), true)?;
                for related in [
                    Family::VectorOrigins,
                    Family::VectorChanges,
                    Family::VectorPopulations,
                    Family::VectorPopulationWitnesses,
                ] {
                    self.snapshot
                        .delete_prefix(batch, related, owner, &[self.field()])?;
                }
            }
            self.snapshot
                .delete_prefix(batch, family, owner, &[self.field()])?;
        }
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use crate::{Catalog, ManagedConnection, SQLiteHNSWIndex, SQLiteIVFIndex, SQLiteVectorIndex};
    use uqa_storage::{mvcc::VersionedSessionOptions, vector_index::VectorIndex};

    #[test]
    fn native_vector_mutations_reject_read_only_record_transactions() {
        let connection = ManagedConnection::open_in_memory().unwrap();
        Catalog::open(connection.clone()).unwrap();
        connection
            .bind_native_records(VersionedSessionOptions::default())
            .unwrap();
        let indexes: [Box<dyn VectorIndex>; 3] = [
            Box::new(SQLiteVectorIndex::new(
                connection.clone(),
                "docs",
                "exact",
                3,
            )),
            Box::new(SQLiteIVFIndex::new(connection.clone(), "docs", "ivf", 3)),
            Box::new(SQLiteHNSWIndex::new(connection.clone(), "docs", "hnsw", 3)),
        ];
        for mut index in indexes {
            connection.begin_record_read().unwrap();
            assert!(index.add(1, vec![1.0, 0.0, 0.0]).is_err());
            assert!(!connection.transaction_has_written().unwrap());
            connection.commit_transaction().unwrap();
            assert_eq!(index.count().unwrap(), 0);
        }
    }
}