uqa-storage-sqlite 0.4.9

SQLite catalog, indexes, compressed storage, graph and key/value providers
Documentation
//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

use super::*;
use crate::{Catalog, SQLiteCompressionOptions, SQLiteRecordStore};
use std::path::Path;
use uqa_storage::diskann_index::{DiskANNCanonicalRead, DiskANNCanonicalScorer};
use uqa_storage::mvcc::{CommitStatus, VersionedPersistence, VersionedSessionOptions};

mod binding;
mod changes;
mod corpus;
mod coverage;
mod field_guards;
mod lifecycle;
mod raw_values;
mod validation;

fn open(path: &Path, mode: u8) -> ManagedConnection {
    let connection = match mode {
        0 => ManagedConnection::open(path),
        1 => ManagedConnection::open_encrypted(path, "canonical-native-key"),
        2 => ManagedConnection::open_compressed(path, SQLiteCompressionOptions::default()),
        _ => ManagedConnection::open_compressed_encrypted(
            path,
            "canonical-native-key",
            SQLiteCompressionOptions::default(),
        ),
    }
    .unwrap();
    bind(&connection);
    connection
}

fn bind(connection: &ManagedConnection) {
    connection
        .bind_native_records(VersionedSessionOptions::default())
        .unwrap();
}

fn memory() -> ManagedConnection {
    let connection = ManagedConnection::open_in_memory().unwrap();
    bind(&connection);
    connection
}

fn canonical(
    connection: &ManagedConnection,
    table: &str,
    field: &str,
    dimensions: u32,
) -> SQLiteDiskANNCanonical {
    SQLiteDiskANNCanonical::new(connection.clone(), table, field, dimensions).unwrap()
}

fn assert_change(
    source: &RetainedSQLiteDiskANNCanonical,
    document: DocId,
    version: DiskANNVectorVersion,
    control: &StorageReadControl,
) {
    assert_eq!(
        source
            .next_change_after(document.checked_sub(1), control)
            .unwrap(),
        Some(DiskANNChangeIdentity::new(document, version))
    );
}

fn change_count(connection: &ManagedConnection) -> i64 {
    connection
        .with_physical(|sqlite| {
            Ok(sqlite.query_row(
                "SELECT count(*) FROM _uqa_mvcc_native_vector_changes",
                [],
                |row| row.get(0),
            )?)
        })
        .unwrap()
}

fn assert_tensor(
    source: &RetainedSQLiteDiskANNCanonical,
    document: DocId,
    expected: &[Vec<f32>],
    control: &StorageReadControl,
) -> Option<DiskANNVectorVersion> {
    let origin = source.origin(document, control).unwrap();
    let mut count = 0;
    let actual = source
        .visit_document(document, control, &mut |ordinal, version, vector| {
            assert_eq!(ordinal as usize, count);
            assert_eq!(Some(version), origin);
            assert_eq!(
                vector.iter().map(|v| v.to_bits()).collect::<Vec<_>>(),
                expected[count]
                    .iter()
                    .map(|v| v.to_bits())
                    .collect::<Vec<_>>()
            );
            count += 1;
            Ok(())
        })
        .unwrap();
    assert_eq!(count, expected.len());
    assert_eq!(actual, origin);
    actual
}

#[test]
fn native_diskann_canonical_tensors_retain_private_origins_and_reopen_in_all_file_modes() {
    let directory = tempfile::tempdir().unwrap();
    for mode in 0..4 {
        let path = directory.path().join(format!("canonical-{mode}.db"));
        let control = StorageReadControl::with_limit(1 << 22);
        let tensor = vec![vec![-0.0, f32::MAX], vec![1.0, f32::from_bits(1)]];
        let version = {
            let connection = open(&path, mode);
            let source = canonical(&connection, "docs", "embedding", 2);
            let version = source.replace(1, &tensor, &control).unwrap();
            source.replace(2, &[], &control).unwrap();
            let persistence = SQLiteRecordStore::for_native(&connection, &control).unwrap();
            assert_eq!(version.writer().database(), persistence.database_id());
            assert!(matches!(
                persistence
                    .commit_status(version.writer(), &control)
                    .unwrap(),
                CommitStatus::Committed(_)
            ));
            let retained = source.retain(&control).unwrap();
            assert_change(&retained, 1, version, &control);
            assert_eq!(
                assert_tensor(&retained, 1, &tensor, &control),
                Some(version)
            );
            connection.begin_transaction().unwrap();
            connection.savepoint("before").unwrap();
            let changed = source.replace(1, &[vec![3.0, 4.0]], &control).unwrap();
            let private = source.retain(&control).unwrap();
            connection.rollback_to_savepoint("before").unwrap();
            let empty = source.replace(1, &[], &control).unwrap();
            assert_eq!(empty.writer(), changed.writer());
            assert!(empty.revision() > changed.revision());
            assert_eq!(
                assert_tensor(&source.retain(&control).unwrap(), 1, &[], &control),
                Some(empty)
            );
            connection.rollback_transaction().unwrap();
            assert_change(&private, 1, changed, &control);
            assert_eq!(change_count(&connection), 2);
            assert_eq!(
                assert_tensor(&private, 1, &[vec![3.0, 4.0]], &control),
                Some(changed)
            );
            assert_canonical_physical_rows(&connection);
            drop(persistence);
            drop(source);
            drop(connection);
            assert_eq!(
                assert_tensor(&retained, 1, &tensor, &control),
                Some(version)
            );
            assert!(assert_tensor(&retained, 2, &[], &control).is_some());
            assert_tensor(&private, 1, &[vec![3.0, 4.0]], &control);
            version
        };
        let connection = open(&path, mode);
        let source = canonical(&connection, "docs", "embedding", 2)
            .retain(&control)
            .unwrap();
        assert_eq!(assert_tensor(&source, 1, &tensor, &control), Some(version));
        assert_change(&source, 1, version, &control);
        assert_change(
            &source,
            2,
            source.origin(2, &control).unwrap().unwrap(),
            &control,
        );
        assert!(source.origin(2, &control).unwrap().is_some());
    }
}

fn assert_canonical_physical_rows(connection: &ManagedConnection) {
    connection
        .with_physical(|sqlite| {
            assert_eq!(
                sqlite.query_row("SELECT count(*) FROM _vectors", [], |r| r.get::<_, i64>(0))?,
                2
            );
            assert_eq!(
                sqlite.query_row(
                    "SELECT count(*) FROM _uqa_mvcc_native_vector_origins",
                    [],
                    |r| r.get::<_, i64>(0)
                )?,
                2
            );
            assert_eq!(
                sqlite.query_row(
                    "SELECT count(*) FROM _uqa_mvcc_native_diskann_records",
                    [],
                    |r| r.get::<_, i64>(0)
                )?,
                0
            );
            let raw: Vec<u8> = sqlite.query_row(
                "SELECT vector FROM _vectors WHERE doc_id=1 AND vector_ordinal=0",
                [],
                |r| r.get(0),
            )?;
            assert_eq!(raw, [0, 0, 0, 128, 255, 255, 127, 127]);
            assert!(sqlite
                .execute("DELETE FROM _uqa_mvcc_native_vector_origins", [])
                .is_err());
            Ok(())
        })
        .unwrap();
}

#[test]
fn native_diskann_canonical_writers_keep_disjoint_commits_and_reject_same_document_conflicts() {
    for reverse in [false, true] {
        let connection = memory();
        let control = StorageReadControl::with_limit(1 << 22);
        let a = canonical(&connection, "docs", "embedding", 2);
        a.replace(0, &[], &control).unwrap();
        let peer = connection.new_session();
        let b = canonical(&peer, "docs", "embedding", 2);
        connection.begin_transaction().unwrap();
        peer.begin_transaction().unwrap();
        let av = a.replace(1, &[vec![1.0, 0.0]], &control).unwrap();
        let bv = b.replace(2, &[vec![0.0, 1.0]], &control).unwrap();
        assert_ne!(av.writer(), bv.writer());
        let (first, last) = if reverse {
            (&peer, &connection)
        } else {
            (&connection, &peer)
        };
        first.commit_transaction().unwrap();
        last.commit_transaction().unwrap();
        let read = a.retain(&control).unwrap();
        assert_eq!(read.origin(1, &control).unwrap(), Some(av));
        assert_eq!(read.origin(2, &control).unwrap(), Some(bv));
        assert_change(&read, 1, av, &control);
        assert_change(&read, 2, bv, &control);
        connection.begin_transaction().unwrap();
        peer.begin_transaction().unwrap();
        let empty = a.replace(1, &[], &control).unwrap();
        let populated = b.replace(1, &[vec![2.0, 0.0]], &control).unwrap();
        first.commit_transaction().unwrap();
        assert!(last.commit_transaction().is_err());
        last.rollback_transaction().unwrap();
        assert_change(
            &a.retain(&control).unwrap(),
            1,
            if reverse { populated } else { empty },
            &control,
        );
        assert_eq!(change_count(&connection), 4);
        assert_tensor(&read, 1, &[vec![1.0, 0.0]], &control);
    }
}

#[test]
fn native_diskann_canonical_streams_tensors_larger_than_query_allowance() {
    let connection = memory();
    let control = StorageReadControl::with_limit(1 << 22);
    let source = canonical(&connection, "docs", "embedding", 128);
    let tensor = vec![vec![1.0; 128]; 32];
    source.replace(1, &tensor, &control).unwrap();
    let retained = source.retain(&control).unwrap();
    let small = StorageReadControl::with_limit(8192);
    assert_tensor(&retained, 1, &tensor, &small);
    let mut ordinals = 0;
    retained
        .visit_all(&small, &mut |doc, ordinal, _, raw| {
            assert_eq!((doc, ordinal), (1, ordinals));
            assert_eq!(raw, [1.0; 128]);
            ordinals += 1;
            Ok(())
        })
        .unwrap();
    assert_eq!(ordinals, 32);
    let scorer = DiskANNCanonicalScorer::new(&retained, &[1.0; 128], &small).unwrap();
    assert_eq!(
        scorer.score_document(1).unwrap().unwrap().vector_count(),
        32
    );
    assert_eq!(
        scorer
            .search_exact_knn(1)
            .unwrap()
            .doc_ids()
            .collect::<Vec<_>>(),
        [1]
    );
    assert_eq!(small.memory().used(), 0);
    let empty = canonical(&connection, "docs", "wide", 16384);
    let version = empty.replace(1, &[], &control).unwrap();
    assert_eq!(
        assert_tensor(&empty.retain(&control).unwrap(), 1, &[], &small),
        Some(version)
    );
    assert_eq!(small.memory().used(), 0);
    let tiny = StorageReadControl::with_limit(1);
    assert!(retained.origin(1, &tiny).is_err());
    assert_eq!(tiny.memory().used(), 0);
    let cancelled = StorageReadControl::with_limit(8192);
    let mut calls = 0;
    assert!(retained
        .visit_document(1, &cancelled, &mut |_, _, _| {
            calls += 1;
            cancelled.cancellation().cancel();
            Ok(())
        })
        .is_err());
    assert_eq!(calls, 1);
    assert_eq!(cancelled.memory().used(), 0);
    let original = StorageReadControl::with_limit(1 << 20);
    let held = source.retain(&original).unwrap();
    original.cancellation().cancel();
    assert!(held.origin(1, &small).is_err());
}