uqa-storage-sqlite 0.5.4

SQLite catalog, indexes, compressed storage, graph and key/value providers
Documentation
//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

use uqa_storage::mvcc::{PreparedRecordCommit, RecordWrite, VersionedPersistence};

use super::{materialization::with, persistence::connection, *};
use crate::SQLiteRecordStore;

#[test]
fn population_origin_prefix_selects_one_native_owner_and_field() {
    use uqa_storage::diskann_index::format::{DiskANNCanonicalOrigin, DiskANNVectorVersion};
    use uqa_storage::mvcc::{DiskANNPopulationRecordLayout, StorageTransactionId};

    let control = StorageReadControl::with_limit(128 << 10);
    let database = DatabaseId::from_bytes([5; 16]);
    let layout = NativeRecordNamespace(database);
    let origin = DiskANNCanonicalOrigin::new(
        DiskANNVectorVersion::new(StorageTransactionId::new(database, 1).unwrap(), 1).unwrap(),
        3,
        1,
    )
    .unwrap()
    .encode();
    let owners = [
        owner(),
        NativeRecordOwner::Object {
            identity: [4; 16],
            generation: [7; 16],
        },
    ];
    let mut records = Vec::new();
    for owner in owners {
        for field in [b"a".as_slice(), b"aa", b"a\0"] {
            for document in [0, 7, i64::MAX] {
                records.push(
                    NativeRecord::encode(
                        NativeRecordFamily::VectorOrigins,
                        owner,
                        &[
                            ValueRef::Text(b"public.table"),
                            ValueRef::Text(field),
                            ValueRef::Integer(document),
                            ValueRef::Blob(&origin),
                        ],
                        &control,
                    )
                    .unwrap(),
                );
            }
        }
    }
    for record in &records {
        let decoded = layout.origin(record.key(), record.row(), &control).unwrap();
        let prefix = layout.origin_prefix(&decoded.field, &control).unwrap();
        let selected: Vec<_> = records
            .iter()
            .filter(|candidate| candidate.key().starts_with(&prefix))
            .collect();
        assert_eq!(selected.len(), 3);
        for selected in selected {
            assert_eq!(
                layout
                    .origin(selected.key(), selected.row(), &control)
                    .unwrap()
                    .field
                    .as_ref(),
                decoded.field.as_ref()
            );
        }
    }
}

pub(in crate::mvcc::native) fn remove_empty_table(
    sqlite: &rusqlite::Connection,
) -> crate::Result<()> {
    remove_empty_origins(sqlite)?;
    assert_eq!(
        sqlite.query_row(
            "SELECT count(*) FROM _uqa_mvcc_native_diskann_records",
            [],
            |row| row.get::<_, i64>(0)
        )?,
        0
    );
    sqlite.execute_batch("DROP TABLE _uqa_mvcc_native_diskann_records")?;
    Ok(())
}

pub(in crate::mvcc::native) fn remove_empty_origins(
    sqlite: &rusqlite::Connection,
) -> crate::Result<()> {
    remove_empty_changes(sqlite)?;
    assert_eq!(
        sqlite.query_row(
            "SELECT count(*) FROM _uqa_mvcc_native_vector_origins",
            [],
            |row| row.get::<_, i64>(0)
        )?,
        0
    );
    sqlite.execute_batch("DROP TABLE _uqa_mvcc_native_vector_origins")?;
    Ok(())
}

pub(in crate::mvcc::native) fn remove_empty_changes(
    sqlite: &rusqlite::Connection,
) -> crate::Result<()> {
    remove_empty_populations(sqlite)?;
    assert_eq!(
        sqlite.query_row(
            "SELECT count(*) FROM _uqa_mvcc_native_vector_changes",
            [],
            |row| row.get::<_, i64>(0)
        )?,
        0
    );
    sqlite.execute_batch("DROP TABLE _uqa_mvcc_native_vector_changes")?;
    Ok(())
}

/// Remove the sequence value records of a database without sequences, as formats before them lacked.
pub(in crate::mvcc::native) fn remove_empty_sequence_values(
    sqlite: &rusqlite::Connection,
) -> crate::Result<()> {
    remove_empty_foreign_server_metadata(sqlite)?;
    let table = NativeRecordFamily::SequenceValues.layout().table;
    assert_eq!(
        sqlite.query_row(&format!("SELECT count(*) FROM {table}"), [], |row| row
            .get::<_, i64>(0))?,
        0
    );
    sqlite.execute_batch(&format!("DROP TABLE {table}"))?;
    Ok(())
}

/// Strip the empty family when constructing a predecessor native format.
pub(in crate::mvcc::native) fn remove_empty_foreign_server_metadata(
    sqlite: &rusqlite::Connection,
) -> crate::Result<()> {
    let table = NativeRecordFamily::ForeignServerMetadata.layout().table;
    assert_eq!(
        sqlite.query_row(&format!("SELECT count(*) FROM {table}"), [], |row| row
            .get::<_, i64>(0))?,
        0
    );
    sqlite.execute_batch(&format!("DROP TABLE {table}"))?;
    Ok(())
}

pub(in crate::mvcc::native) fn remove_empty_populations(
    sqlite: &rusqlite::Connection,
) -> crate::Result<()> {
    remove_empty_sequence_values(sqlite)?;
    for (family, _) in super::super::populations::schema::TABLES {
        let table = family.layout().table;
        assert_eq!(
            sqlite.query_row(&format!("SELECT count(*) FROM {table}"), [], |row| row
                .get::<_, i64>(0))?,
            0
        );
        sqlite.execute_batch(&format!("DROP TABLE {table}"))?;
    }
    Ok(())
}

#[test]
fn native_diskann_blob_records_preserve_bytes_snapshots_guards_and_cold_reopen() {
    let directory = tempfile::tempdir().unwrap();
    let control = StorageReadControl::with_limit(1 << 22);
    for mode in 0..4 {
        let path = directory.path().join(format!("native-diskann-{mode}.db"));
        let namespace;
        let mut payload = [255; 4096];
        payload[..4].copy_from_slice(&[0, 1, 0, 128]);
        let key = [0, 255, 0, 3];
        {
            let connection = connection(&path, mode);
            let persistence = SQLiteRecordStore::for_native(&connection, &control).unwrap();
            namespace = persistence.native_namespace().unwrap();
            let record = NativeRecord::encode(
                NativeRecordFamily::DiskANNRecords,
                NativeRecordOwner::Database(namespace),
                &[ValueRef::Blob(&key), ValueRef::Blob(&payload)],
                &control,
            )
            .unwrap();
            let before = persistence.snapshot(&control).unwrap();
            let transaction = persistence.allocate_transaction(&control).unwrap();
            let receipt = persistence
                .commit(
                    transaction,
                    &PreparedRecordCommit::new(&[record.write(None)], &control).unwrap(),
                    &control,
                )
                .unwrap();
            assert!(before.get(record.key(), &control).unwrap().is_none());
            let retained = persistence.snapshot(&control).unwrap();
            assert_retained(&*retained, &record, &control);
            assert_physical(&connection, &key, &payload);
            let transaction = persistence.allocate_transaction(&control).unwrap();
            persistence
                .commit(
                    transaction,
                    &PreparedRecordCommit::new(
                        &[RecordWrite {
                            key: record.key(),
                            expected: Some(receipt.sequence),
                            value: None,
                        }],
                        &control,
                    )
                    .unwrap(),
                    &control,
                )
                .unwrap();
            persistence.reclaim_versions(&control).unwrap();
            assert!(persistence
                .snapshot(&control)
                .unwrap()
                .get(record.key(), &control)
                .unwrap()
                .unwrap()
                .value()
                .is_none());
            assert_retained(&*retained, &record, &control);
            let head = persistence
                .snapshot(&control)
                .unwrap()
                .get(record.key(), &control)
                .unwrap()
                .unwrap()
                .sequence();
            let transaction = persistence.allocate_transaction(&control).unwrap();
            persistence
                .commit(
                    transaction,
                    &PreparedRecordCommit::new(&[record.write(Some(head))], &control).unwrap(),
                    &control,
                )
                .unwrap();
        }
        let connection = connection(&path, mode);
        let reopened = SQLiteRecordStore::for_native(&connection, &control).unwrap();
        assert_eq!(reopened.native_namespace(), Some(namespace));
        assert_physical(&connection, &key, &payload);
    }
}

#[test]
fn native_diskann_family_is_distinct_from_standalone_graph_and_keeps_stable_ids() {
    assert_eq!(NativeRecordFamily::DiskANNRecords.id(), 57);
    assert_eq!(NativeRecordFamily::VectorOrigins.id(), 58);
    assert_eq!(NativeRecordFamily::VectorChanges.id(), 59);
    assert!(NativeRecordFamily::VectorChanges.layout().object_owned);
    assert_eq!(NativeRecordFamily::StandaloneGraphLookups.id(), 56);
    assert!(!NativeRecordFamily::DiskANNRecords.is_standalone_graph());
    assert_eq!(
        NativeRecordFamily::DiskANNRecords.layout().columns,
        ["key", "value"]
    );
    assert!(!NativeRecordFamily::DiskANNRecords.layout().object_owned);
    for family in NativeRecordFamily::all() {
        assert_eq!(NativeRecordFamily::from_id(family.id()), Some(family));
        assert_eq!(family.layout().family, family);
    }
}

fn assert_retained(
    snapshot: &dyn uqa_storage::mvcc::CommittedRecordSnapshot,
    record: &NativeRecord,
    control: &StorageReadControl,
) {
    assert_eq!(
        &***snapshot
            .get(record.key(), control)
            .unwrap()
            .unwrap()
            .value()
            .unwrap(),
        record.row()
    );
}

fn assert_physical(connection: &ManagedConnection, key: &[u8], payload: &[u8]) {
    with(connection, |sqlite| {
        let (kind, bytes): (String, Vec<u8>) = sqlite.query_row(
            "SELECT typeof(value),value FROM _uqa_mvcc_native_diskann_records WHERE key=?1",
            [key],
            |row| Ok((row.get(0)?, row.get(1)?)),
        )?;
        assert_eq!(kind, "blob");
        assert_eq!(bytes, payload);
        assert!(sqlite
            .execute("DELETE FROM _uqa_mvcc_native_diskann_records", [])
            .is_err());
        Ok(())
    });
}