uqa-storage-sqlite 0.5.1

SQLite catalog, indexes, compressed storage, graph and key/value providers
Documentation
//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

use super::*;
use uqa_storage::{hnsw_index::HNSWIndex, read_control::StorageReadControl};

const DOCUMENTS: u64 = 20;

fn vector(document: u64, dimensions: usize) -> Vec<f32> {
    let mut vector = vec![0.0; dimensions];
    vector[document as usize % dimensions] = 1.0;
    vector[0] = 0.125;
    vector
}

#[test]
fn physical_hnsw_spill_builds_mutates_rolls_back_and_reopens() {
    lifecycle(false);
}

#[test]
fn native_hnsw_spill_builds_mutates_rolls_back_and_reopens() {
    lifecycle(true);
}

#[test]
fn native_hnsw_loading_spills_a_decoded_graph_beyond_the_session_allowance() {
    let connection = ManagedConnection::open_in_memory().unwrap();
    Catalog::open(connection.clone()).unwrap();
    let mut index = SQLiteHNSWIndex::new(connection.clone(), "docs", "embedding", 64);
    for doc in 0..256 {
        index.add(doc, vec![1.0; 64]).unwrap();
    }
    index.initialize().unwrap();
    crate::SQLiteRecordStore::for_native(&connection, &StorageReadControl::with_limit(16 << 20))
        .unwrap();
    connection
        .bind_native_records(uqa_storage::mvcc::VersionedSessionOptions {
            retained_bytes: 64 << 10,
        })
        .unwrap();
    let control = connection.retention_control().unwrap();
    assert!(256 * 64 * size_of::<f32>() >= control.memory().limit());
    assert_eq!(index.count().unwrap(), 256);
    let found = index.search_knn(&[1.0; 64], 1).unwrap();
    assert_eq!(found.doc_ids().count(), 1);
    assert!(found.doc_ids().all(|doc| doc < 256));
    let snapshot = index.snapshot().unwrap();
    assert_eq!(snapshot.count().unwrap(), 256);
    drop(snapshot);
    assert_eq!(index.count().unwrap(), 256);
    assert!(control.memory().peak() <= control.memory().limit());
    assert!(!connection.in_transaction());
}

fn lifecycle(native: bool) {
    let dimensions = if native { 4096 } else { 1024 };
    let vector = |document| vector(document, dimensions);
    let directory = tempdir().unwrap();
    let path = directory.path().join("bounded-hnsw.db");
    let params = HNSWIndexParams {
        m: 2,
        ef_construction: 4,
        ef_search: 24,
        rebuild_threshold: 3,
        ..HNSWIndexParams::default()
    };
    let connection = ManagedConnection::open(&path).unwrap();
    Catalog::open(connection.clone()).unwrap();
    let control = bind(&connection, native);
    assert!(DOCUMENTS as usize * dimensions * size_of::<f32>() > control.memory().limit());
    let mut index = SQLiteHNSWIndex::with_params(
        connection.clone(),
        "docs",
        "embedding",
        dimensions as u32,
        params,
    );
    index.physical_control = control.clone();
    let mut reference = HNSWIndex::with_params(dimensions as u32, params).unwrap();
    for document in 1..=DOCUMENTS {
        index.add(document, vector(document)).unwrap();
        reference.add(document, vector(document)).unwrap();
    }
    index.initialize().unwrap();
    let query = vector(17);
    let expected = reference.search_knn(&query, 5).unwrap();
    assert_eq!(index.search_knn(&query, 5).unwrap(), expected);
    let retained = index.snapshot().unwrap();

    connection.begin_transaction().unwrap();
    connection.savepoint("before").unwrap();
    index.add(90, query.clone()).unwrap();
    let discarded = index.snapshot().unwrap();
    connection.rollback_to_savepoint("before").unwrap();
    connection.release_savepoint("before").unwrap();
    assert_eq!(index.search_knn(&query, 5).unwrap(), expected);
    assert_eq!(discarded.count().unwrap(), DOCUMENTS as usize + 1);
    drop(discarded);
    index.delete(1).unwrap();
    connection.rollback_transaction().unwrap();
    assert_eq!(index.count().unwrap(), DOCUMENTS as usize);

    index.add(17, vector(250)).unwrap();
    index.delete(8).unwrap();
    index.delete(9).unwrap();
    reference.add(17, vector(250)).unwrap();
    reference.delete(8).unwrap();
    reference.delete(9).unwrap();
    let changed = reference.search_knn(&query, 5).unwrap();
    assert_eq!(index.search_knn(&query, 5).unwrap(), changed);
    assert_eq!(retained.search_knn(&query, 5).unwrap(), expected);
    assert!(control.memory().peak() <= control.memory().limit());
    drop((index, connection));
    assert!(control.memory().used() > 0);
    assert_eq!(retained.search_knn(&query, 5).unwrap(), expected);
    drop(retained);
    assert_eq!(control.memory().used(), 0);

    let connection = ManagedConnection::open(&path).unwrap();
    let control = bind(&connection, native);
    let mut reopened =
        SQLiteHNSWIndex::open_existing(connection, "docs", "embedding", dimensions as u32, params);
    reopened.physical_control = control.clone();
    assert_eq!(reopened.search_knn(&query, 5).unwrap(), changed);
    assert_eq!(reopened.count().unwrap(), DOCUMENTS as usize - 2);
    drop(reopened);
    assert_eq!(control.memory().used(), 0);
}

fn bind(connection: &ManagedConnection, native: bool) -> StorageReadControl {
    if native {
        connection
            .bind_native_records(uqa_storage::mvcc::VersionedSessionOptions {
                retained_bytes: 256 * 1024,
            })
            .unwrap();
        connection.retention_control().unwrap()
    } else {
        StorageReadControl::with_limit(64 * 1024)
    }
}