uqa-storage-sqlite 0.6.3

SQLite catalog, indexes, compressed storage, graph and key/value providers
Documentation
//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

use std::fmt::Write as _;

use super::*;
use crate::SQLiteInvertedIndex;
use uqa_storage::mvcc::VersionedSessionOptions;
use uqa_storage::InvertedIndex;

fn fields<const N: usize>(pairs: [(&str, &str); N]) -> std::collections::BTreeMap<String, String> {
    pairs
        .into_iter()
        .map(|(key, value)| (key.into(), value.into()))
        .collect()
}

#[test]
fn large_text_replacements_remain_bounded_across_one_transaction() {
    let directory = tempfile::tempdir().unwrap();
    let path = directory.path().join("text-replacements.db");
    let mut body = String::new();
    for term in 0..4096 {
        write!(body, "term{term:05} ").unwrap();
    }
    {
        let connection = ManagedConnection::open(&path).unwrap();
        Catalog::open(connection.clone()).unwrap();
        connection
            .bind_native_records(VersionedSessionOptions::default())
            .unwrap();
        let mut index = SQLiteInvertedIndex::new(
            connection.clone(),
            "records",
            uqa_analysis::whitespace_analyzer(),
        );
        connection.begin_transaction().unwrap();
        index.add_document(1, fields([("body", &body)])).unwrap();
        index.add_document(2, fields([("body", &body)])).unwrap();
        connection.commit_transaction().unwrap();
    }
    body.push_str("additional");
    {
        let connection = ManagedConnection::open(&path).unwrap();
        connection
            .bind_native_records(VersionedSessionOptions {
                retained_bytes: 2 << 20,
            })
            .unwrap();
        let mut index = SQLiteInvertedIndex::new(
            connection.clone(),
            "records",
            uqa_analysis::whitespace_analyzer(),
        );
        connection.begin_transaction().unwrap();
        index
            .add_document(1, fields([("body", &body)]))
            .expect("first replacement");
        index
            .add_document(2, fields([("body", &body)]))
            .expect("second replacement in the same transaction");
        connection.commit_transaction().unwrap();
    }
    let connection = ManagedConnection::open(&path).unwrap();
    connection
        .bind_native_records(VersionedSessionOptions {
            retained_bytes: 2 << 20,
        })
        .unwrap();
    let index =
        SQLiteInvertedIndex::new(connection, "records", uqa_analysis::whitespace_analyzer());
    for document in [1, 2] {
        assert_eq!(index.get_doc_length(document, "body").unwrap(), 4097);
        assert_eq!(
            index.get_term_freq(document, "body", "additional").unwrap(),
            1
        );
        assert_eq!(
            index.get_term_freq(document, "body", "term04095").unwrap(),
            1
        );
    }
}

#[test]
fn warm_text_replacements_remain_bounded_across_commits() {
    let directory = tempfile::tempdir().unwrap();
    let path = directory.path().join("warm-text.db");
    let connection = ManagedConnection::open(&path).unwrap();
    Catalog::open(connection.clone()).unwrap();
    connection
        .bind_native_records(VersionedSessionOptions {
            retained_bytes: 2 << 20,
        })
        .unwrap();
    let mut index = SQLiteInvertedIndex::new(
        connection.clone(),
        "records",
        uqa_analysis::whitespace_analyzer(),
    );
    let mut body = String::new();
    for term in 0..4096 {
        write!(body, "term{term:05} ").unwrap();
    }
    for document in [1, 2] {
        index
            .add_document(document, fields([("body", &body)]))
            .unwrap();
    }
    body.push_str("additional");
    connection.begin_transaction().unwrap();
    for document in [1, 2] {
        index
            .add_document(document, fields([("body", &body)]))
            .unwrap();
    }
    connection.commit_transaction().unwrap();
    drop((index, connection));
    let connection = ManagedConnection::open(&path).unwrap();
    connection
        .bind_native_records(VersionedSessionOptions {
            retained_bytes: 2 << 20,
        })
        .unwrap();
    let index =
        SQLiteInvertedIndex::new(connection, "records", uqa_analysis::whitespace_analyzer());
    for document in [1, 2] {
        assert_eq!(index.get_doc_length(document, "body").unwrap(), 4097);
        assert_eq!(
            index.get_term_freq(document, "body", "additional").unwrap(),
            1
        );
        assert_eq!(
            index.get_term_freq(document, "body", "term04095").unwrap(),
            1
        );
    }
}

#[test]
fn native_occurrence_commit_batches_unchanged_snapshot_reads() {
    for terms in [64, 256, 1024] {
        let directory = tempfile::tempdir().unwrap();
        let path = directory.path().join("commit.db");
        let connection = ManagedConnection::open(&path).unwrap();
        Catalog::open(connection.clone()).unwrap();
        connection
            .bind_native_records(VersionedSessionOptions {
                retained_bytes: 2 << 20,
            })
            .unwrap();
        let mut index = SQLiteInvertedIndex::new(
            connection.clone(),
            "notes",
            uqa_analysis::whitespace_analyzer(),
        );
        let mut text = String::new();
        for id in 0..terms {
            write!(text, "term{id:05} ").unwrap();
        }
        index.add_document(1, fields([("body", &text)])).unwrap();
        let retained = index.snapshot().unwrap();
        connection.begin_transaction().unwrap();
        index.add_document(2, fields([("body", &text)])).unwrap();
        let reads = crate::mvcc::RECORD_READS.get();
        connection.commit_transaction().unwrap();
        let reads = crate::mvcc::RECORD_READS.get() - reads;
        eprintln!("terms={terms} physical_read_admissions={reads}");
        assert!(reads <= 32, "occurrence validation must share admission: {terms} terms required {reads} physical reads");
        assert_eq!(index.doc_count().unwrap(), 2);
        assert_eq!(index.total_field_length("body").unwrap(), 2 * terms);
        assert_eq!(retained.doc_count().unwrap(), 1);
        for id in 0..terms {
            assert_eq!(
                index
                    .get_term_freq(1, "body", &format!("term{id:05}"))
                    .unwrap(),
                1
            );
            assert_eq!(
                index
                    .get_term_freq(2, "body", &format!("term{id:05}"))
                    .unwrap(),
                1
            );
        }
        connection.begin_transaction().unwrap();
        index
            .add_document(3, fields([("body", "discarded")]))
            .unwrap();
        connection.rollback_transaction().unwrap();
        assert_eq!(index.doc_count().unwrap(), 2);
        drop((retained, index, connection));
        let connection = ManagedConnection::open(&path).unwrap();
        connection
            .bind_native_records(VersionedSessionOptions {
                retained_bytes: 2 << 20,
            })
            .unwrap();
        let index =
            SQLiteInvertedIndex::new(connection, "notes", uqa_analysis::whitespace_analyzer());
        assert_eq!(index.doc_count().unwrap(), 2);
        assert_eq!(index.total_field_length("body").unwrap(), 2 * terms);
        assert_eq!(index.get_term_freq(1, "body", "term00000").unwrap(), 1);
        assert_eq!(
            index
                .get_term_freq(2, "body", &format!("term{:05}", terms - 1))
                .unwrap(),
            1
        );
    }
}