uqa-storage 0.3.7

Document store, inverted index, IVF/HNSW vectors, B-tree, R*Tree, catalog
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! Batch atomicity, untouched allocation ownership, and ordered update equivalence.

use super::*;
use crate::InvertedIndex;
use proptest::prelude::*;
use uqa_analysis::whitespace_analyzer;

fn fields(text: &str) -> BTreeMap<String, String> {
    BTreeMap::from([("body".into(), text.into())])
}

fn seeded() -> MemoryInvertedIndex {
    let mut index = MemoryInvertedIndex::new(whitespace_analyzer());
    index.add_document(1, fields("old shared")).unwrap();
    index.add_document(2, fields("other shared")).unwrap();
    index.add_document(99, fields("unrelated shared")).unwrap();
    index
}

fn assert_state(left: &MemoryInvertedIndex, right: &MemoryInvertedIndex) {
    assert_eq!(left.state.doc_count, right.state.doc_count);
    assert_eq!(left.state.total_length, right.state.total_length);
    assert_eq!(left.state.field_doc_counts, right.state.field_doc_counts);
    assert_eq!(left.state.doc_fields, right.state.doc_fields);
    assert_eq!(left.state.doc_terms, right.state.doc_terms);
    assert_eq!(
        format!("{:?}", left.state.index),
        format!("{:?}", right.state.index)
    );
}

#[test]
fn batches_preserve_untouched_posting_allocations() {
    let mut index = seeded();
    let key = ("body".into(), crate::TokenTermKey::from_text("shared"));
    let occurrences = index.state.index[&key][&99].occurrences.as_ptr();
    let positions = index.state.index[&key][&99]
        .projection
        .payload
        .positions
        .as_ptr();
    index.try_add_documents(Vec::new()).unwrap();
    assert_eq!(
        index.state.index[&key][&99].occurrences.as_ptr(),
        occurrences
    );
    index
        .try_add_documents(vec![(1, fields("new shared")), (3, fields("new"))])
        .unwrap();
    assert_eq!(
        index.state.index[&key][&99].occurrences.as_ptr(),
        occurrences
    );
    assert_eq!(
        index.state.index[&key][&99]
            .projection
            .payload
            .positions
            .as_ptr(),
        positions
    );
    assert_eq!(index.doc_count().unwrap(), 4);
    assert_eq!(index.doc_freq("body", "old").unwrap(), 0);
    assert_eq!(index.doc_freq("body", "new").unwrap(), 2);
}

#[test]
fn late_document_and_field_counter_failures_publish_nothing() {
    for field_overflow in [false, true] {
        let mut index = seeded();
        if field_overflow {
            Arc::make_mut(&mut index.state)
                .total_length
                .insert("body".into(), u64::MAX - 1);
        } else {
            Arc::make_mut(&mut index.state).doc_count = u64::MAX - 1;
        }
        let before = index.clone();
        let error = index
            .try_add_documents(vec![(3, fields("next")), (4, fields("overflow"))])
            .unwrap_err();
        assert!(error.to_string().contains(if field_overflow {
            "total field length"
        } else {
            "document count"
        }));
        assert_state(&index, &before);
    }
}

#[test]
fn corrupt_affected_postings_or_reverse_metadata_publish_nothing() {
    for missing_field in [false, true] {
        let mut index = seeded();
        if missing_field {
            Arc::make_mut(&mut index.state).doc_fields.remove(&2);
        } else {
            Arc::make_mut(&mut index.state)
                .index
                .remove(&("body".into(), crate::TokenTermKey::from_text("other")));
        }
        let before = index.clone();
        assert!(index
            .try_add_documents(vec![(1, fields("new")), (2, fields("new"))])
            .is_err());
        assert_state(&index, &before);
    }
}

proptest! {
    #[test]
    fn repeated_ids_deletions_and_field_moves_match_ordered_point_updates(
        operations in prop::collection::vec((0_u8..12, 0_u8..4, 0_u8..8), 0..40)
    ) {
        let mut index = seeded();
        let mut expected = index.clone();
        let documents: Vec<_> = operations.into_iter().map(|(id, mask, length)| {
            let mut values = BTreeMap::new();
            if mask & 1 != 0 { values.insert("body".into(), "a b ".repeat(usize::from(length))); }
            if mask & 2 != 0 { values.insert("title".into(), "c ".repeat(usize::from(length))); }
            (u64::from(id), values)
        }).collect();
        for (id, fields) in &documents {
            expected.add_document(*id, fields.clone()).unwrap();
        }
        index.try_add_documents(documents).unwrap();
        assert_state(&index, &expected);
    }
}