uqa-storage 0.4.8

Document store, inverted index, IVF/HNSW vectors, B-tree, R*Tree, catalog
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148

//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

use super::*;
use crate::{HNSWIndex, IVFIndex};

#[test]
fn immutable_vector_facets_keep_raw_tensors_identity_and_original_controls() {
    for mut live in [
        Box::new(HNSWIndex::new(2)) as Box<dyn VectorIndex>,
        Box::new(IVFIndex::with_params(2, 1, 1, 1)),
    ] {
        live.add_many(7, vec![vec![8.0, 0.0], vec![-0.0, 4.0]])
            .unwrap();
        live.add(9, vec![3.0, 0.0]).unwrap();
        live.add(u64::MAX, vec![0.0, -2.0]).unwrap();
        live.delete(9).unwrap();
        let owner = StorageReadControl::with_limit(1 << 20);
        let retained = live.snapshot_with_control(&owner).unwrap();
        let source = retained.vector_read_snapshot(&owner).unwrap().unwrap();
        let fingerprint = source.corpus_fingerprint(&owner).unwrap();
        assert!(fingerprint.is_some());
        live.add(7, vec![-1.0, 0.0]).unwrap();
        live.delete(u64::MAX).unwrap();
        assert_eq!(source.next_document_after(None, &owner).unwrap(), Some(7));
        assert_eq!(
            source.next_document_after(Some(7), &owner).unwrap(),
            Some(u64::MAX)
        );
        assert_eq!(
            source.next_document_after(Some(u64::MAX), &owner).unwrap(),
            None
        );
        assert_eq!(source.document_vector_count(7, &owner).unwrap(), 2);
        assert_eq!(source.document_vector_count(9, &owner).unwrap(), 0);
        assert!(source.read_vector(7, 2, &owner).unwrap().is_none());
        assert_eq!(
            &*source.read_vector(7, 0, &owner).unwrap().unwrap(),
            &[8.0, 0.0]
        );
        assert_eq!(
            source.read_vector(7, 1, &owner).unwrap().unwrap()[0].to_bits(),
            (-0.0_f32).to_bits()
        );
        let nested = retained.snapshot_with_control(&owner).unwrap();
        let nested_source = nested.vector_read_snapshot(&owner).unwrap().unwrap();
        assert_eq!(
            nested_source.corpus_fingerprint(&owner).unwrap(),
            fingerprint
        );
        drop((retained, nested, source, live));
        assert_eq!(
            &*nested_source
                .read_vector(u64::MAX, 0, &owner)
                .unwrap()
                .unwrap(),
            &[0.0, -2.0]
        );
        let invocation = StorageReadControl::with_limit(1024);
        invocation.cancellation().cancel();
        assert!(nested_source.corpus_fingerprint(&invocation).is_err());
        owner.cancellation().cancel();
        assert!(nested_source
            .read_vector(7, 0, &StorageReadControl::with_limit(1024))
            .is_err());
        drop(nested_source);
        assert_eq!(owner.memory().used(), 0);
    }
}

#[test]
fn canonical_capture_shares_existing_coordinates_under_a_header_only_allowance() {
    for mut live in [
        Box::new(HNSWIndex::new(16_384)) as Box<dyn VectorIndex>,
        Box::new(IVFIndex::with_params(16_384, 1, 1, 1)),
    ] {
        live.add(3, vec![2.0; 16_384]).unwrap();
        let owner = StorageReadControl::with_limit(2 << 20);
        let snapshot = live.snapshot_with_control(&owner).unwrap();
        let pressure = owner
            .memory()
            .reserve((2 << 20) - owner.memory().used() - 2048)
            .unwrap();
        let source = snapshot.vector_read_snapshot(&owner).unwrap().unwrap();
        assert_eq!(source.document_vector_count(3, &owner).unwrap(), 1);
        assert!(source.corpus_fingerprint(&owner).unwrap().is_some());
        drop((pressure, snapshot, live));
        assert_eq!(
            source.read_vector(3, 0, &owner).unwrap().unwrap()[16_383],
            2.0
        );
        drop(source);
        assert_eq!(owner.memory().used(), 0);
    }
}

#[test]
fn canonical_readers_bind_original_controls_when_the_snapshot_is_unbound() {
    for mut live in [
        Box::new(HNSWIndex::new(2)) as Box<dyn VectorIndex>,
        Box::new(IVFIndex::with_params(2, 1, 1, 1)),
    ] {
        live.add(3, vec![8.0, 0.0]).unwrap();
        let snapshot = live.snapshot().unwrap();
        let original = StorageReadControl::with_limit(4096);
        let independent = StorageReadControl::with_limit(4096);
        let bound = snapshot.snapshot_with_control(&original).unwrap();
        let nested = bound.snapshot_with_control(&independent).unwrap();
        let first = snapshot.vector_read_snapshot(&original).unwrap().unwrap();
        let second = snapshot
            .vector_read_snapshot(&independent)
            .unwrap()
            .unwrap();
        assert_eq!(
            first.corpus_fingerprint(&independent).unwrap(),
            second.corpus_fingerprint(&independent).unwrap()
        );
        assert_eq!(
            first.corpus_fingerprint(&independent).unwrap(),
            bound
                .vector_read_snapshot(&independent)
                .unwrap()
                .unwrap()
                .corpus_fingerprint(&independent)
                .unwrap()
        );
        drop((snapshot, live));
        original.cancellation().cancel();
        assert!(bound
            .search_knn_with_control(&[1.0, 0.0], 1, &independent)
            .is_err());
        assert!(nested.search_knn(&[1.0, 0.0], 1).is_err());
        assert!(first.corpus_fingerprint(&independent).is_err());
        assert!(first.next_document_after(None, &independent).is_err());
        assert!(first.document_vector_count(3, &independent).is_err());
        assert!(first.read_vector(3, 0, &independent).is_err());
        assert_eq!(
            &*second.read_vector(3, 0, &independent).unwrap().unwrap(),
            &[8.0, 0.0]
        );
        drop((first, second, bound, nested));
        assert_eq!(original.memory().used(), 0);
        assert_eq!(independent.memory().used(), 0);
    }
}