1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//
//! Ordered canonical observations on a retained provider boundary, independent of graph coverage.
use super::{
format::{
DiskANNCanonicalOrigin, DiskANNChangeIdentity, DiskANNGeneration, DiskANNVectorVersion,
},
DiskANNCanonicalVectorVisitor,
};
use crate::{mvcc::VersionError, read_control::StorageReadControl, StorageBackendResult};
use uqa_core::DocId;
mod populations;
pub use populations::{DiskANNCanonicalCounts, DiskANNPopulationState, DiskANNPopulationWitness};
/// Borrow one canonical ordinal in document order. Failure invalidates the consumer's partial output.
pub type DiskANNCanonicalCorpusVisitor<'a> =
dyn FnMut(DocId, u32, DiskANNVectorVersion, &[f32]) -> StorageBackendResult<()> + 'a;
/// A retained canonical source with the actual versioned change journal on that same view. The lifecycle owner must establish complete coverage before opening a query; absence from this journal alone is not proof of build membership.
pub trait DiskANNQueryRead: DiskANNCanonicalRead {
/// An independently retained complete value view used when an index definition changed. Origins exposed by this source cover only tensors verified equal to its actual versioned source; the remaining tensors join the ordinary uncovered-candidate stream without fabricated mutation versions.
fn unversioned_vectors(&self) -> Option<&crate::vector_index::VectorReadSnapshot> {
None
}
/// Count visible ordinals from complete origin metadata without decoding coordinates or preparing a graph reader.
fn vector_count(&self, control: &StorageReadControl) -> StorageBackendResult<usize> {
self.check_control(control)?;
let mut after = None;
let mut count = 0_usize;
while let Some(document) = self.next_document_after(after, control)? {
self.check_control(control)?;
if after.is_some_and(|previous| document <= previous) {
return Err(VersionError::InvalidEncoding(
"canonical vector count cursor did not advance",
)
.into_storage_error());
}
let origin = self.document_origin(document, control)?.ok_or_else(|| {
VersionError::InvalidEncoding("enumerated vector document has no origin")
.into_storage_error()
})?;
let vectors = usize::try_from(origin.count())
.map_err(|_| uqa_core::memory::MemoryError::SizeOverflow)?;
count = count
.checked_add(vectors)
.ok_or(uqa_core::memory::MemoryError::SizeOverflow)?;
after = Some(document);
}
self.check_control(control)?;
Ok(count)
}
/// Test nonempty membership using complete canonical metadata on this exact view.
fn contains_vectors(
&self,
document: DocId,
control: &StorageReadControl,
) -> StorageBackendResult<bool> {
self.check_control(control)?;
let origin = self.document_origin(document, control)?;
self.check_control(control)?;
Ok(origin.is_some_and(|origin| origin.count() != 0))
}
/// Return the selected mutation and complete tensor cardinality after validating its ordinal-key coverage. This metadata probe must not decode coordinate values or inspect graph pages; empty replacements return Some with count zero.
fn document_origin(
&self,
document: DocId,
control: &StorageReadControl,
) -> StorageBackendResult<Option<DiskANNCanonicalOrigin>>;
/// Return current journaled documents in strictly increasing order, including empty replacements. Validate the immutable change envelope against its current canonical origin; skip obsolete versions without loading their values.
fn next_change_after(
&self,
after: Option<DocId>,
control: &StorageReadControl,
) -> StorageBackendResult<Option<DiskANNChangeIdentity>>;
}
/// One fixed committed/private canonical source. Implementations retain their original visibility and controls; callbacks must not reenter the source.
pub trait DiskANNCanonicalRead {
/// Exact owner-maintained counts for this canonical view and the requested build coverage. Unsupported associations return `None`; this operation must not enumerate documents, journals or origin pages to reconstruct counts.
fn population_counts(
&self,
_generation: DiskANNGeneration,
control: &StorageReadControl,
) -> StorageBackendResult<Option<DiskANNCanonicalCounts>> {
self.check_control(control)?;
Ok(None)
}
/// Identity of the complete selected canonical field, independent of graph generations. Unsupported sources cannot authorize fixed-model reuse.
fn corpus_fingerprint(
&self,
control: &StorageReadControl,
) -> StorageBackendResult<Option<[u8; 32]>> {
self.check_control(control)?;
Ok(None)
}
/// Check the retained source's original controls and the invoking control, including queries that need no provider reads.
fn check_control(&self, control: &StorageReadControl) -> StorageBackendResult<()>;
/// The canonical width, shared by every ordinal on this source.
fn dimensions(&self) -> u32;
/// Return the least live document strictly after `after` from the union of canonical keys and origin keys. Include empty replacements and unstamped canonical values; read key metadata only, with bounded workspace.
fn next_document_after(
&self,
after: Option<DocId>,
control: &StorageReadControl,
) -> StorageBackendResult<Option<DocId>>;
/// Validate the complete ordinal set before returning its origin. Absence and an explicit empty replacement remain distinct.
fn origin(
&self,
document: DocId,
control: &StorageReadControl,
) -> StorageBackendResult<Option<DiskANNVectorVersion>>;
/// Stream every visible ordinal, preserving coordinate bits and checking the complete tensor's origin and shape. Use bounded point reads and a reusable charged vector buffer.
fn visit_document(
&self,
document: DocId,
control: &StorageReadControl,
visit: &mut DiskANNCanonicalVectorVisitor<'_>,
) -> StorageBackendResult<Option<DiskANNVectorVersion>>;
/// Own one canonical ordinal so consumers can compare independent fixed sources without nested provider callbacks. Physical owners override this default with bounded point reads.
fn read_vector(
&self,
document: DocId,
ordinal: u32,
control: &StorageReadControl,
) -> StorageBackendResult<Option<uqa_core::memory::BudgetedVec<f32>>> {
let mut output = None;
self.visit_document(document, control, &mut |found, _, raw| {
if found == ordinal {
let mut vector = uqa_core::memory::BudgetedVec::new(control.memory());
for chunk in raw.chunks(1024) {
self.check_control(control)?;
vector.extend_from_slice(chunk)?;
}
output = Some(vector);
}
Ok(())
})?;
self.check_control(control)?;
Ok(output)
}
/// Stream the entire fixed source in document/ordinal order without retaining a corpus directory. Empty replacements are validated but emit no vectors; unstamped values fail instead of disappearing from the corpus.
fn visit_all(
&self,
control: &StorageReadControl,
visit: &mut DiskANNCanonicalCorpusVisitor<'_>,
) -> StorageBackendResult<()> {
self.check_control(control)?;
let mut after = None;
while let Some(document) = self.next_document_after(after, control)? {
control.check()?;
if after.is_some_and(|previous| document <= previous) {
return Err(VersionError::InvalidEncoding(
"canonical corpus cursor did not advance",
)
.into_storage_error());
}
let origin = self.visit_document(document, control, &mut |ordinal, version, raw| {
visit(document, ordinal, version, raw)
})?;
if origin.is_none() {
return Err(VersionError::InvalidEncoding(
"enumerated canonical document has no origin",
)
.into_storage_error());
}
after = Some(document);
}
self.check_control(control)
}
}