uqa_storage/diskann_index/
canonical.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Ordered canonical observations on a retained provider boundary, independent of graph coverage.
8
9use super::{
10    format::{
11        DiskANNCanonicalOrigin, DiskANNChangeIdentity, DiskANNGeneration, DiskANNVectorVersion,
12    },
13    DiskANNCanonicalVectorVisitor,
14};
15use crate::{mvcc::VersionError, read_control::StorageReadControl, StorageBackendResult};
16use uqa_core::DocId;
17
18mod populations;
19pub use populations::{DiskANNCanonicalCounts, DiskANNPopulationState, DiskANNPopulationWitness};
20
21/// Borrow one canonical ordinal in document order. Failure invalidates the consumer's partial output.
22pub type DiskANNCanonicalCorpusVisitor<'a> =
23    dyn FnMut(DocId, u32, DiskANNVectorVersion, &[f32]) -> StorageBackendResult<()> + 'a;
24
25/// A retained canonical source with the actual versioned change journal on that same view. The lifecycle owner must establish complete coverage before opening a query; absence from this journal alone is not proof of build membership.
26pub trait DiskANNQueryRead: DiskANNCanonicalRead {
27    /// An independently retained complete value view used when an index definition changed. Origins exposed by this source cover only tensors verified equal to its actual versioned source; the remaining tensors join the ordinary uncovered-candidate stream without fabricated mutation versions.
28    fn unversioned_vectors(&self) -> Option<&crate::vector_index::VectorReadSnapshot> {
29        None
30    }
31
32    /// Count visible ordinals from complete origin metadata without decoding coordinates or preparing a graph reader.
33    fn vector_count(&self, control: &StorageReadControl) -> StorageBackendResult<usize> {
34        self.check_control(control)?;
35        let mut after = None;
36        let mut count = 0_usize;
37        while let Some(document) = self.next_document_after(after, control)? {
38            self.check_control(control)?;
39            if after.is_some_and(|previous| document <= previous) {
40                return Err(VersionError::InvalidEncoding(
41                    "canonical vector count cursor did not advance",
42                )
43                .into_storage_error());
44            }
45            let origin = self.document_origin(document, control)?.ok_or_else(|| {
46                VersionError::InvalidEncoding("enumerated vector document has no origin")
47                    .into_storage_error()
48            })?;
49            let vectors = usize::try_from(origin.count())
50                .map_err(|_| uqa_core::memory::MemoryError::SizeOverflow)?;
51            count = count
52                .checked_add(vectors)
53                .ok_or(uqa_core::memory::MemoryError::SizeOverflow)?;
54            after = Some(document);
55        }
56        self.check_control(control)?;
57        Ok(count)
58    }
59
60    /// Test nonempty membership using complete canonical metadata on this exact view.
61    fn contains_vectors(
62        &self,
63        document: DocId,
64        control: &StorageReadControl,
65    ) -> StorageBackendResult<bool> {
66        self.check_control(control)?;
67        let origin = self.document_origin(document, control)?;
68        self.check_control(control)?;
69        Ok(origin.is_some_and(|origin| origin.count() != 0))
70    }
71
72    /// Return the selected mutation and complete tensor cardinality after validating its ordinal-key coverage. This metadata probe must not decode coordinate values or inspect graph pages; empty replacements return Some with count zero.
73    fn document_origin(
74        &self,
75        document: DocId,
76        control: &StorageReadControl,
77    ) -> StorageBackendResult<Option<DiskANNCanonicalOrigin>>;
78
79    /// Return current journaled documents in strictly increasing order, including empty replacements. Validate the immutable change envelope against its current canonical origin; skip obsolete versions without loading their values.
80    fn next_change_after(
81        &self,
82        after: Option<DocId>,
83        control: &StorageReadControl,
84    ) -> StorageBackendResult<Option<DiskANNChangeIdentity>>;
85}
86
87/// One fixed committed/private canonical source. Implementations retain their original visibility and controls; callbacks must not reenter the source.
88pub trait DiskANNCanonicalRead {
89    /// Exact owner-maintained counts for this canonical view and the requested build coverage. Unsupported associations return `None`; this operation must not enumerate documents, journals or origin pages to reconstruct counts.
90    fn population_counts(
91        &self,
92        _generation: DiskANNGeneration,
93        control: &StorageReadControl,
94    ) -> StorageBackendResult<Option<DiskANNCanonicalCounts>> {
95        self.check_control(control)?;
96        Ok(None)
97    }
98
99    /// Identity of the complete selected canonical field, independent of graph generations. Unsupported sources cannot authorize fixed-model reuse.
100    fn corpus_fingerprint(
101        &self,
102        control: &StorageReadControl,
103    ) -> StorageBackendResult<Option<[u8; 32]>> {
104        self.check_control(control)?;
105        Ok(None)
106    }
107    /// Check the retained source's original controls and the invoking control, including queries that need no provider reads.
108    fn check_control(&self, control: &StorageReadControl) -> StorageBackendResult<()>;
109
110    /// The canonical width, shared by every ordinal on this source.
111    fn dimensions(&self) -> u32;
112
113    /// Return the least live document strictly after `after` from the union of canonical keys and origin keys. Include empty replacements and unstamped canonical values; read key metadata only, with bounded workspace.
114    fn next_document_after(
115        &self,
116        after: Option<DocId>,
117        control: &StorageReadControl,
118    ) -> StorageBackendResult<Option<DocId>>;
119
120    /// Validate the complete ordinal set before returning its origin. Absence and an explicit empty replacement remain distinct.
121    fn origin(
122        &self,
123        document: DocId,
124        control: &StorageReadControl,
125    ) -> StorageBackendResult<Option<DiskANNVectorVersion>>;
126
127    /// Stream every visible ordinal, preserving coordinate bits and checking the complete tensor's origin and shape. Use bounded point reads and a reusable charged vector buffer.
128    fn visit_document(
129        &self,
130        document: DocId,
131        control: &StorageReadControl,
132        visit: &mut DiskANNCanonicalVectorVisitor<'_>,
133    ) -> StorageBackendResult<Option<DiskANNVectorVersion>>;
134
135    /// Own one canonical ordinal so consumers can compare independent fixed sources without nested provider callbacks. Physical owners override this default with bounded point reads.
136    fn read_vector(
137        &self,
138        document: DocId,
139        ordinal: u32,
140        control: &StorageReadControl,
141    ) -> StorageBackendResult<Option<uqa_core::memory::BudgetedVec<f32>>> {
142        let mut output = None;
143        self.visit_document(document, control, &mut |found, _, raw| {
144            if found == ordinal {
145                let mut vector = uqa_core::memory::BudgetedVec::new(control.memory());
146                for chunk in raw.chunks(1024) {
147                    self.check_control(control)?;
148                    vector.extend_from_slice(chunk)?;
149                }
150                output = Some(vector);
151            }
152            Ok(())
153        })?;
154        self.check_control(control)?;
155        Ok(output)
156    }
157
158    /// Stream the entire fixed source in document/ordinal order without retaining a corpus directory. Empty replacements are validated but emit no vectors; unstamped values fail instead of disappearing from the corpus.
159    fn visit_all(
160        &self,
161        control: &StorageReadControl,
162        visit: &mut DiskANNCanonicalCorpusVisitor<'_>,
163    ) -> StorageBackendResult<()> {
164        self.check_control(control)?;
165        let mut after = None;
166        while let Some(document) = self.next_document_after(after, control)? {
167            control.check()?;
168            if after.is_some_and(|previous| document <= previous) {
169                return Err(VersionError::InvalidEncoding(
170                    "canonical corpus cursor did not advance",
171                )
172                .into_storage_error());
173            }
174            let origin = self.visit_document(document, control, &mut |ordinal, version, raw| {
175                visit(document, ordinal, version, raw)
176            })?;
177            if origin.is_none() {
178                return Err(VersionError::InvalidEncoding(
179                    "enumerated canonical document has no origin",
180                )
181                .into_storage_error());
182            }
183            after = Some(document);
184        }
185        self.check_control(control)
186    }
187}
uqa_storage/diskann_index/canonical.rs

uqa_storage/diskann_index/
canonical.rs
