Skip to main content

uqa_storage_sqlite/vector_index/diskann/retained/
changes.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Native change identities and canonical tensors share one retained object/field boundary.
8
9use super::{invalid, DiskANNCanonicalOrigin, Family, Identity, RetainedSQLiteDiskANNCanonical};
10use crate::mvcc::native::{decode_record, variable_fields_limit};
11use rusqlite::types::ValueRef;
12use uqa_core::DocId;
13use uqa_storage::{
14    diskann_index::format::{DiskANNChangeIdentity, CANONICAL_ORIGIN_BYTES, CHANGE_IDENTITY_BYTES},
15    mvcc::VersionError,
16    read_control::StorageReadControl,
17    StorageBackendResult,
18};
19
20impl RetainedSQLiteDiskANNCanonical {
21    /// Return each currently journaled document once on this canonical view, including an empty replacement. This does not establish the coverage of a published generation.
22    pub fn next_change_after(
23        &self,
24        mut after: Option<DocId>,
25        control: &StorageReadControl,
26    ) -> StorageBackendResult<Option<DiskANNChangeIdentity>> {
27        loop {
28            self.check(control)?;
29            if after.is_some_and(|document| document >= i64::MAX as DocId) {
30                return Ok(None);
31            }
32            let Some(document) = self.next_changed_document(after, control)? else {
33                self.check(control)?;
34                return Ok(None);
35            };
36            after = Some(document);
37            let Some(canonical) = self.record(document, control)? else {
38                continue;
39            };
40            let identity = DiskANNChangeIdentity::new(document, canonical.version());
41            if self.matches_change(identity, canonical, control)? {
42                self.check(control)?;
43                return Ok(Some(identity));
44            }
45        }
46    }
47
48    fn next_changed_document(
49        &self,
50        after: Option<DocId>,
51        control: &StorageReadControl,
52    ) -> StorageBackendResult<Option<DocId>> {
53        let Some(owner) = self.owner else {
54            return Ok(None);
55        };
56        let identity = Identity::new(Family::VectorChanges, owner)
57            .map_err(VersionError::into_storage_error)?;
58        let prefix = identity
59            .encode_prefix(&[ValueRef::Text(&self.field)], control)
60            .map_err(VersionError::into_storage_error)?;
61        let cursor = after
62            .map(|document| {
63                identity.encode_key(
64                    &[
65                        ValueRef::Text(&self.field),
66                        ValueRef::Blob(&DiskANNChangeIdentity::document_end(document)),
67                    ],
68                    control,
69                )
70            })
71            .transpose()
72            .map_err(VersionError::into_storage_error)?;
73        let mut selected = None;
74        self.snapshot.view.visit_keys(&prefix, cursor.as_deref(), usize::MAX, control, &mut |key, metadata| {
75            self.check(control).map_err(VersionError::Storage)?;
76            if !metadata.live {
77                return Ok(true);
78            }
79            let actual = Identity::visit_key_components(key, control, |position, value| {
80                match position {
81                    0 if value == ValueRef::Text(&self.field) => {}
82                    1 => {
83                        let bytes = value.as_blob().map_err(|_| VersionError::InvalidEncoding("invalid native change identity"))?;
84                        let change = DiskANNChangeIdentity::decode(bytes).map_err(VersionError::Storage)?;
85                        if change.document() > i64::MAX as DocId || after.is_some_and(|after| change.document() <= after) {
86                            return Err(VersionError::InvalidEncoding("native change cursor did not advance within its document range"));
87                        }
88                        selected = Some(change.document());
89                    }
90                    _ => return Err(VersionError::InvalidEncoding("native change key escaped its field")),
91                }
92                Ok(())
93            })?;
94            if actual != identity {
95                return Err(VersionError::InvalidEncoding("native change key escaped its owner"));
96            }
97            Ok(false)
98        }).map_err(VersionError::into_storage_error)?;
99        self.check(control)?;
100        Ok(selected)
101    }
102
103    fn matches_change(
104        &self,
105        change: DiskANNChangeIdentity,
106        canonical: DiskANNCanonicalOrigin,
107        control: &StorageReadControl,
108    ) -> StorageBackendResult<bool> {
109        let owner = self
110            .owner
111            .ok_or_else(|| invalid("native change has no canonical owner"))?;
112        let identity = Identity::new(Family::VectorChanges, owner)
113            .map_err(VersionError::into_storage_error)?;
114        let key = identity
115            .encode_key(
116                &[
117                    ValueRef::Text(&self.field),
118                    ValueRef::Blob(&change.encode()),
119                ],
120                control,
121            )
122            .map_err(VersionError::into_storage_error)?;
123        let limit = variable_fields_limit(&[
124            self.table.len(),
125            self.field.len(),
126            CHANGE_IDENTITY_BYTES,
127            CANONICAL_ORIGIN_BYTES,
128        ])
129        .map_err(VersionError::into_storage_error)?;
130        let mut found = false;
131        self.snapshot
132            .view
133            .visit_value_bounded(&key, limit, control, &mut |record| {
134                self.check(control).map_err(VersionError::Storage)?;
135                let Some(bytes) = record.and_then(|record| record.value) else {
136                    return Ok(());
137                };
138                let (actual, row) = decode_record(&key, bytes, control)?;
139                if actual != identity || row[0] != ValueRef::Text(&self.table) {
140                    return Err(VersionError::InvalidEncoding(
141                        "native change row scope mismatch",
142                    ));
143                }
144                let bytes = row[3]
145                    .as_blob()
146                    .map_err(|_| VersionError::InvalidEncoding("invalid native change payload"))?;
147                let origin = DiskANNCanonicalOrigin::decode(bytes, self.dimensions)
148                    .map_err(VersionError::Storage)?;
149                if origin != canonical {
150                    return Err(VersionError::InvalidEncoding(
151                        "native change differs from its canonical origin",
152                    ));
153                }
154                found = true;
155                Ok(())
156            })
157            .map_err(VersionError::into_storage_error)?;
158        self.check(control)?;
159        Ok(found)
160    }
161}