uqa_storage/
key_value.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Backend-neutral Key/Value storage.
8//!
9//! This module is the logical storage boundary for non-relational
10//! persistence. Concrete stores only need ordered byte keys, atomic
11//! batches, prefix scans, and transaction hooks. Catalog, document,
12//! inverted-index, and vector-index behavior stays above that boundary.
13
14use std::collections::{BTreeMap, BTreeSet};
15use std::sync::Arc;
16
17use parking_lot::Mutex;
18use serde::{Deserialize, Serialize};
19use uqa_analysis::Analyzer;
20use uqa_core::{DocId, FieldName, IndexStats, Payload, PostingEntry, PostingList, Value};
21
22use crate::backend::{PersistentStorageBackend, PersistentStorageIdentity};
23use crate::document_store::{Document, DocumentStore};
24use crate::inverted_index::{AnalyzerPhase, InvertedIndex};
25use crate::vector_index::{
26    validate_vector_values, VectorIndex, VectorIndexOpenMode, VectorIndexSpec,
27};
28use crate::{StorageBackendError, StorageBackendResult};
29
30mod catalog;
31pub use catalog::KeyValueCatalog;
32mod diskann;
33pub use diskann::identifiers as diskann_identifiers;
34pub use diskann::{
35    publication, DiskANNMaintenanceStatus, DiskANNMaintenanceStep, DiskANNStageStatus,
36    KeyValueDiskANNMaintenance, KeyValueDiskANNMappingMaintenance, KeyValueDiskANNPruner,
37    KeyValueDiskANNSource, KeyValueDiskANNStage, KeyValueDiskANNStore,
38};
39mod graph_commit;
40mod table_owners;
41pub use graph_commit::KeyValueGraphRecords;
42mod hnsw_records;
43mod index_view;
44mod ivf_records;
45mod maintenance_records;
46mod notification_records;
47pub use notification_records::KeyValueNotificationRecords;
48pub(crate) mod record_json;
49pub use maintenance_records::KeyValueMaintenanceRecords;
50mod vector_records;
51pub use hnsw_records::KeyValueHNSWRecords;
52pub use ivf_records::KeyValueIVFRecords;
53pub(crate) mod occurrence_records;
54pub use occurrence_records::KeyValueOccurrenceRecords;
55mod view;
56pub use view::{
57    KeyValueMutation, KeyValueRead, KeyValueReadRevision, KeyValueReadScope,
58    KeyValueVersionedMutation,
59};
60
61const TAG_METADATA: u8 = b'm';
62const TAG_TABLE: u8 = b't';
63const TAG_MODEL: u8 = b'M';
64const TAG_SCORING_PARAMS: u8 = b'S';
65const TAG_NAMED_GRAPH: u8 = b'g';
66const TAG_VERTEX: u8 = b'V';
67const TAG_EDGE: u8 = b'E';
68const TAG_GRAPH_MEMBERSHIP: u8 = b'G';
69const TAG_GRAPH_LOOKUP: u8 = b'J';
70const TAG_ANALYZER: u8 = b'a';
71const TAG_ANALYZER_DESCRIPTOR: u8 = b'D';
72const TAG_FIELD_ANALYZER_BINDING: u8 = b'U';
73const TAG_TABLE_FIELD_ANALYZER: u8 = b'A';
74const TAG_FOREIGN_SERVER: u8 = b'F';
75const TAG_FOREIGN_TABLE: u8 = b'T';
76const TAG_CATALOG_INDEX: u8 = b'C';
77const TAG_PATH_INDEX: u8 = b'P';
78const TAG_PATH_INDEX_DATA: u8 = b'Q';
79const TAG_COLUMN_STATS: u8 = b'c';
80const TAG_SCHEMA: u8 = b's';
81const TAG_SEQUENCE: u8 = b'q';
82const TAG_RELATION: u8 = b'R';
83const TAG_VIEW: u8 = b'w';
84const TAG_DOCUMENT: u8 = b'd';
85const TAG_POSTING: u8 = b'p';
86const TAG_OCCURRENCE_INDEX: u8 = b'e';
87const TAG_POSTING_CLUSTER_SCORE: u8 = b'k';
88const TAG_POSTING_CLUSTER_POSITIONS: u8 = b'o';
89const TAG_POSTING_DOCUMENT: u8 = b'x';
90const TAG_DOC_LENGTH: u8 = b'l';
91const TAG_FIELD_STATS: u8 = b'f';
92const TAG_REVERSE_POSTING: u8 = b'r';
93const TAG_VECTOR: u8 = b'v';
94const TAG_BTREE_INDEX: u8 = b'B';
95const TAG_BTREE_ENTRY: u8 = b'b';
96const TAG_NAMED_BTREE_INDEX: u8 = b'N';
97const TAG_NAMED_BTREE_ENTRY: u8 = b'n';
98const TAG_IVF_METADATA: u8 = b'I';
99const TAG_IVF_CENTROID: u8 = b'i';
100const TAG_IVF_ASSIGNMENT: u8 = b'j';
101const TAG_HNSW_METADATA: u8 = b'H';
102const TAG_HNSW_NODE: u8 = b'h';
103
104/// Prefix for the unambiguous document encoding introduced after JSON arrays
105/// became ordinary [`Value::List`] values. A legacy document is plain JSON and
106/// therefore cannot start with NUL; the prefix lets reads preserve the old
107/// `Bytes`-before-`List` interpretation without misreading newly written lists.
108const DOCUMENT_VALUE_V1_PREFIX: &[u8] = b"\0uqa-document-json-v1\0";
109const DOCUMENT_VALUE_V2_PREFIX: &[u8] = b"\0uqa-document-record-v2\0";
110
111#[derive(Debug, Clone)]
112enum KeyValueBatchOperation {
113    Put(Vec<u8>, Vec<u8>),
114    Delete(Vec<u8>),
115    DeletePrefix(Vec<u8>),
116}
117
118/// Atomic mutation buffer for a [`KeyValueStore`].
119pub trait KeyValueBatch {
120    /// Original SSI participant captured with this evaluated mutation. Wrappers must forward this capability; a batch cannot move to another participant before publication.
121    fn serializable_participant(&self) -> Option<crate::mvcc::SerializableTransactionId> {
122        None
123    }
124    /// Stage a logical write with the evaluated records. Failed application discards its new intents and records together while retaining already established dependencies. Providers without this capability reject the request.
125    fn observe_serializable_write(
126        &mut self,
127        _predicate: crate::mvcc::SerializablePredicate<'_>,
128    ) -> StorageBackendResult<()> {
129        Err(StorageBackendError::Other(
130            "atomic serializable write observations are not supported".into(),
131        ))
132    }
133    /// Write a record at a key that never had one, as the caller has established. Such a key has no committed revision, so the write expects none without reading one, and a record that exists after all fails the commit as a write conflict instead of being replaced. The default reads the revision as `put` does; capable wrappers must forward this method to keep the write blind.
134    fn put_unused(&mut self, key: &[u8], value: &[u8]) -> StorageBackendResult<()> {
135        self.put(key, value)
136    }
137    /// Require this record's original committed revision at publication without replacing it. This permits independent data writers to share a definition. Stores without commit-time read validation reject this operation; capable wrappers must forward it.
138    fn require_unchanged(&mut self, _key: &[u8]) -> StorageBackendResult<()> {
139        Err(StorageBackendError::Other(
140            "commit-time record requirements are not supported".into(),
141        ))
142    }
143    /// Require an actual committed record observed through an independently retained metadata view, without advancing the transaction's data snapshot. The revision must identify this same key in this database; private and unversioned provenance is rejected. Reserved for storage-owned independently staged resources, not SQL row writes.
144    fn require_observed(
145        &mut self,
146        _key: &[u8],
147        _revision: &KeyValueReadRevision,
148    ) -> StorageBackendResult<()> {
149        Err(StorageBackendError::Other(
150            "observed metadata requirements are not supported".into(),
151        ))
152    }
153    /// Replace an independently staged metadata record using its observed committed precondition. Existing private changes to the key are rejected. Commit conflict, savepoint and receipt semantics are the same as other evaluated writes; this does not refresh the caller's snapshot.
154    fn put_observed(
155        &mut self,
156        _key: &[u8],
157        _value: &[u8],
158        _revision: &KeyValueReadRevision,
159    ) -> StorageBackendResult<()> {
160        Err(StorageBackendError::Other(
161            "observed metadata replacements are not supported".into(),
162        ))
163    }
164    /// Replace metadata and retain its independently staged read source in the same evaluated private write. The source follows savepoint undo, retained views and original receipt resolution; replacing or deleting the key releases it from the new view. Only the bytes are persisted. Capable wrappers must forward the source unchanged.
165    fn put_with_retained_source(
166        &mut self,
167        _key: &[u8],
168        _value: &[u8],
169        _source: Arc<dyn KeyValueRead + Send + Sync>,
170    ) -> StorageBackendResult<()> {
171        Err(StorageBackendError::Other(
172            "private metadata read sources are not supported".into(),
173        ))
174    }
175    /// Publish a new revision of an immutable marker, merging concurrent touches of the same value. Structural changes fence the marker and replace their definition; data changes require the definition and touch the marker. The payload must contain only immutable owner/format data. Capable wrappers must forward this operation.
176    fn touch_marker(&mut self, _key: &[u8], _value: &[u8]) -> StorageBackendResult<()> {
177        Err(StorageBackendError::Other(
178            "mergeable revision markers are not supported".into(),
179        ))
180    }
181
182    /// Stage an actual canonical origin with its evaluated complete tensor. Its original write precondition remains strict; common MVCC maintains all selected-generation populations without replaying the mutation. Capable wrappers must forward this operation.
183    fn replace_diskann_origin(&mut self, _key: &[u8], _value: &[u8]) -> StorageBackendResult<()> {
184        Err(StorageBackendError::Other(
185            "DiskANN population mutations are not supported".into(),
186        ))
187    }
188
189    /// Remove the canonical origin invalidated by a raw writer. Concurrent stores reject an existing population unless their complete publication census validates the final canonical view. Validation follows private application, command refresh and final publication, including an originally absent origin. Capable wrappers must forward this operation.
190    fn invalidate_diskann_origin(&mut self, key: &[u8]) -> StorageBackendResult<()> {
191        self.delete(key)
192    }
193
194    /// Retain already verified complete build origins for one authorized generation publication. The template supplies physical row encoding, not trusted counts; common MVCC computes the private and committed census on their actual canonical views. The catalog/head owner must authorize and guard this publication in the same batch.
195    fn publish_diskann_population(
196        &mut self,
197        _key: &[u8],
198        _template: &[u8],
199        _origins: crate::diskann_index::pages::DiskANNOriginReader,
200    ) -> StorageBackendResult<()> {
201        Err(StorageBackendError::Other(
202            "DiskANN population publication is not supported".into(),
203        ))
204    }
205
206    /// Retire a generation's population header and witnesses, including witnesses added by intervening disjoint writers. The generation/head owner must retain its original publication guards. This operation follows private undo and original receipt resolution.
207    fn retire_diskann_population(&mut self, _key: &[u8]) -> StorageBackendResult<()> {
208        Err(StorageBackendError::Other(
209            "DiskANN population retirement is not supported".into(),
210        ))
211    }
212    /// Stage a durable identifier observation before publishing this batch's records. Successful observations survive later transaction/savepoint rollback. Dropping an unevaluated batch consumes nothing. Stores without autonomous allocation reject this operation; capable wrappers must forward it.
213    fn observe_identifier(&mut self, _namespace: &[u8], _value: u64) -> StorageBackendResult<()> {
214        Err(StorageBackendError::Other(
215            "durable identifier observations are not supported".into(),
216        ))
217    }
218    /// Carry the source's durable identifier watermark into the target before this batch publishes its records. Successful watermark updates survive undo, just like observations. Owning lifecycle code must coordinate the accompanying definition change with data writers. Capable wrappers must forward this namespace operation.
219    fn inherit_identifiers(&mut self, _from: &[u8], _to: &[u8]) -> StorageBackendResult<()> {
220        Err(StorageBackendError::Other(
221            "durable identifier inheritance is not supported".into(),
222        ))
223    }
224    fn put(&mut self, key: &[u8], value: &[u8]) -> StorageBackendResult<()>;
225    fn delete(&mut self, key: &[u8]) -> StorageBackendResult<()>;
226    fn delete_prefix(&mut self, prefix: &[u8]) -> StorageBackendResult<()>;
227    /// Delete only the keys selected by this evaluated prefix, accepting concurrent deletion of those keys. A concurrent live replacement still conflicts, and explicit read requirements remain strict. Reserved for obsolete internal records; callers must separately fence structural changes and new insertions. Concurrent wrappers must forward this operation.
228    fn delete_prefix_allow_absent(&mut self, prefix: &[u8]) -> StorageBackendResult<()> {
229        self.delete_prefix(prefix)
230    }
231    /// Retain already evaluated document input in the same atomic batch as its canonical values and IVF preview. Concurrent wrappers must forward this call.
232    fn ivf_mutation(
233        &mut self,
234        _metadata: &[u8],
235        _mutation: crate::ivf_index::IVFMutation<'_>,
236    ) -> StorageBackendResult<()> {
237        Ok(())
238    }
239    /// The immutable preview serves private reads; concurrent commit preparation may replace it with an internally recalculated generation.
240    fn preview_ivf_record(&mut self, key: &[u8], value: Option<&[u8]>) -> StorageBackendResult<()> {
241        match value {
242            Some(value) => self.put(key, value),
243            None => self.delete(key),
244        }
245    }
246    fn preview_ivf_prefix(&mut self, prefix: &[u8]) -> StorageBackendResult<()> {
247        self.delete_prefix(prefix)
248    }
249    /// Fence each existing IVF definition under this metadata prefix, including definitions with no vectors. Serialized stores already exclude competing writers.
250    fn fence_ivf_prefix(&mut self, _prefix: &[u8]) -> StorageBackendResult<()> {
251        Ok(())
252    }
253    /// Retain already evaluated document input in the same atomic batch as its canonical values and HNSW preview. Concurrent wrappers must forward this call.
254    fn hnsw_mutation(
255        &mut self,
256        _metadata: &[u8],
257        _mutation: crate::hnsw_index::HNSWMutation<'_>,
258    ) -> StorageBackendResult<()> {
259        Ok(())
260    }
261    /// The immutable preview serves private reads; concurrent commit preparation may replace it with an internally recalculated generation.
262    fn preview_hnsw_record(
263        &mut self,
264        key: &[u8],
265        value: Option<&[u8]>,
266    ) -> StorageBackendResult<()> {
267        match value {
268            Some(value) => self.put(key, value),
269            None => self.delete(key),
270        }
271    }
272    fn preview_hnsw_prefix(&mut self, prefix: &[u8]) -> StorageBackendResult<()> {
273        self.delete_prefix(prefix)
274    }
275    /// Fence each existing HNSW definition under this metadata prefix, including definitions with no vectors. Serialized stores already exclude competing writers.
276    fn fence_hnsw_prefix(&mut self, _prefix: &[u8]) -> StorageBackendResult<()> {
277        Ok(())
278    }
279    /// Stage an evaluated occurrence cluster, field total or source marker using the persistence's declared record layout. Source mutations must include their format marker and document guards. Concurrent stores merge only these explicitly typed replacements; ordinary byte writes remain conditional replacements.
280    fn replace_occurrence_record(
281        &mut self,
282        key: &[u8],
283        value: Option<&[u8]>,
284    ) -> StorageBackendResult<()> {
285        match value {
286            Some(value) => self.put(key, value),
287            None => self.delete(key),
288        }
289    }
290    /// Stage source-owned occurrence-cache invalidation. The source marker in the same batch also drives discovery of caches published after the retained source view.
291    fn invalidate_occurrence_prefix(&mut self, prefix: &[u8]) -> StorageBackendResult<()> {
292        self.delete_prefix(prefix)
293    }
294    /// Protect whole-document replacement even when two writers select different fields of an originally absent document.
295    fn occurrence_document(&mut self, _table: &str, _document: DocId) -> StorageBackendResult<()> {
296        Ok(())
297    }
298    /// Fence a structural occurrence change, including an empty rebuild, drop, purge or rename. Serialized projections already retain their native mutation boundary.
299    fn reset_occurrences(&mut self, _table: &str) -> StorageBackendResult<()> {
300        Ok(())
301    }
302    /// Retain the original precondition and current value of a record, including an absent tombstone, before later replacements. Concurrent wrappers must forward this call; serialized stores already exclude intervening writers.
303    fn fence_record(&mut self, _key: &[u8]) -> StorageBackendResult<()> {
304        Ok(())
305    }
306    /// Stage an evaluated statistics-maintenance replacement using the provider's declared layout. Keep ordinary metadata writes canonical.
307    fn replace_statistics_maintenance(
308        &mut self,
309        key: &[u8],
310        value: &[u8],
311    ) -> StorageBackendResult<()> {
312        self.put(key, value)
313    }
314    /// Record graph-cache dependencies in the same atomic batch. Concurrent MVCC stores must resolve these logical effects before admitting a commit; serialized legacy stores use the ordinary preview writes already included by the catalog.
315    fn graph_mutation(
316        &mut self,
317        _mutation: crate::mvcc::GraphMutation<'_>,
318    ) -> StorageBackendResult<()> {
319        Ok(())
320    }
321    /// Maintain read-your-writes for a graph invalidation. MVCC replaces this preview with effects selected from current graph ownership at commit.
322    fn preview_graph_invalidation(
323        &mut self,
324        key: &[u8],
325        value: Option<&[u8]>,
326    ) -> StorageBackendResult<()> {
327        match value {
328            Some(value) => self.put(key, value),
329            None => self.delete(key),
330        }
331    }
332    /// Replace or clear derived graph cache state. Publishing a completed build must also record `GraphMutation::PublishPath` so its source dependencies are validated.
333    fn replace_graph_cache(
334        &mut self,
335        key: &[u8],
336        value: Option<&[u8]>,
337    ) -> StorageBackendResult<()> {
338        match value {
339            Some(value) => self.put(key, value),
340            None => self.delete(key),
341        }
342    }
343    fn commit(self: Box<Self>) -> StorageBackendResult<()>;
344}
345
346pub use vector_index::guards::KeyValueVectorFieldGuards;
347
348pub(crate) fn diskann_tombstone_prefixes() -> [&'static [u8]; 5] {
349    [
350        diskann::READ_PREFIX,
351        vector_index::origin::ROOT,
352        vector_index::origin::journal::ROOT,
353        vector_index::guards::ROOT,
354        vector_index::origin::populations::ROOT,
355    ]
356}
357
358/// Ordered byte-key storage used by Key/Value catalog and index backends.
359pub trait KeyValueStore: Send + Sync {
360    /// Physical resource ownership for retained builds and recovery. Versioned wrappers must forward this capability.
361    fn resource_leases(&self) -> Option<&dyn crate::mvcc::ResourceLeaseProvider> {
362        None
363    }
364
365    /// Credential for database-owned auxiliary files. Wrappers over an encrypted store must forward it; `None` permits unencrypted auxiliary storage.
366    fn auxiliary_encryption_key(&self) -> Option<crate::StorageEncryptionKey> {
367        None
368    }
369
370    /// Shared session allowance, including resources retained by readers after their producer finishes. Versioned wrappers must forward this capability.
371    fn retention_control(&self) -> Option<crate::read_control::StorageReadControl> {
372        None
373    }
374
375    /// Atomic auxiliary notification publication and fresh committed recovery. Session wrappers must forward this capability.
376    fn notification_publications(
377        &self,
378    ) -> Option<&dyn crate::notifications::NotificationPublicationStore> {
379        None
380    }
381
382    /// Original participant admission and logical observations. Wrappers must forward this capability, including original attribution on retained read-only sessions; capability presence does not establish complete SQL SSI support.
383    fn serializable_session(&self) -> Option<&dyn crate::mvcc::SerializableSession> {
384        None
385    }
386
387    /// Reclaim obsolete records and committed history without a whole-file rewrite. Versioned wrappers must forward maintenance; serialized stores that eagerly remove obsolete values may keep the default.
388    fn reclaim_obsolete(&self) -> StorageBackendResult<()> {
389        if self.transaction_model().is_versioned() {
390            return Err(StorageBackendError::Other(
391                "versioned KeyValue maintenance is not implemented by this store".into(),
392            ));
393        }
394        Ok(())
395    }
396
397    /// Reclaim obsolete records and compact physical storage where supported, outside this session's transaction.
398    fn vacuum(&self) -> StorageBackendResult<()> {
399        self.reclaim_obsolete()
400    }
401
402    /// Token shared with the execution that owns this session's writes. Cancellation must not prevent rollback cleanup or diagnostic reads. Versioned wrappers must forward this capability.
403    fn write_cancellation(&self) -> Option<uqa_core::CancellationToken> {
404        None
405    }
406
407    /// Open an independent writer sharing the invoking maintenance operation's allowance and write cancellation. Retained reads and cleanup keep independent cancellation. Capable wrappers must forward this without granting another memory limit.
408    fn open_controlled_session(
409        &self,
410        _control: &crate::read_control::StorageReadControl,
411    ) -> StorageBackendResult<Arc<dyn KeyValueStore>> {
412        Err(StorageBackendError::Other(
413            "controlled independent sessions are not implemented for this store".into(),
414        ))
415    }
416
417    /// Create a transaction-isolated session with the caller's write cancellation and a fresh retention budget.
418    fn open_session_with_cancellation(
419        &self,
420        cancellation: &uqa_core::CancellationToken,
421    ) -> StorageBackendResult<Arc<dyn KeyValueStore>> {
422        cancellation.check()?;
423        if self.transaction_model().is_versioned() {
424            return Err(StorageBackendError::Other(
425                "cancellable independent KeyValue sessions are not implemented by this store"
426                    .into(),
427            ));
428        }
429        self.open_session()
430    }
431
432    /// Retain this exact committed/private view in a separate read-only session, including its original logical reader attribution. The returned session cannot publish records or complete the source transaction. Versioned wrappers must forward this capability; opening a newer independent snapshot is not equivalent.
433    fn open_retained_read_session(
434        &self,
435        cancellation: &uqa_core::CancellationToken,
436    ) -> StorageBackendResult<Arc<dyn KeyValueStore>> {
437        cancellation.check()?;
438        Err(StorageBackendError::Other(
439            "retained read sessions are not implemented by this KeyValue store".into(),
440        ))
441    }
442
443    /// Whether writes remain private while independent sessions read and write. Versioned stores must provide command refresh, retained reads, savepoint undo and conditional publication under one reported affinity.
444    fn transaction_model(&self) -> crate::StorageTransactionModel {
445        crate::StorageTransactionModel::ProviderSerialized
446    }
447
448    /// Durable reservations outside private record undo. Serialized custom stores may omit this capability; wrappers over a capable store must forward it.
449    fn identifier_allocator(&self) -> Option<&dyn crate::mvcc::IdentifierAllocator> {
450        None
451    }
452
453    /// Evaluate a compound read once against one fixed committed/private view. The callback must use the supplied reader and must not reenter this session. Stores without this capability reject it explicitly.
454    fn with_read_view(&self, _read: &mut KeyValueReadScope<'_>) -> StorageBackendResult<()> {
455        Err(StorageBackendError::Other(
456            "compound KeyValue reads are not supported by this store".into(),
457        ))
458    }
459
460    /// Evaluate once against the same view that supplies write preconditions, then atomically stage the batch. Errors and unwinds discard that batch; failed publication retains the evaluated attempt. The callback must not reenter this session or perform transaction control.
461    fn with_mutation(&self, _mutate: &mut KeyValueMutation<'_>) -> StorageBackendResult<()> {
462        Err(StorageBackendError::Other(
463            "atomic KeyValue evaluation is not supported by this store".into(),
464        ))
465    }
466
467    /// Evaluate and stage once with the actual durable writer and a non-reused mutation revision. Origins may be persisted alongside canonical values, but do not prove publication or visibility. The callback has the same non-reentrancy rules as `with_mutation`; capable wrappers must preserve the original origin and atomic batch.
468    fn with_versioned_mutation(
469        &self,
470        _mutate: &mut KeyValueVersionedMutation<'_>,
471    ) -> StorageBackendResult<()> {
472        Err(StorageBackendError::Other(
473            "versioned KeyValue mutation origins are not supported by this store".into(),
474        ))
475    }
476
477    /// Identity of the transaction context shared by this store's catalog and data handles. Independent sessions must report different identities, even over the same file.
478    fn transaction_affinity(&self) -> Option<crate::StorageSessionAffinity> {
479        None
480    }
481
482    fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
483        Ok(None)
484    }
485
486    /// Open an independent transaction session over the same logical store. The default keeps simple test/custom stores source-compatible while making the missing MVCC capability explicit when a persistent engine needs a committed reader alongside a pinned statement snapshot.
487    fn open_session(&self) -> StorageBackendResult<Arc<dyn KeyValueStore>> {
488        Err(StorageBackendError::Other(
489            "independent sessions are not implemented for this KeyValue store".into(),
490        ))
491    }
492
493    fn get(&self, key: &[u8]) -> StorageBackendResult<Option<Vec<u8>>>;
494    /// Visit one borrowed value under the retained read. Providers reserve any temporary encoded payload before fetching it; callbacks must not reenter the store.
495    fn visit_value(
496        &self,
497        _key: &[u8],
498        control: &crate::read_control::StorageReadControl,
499        _visit: &mut crate::read_control::ValueReadVisitor<'_>,
500    ) -> StorageBackendResult<()> {
501        control.check()?;
502        Err(StorageBackendError::Other(
503            "controlled value reads are not supported by this KeyValue store".into(),
504        ))
505    }
506    /// Visit one value with an encoded-size cap checked before provider materialization. This is distinct from the shared workspace allowance; unsupported stores must not fetch an unbounded value as a fallback.
507    fn visit_value_bounded(
508        &self,
509        _key: &[u8],
510        _max_bytes: usize,
511        control: &crate::read_control::StorageReadControl,
512        _visit: &mut crate::read_control::ValueReadVisitor<'_>,
513    ) -> StorageBackendResult<()> {
514        control.check()?;
515        Err(StorageBackendError::Other(
516            "size-bounded value reads are not supported by this KeyValue store".into(),
517        ))
518    }
519
520    /// Visit at most `limit` entries in key order, strictly after `after` when supplied. Encoded values remain borrowed from their provider owner and callbacks must not reenter the store.
521    fn visit_prefix_after(
522        &self,
523        _prefix: &[u8],
524        _after: Option<&[u8]>,
525        _limit: usize,
526        control: &crate::read_control::StorageReadControl,
527        _visit: &mut crate::read_control::KeyValueReadVisitor<'_>,
528    ) -> StorageBackendResult<()> {
529        control.check()?;
530        Err(StorageBackendError::Other(
531            "controlled prefix reads are not supported by this KeyValue store".into(),
532        ))
533    }
534    /// Test prefix existence under the read allowance. Providers that materialize values should implement a key-only probe.
535    fn contains_prefix_budgeted(
536        &self,
537        prefix: &[u8],
538        control: &crate::read_control::StorageReadControl,
539    ) -> StorageBackendResult<bool> {
540        let mut found = false;
541        self.visit_prefix_after(prefix, None, 1, control, &mut |_, _| {
542            found = true;
543            Ok(())
544        })?;
545        control.check()?;
546        Ok(found)
547    }
548
549    fn contains_key(&self, key: &[u8]) -> StorageBackendResult<bool> {
550        self.get(key).map(|value| value.is_some())
551    }
552    fn put(&self, key: &[u8], value: &[u8]) -> StorageBackendResult<()>;
553    fn delete(&self, key: &[u8]) -> StorageBackendResult<()>;
554    fn scan_prefix(&self, prefix: &[u8]) -> StorageBackendResult<Vec<(Vec<u8>, Vec<u8>)>>;
555    /// Return at most `limit` key/value pairs in key order, strictly after
556    /// `after` when supplied. This is the bounded value cursor used by large
557    /// format migrations and other paged consumers.
558    fn scan_prefix_after(
559        &self,
560        prefix: &[u8],
561        after: Option<&[u8]>,
562        limit: usize,
563    ) -> StorageBackendResult<Vec<(Vec<u8>, Vec<u8>)>> {
564        if limit == 0 {
565            return Ok(Vec::new());
566        }
567        Ok(self
568            .scan_prefix(prefix)?
569            .into_iter()
570            .filter(|(key, _)| after.is_none_or(|after| key.as_slice() > after))
571            .take(limit)
572            .collect())
573    }
574    /// Return at most `limit` keys in key order, strictly after `after` when
575    /// it is present. Backends should override this method with a key-only,
576    /// bounded range scan so cursor consumers neither materialize the entire
577    /// prefix nor read values they do not need on every page.
578    fn scan_prefix_keys_after(
579        &self,
580        prefix: &[u8],
581        after: Option<&[u8]>,
582        limit: usize,
583    ) -> StorageBackendResult<Vec<Vec<u8>>> {
584        if limit == 0 {
585            return Ok(Vec::new());
586        }
587        Ok(self
588            .scan_prefix(prefix)?
589            .into_iter()
590            .filter(|(key, _)| after.is_none_or(|after| key.as_slice() > after))
591            .take(limit)
592            .map(|(key, _)| key)
593            .collect())
594    }
595    fn first_prefix_after(
596        &self,
597        prefix: &[u8],
598        after: Option<&[u8]>,
599    ) -> StorageBackendResult<Option<(Vec<u8>, Vec<u8>)>> {
600        Ok(self
601            .scan_prefix(prefix)?
602            .into_iter()
603            .find(|(key, _)| after.is_none_or(|after| key.as_slice() > after)))
604    }
605    fn delete_prefix(&self, prefix: &[u8]) -> StorageBackendResult<usize>;
606    fn batch(&self) -> Box<dyn KeyValueBatch + '_>;
607
608    fn begin_transaction(&self) -> StorageBackendResult<()> {
609        Err(StorageBackendError::Other(
610            "KeyValue transaction begin is not implemented for this store".into(),
611        ))
612    }
613
614    fn begin_read_transaction(&self) -> StorageBackendResult<()> {
615        self.begin_transaction()
616    }
617
618    fn begin_upgradeable_transaction(&self) -> StorageBackendResult<()> {
619        self.begin_transaction()
620    }
621
622    fn in_transaction(&self) -> bool;
623
624    /// Report ordinary record and derived-index writes, excluding typed auxiliary notification effects. Ordinary writes remain forbidden in read-only transactions.
625    fn transaction_has_written(&self) -> StorageBackendResult<bool>;
626
627    fn change_version(&self) -> StorageBackendResult<Option<u64>> {
628        Ok(None)
629    }
630
631    /// Complete committed/private visibility identity, including savepoint undo. `None` requires consumers to use their existing refresh path. Tokens are process-local and do not replace transaction isolation or logical read observations.
632    fn read_view_revision(&self) -> StorageBackendResult<Option<KeyValueReadRevision>> {
633        Ok(None)
634    }
635
636    fn change_version_monitor_is_nonblocking(&self) -> StorageBackendResult<bool> {
637        Ok(true)
638    }
639
640    fn pin_transaction_snapshot(&self) -> StorageBackendResult<()> {
641        Ok(())
642    }
643
644    /// Advance an active logical transaction's committed view while preserving its private writes and retained readers. The caller chooses command boundaries allowed by SQL isolation. Stores without this capability must reject the request; ending or replaying the caller's transaction is not a valid fallback.
645    fn refresh_transaction_snapshot(
646        &self,
647        _cancellation: &uqa_core::CancellationToken,
648    ) -> StorageBackendResult<()> {
649        Err(StorageBackendError::Other(
650            "transaction snapshot refresh is not supported by this KeyValue store".into(),
651        ))
652    }
653
654    fn commit_transaction(&self) -> StorageBackendResult<()> {
655        Err(StorageBackendError::Other(
656            "KeyValue transaction commit is not implemented for this store".into(),
657        ))
658    }
659
660    fn rollback_transaction(&self) -> StorageBackendResult<()> {
661        Err(StorageBackendError::Other(
662            "KeyValue transaction rollback is not implemented for this store".into(),
663        ))
664    }
665
666    fn savepoint(&self, _name: &str) -> StorageBackendResult<()> {
667        Err(StorageBackendError::Other(
668            "KeyValue savepoints are not implemented for this store".into(),
669        ))
670    }
671
672    fn release_savepoint(&self, _name: &str) -> StorageBackendResult<()> {
673        Err(StorageBackendError::Other(
674            "KeyValue savepoint release is not implemented for this store".into(),
675        ))
676    }
677
678    fn rollback_to_savepoint(&self, _name: &str) -> StorageBackendResult<()> {
679        Err(StorageBackendError::Other(
680            "KeyValue savepoint rollback is not implemented for this store".into(),
681        ))
682    }
683}
684
685mod btree_index;
686mod codec;
687pub mod conformance;
688mod document_store;
689mod hnsw_index;
690mod hnsw_persistence;
691mod index_keys;
692mod inverted_index;
693mod ivf_index;
694mod ivf_persistence;
695mod memory_store;
696pub mod occurrence_format;
697mod occurrence_keys;
698mod storage_backend;
699mod vector_index;
700
701pub use codec::prefix_upper_bound;
702pub use document_store::KeyValueDocumentStore;
703pub use hnsw_index::KeyValueHNSWIndex;
704pub use inverted_index::{KeyValueInvertedIndex, OccurrenceStorage};
705pub use ivf_index::KeyValueIVFIndex;
706pub use memory_store::MemoryKeyValueStore;
707pub use storage_backend::KeyValueStorageBackend;
708pub use vector_index::{
709    DiskANNCanonicalVectorVisitor, KeyValueDiskANNCanonical, KeyValueDiskANNHandle,
710    KeyValueDiskANNPopulationRecords, KeyValueVectorIndex, RetainedDiskANNCanonical,
711};
712
713#[cfg(test)]
714mod tests;
uqa_storage/key_value.rs

uqa_storage/
key_value.rs
