uqa_storage/
key_value.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Backend-neutral Key/Value storage.
8//!
9//! This module is the logical storage boundary for non-relational
10//! persistence. Concrete stores only need ordered byte keys, atomic
11//! batches, prefix scans, and transaction hooks. Catalog, document,
12//! inverted-index, and vector-index behavior stays above that boundary.
13
14use std::collections::{BTreeMap, BTreeSet};
15use std::sync::Arc;
16
17use parking_lot::Mutex;
18use serde::{Deserialize, Serialize};
19use uqa_analysis::Analyzer;
20use uqa_core::{DocId, FieldName, IndexStats, Payload, PostingEntry, PostingList, Value};
21
22use crate::backend::{PersistentStorageBackend, PersistentStorageIdentity};
23use crate::document_store::{Document, DocumentStore};
24use crate::inverted_index::{AnalyzerPhase, InvertedIndex};
25use crate::vector_index::{
26    validate_vector_values, VectorIndex, VectorIndexOpenMode, VectorIndexSpec,
27};
28use crate::{StorageBackendError, StorageBackendResult};
29
30mod catalog;
31pub use catalog::KeyValueCatalog;
32mod diskann;
33pub use diskann::identifiers as diskann_identifiers;
34pub use diskann::{
35    publication, DiskANNMaintenanceStatus, DiskANNMaintenanceStep, DiskANNStageStatus,
36    KeyValueDiskANNMaintenance, KeyValueDiskANNMappingMaintenance, KeyValueDiskANNPruner,
37    KeyValueDiskANNSource, KeyValueDiskANNStage, KeyValueDiskANNStore,
38};
39mod graph_commit;
40mod table_owners;
41pub use graph_commit::KeyValueGraphRecords;
42mod hnsw_records;
43mod index_view;
44mod ivf_records;
45mod maintenance_records;
46mod notification_records;
47pub use notification_records::KeyValueNotificationRecords;
48pub(crate) mod record_json;
49pub use maintenance_records::KeyValueMaintenanceRecords;
50mod vector_records;
51pub use hnsw_records::KeyValueHNSWRecords;
52pub use ivf_records::KeyValueIVFRecords;
53pub(crate) mod occurrence_records;
54pub use occurrence_records::KeyValueOccurrenceRecords;
55mod view;
56pub use view::{
57    KeyValueMutation, KeyValueRead, KeyValueReadRevision, KeyValueReadScope,
58    KeyValueVersionedMutation,
59};
60
61const TAG_METADATA: u8 = b'm';
62const TAG_TABLE: u8 = b't';
63const TAG_MODEL: u8 = b'M';
64const TAG_SCORING_PARAMS: u8 = b'S';
65const TAG_NAMED_GRAPH: u8 = b'g';
66const TAG_VERTEX: u8 = b'V';
67const TAG_EDGE: u8 = b'E';
68const TAG_GRAPH_MEMBERSHIP: u8 = b'G';
69const TAG_GRAPH_LOOKUP: u8 = b'J';
70const TAG_ANALYZER: u8 = b'a';
71const TAG_ANALYZER_DESCRIPTOR: u8 = b'D';
72const TAG_FIELD_ANALYZER_BINDING: u8 = b'U';
73const TAG_TABLE_FIELD_ANALYZER: u8 = b'A';
74const TAG_FOREIGN_SERVER: u8 = b'F';
75const TAG_FOREIGN_TABLE: u8 = b'T';
76const TAG_CATALOG_INDEX: u8 = b'C';
77const TAG_PATH_INDEX: u8 = b'P';
78const TAG_PATH_INDEX_DATA: u8 = b'Q';
79const TAG_COLUMN_STATS: u8 = b'c';
80const TAG_SCHEMA: u8 = b's';
81const TAG_SEQUENCE: u8 = b'q';
82const TAG_RELATION: u8 = b'R';
83const TAG_VIEW: u8 = b'w';
84const TAG_DOCUMENT: u8 = b'd';
85const TAG_POSTING: u8 = b'p';
86const TAG_OCCURRENCE_INDEX: u8 = b'e';
87const TAG_POSTING_CLUSTER_SCORE: u8 = b'k';
88const TAG_POSTING_CLUSTER_POSITIONS: u8 = b'o';
89const TAG_POSTING_DOCUMENT: u8 = b'x';
90const TAG_DOC_LENGTH: u8 = b'l';
91const TAG_FIELD_STATS: u8 = b'f';
92const TAG_REVERSE_POSTING: u8 = b'r';
93const TAG_VECTOR: u8 = b'v';
94const TAG_BTREE_INDEX: u8 = b'B';
95const TAG_BTREE_ENTRY: u8 = b'b';
96const TAG_NAMED_BTREE_INDEX: u8 = b'N';
97const TAG_NAMED_BTREE_ENTRY: u8 = b'n';
98const TAG_IVF_METADATA: u8 = b'I';
99const TAG_IVF_CENTROID: u8 = b'i';
100const TAG_IVF_ASSIGNMENT: u8 = b'j';
101const TAG_HNSW_METADATA: u8 = b'H';
102const TAG_HNSW_NODE: u8 = b'h';
103
104/// Prefix for the unambiguous document encoding introduced after JSON arrays
105/// became ordinary [`Value::List`] values. A legacy document is plain JSON and
106/// therefore cannot start with NUL; the prefix lets reads preserve the old
107/// `Bytes`-before-`List` interpretation without misreading newly written lists.
108const DOCUMENT_VALUE_V1_PREFIX: &[u8] = b"\0uqa-document-json-v1\0";
109const DOCUMENT_VALUE_V2_PREFIX: &[u8] = b"\0uqa-document-record-v2\0";
110
111#[derive(Debug, Clone)]
112enum KeyValueBatchOperation {
113    Put(Vec<u8>, Vec<u8>),
114    Delete(Vec<u8>),
115    DeletePrefix(Vec<u8>),
116}
117
118/// Atomic mutation buffer for a [`KeyValueStore`].
119pub trait KeyValueBatch {
120    /// Original SSI participant captured with this evaluated mutation. Wrappers must forward this capability; a batch cannot move to another participant before publication.
121    fn serializable_participant(&self) -> Option<crate::mvcc::SerializableTransactionId> {
122        None
123    }
124    /// Stage a logical write with the evaluated records. Failed application discards its new intents and records together while retaining already established dependencies. Providers without this capability reject the request.
125    fn observe_serializable_write(
126        &mut self,
127        _predicate: crate::mvcc::SerializablePredicate<'_>,
128    ) -> StorageBackendResult<()> {
129        Err(StorageBackendError::Other(
130            "atomic serializable write observations are not supported".into(),
131        ))
132    }
133    /// Require this record's original committed revision at publication without replacing it. This permits independent data writers to share a definition. Stores without commit-time read validation reject this operation; capable wrappers must forward it.
134    fn require_unchanged(&mut self, _key: &[u8]) -> StorageBackendResult<()> {
135        Err(StorageBackendError::Other(
136            "commit-time record requirements are not supported".into(),
137        ))
138    }
139    /// Require an actual committed record observed through an independently retained metadata view, without advancing the transaction's data snapshot. The revision must identify this same key in this database; private and unversioned provenance is rejected. Reserved for storage-owned independently staged resources, not SQL row writes.
140    fn require_observed(
141        &mut self,
142        _key: &[u8],
143        _revision: &KeyValueReadRevision,
144    ) -> StorageBackendResult<()> {
145        Err(StorageBackendError::Other(
146            "observed metadata requirements are not supported".into(),
147        ))
148    }
149    /// Replace an independently staged metadata record using its observed committed precondition. Existing private changes to the key are rejected. Commit conflict, savepoint and receipt semantics are the same as other evaluated writes; this does not refresh the caller's snapshot.
150    fn put_observed(
151        &mut self,
152        _key: &[u8],
153        _value: &[u8],
154        _revision: &KeyValueReadRevision,
155    ) -> StorageBackendResult<()> {
156        Err(StorageBackendError::Other(
157            "observed metadata replacements are not supported".into(),
158        ))
159    }
160    /// Replace metadata and retain its independently staged read source in the same evaluated private write. The source follows savepoint undo, retained views and original receipt resolution; replacing or deleting the key releases it from the new view. Only the bytes are persisted. Capable wrappers must forward the source unchanged.
161    fn put_with_retained_source(
162        &mut self,
163        _key: &[u8],
164        _value: &[u8],
165        _source: Arc<dyn KeyValueRead + Send + Sync>,
166    ) -> StorageBackendResult<()> {
167        Err(StorageBackendError::Other(
168            "private metadata read sources are not supported".into(),
169        ))
170    }
171    /// Publish a new revision of an immutable marker, merging concurrent touches of the same value. Structural changes fence the marker and replace their definition; data changes require the definition and touch the marker. The payload must contain only immutable owner/format data. Capable wrappers must forward this operation.
172    fn touch_marker(&mut self, _key: &[u8], _value: &[u8]) -> StorageBackendResult<()> {
173        Err(StorageBackendError::Other(
174            "mergeable revision markers are not supported".into(),
175        ))
176    }
177
178    /// Stage an actual canonical origin with its evaluated complete tensor. Its original write precondition remains strict; common MVCC maintains all selected-generation populations without replaying the mutation. Capable wrappers must forward this operation.
179    fn replace_diskann_origin(&mut self, _key: &[u8], _value: &[u8]) -> StorageBackendResult<()> {
180        Err(StorageBackendError::Other(
181            "DiskANN population mutations are not supported".into(),
182        ))
183    }
184
185    /// Remove the canonical origin invalidated by a raw writer. Concurrent stores reject an existing population unless their complete publication census validates the final canonical view. Validation follows private application, command refresh and final publication, including an originally absent origin. Capable wrappers must forward this operation.
186    fn invalidate_diskann_origin(&mut self, key: &[u8]) -> StorageBackendResult<()> {
187        self.delete(key)
188    }
189
190    /// Retain already verified complete build origins for one authorized generation publication. The template supplies physical row encoding, not trusted counts; common MVCC computes the private and committed census on their actual canonical views. The catalog/head owner must authorize and guard this publication in the same batch.
191    fn publish_diskann_population(
192        &mut self,
193        _key: &[u8],
194        _template: &[u8],
195        _origins: crate::diskann_index::pages::DiskANNOriginReader,
196    ) -> StorageBackendResult<()> {
197        Err(StorageBackendError::Other(
198            "DiskANN population publication is not supported".into(),
199        ))
200    }
201
202    /// Retire a generation's population header and witnesses, including witnesses added by intervening disjoint writers. The generation/head owner must retain its original publication guards. This operation follows private undo and original receipt resolution.
203    fn retire_diskann_population(&mut self, _key: &[u8]) -> StorageBackendResult<()> {
204        Err(StorageBackendError::Other(
205            "DiskANN population retirement is not supported".into(),
206        ))
207    }
208    /// Stage a durable identifier observation before publishing this batch's records. Successful observations survive later transaction/savepoint rollback. Dropping an unevaluated batch consumes nothing. Stores without autonomous allocation reject this operation; capable wrappers must forward it.
209    fn observe_identifier(&mut self, _namespace: &[u8], _value: u64) -> StorageBackendResult<()> {
210        Err(StorageBackendError::Other(
211            "durable identifier observations are not supported".into(),
212        ))
213    }
214    /// Carry the source's durable identifier watermark into the target before this batch publishes its records. Successful watermark updates survive undo, just like observations. Owning lifecycle code must coordinate the accompanying definition change with data writers. Capable wrappers must forward this namespace operation.
215    fn inherit_identifiers(&mut self, _from: &[u8], _to: &[u8]) -> StorageBackendResult<()> {
216        Err(StorageBackendError::Other(
217            "durable identifier inheritance is not supported".into(),
218        ))
219    }
220    fn put(&mut self, key: &[u8], value: &[u8]) -> StorageBackendResult<()>;
221    fn delete(&mut self, key: &[u8]) -> StorageBackendResult<()>;
222    fn delete_prefix(&mut self, prefix: &[u8]) -> StorageBackendResult<()>;
223    /// Delete only the keys selected by this evaluated prefix, accepting concurrent deletion of those keys. A concurrent live replacement still conflicts, and explicit read requirements remain strict. Reserved for obsolete internal records; callers must separately fence structural changes and new insertions. Concurrent wrappers must forward this operation.
224    fn delete_prefix_allow_absent(&mut self, prefix: &[u8]) -> StorageBackendResult<()> {
225        self.delete_prefix(prefix)
226    }
227    /// Retain already evaluated document input in the same atomic batch as its canonical values and IVF preview. Concurrent wrappers must forward this call.
228    fn ivf_mutation(
229        &mut self,
230        _metadata: &[u8],
231        _mutation: crate::ivf_index::IVFMutation<'_>,
232    ) -> StorageBackendResult<()> {
233        Ok(())
234    }
235    /// The immutable preview serves private reads; concurrent commit preparation may replace it with an internally recalculated generation.
236    fn preview_ivf_record(&mut self, key: &[u8], value: Option<&[u8]>) -> StorageBackendResult<()> {
237        match value {
238            Some(value) => self.put(key, value),
239            None => self.delete(key),
240        }
241    }
242    fn preview_ivf_prefix(&mut self, prefix: &[u8]) -> StorageBackendResult<()> {
243        self.delete_prefix(prefix)
244    }
245    /// Fence each existing IVF definition under this metadata prefix, including definitions with no vectors. Serialized stores already exclude competing writers.
246    fn fence_ivf_prefix(&mut self, _prefix: &[u8]) -> StorageBackendResult<()> {
247        Ok(())
248    }
249    /// Retain already evaluated document input in the same atomic batch as its canonical values and HNSW preview. Concurrent wrappers must forward this call.
250    fn hnsw_mutation(
251        &mut self,
252        _metadata: &[u8],
253        _mutation: crate::hnsw_index::HNSWMutation<'_>,
254    ) -> StorageBackendResult<()> {
255        Ok(())
256    }
257    /// The immutable preview serves private reads; concurrent commit preparation may replace it with an internally recalculated generation.
258    fn preview_hnsw_record(
259        &mut self,
260        key: &[u8],
261        value: Option<&[u8]>,
262    ) -> StorageBackendResult<()> {
263        match value {
264            Some(value) => self.put(key, value),
265            None => self.delete(key),
266        }
267    }
268    fn preview_hnsw_prefix(&mut self, prefix: &[u8]) -> StorageBackendResult<()> {
269        self.delete_prefix(prefix)
270    }
271    /// Fence each existing HNSW definition under this metadata prefix, including definitions with no vectors. Serialized stores already exclude competing writers.
272    fn fence_hnsw_prefix(&mut self, _prefix: &[u8]) -> StorageBackendResult<()> {
273        Ok(())
274    }
275    /// Stage an evaluated occurrence cluster, field total or source marker using the persistence's declared record layout. Source mutations must include their format marker and document guards. Concurrent stores merge only these explicitly typed replacements; ordinary byte writes remain conditional replacements.
276    fn replace_occurrence_record(
277        &mut self,
278        key: &[u8],
279        value: Option<&[u8]>,
280    ) -> StorageBackendResult<()> {
281        match value {
282            Some(value) => self.put(key, value),
283            None => self.delete(key),
284        }
285    }
286    /// Stage source-owned occurrence-cache invalidation. The source marker in the same batch also drives discovery of caches published after the retained source view.
287    fn invalidate_occurrence_prefix(&mut self, prefix: &[u8]) -> StorageBackendResult<()> {
288        self.delete_prefix(prefix)
289    }
290    /// Protect whole-document replacement even when two writers select different fields of an originally absent document.
291    fn occurrence_document(&mut self, _table: &str, _document: DocId) -> StorageBackendResult<()> {
292        Ok(())
293    }
294    /// Fence a structural occurrence change, including an empty rebuild, drop, purge or rename. Serialized projections already retain their native mutation boundary.
295    fn reset_occurrences(&mut self, _table: &str) -> StorageBackendResult<()> {
296        Ok(())
297    }
298    /// Retain the original precondition and current value of a record, including an absent tombstone, before later replacements. Concurrent wrappers must forward this call; serialized stores already exclude intervening writers.
299    fn fence_record(&mut self, _key: &[u8]) -> StorageBackendResult<()> {
300        Ok(())
301    }
302    /// Stage an evaluated statistics-maintenance replacement using the provider's declared layout. Keep ordinary metadata writes canonical.
303    fn replace_statistics_maintenance(
304        &mut self,
305        key: &[u8],
306        value: &[u8],
307    ) -> StorageBackendResult<()> {
308        self.put(key, value)
309    }
310    /// Record graph-cache dependencies in the same atomic batch. Concurrent MVCC stores must resolve these logical effects before admitting a commit; serialized legacy stores use the ordinary preview writes already included by the catalog.
311    fn graph_mutation(
312        &mut self,
313        _mutation: crate::mvcc::GraphMutation<'_>,
314    ) -> StorageBackendResult<()> {
315        Ok(())
316    }
317    /// Maintain read-your-writes for a graph invalidation. MVCC replaces this preview with effects selected from current graph ownership at commit.
318    fn preview_graph_invalidation(
319        &mut self,
320        key: &[u8],
321        value: Option<&[u8]>,
322    ) -> StorageBackendResult<()> {
323        match value {
324            Some(value) => self.put(key, value),
325            None => self.delete(key),
326        }
327    }
328    /// Replace or clear derived graph cache state. Publishing a completed build must also record `GraphMutation::PublishPath` so its source dependencies are validated.
329    fn replace_graph_cache(
330        &mut self,
331        key: &[u8],
332        value: Option<&[u8]>,
333    ) -> StorageBackendResult<()> {
334        match value {
335            Some(value) => self.put(key, value),
336            None => self.delete(key),
337        }
338    }
339    fn commit(self: Box<Self>) -> StorageBackendResult<()>;
340}
341
342pub use vector_index::guards::KeyValueVectorFieldGuards;
343
344pub(crate) fn diskann_tombstone_prefixes() -> [&'static [u8]; 5] {
345    [
346        diskann::READ_PREFIX,
347        vector_index::origin::ROOT,
348        vector_index::origin::journal::ROOT,
349        vector_index::guards::ROOT,
350        vector_index::origin::populations::ROOT,
351    ]
352}
353
354/// Ordered byte-key storage used by Key/Value catalog and index backends.
355pub trait KeyValueStore: Send + Sync {
356    /// Physical resource ownership for retained builds and recovery. Versioned wrappers must forward this capability.
357    fn resource_leases(&self) -> Option<&dyn crate::mvcc::ResourceLeaseProvider> {
358        None
359    }
360
361    /// Credential for database-owned auxiliary files. Wrappers over an encrypted store must forward it; `None` permits unencrypted auxiliary storage.
362    fn auxiliary_encryption_key(&self) -> Option<crate::StorageEncryptionKey> {
363        None
364    }
365
366    /// Shared session allowance, including resources retained by readers after their producer finishes. Versioned wrappers must forward this capability.
367    fn retention_control(&self) -> Option<crate::read_control::StorageReadControl> {
368        None
369    }
370
371    /// Atomic auxiliary notification publication and fresh committed recovery. Session wrappers must forward this capability.
372    fn notification_publications(
373        &self,
374    ) -> Option<&dyn crate::notifications::NotificationPublicationStore> {
375        None
376    }
377
378    /// Original participant admission and logical observations. Wrappers must forward this capability, including original attribution on retained read-only sessions; capability presence does not establish complete SQL SSI support.
379    fn serializable_session(&self) -> Option<&dyn crate::mvcc::SerializableSession> {
380        None
381    }
382
383    /// Reclaim obsolete records and committed history without a whole-file rewrite. Versioned wrappers must forward maintenance; serialized stores that eagerly remove obsolete values may keep the default.
384    fn reclaim_obsolete(&self) -> StorageBackendResult<()> {
385        if self.transaction_model().is_versioned() {
386            return Err(StorageBackendError::Other(
387                "versioned KeyValue maintenance is not implemented by this store".into(),
388            ));
389        }
390        Ok(())
391    }
392
393    /// Reclaim obsolete records and compact physical storage where supported, outside this session's transaction.
394    fn vacuum(&self) -> StorageBackendResult<()> {
395        self.reclaim_obsolete()
396    }
397
398    /// Token shared with the execution that owns this session's writes. Cancellation must not prevent rollback cleanup or diagnostic reads. Versioned wrappers must forward this capability.
399    fn write_cancellation(&self) -> Option<uqa_core::CancellationToken> {
400        None
401    }
402
403    /// Open an independent writer sharing the invoking maintenance operation's allowance and write cancellation. Retained reads and cleanup keep independent cancellation. Capable wrappers must forward this without granting another memory limit.
404    fn open_controlled_session(
405        &self,
406        _control: &crate::read_control::StorageReadControl,
407    ) -> StorageBackendResult<Arc<dyn KeyValueStore>> {
408        Err(StorageBackendError::Other(
409            "controlled independent sessions are not implemented for this store".into(),
410        ))
411    }
412
413    /// Create a transaction-isolated session with the caller's write cancellation and a fresh retention budget.
414    fn open_session_with_cancellation(
415        &self,
416        cancellation: &uqa_core::CancellationToken,
417    ) -> StorageBackendResult<Arc<dyn KeyValueStore>> {
418        cancellation.check()?;
419        if self.transaction_model().is_versioned() {
420            return Err(StorageBackendError::Other(
421                "cancellable independent KeyValue sessions are not implemented by this store"
422                    .into(),
423            ));
424        }
425        self.open_session()
426    }
427
428    /// Retain this exact committed/private view in a separate read-only session, including its original logical reader attribution. The returned session cannot publish records or complete the source transaction. Versioned wrappers must forward this capability; opening a newer independent snapshot is not equivalent.
429    fn open_retained_read_session(
430        &self,
431        cancellation: &uqa_core::CancellationToken,
432    ) -> StorageBackendResult<Arc<dyn KeyValueStore>> {
433        cancellation.check()?;
434        Err(StorageBackendError::Other(
435            "retained read sessions are not implemented by this KeyValue store".into(),
436        ))
437    }
438
439    /// Whether writes remain private while independent sessions read and write. Versioned stores must provide command refresh, retained reads, savepoint undo and conditional publication under one reported affinity.
440    fn transaction_model(&self) -> crate::StorageTransactionModel {
441        crate::StorageTransactionModel::ProviderSerialized
442    }
443
444    /// Durable reservations outside private record undo. Serialized custom stores may omit this capability; wrappers over a capable store must forward it.
445    fn identifier_allocator(&self) -> Option<&dyn crate::mvcc::IdentifierAllocator> {
446        None
447    }
448
449    /// Evaluate a compound read once against one fixed committed/private view. The callback must use the supplied reader and must not reenter this session. Stores without this capability reject it explicitly.
450    fn with_read_view(&self, _read: &mut KeyValueReadScope<'_>) -> StorageBackendResult<()> {
451        Err(StorageBackendError::Other(
452            "compound KeyValue reads are not supported by this store".into(),
453        ))
454    }
455
456    /// Evaluate once against the same view that supplies write preconditions, then atomically stage the batch. Errors and unwinds discard that batch; failed publication retains the evaluated attempt. The callback must not reenter this session or perform transaction control.
457    fn with_mutation(&self, _mutate: &mut KeyValueMutation<'_>) -> StorageBackendResult<()> {
458        Err(StorageBackendError::Other(
459            "atomic KeyValue evaluation is not supported by this store".into(),
460        ))
461    }
462
463    /// Evaluate and stage once with the actual durable writer and a non-reused mutation revision. Origins may be persisted alongside canonical values, but do not prove publication or visibility. The callback has the same non-reentrancy rules as `with_mutation`; capable wrappers must preserve the original origin and atomic batch.
464    fn with_versioned_mutation(
465        &self,
466        _mutate: &mut KeyValueVersionedMutation<'_>,
467    ) -> StorageBackendResult<()> {
468        Err(StorageBackendError::Other(
469            "versioned KeyValue mutation origins are not supported by this store".into(),
470        ))
471    }
472
473    /// Identity of the transaction context shared by this store's catalog and data handles. Independent sessions must report different identities, even over the same file.
474    fn transaction_affinity(&self) -> Option<crate::StorageSessionAffinity> {
475        None
476    }
477
478    fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
479        Ok(None)
480    }
481
482    /// Open an independent transaction session over the same logical store. The default keeps simple test/custom stores source-compatible while making the missing MVCC capability explicit when a persistent engine needs a committed reader alongside a pinned statement snapshot.
483    fn open_session(&self) -> StorageBackendResult<Arc<dyn KeyValueStore>> {
484        Err(StorageBackendError::Other(
485            "independent sessions are not implemented for this KeyValue store".into(),
486        ))
487    }
488
489    fn get(&self, key: &[u8]) -> StorageBackendResult<Option<Vec<u8>>>;
490    /// Visit one borrowed value under the retained read. Providers reserve any temporary encoded payload before fetching it; callbacks must not reenter the store.
491    fn visit_value(
492        &self,
493        _key: &[u8],
494        control: &crate::read_control::StorageReadControl,
495        _visit: &mut crate::read_control::ValueReadVisitor<'_>,
496    ) -> StorageBackendResult<()> {
497        control.check()?;
498        Err(StorageBackendError::Other(
499            "controlled value reads are not supported by this KeyValue store".into(),
500        ))
501    }
502    /// Visit one value with an encoded-size cap checked before provider materialization. This is distinct from the shared workspace allowance; unsupported stores must not fetch an unbounded value as a fallback.
503    fn visit_value_bounded(
504        &self,
505        _key: &[u8],
506        _max_bytes: usize,
507        control: &crate::read_control::StorageReadControl,
508        _visit: &mut crate::read_control::ValueReadVisitor<'_>,
509    ) -> StorageBackendResult<()> {
510        control.check()?;
511        Err(StorageBackendError::Other(
512            "size-bounded value reads are not supported by this KeyValue store".into(),
513        ))
514    }
515
516    /// Visit at most `limit` entries in key order, strictly after `after` when supplied. Encoded values remain borrowed from their provider owner and callbacks must not reenter the store.
517    fn visit_prefix_after(
518        &self,
519        _prefix: &[u8],
520        _after: Option<&[u8]>,
521        _limit: usize,
522        control: &crate::read_control::StorageReadControl,
523        _visit: &mut crate::read_control::KeyValueReadVisitor<'_>,
524    ) -> StorageBackendResult<()> {
525        control.check()?;
526        Err(StorageBackendError::Other(
527            "controlled prefix reads are not supported by this KeyValue store".into(),
528        ))
529    }
530    /// Test prefix existence under the read allowance. Providers that materialize values should implement a key-only probe.
531    fn contains_prefix_budgeted(
532        &self,
533        prefix: &[u8],
534        control: &crate::read_control::StorageReadControl,
535    ) -> StorageBackendResult<bool> {
536        let mut found = false;
537        self.visit_prefix_after(prefix, None, 1, control, &mut |_, _| {
538            found = true;
539            Ok(())
540        })?;
541        control.check()?;
542        Ok(found)
543    }
544
545    fn contains_key(&self, key: &[u8]) -> StorageBackendResult<bool> {
546        self.get(key).map(|value| value.is_some())
547    }
548    fn put(&self, key: &[u8], value: &[u8]) -> StorageBackendResult<()>;
549    fn delete(&self, key: &[u8]) -> StorageBackendResult<()>;
550    fn scan_prefix(&self, prefix: &[u8]) -> StorageBackendResult<Vec<(Vec<u8>, Vec<u8>)>>;
551    /// Return at most `limit` key/value pairs in key order, strictly after
552    /// `after` when supplied. This is the bounded value cursor used by large
553    /// format migrations and other paged consumers.
554    fn scan_prefix_after(
555        &self,
556        prefix: &[u8],
557        after: Option<&[u8]>,
558        limit: usize,
559    ) -> StorageBackendResult<Vec<(Vec<u8>, Vec<u8>)>> {
560        if limit == 0 {
561            return Ok(Vec::new());
562        }
563        Ok(self
564            .scan_prefix(prefix)?
565            .into_iter()
566            .filter(|(key, _)| after.is_none_or(|after| key.as_slice() > after))
567            .take(limit)
568            .collect())
569    }
570    /// Return at most `limit` keys in key order, strictly after `after` when
571    /// it is present. Backends should override this method with a key-only,
572    /// bounded range scan so cursor consumers neither materialize the entire
573    /// prefix nor read values they do not need on every page.
574    fn scan_prefix_keys_after(
575        &self,
576        prefix: &[u8],
577        after: Option<&[u8]>,
578        limit: usize,
579    ) -> StorageBackendResult<Vec<Vec<u8>>> {
580        if limit == 0 {
581            return Ok(Vec::new());
582        }
583        Ok(self
584            .scan_prefix(prefix)?
585            .into_iter()
586            .filter(|(key, _)| after.is_none_or(|after| key.as_slice() > after))
587            .take(limit)
588            .map(|(key, _)| key)
589            .collect())
590    }
591    fn first_prefix_after(
592        &self,
593        prefix: &[u8],
594        after: Option<&[u8]>,
595    ) -> StorageBackendResult<Option<(Vec<u8>, Vec<u8>)>> {
596        Ok(self
597            .scan_prefix(prefix)?
598            .into_iter()
599            .find(|(key, _)| after.is_none_or(|after| key.as_slice() > after)))
600    }
601    fn delete_prefix(&self, prefix: &[u8]) -> StorageBackendResult<usize>;
602    fn batch(&self) -> Box<dyn KeyValueBatch + '_>;
603
604    fn begin_transaction(&self) -> StorageBackendResult<()> {
605        Err(StorageBackendError::Other(
606            "KeyValue transaction begin is not implemented for this store".into(),
607        ))
608    }
609
610    fn begin_read_transaction(&self) -> StorageBackendResult<()> {
611        self.begin_transaction()
612    }
613
614    fn begin_upgradeable_transaction(&self) -> StorageBackendResult<()> {
615        self.begin_transaction()
616    }
617
618    fn in_transaction(&self) -> bool;
619
620    /// Report ordinary record and derived-index writes, excluding typed auxiliary notification effects. Ordinary writes remain forbidden in read-only transactions.
621    fn transaction_has_written(&self) -> StorageBackendResult<bool>;
622
623    fn change_version(&self) -> StorageBackendResult<Option<u64>> {
624        Ok(None)
625    }
626
627    fn change_version_monitor_is_nonblocking(&self) -> StorageBackendResult<bool> {
628        Ok(true)
629    }
630
631    fn pin_transaction_snapshot(&self) -> StorageBackendResult<()> {
632        Ok(())
633    }
634
635    /// Advance an active logical transaction's committed view while preserving its private writes and retained readers. The caller chooses command boundaries allowed by SQL isolation. Stores without this capability must reject the request; ending or replaying the caller's transaction is not a valid fallback.
636    fn refresh_transaction_snapshot(
637        &self,
638        _cancellation: &uqa_core::CancellationToken,
639    ) -> StorageBackendResult<()> {
640        Err(StorageBackendError::Other(
641            "transaction snapshot refresh is not supported by this KeyValue store".into(),
642        ))
643    }
644
645    fn commit_transaction(&self) -> StorageBackendResult<()> {
646        Err(StorageBackendError::Other(
647            "KeyValue transaction commit is not implemented for this store".into(),
648        ))
649    }
650
651    fn rollback_transaction(&self) -> StorageBackendResult<()> {
652        Err(StorageBackendError::Other(
653            "KeyValue transaction rollback is not implemented for this store".into(),
654        ))
655    }
656
657    fn savepoint(&self, _name: &str) -> StorageBackendResult<()> {
658        Err(StorageBackendError::Other(
659            "KeyValue savepoints are not implemented for this store".into(),
660        ))
661    }
662
663    fn release_savepoint(&self, _name: &str) -> StorageBackendResult<()> {
664        Err(StorageBackendError::Other(
665            "KeyValue savepoint release is not implemented for this store".into(),
666        ))
667    }
668
669    fn rollback_to_savepoint(&self, _name: &str) -> StorageBackendResult<()> {
670        Err(StorageBackendError::Other(
671            "KeyValue savepoint rollback is not implemented for this store".into(),
672        ))
673    }
674}
675
676mod btree_index;
677mod codec;
678pub mod conformance;
679mod document_store;
680mod hnsw_index;
681mod hnsw_persistence;
682mod index_keys;
683mod inverted_index;
684mod ivf_index;
685mod ivf_persistence;
686mod memory_store;
687pub mod occurrence_format;
688mod occurrence_keys;
689mod storage_backend;
690mod vector_index;
691
692pub use codec::prefix_upper_bound;
693pub use document_store::KeyValueDocumentStore;
694pub use hnsw_index::KeyValueHNSWIndex;
695pub use inverted_index::{KeyValueInvertedIndex, OccurrenceStorage};
696pub use ivf_index::KeyValueIVFIndex;
697pub use memory_store::MemoryKeyValueStore;
698pub use storage_backend::KeyValueStorageBackend;
699pub use vector_index::{
700    DiskANNCanonicalVectorVisitor, KeyValueDiskANNCanonical, KeyValueDiskANNHandle,
701    KeyValueDiskANNPopulationRecords, KeyValueVectorIndex, RetainedDiskANNCanonical,
702};
703
704#[cfg(test)]
705mod tests;
uqa_storage/key_value.rs

uqa_storage/
key_value.rs
