1use std::collections::BTreeMap;
14use std::path::{Path, PathBuf};
15use std::sync::atomic::{AtomicU64, Ordering};
16use std::sync::Arc;
17
18use uqa_analysis::Analyzer;
19use uqa_core::{DocId, Value};
20
21use crate::document_store::DocumentStore;
22use crate::inverted_index::InvertedIndex;
23use crate::vector_index::{VectorIndex, VectorIndexOpenMode, VectorIndexSpec};
24use crate::CatalogFacade;
25
26#[derive(Clone)]
28pub struct CommittedDataRevision {
29 pub revision: crate::key_value::KeyValueReadRevision,
30 pub change_version: u64,
31}
32
33impl CommittedDataRevision {
34 pub fn from_receipt(receipt: crate::mvcc::CommitReceipt) -> Self {
35 Self {
36 revision: crate::key_value::KeyValueReadRevision::records(
37 receipt.transaction.database(),
38 receipt.sequence,
39 None,
40 ),
41 change_version: receipt.sequence.as_u64(),
42 }
43 }
44}
45
46#[derive(Debug, thiserror::Error)]
47pub enum StorageBackendError {
48 #[error(transparent)]
49 Memory(#[from] uqa_core::memory::MemoryError),
50 #[error(transparent)]
51 Cancelled(#[from] uqa_core::QueryCancelled),
52 #[error("text analysis failed: {0}")]
53 Analysis(#[from] uqa_analysis::AnalysisError),
54 #[error("payload serialization failed: {0}")]
55 Serde(#[from] serde_json::Error),
56 #[error("{backend} storage failed: {source}")]
57 Backend {
58 backend: &'static str,
59 #[source]
60 source: Box<dyn std::error::Error + Send + Sync>,
61 },
62 #[error("{0}")]
63 Other(String),
64}
65
66impl StorageBackendError {
67 pub fn backend(
68 backend: &'static str,
69 source: impl std::error::Error + Send + Sync + 'static,
70 ) -> Self {
71 Self::Backend {
72 backend,
73 source: Box::new(source),
74 }
75 }
76}
77
78impl From<uqa_core::ValueRetentionError> for StorageBackendError {
79 fn from(error: uqa_core::ValueRetentionError) -> Self {
80 match error {
81 uqa_core::ValueRetentionError::Memory(error) => Self::Memory(error),
82 uqa_core::ValueRetentionError::Cancelled(error) => Self::Cancelled(error),
83 error @ uqa_core::ValueRetentionError::Malformed { .. } => {
84 Self::backend("value", error)
85 }
86 }
87 }
88}
89
90pub type StorageBackendResult<T> = std::result::Result<T, StorageBackendError>;
91
92#[derive(Clone, Debug)]
94pub struct StorageSessionAffinity(Arc<()>);
95
96impl StorageSessionAffinity {
97 #[must_use]
98 pub fn new() -> Self {
99 Self(Arc::new(()))
100 }
101}
102
103impl Default for StorageSessionAffinity {
104 fn default() -> Self {
105 Self::new()
106 }
107}
108
109impl PartialEq for StorageSessionAffinity {
110 fn eq(&self, other: &Self) -> bool {
111 Arc::ptr_eq(&self.0, &other.0)
112 }
113}
114
115impl Eq for StorageSessionAffinity {}
116
117#[derive(Debug, thiserror::Error)]
118#[error("catalog and data backend must share one storage transaction context")]
119pub struct StorageSessionMismatch;
120
121#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
123pub struct StorageSavepointId(u64);
124
125impl StorageSavepointId {
126 #[must_use]
127 pub fn allocate() -> Self {
128 static NEXT_ID: AtomicU64 = AtomicU64::new(1);
129 let id = uqa_core::atomic::try_update_u64(
130 &NEXT_ID,
131 Ordering::Relaxed,
132 Ordering::Relaxed,
133 |current| current.checked_add(1),
134 )
135 .expect("storage savepoint identity space exhausted");
136 Self(id)
137 }
138
139 pub fn backend_name(self) -> String {
140 self.0.to_string()
141 }
142}
143
144pub struct PersistentStorageSession {
150 pub catalog: Arc<dyn CatalogFacade>,
151 pub backend: Arc<dyn PersistentStorageBackend>,
152}
153
154#[derive(Clone, Debug, PartialEq, Eq, Hash)]
156pub enum PersistentStorageIdentity {
157 File(PathBuf),
158 Opaque(String),
159}
160
161impl PersistentStorageIdentity {
162 pub fn for_database_path(path: &Path) -> StorageBackendResult<Self> {
164 let path = resolve_final_symlinks(path)?;
165 match std::fs::canonicalize(&path) {
166 Ok(canonical) => Ok(Self::File(canonical)),
167 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
168 let file_name = path.file_name().ok_or_else(|| {
169 StorageBackendError::Other(format!(
170 "database path `{}` has no file name",
171 path.display()
172 ))
173 })?;
174 let parent = path
175 .parent()
176 .filter(|parent| !parent.as_os_str().is_empty())
177 .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
178 let parent = std::fs::canonicalize(&parent).map_err(|error| {
179 StorageBackendError::Other(format!(
180 "canonicalize database directory `{}`: {error}",
181 parent.display()
182 ))
183 })?;
184 Ok(Self::File(parent.join(file_name)))
185 }
186 Err(error) => Err(StorageBackendError::Other(format!(
187 "canonicalize database `{}`: {error}",
188 path.display()
189 ))),
190 }
191 }
192}
193
194fn resolve_final_symlinks(path: &Path) -> StorageBackendResult<PathBuf> {
195 let mut current = path.to_path_buf();
196 let mut followed = 0usize;
197 loop {
198 match std::fs::symlink_metadata(¤t) {
199 Ok(metadata) if metadata.file_type().is_symlink() => {
200 followed += 1;
201 if followed > 40 {
202 return Err(StorageBackendError::Other(format!(
203 "database path `{}` has too many symbolic-link levels",
204 path.display()
205 )));
206 }
207 let target = std::fs::read_link(¤t).map_err(|error| {
208 StorageBackendError::Other(format!(
209 "read database symbolic link `{}`: {error}",
210 current.display()
211 ))
212 })?;
213 current = if target.is_absolute() {
214 target
215 } else {
216 current
217 .parent()
218 .filter(|parent| !parent.as_os_str().is_empty())
219 .unwrap_or_else(|| Path::new("."))
220 .join(target)
221 };
222 }
223 Ok(_) => return Ok(current),
224 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(current),
225 Err(error) => {
226 return Err(StorageBackendError::Other(format!(
227 "inspect database path `{}`: {error}",
228 current.display()
229 )))
230 }
231 }
232 }
233}
234
235#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
237pub enum StorageTransactionModel {
238 #[default]
239 ProviderSerialized,
240 VersionedConcurrent { database: crate::mvcc::DatabaseId },
242}
243
244impl StorageTransactionModel {
245 pub fn is_versioned(self) -> bool {
246 matches!(self, Self::VersionedConcurrent { .. })
247 }
248}
249
250impl PersistentStorageSession {
251 pub fn new(
252 catalog: Arc<dyn CatalogFacade>,
253 backend: Arc<dyn PersistentStorageBackend>,
254 ) -> Self {
255 Self { catalog, backend }
256 }
257
258 pub fn validate_transaction_affinity(&self) -> StorageBackendResult<()> {
260 let affinity = self.backend.transaction_affinity();
261 let model = self.backend.transaction_model();
262 if self.catalog.transaction_affinity() != affinity
263 || self.catalog.transaction_model() != model
264 || (model.is_versioned() && affinity.is_none())
265 {
266 return Err(StorageBackendError::backend(
267 "session",
268 StorageSessionMismatch,
269 ));
270 }
271 if model.is_versioned() && self.backend.write_cancellation().is_none() {
272 return Err(StorageBackendError::Other(
273 "versioned storage backend does not expose write cancellation".into(),
274 ));
275 }
276 Ok(())
277 }
278}
279
280pub trait PersistentStorageProvider: Send + Sync {
286 fn open_session(&self) -> StorageBackendResult<PersistentStorageSession>;
287
288 fn open_session_with_cancellation(
290 &self,
291 cancellation: &uqa_core::CancellationToken,
292 ) -> StorageBackendResult<PersistentStorageSession> {
293 cancellation.check()?;
294 let session = self.open_session()?;
295 if session.backend.transaction_model().is_versioned() {
296 return Err(StorageBackendError::Other(
297 "cancellable independent storage sessions are not implemented by this provider"
298 .into(),
299 ));
300 }
301 Ok(session)
302 }
303
304 fn open_initial_session(&self) -> StorageBackendResult<PersistentStorageSession> {
306 self.open_session()
307 }
308
309 fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
311 Ok(None)
312 }
313
314 fn auxiliary_encryption_key(&self) -> Option<crate::StorageEncryptionKey> {
318 None
319 }
320}
321
322pub trait PersistentStorageBackend: Send + Sync {
324 fn require_sequence_value_durability(&self) -> StorageBackendResult<()> {
326 Ok(())
327 }
328
329 fn open_sequence_value_session(
331 &self,
332 cancellation: &uqa_core::CancellationToken,
333 ) -> StorageBackendResult<PersistentStorageSession> {
334 self.open_session_with_cancellation(cancellation)
335 }
336
337 fn retention_control(&self) -> Option<crate::read_control::StorageReadControl> {
339 None
340 }
341
342 fn notification_publications(
344 &self,
345 ) -> Option<&dyn crate::notifications::NotificationPublicationStore> {
346 None
347 }
348
349 fn serializable_session(&self) -> Option<&dyn crate::mvcc::SerializableSession> {
351 None
352 }
353
354 fn write_cancellation(&self) -> Option<uqa_core::CancellationToken> {
356 None
357 }
358
359 fn open_session_with_cancellation(
361 &self,
362 cancellation: &uqa_core::CancellationToken,
363 ) -> StorageBackendResult<PersistentStorageSession> {
364 cancellation.check()?;
365 if self.transaction_model().is_versioned() {
366 return Err(StorageBackendError::Other(
367 "cancellable independent storage sessions are not implemented by this backend"
368 .into(),
369 ));
370 }
371 self.open_session()
372 }
373
374 fn open_controlled_session(
376 &self,
377 _control: &crate::read_control::StorageReadControl,
378 ) -> StorageBackendResult<PersistentStorageSession> {
379 Err(StorageBackendError::Other(
380 "controlled independent sessions are not implemented by this backend".into(),
381 ))
382 }
383
384 fn open_retained_read_session(
386 &self,
387 cancellation: &uqa_core::CancellationToken,
388 ) -> StorageBackendResult<PersistentStorageSession> {
389 cancellation.check()?;
390 Err(StorageBackendError::Other(
391 "retained read sessions are not implemented by this storage backend".into(),
392 ))
393 }
394
395 fn capture_committed_read_boundary(
397 &self,
398 ) -> StorageBackendResult<Arc<crate::mvcc::CommittedReadBoundary>> {
399 Err(StorageBackendError::Other(
400 "fixed committed read boundaries are not supported by this backend".into(),
401 ))
402 }
403
404 fn open_read_session_with_private_changes(
406 &self,
407 _boundary: &crate::mvcc::CommittedReadBoundary,
408 cancellation: &uqa_core::CancellationToken,
409 ) -> StorageBackendResult<PersistentStorageSession> {
410 cancellation.check()?;
411 Err(StorageBackendError::Other(
412 "fixed committed/private read composition is not supported by this backend".into(),
413 ))
414 }
415
416 fn transaction_model(&self) -> StorageTransactionModel {
418 StorageTransactionModel::ProviderSerialized
419 }
420
421 fn identifier_allocator(&self) -> Option<&dyn crate::mvcc::IdentifierAllocator> {
423 None
424 }
425
426 fn transaction_affinity(&self) -> Option<StorageSessionAffinity> {
428 None
429 }
430
431 fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
433 Ok(None)
434 }
435
436 fn auxiliary_encryption_key(&self) -> Option<crate::StorageEncryptionKey> {
439 None
440 }
441
442 fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
444 Err(StorageBackendError::Other(
445 "independent sessions are not implemented for this persistent backend".into(),
446 ))
447 }
448
449 fn supports_concurrent_pinned_read_and_write(&self) -> bool {
451 false
452 }
453
454 fn document_store(&self, table: &str) -> Box<dyn DocumentStore>;
455
456 fn migrate_document_storage(&self) -> StorageBackendResult<()> {
458 Ok(())
459 }
460
461 fn inverted_index(&self, table: &str, analyzer: Analyzer) -> Box<dyn InvertedIndex>;
462
463 fn migrate_inverted_index_storage(&self) -> StorageBackendResult<()> {
466 Ok(())
467 }
468
469 fn vector_index(
470 &self,
471 table: &str,
472 field: &str,
473 dimensions: u32,
474 spec: VectorIndexSpec,
475 mode: VectorIndexOpenMode,
476 ) -> StorageBackendResult<Box<dyn VectorIndex>>;
477
478 fn diskann_index(
480 &self,
481 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
482 _options: crate::diskann_index::DiskANNIndexOptions,
483 _temporary: &crate::diskann_index::build::DiskANNTemporaryBudget,
484 _mode: VectorIndexOpenMode,
485 ) -> StorageBackendResult<Box<dyn VectorIndex>> {
486 Err(StorageBackendError::Other(
487 "this backend does not support catalog-bound DiskANN indexes".into(),
488 ))
489 }
490
491 fn diskann_journal_pruner(
493 &self,
494 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
495 _max_record_bytes: usize,
496 ) -> StorageBackendResult<Box<dyn crate::diskann_index::changes::DiskANNJournalPruner>> {
497 Err(StorageBackendError::Other(
498 "this backend does not support DiskANN journal maintenance".into(),
499 ))
500 }
501
502 fn diskann_maintenance_source(
504 &self,
505 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
506 _max_record_bytes: usize,
507 ) -> StorageBackendResult<Box<dyn crate::diskann_index::maintenance::DiskANNMaintenanceSource>>
508 {
509 Err(StorageBackendError::Other(
510 "this backend does not support DiskANN rebuild maintenance".into(),
511 ))
512 }
513
514 fn retire_diskann_index(
515 &self,
516 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
517 ) -> StorageBackendResult<()> {
518 Err(StorageBackendError::Other(
519 "this backend does not support DiskANN retirement".into(),
520 ))
521 }
522
523 fn drop_vector_index_metadata(&self, _table: &str, _field: &str) -> StorageBackendResult<()> {
524 Ok(())
525 }
526
527 fn persists_btree_indexes(&self) -> bool {
529 false
530 }
531
532 fn load_btree_index(
535 &self,
536 _table: &str,
537 _field: &crate::ValueIndexKey,
538 ) -> StorageBackendResult<Option<Vec<(DocId, Value)>>> {
539 Ok(None)
540 }
541
542 fn visit_btree_index_entries(
544 &self,
545 _table: &str,
546 _field: &crate::ValueIndexKey,
547 _visit: &mut dyn FnMut(DocId, Value) -> StorageBackendResult<()>,
548 ) -> StorageBackendResult<bool> {
549 Err(StorageBackendError::Other(
550 "persistent provider does not support ordered stored-key reads".into(),
551 ))
552 }
553
554 fn btree_index_fields(&self, _table: &str) -> StorageBackendResult<Vec<crate::ValueIndexKey>> {
555 Ok(Vec::new())
556 }
557
558 fn probe_btree_index_equal(
560 &self,
561 _table: &str,
562 _field: &crate::ValueIndexKey,
563 _value: &Value,
564 ) -> StorageBackendResult<Option<Vec<DocId>>> {
565 Ok(None)
566 }
567
568 fn visit_btree_index_equal(
570 &self,
571 _table: &str,
572 _field: &crate::ValueIndexKey,
573 _value: &Value,
574 _visit: &mut dyn FnMut(DocId) -> bool,
575 ) -> StorageBackendResult<bool> {
576 Ok(false)
577 }
578
579 fn read_btree_index_entry(
581 &self,
582 _table: &str,
583 _field: &crate::ValueIndexKey,
584 _doc_id: DocId,
585 ) -> StorageBackendResult<crate::ValueIndexEntry> {
586 if self.persists_btree_indexes() {
587 return Err(crate::StorageBackendError::Other(
588 "point reads of stored B-tree entries are not implemented for this backend".into(),
589 ));
590 }
591 Ok(crate::ValueIndexEntry::Unbuilt)
592 }
593
594 fn btree_index_repairs(&self) -> StorageBackendResult<Vec<(String, crate::ValueIndexKey)>> {
598 Ok(Vec::new())
599 }
600
601 fn clear_btree_index_repair(
602 &self,
603 _table: &str,
604 _field: &crate::ValueIndexKey,
605 ) -> StorageBackendResult<()> {
606 Ok(())
607 }
608
609 fn replace_btree_index(
610 &self,
611 _table: &str,
612 _field: &crate::ValueIndexKey,
613 _values: &[(DocId, Value)],
614 ) -> StorageBackendResult<()> {
615 Ok(())
616 }
617
618 fn repair_btree_index(
622 &self,
623 table: &str,
624 field: &crate::ValueIndexKey,
625 complete: &[(DocId, Value)],
626 _stale_doc_ids: &[DocId],
627 _missing: &[(DocId, Value)],
628 ) -> StorageBackendResult<()> {
629 self.replace_btree_index(table, field, complete)
630 }
631
632 fn replace_btree_indexes(
635 &self,
636 table: &str,
637 indexes: &[(&crate::ValueIndexKey, &[(DocId, Value)])],
638 ) -> StorageBackendResult<()> {
639 for (field, values) in indexes {
640 self.replace_btree_index(table, field, values)?;
641 }
642 Ok(())
643 }
644
645 fn apply_btree_index_write(
646 &self,
647 _table: &str,
648 _doc_id: DocId,
649 _values: Option<&BTreeMap<crate::ValueIndexKey, Value>>,
650 ) -> StorageBackendResult<()> {
651 Ok(())
652 }
653
654 fn apply_unused_btree_index_write(
656 &self,
657 table: &str,
658 doc_id: DocId,
659 values: &BTreeMap<crate::ValueIndexKey, Value>,
660 namespace: crate::document_store::identifiers::DocumentIdNamespace,
661 ) -> StorageBackendResult<()> {
662 let _ = namespace;
663 self.apply_btree_index_write(table, doc_id, Some(values))
664 }
665
666 fn drop_btree_index(
667 &self,
668 _table: &str,
669 _field: &crate::ValueIndexKey,
670 ) -> StorageBackendResult<()> {
671 Ok(())
672 }
673
674 fn clear_btree_indexes(&self, _table: &str) -> StorageBackendResult<()> {
675 Ok(())
676 }
677
678 fn reclaim_obsolete(&self) -> StorageBackendResult<()> {
680 if self.transaction_model().is_versioned() {
681 return Err(StorageBackendError::Other(
682 "versioned storage reclamation is not implemented by this backend".into(),
683 ));
684 }
685 Ok(())
686 }
687
688 fn vacuum(&self) -> StorageBackendResult<()> {
690 Ok(())
691 }
692
693 fn begin_transaction(&self) -> StorageBackendResult<()>;
694
695 fn begin_read_transaction(&self) -> StorageBackendResult<()> {
699 self.begin_transaction()
700 }
701
702 fn begin_upgradeable_transaction(&self) -> StorageBackendResult<()> {
704 self.begin_transaction()
705 }
706
707 fn refresh_transaction_snapshot(
709 &self,
710 _cancellation: &uqa_core::CancellationToken,
711 ) -> StorageBackendResult<()> {
712 Err(StorageBackendError::Other(
713 "transaction snapshot refresh is not supported by this backend".into(),
714 ))
715 }
716
717 fn in_transaction(&self) -> bool;
719
720 fn transaction_has_written(&self) -> StorageBackendResult<bool>;
722
723 fn change_version(&self) -> StorageBackendResult<Option<u64>> {
726 Ok(None)
727 }
728
729 fn read_view_revision(
731 &self,
732 ) -> StorageBackendResult<Option<crate::key_value::KeyValueReadRevision>> {
733 Ok(None)
734 }
735
736 fn committed_data_revision(&self) -> StorageBackendResult<Option<CommittedDataRevision>> {
738 Ok(None)
739 }
740
741 fn commit_monitor_version(&self) -> StorageBackendResult<Option<u64>> {
743 Ok(None)
744 }
745
746 fn change_version_monitor_is_nonblocking(&self) -> StorageBackendResult<bool> {
748 Ok(true)
749 }
750
751 fn pin_transaction_snapshot(&self) -> StorageBackendResult<()> {
753 Ok(())
754 }
755
756 fn prepare_transaction_commit(&self) -> StorageBackendResult<()> {
758 Ok(())
759 }
760
761 fn commit_transaction(&self) -> StorageBackendResult<()>;
762
763 fn rollback_transaction(&self) -> StorageBackendResult<()>;
764
765 fn savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()>;
766
767 fn release_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()>;
768
769 fn rollback_to_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()>;
770}
771
772#[cfg(all(test, unix))]
773mod identity_tests {
774 use super::*;
775
776 #[test]
777 fn dangling_database_symlink_keeps_the_target_identity_after_creation() {
778 use std::os::unix::fs::symlink;
779
780 let directory = tempfile::tempdir().unwrap();
781 let target = directory.path().join("target.db");
782 let link = directory.path().join("database.db");
783 symlink("target.db", &link).unwrap();
784
785 let before = PersistentStorageIdentity::for_database_path(&link).unwrap();
786 std::fs::File::create(&target).unwrap();
787 let after = PersistentStorageIdentity::for_database_path(&link).unwrap();
788
789 assert_eq!(before, after);
790 assert_eq!(
791 after,
792 PersistentStorageIdentity::File(target.canonicalize().unwrap())
793 );
794 }
795}