1use std::collections::BTreeMap;
14use std::path::{Path, PathBuf};
15use std::sync::atomic::{AtomicU64, Ordering};
16use std::sync::Arc;
17
18use uqa_analysis::Analyzer;
19use uqa_core::{DocId, Value};
20
21use crate::document_store::DocumentStore;
22use crate::inverted_index::InvertedIndex;
23use crate::vector_index::{VectorIndex, VectorIndexOpenMode, VectorIndexSpec};
24use crate::CatalogFacade;
25
26#[derive(Clone)]
28pub struct CommittedDataRevision {
29 pub revision: crate::key_value::KeyValueReadRevision,
30 pub change_version: u64,
31}
32
33impl CommittedDataRevision {
34 pub fn from_receipt(receipt: crate::mvcc::CommitReceipt) -> Self {
35 Self {
36 revision: crate::key_value::KeyValueReadRevision::records(
37 receipt.transaction.database(),
38 receipt.sequence,
39 None,
40 ),
41 change_version: receipt.sequence.as_u64(),
42 }
43 }
44}
45
46#[derive(Debug, thiserror::Error)]
47pub enum StorageBackendError {
48 #[error(transparent)]
49 Memory(#[from] uqa_core::memory::MemoryError),
50 #[error(transparent)]
51 Cancelled(#[from] uqa_core::QueryCancelled),
52 #[error("text analysis failed: {0}")]
53 Analysis(#[from] uqa_analysis::AnalysisError),
54 #[error("payload serialization failed: {0}")]
55 Serde(#[from] serde_json::Error),
56 #[error("{backend} storage failed: {source}")]
57 Backend {
58 backend: &'static str,
59 #[source]
60 source: Box<dyn std::error::Error + Send + Sync>,
61 },
62 #[error("{0}")]
63 Other(String),
64}
65
66impl StorageBackendError {
67 pub fn backend(
68 backend: &'static str,
69 source: impl std::error::Error + Send + Sync + 'static,
70 ) -> Self {
71 Self::Backend {
72 backend,
73 source: Box::new(source),
74 }
75 }
76}
77
78pub type StorageBackendResult<T> = std::result::Result<T, StorageBackendError>;
79
80#[derive(Clone, Debug)]
82pub struct StorageSessionAffinity(Arc<()>);
83
84impl StorageSessionAffinity {
85 #[must_use]
86 pub fn new() -> Self {
87 Self(Arc::new(()))
88 }
89}
90
91impl Default for StorageSessionAffinity {
92 fn default() -> Self {
93 Self::new()
94 }
95}
96
97impl PartialEq for StorageSessionAffinity {
98 fn eq(&self, other: &Self) -> bool {
99 Arc::ptr_eq(&self.0, &other.0)
100 }
101}
102
103impl Eq for StorageSessionAffinity {}
104
105#[derive(Debug, thiserror::Error)]
106#[error("catalog and data backend must share one storage transaction context")]
107pub struct StorageSessionMismatch;
108
109#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
111pub struct StorageSavepointId(u64);
112
113impl StorageSavepointId {
114 #[must_use]
115 pub fn allocate() -> Self {
116 static NEXT_ID: AtomicU64 = AtomicU64::new(1);
117 let id = uqa_core::atomic::try_update_u64(
118 &NEXT_ID,
119 Ordering::Relaxed,
120 Ordering::Relaxed,
121 |current| current.checked_add(1),
122 )
123 .expect("storage savepoint identity space exhausted");
124 Self(id)
125 }
126
127 pub fn backend_name(self) -> String {
128 self.0.to_string()
129 }
130}
131
132pub struct PersistentStorageSession {
138 pub catalog: Arc<dyn CatalogFacade>,
139 pub backend: Arc<dyn PersistentStorageBackend>,
140}
141
142#[derive(Clone, Debug, PartialEq, Eq, Hash)]
144pub enum PersistentStorageIdentity {
145 File(PathBuf),
146 Opaque(String),
147}
148
149impl PersistentStorageIdentity {
150 pub fn for_database_path(path: &Path) -> StorageBackendResult<Self> {
152 let path = resolve_final_symlinks(path)?;
153 match std::fs::canonicalize(&path) {
154 Ok(canonical) => Ok(Self::File(canonical)),
155 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
156 let file_name = path.file_name().ok_or_else(|| {
157 StorageBackendError::Other(format!(
158 "database path `{}` has no file name",
159 path.display()
160 ))
161 })?;
162 let parent = path
163 .parent()
164 .filter(|parent| !parent.as_os_str().is_empty())
165 .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
166 let parent = std::fs::canonicalize(&parent).map_err(|error| {
167 StorageBackendError::Other(format!(
168 "canonicalize database directory `{}`: {error}",
169 parent.display()
170 ))
171 })?;
172 Ok(Self::File(parent.join(file_name)))
173 }
174 Err(error) => Err(StorageBackendError::Other(format!(
175 "canonicalize database `{}`: {error}",
176 path.display()
177 ))),
178 }
179 }
180}
181
182fn resolve_final_symlinks(path: &Path) -> StorageBackendResult<PathBuf> {
183 let mut current = path.to_path_buf();
184 let mut followed = 0usize;
185 loop {
186 match std::fs::symlink_metadata(¤t) {
187 Ok(metadata) if metadata.file_type().is_symlink() => {
188 followed += 1;
189 if followed > 40 {
190 return Err(StorageBackendError::Other(format!(
191 "database path `{}` has too many symbolic-link levels",
192 path.display()
193 )));
194 }
195 let target = std::fs::read_link(¤t).map_err(|error| {
196 StorageBackendError::Other(format!(
197 "read database symbolic link `{}`: {error}",
198 current.display()
199 ))
200 })?;
201 current = if target.is_absolute() {
202 target
203 } else {
204 current
205 .parent()
206 .filter(|parent| !parent.as_os_str().is_empty())
207 .unwrap_or_else(|| Path::new("."))
208 .join(target)
209 };
210 }
211 Ok(_) => return Ok(current),
212 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(current),
213 Err(error) => {
214 return Err(StorageBackendError::Other(format!(
215 "inspect database path `{}`: {error}",
216 current.display()
217 )))
218 }
219 }
220 }
221}
222
223#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
225pub enum StorageTransactionModel {
226 #[default]
227 ProviderSerialized,
228 VersionedConcurrent { database: crate::mvcc::DatabaseId },
230}
231
232impl StorageTransactionModel {
233 pub fn is_versioned(self) -> bool {
234 matches!(self, Self::VersionedConcurrent { .. })
235 }
236}
237
238impl PersistentStorageSession {
239 pub fn new(
240 catalog: Arc<dyn CatalogFacade>,
241 backend: Arc<dyn PersistentStorageBackend>,
242 ) -> Self {
243 Self { catalog, backend }
244 }
245
246 pub fn validate_transaction_affinity(&self) -> StorageBackendResult<()> {
248 let affinity = self.backend.transaction_affinity();
249 let model = self.backend.transaction_model();
250 if self.catalog.transaction_affinity() != affinity
251 || self.catalog.transaction_model() != model
252 || (model.is_versioned() && affinity.is_none())
253 {
254 return Err(StorageBackendError::backend(
255 "session",
256 StorageSessionMismatch,
257 ));
258 }
259 if model.is_versioned() && self.backend.write_cancellation().is_none() {
260 return Err(StorageBackendError::Other(
261 "versioned storage backend does not expose write cancellation".into(),
262 ));
263 }
264 Ok(())
265 }
266}
267
268pub trait PersistentStorageProvider: Send + Sync {
274 fn open_session(&self) -> StorageBackendResult<PersistentStorageSession>;
275
276 fn open_session_with_cancellation(
278 &self,
279 cancellation: &uqa_core::CancellationToken,
280 ) -> StorageBackendResult<PersistentStorageSession> {
281 cancellation.check()?;
282 let session = self.open_session()?;
283 if session.backend.transaction_model().is_versioned() {
284 return Err(StorageBackendError::Other(
285 "cancellable independent storage sessions are not implemented by this provider"
286 .into(),
287 ));
288 }
289 Ok(session)
290 }
291
292 fn open_initial_session(&self) -> StorageBackendResult<PersistentStorageSession> {
294 self.open_session()
295 }
296
297 fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
299 Ok(None)
300 }
301
302 fn auxiliary_encryption_key(&self) -> Option<crate::StorageEncryptionKey> {
306 None
307 }
308}
309
310pub trait PersistentStorageBackend: Send + Sync {
312 fn require_sequence_value_durability(&self) -> StorageBackendResult<()> {
314 Ok(())
315 }
316
317 fn open_sequence_value_session(
319 &self,
320 cancellation: &uqa_core::CancellationToken,
321 ) -> StorageBackendResult<PersistentStorageSession> {
322 self.open_session_with_cancellation(cancellation)
323 }
324
325 fn retention_control(&self) -> Option<crate::read_control::StorageReadControl> {
327 None
328 }
329
330 fn notification_publications(
332 &self,
333 ) -> Option<&dyn crate::notifications::NotificationPublicationStore> {
334 None
335 }
336
337 fn serializable_session(&self) -> Option<&dyn crate::mvcc::SerializableSession> {
339 None
340 }
341
342 fn write_cancellation(&self) -> Option<uqa_core::CancellationToken> {
344 None
345 }
346
347 fn open_session_with_cancellation(
349 &self,
350 cancellation: &uqa_core::CancellationToken,
351 ) -> StorageBackendResult<PersistentStorageSession> {
352 cancellation.check()?;
353 if self.transaction_model().is_versioned() {
354 return Err(StorageBackendError::Other(
355 "cancellable independent storage sessions are not implemented by this backend"
356 .into(),
357 ));
358 }
359 self.open_session()
360 }
361
362 fn open_controlled_session(
364 &self,
365 _control: &crate::read_control::StorageReadControl,
366 ) -> StorageBackendResult<PersistentStorageSession> {
367 Err(StorageBackendError::Other(
368 "controlled independent sessions are not implemented by this backend".into(),
369 ))
370 }
371
372 fn open_retained_read_session(
374 &self,
375 cancellation: &uqa_core::CancellationToken,
376 ) -> StorageBackendResult<PersistentStorageSession> {
377 cancellation.check()?;
378 Err(StorageBackendError::Other(
379 "retained read sessions are not implemented by this storage backend".into(),
380 ))
381 }
382
383 fn capture_committed_read_boundary(
385 &self,
386 ) -> StorageBackendResult<Arc<crate::mvcc::CommittedReadBoundary>> {
387 Err(StorageBackendError::Other(
388 "fixed committed read boundaries are not supported by this backend".into(),
389 ))
390 }
391
392 fn open_read_session_with_private_changes(
394 &self,
395 _boundary: &crate::mvcc::CommittedReadBoundary,
396 cancellation: &uqa_core::CancellationToken,
397 ) -> StorageBackendResult<PersistentStorageSession> {
398 cancellation.check()?;
399 Err(StorageBackendError::Other(
400 "fixed committed/private read composition is not supported by this backend".into(),
401 ))
402 }
403
404 fn transaction_model(&self) -> StorageTransactionModel {
406 StorageTransactionModel::ProviderSerialized
407 }
408
409 fn identifier_allocator(&self) -> Option<&dyn crate::mvcc::IdentifierAllocator> {
411 None
412 }
413
414 fn transaction_affinity(&self) -> Option<StorageSessionAffinity> {
416 None
417 }
418
419 fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
421 Ok(None)
422 }
423
424 fn auxiliary_encryption_key(&self) -> Option<crate::StorageEncryptionKey> {
427 None
428 }
429
430 fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
432 Err(StorageBackendError::Other(
433 "independent sessions are not implemented for this persistent backend".into(),
434 ))
435 }
436
437 fn supports_concurrent_pinned_read_and_write(&self) -> bool {
439 false
440 }
441
442 fn document_store(&self, table: &str) -> Box<dyn DocumentStore>;
443
444 fn migrate_document_storage(&self) -> StorageBackendResult<()> {
446 Ok(())
447 }
448
449 fn inverted_index(&self, table: &str, analyzer: Analyzer) -> Box<dyn InvertedIndex>;
450
451 fn migrate_inverted_index_storage(&self) -> StorageBackendResult<()> {
454 Ok(())
455 }
456
457 fn vector_index(
458 &self,
459 table: &str,
460 field: &str,
461 dimensions: u32,
462 spec: VectorIndexSpec,
463 mode: VectorIndexOpenMode,
464 ) -> StorageBackendResult<Box<dyn VectorIndex>>;
465
466 fn diskann_index(
468 &self,
469 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
470 _options: crate::diskann_index::DiskANNIndexOptions,
471 _temporary: &crate::diskann_index::build::DiskANNTemporaryBudget,
472 _mode: VectorIndexOpenMode,
473 ) -> StorageBackendResult<Box<dyn VectorIndex>> {
474 Err(StorageBackendError::Other(
475 "this backend does not support catalog-bound DiskANN indexes".into(),
476 ))
477 }
478
479 fn diskann_journal_pruner(
481 &self,
482 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
483 _max_record_bytes: usize,
484 ) -> StorageBackendResult<Box<dyn crate::diskann_index::changes::DiskANNJournalPruner>> {
485 Err(StorageBackendError::Other(
486 "this backend does not support DiskANN journal maintenance".into(),
487 ))
488 }
489
490 fn diskann_maintenance_source(
492 &self,
493 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
494 _max_record_bytes: usize,
495 ) -> StorageBackendResult<Box<dyn crate::diskann_index::maintenance::DiskANNMaintenanceSource>>
496 {
497 Err(StorageBackendError::Other(
498 "this backend does not support DiskANN rebuild maintenance".into(),
499 ))
500 }
501
502 fn retire_diskann_index(
503 &self,
504 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
505 ) -> StorageBackendResult<()> {
506 Err(StorageBackendError::Other(
507 "this backend does not support DiskANN retirement".into(),
508 ))
509 }
510
511 fn drop_vector_index_metadata(&self, _table: &str, _field: &str) -> StorageBackendResult<()> {
512 Ok(())
513 }
514
515 fn persists_btree_indexes(&self) -> bool {
517 false
518 }
519
520 fn load_btree_index(
523 &self,
524 _table: &str,
525 _field: &crate::ValueIndexKey,
526 ) -> StorageBackendResult<Option<Vec<(DocId, Value)>>> {
527 Ok(None)
528 }
529
530 fn visit_btree_index_entries(
532 &self,
533 _table: &str,
534 _field: &crate::ValueIndexKey,
535 _visit: &mut dyn FnMut(DocId, Value) -> StorageBackendResult<()>,
536 ) -> StorageBackendResult<bool> {
537 Err(StorageBackendError::Other(
538 "persistent provider does not support ordered stored-key reads".into(),
539 ))
540 }
541
542 fn btree_index_fields(&self, _table: &str) -> StorageBackendResult<Vec<crate::ValueIndexKey>> {
543 Ok(Vec::new())
544 }
545
546 fn probe_btree_index_equal(
548 &self,
549 _table: &str,
550 _field: &crate::ValueIndexKey,
551 _value: &Value,
552 ) -> StorageBackendResult<Option<Vec<DocId>>> {
553 Ok(None)
554 }
555
556 fn visit_btree_index_equal(
558 &self,
559 _table: &str,
560 _field: &crate::ValueIndexKey,
561 _value: &Value,
562 _visit: &mut dyn FnMut(DocId) -> bool,
563 ) -> StorageBackendResult<bool> {
564 Ok(false)
565 }
566
567 fn read_btree_index_entry(
569 &self,
570 _table: &str,
571 _field: &crate::ValueIndexKey,
572 _doc_id: DocId,
573 ) -> StorageBackendResult<crate::ValueIndexEntry> {
574 if self.persists_btree_indexes() {
575 return Err(crate::StorageBackendError::Other(
576 "point reads of stored B-tree entries are not implemented for this backend".into(),
577 ));
578 }
579 Ok(crate::ValueIndexEntry::Unbuilt)
580 }
581
582 fn btree_index_repairs(&self) -> StorageBackendResult<Vec<(String, crate::ValueIndexKey)>> {
586 Ok(Vec::new())
587 }
588
589 fn clear_btree_index_repair(
590 &self,
591 _table: &str,
592 _field: &crate::ValueIndexKey,
593 ) -> StorageBackendResult<()> {
594 Ok(())
595 }
596
597 fn replace_btree_index(
598 &self,
599 _table: &str,
600 _field: &crate::ValueIndexKey,
601 _values: &[(DocId, Value)],
602 ) -> StorageBackendResult<()> {
603 Ok(())
604 }
605
606 fn repair_btree_index(
610 &self,
611 table: &str,
612 field: &crate::ValueIndexKey,
613 complete: &[(DocId, Value)],
614 _stale_doc_ids: &[DocId],
615 _missing: &[(DocId, Value)],
616 ) -> StorageBackendResult<()> {
617 self.replace_btree_index(table, field, complete)
618 }
619
620 fn replace_btree_indexes(
623 &self,
624 table: &str,
625 indexes: &[(&crate::ValueIndexKey, &[(DocId, Value)])],
626 ) -> StorageBackendResult<()> {
627 for (field, values) in indexes {
628 self.replace_btree_index(table, field, values)?;
629 }
630 Ok(())
631 }
632
633 fn apply_btree_index_write(
634 &self,
635 _table: &str,
636 _doc_id: DocId,
637 _values: Option<&BTreeMap<crate::ValueIndexKey, Value>>,
638 ) -> StorageBackendResult<()> {
639 Ok(())
640 }
641
642 fn apply_unused_btree_index_write(
644 &self,
645 table: &str,
646 doc_id: DocId,
647 values: &BTreeMap<crate::ValueIndexKey, Value>,
648 namespace: crate::document_store::identifiers::DocumentIdNamespace,
649 ) -> StorageBackendResult<()> {
650 let _ = namespace;
651 self.apply_btree_index_write(table, doc_id, Some(values))
652 }
653
654 fn drop_btree_index(
655 &self,
656 _table: &str,
657 _field: &crate::ValueIndexKey,
658 ) -> StorageBackendResult<()> {
659 Ok(())
660 }
661
662 fn clear_btree_indexes(&self, _table: &str) -> StorageBackendResult<()> {
663 Ok(())
664 }
665
666 fn reclaim_obsolete(&self) -> StorageBackendResult<()> {
668 if self.transaction_model().is_versioned() {
669 return Err(StorageBackendError::Other(
670 "versioned storage reclamation is not implemented by this backend".into(),
671 ));
672 }
673 Ok(())
674 }
675
676 fn vacuum(&self) -> StorageBackendResult<()> {
678 Ok(())
679 }
680
681 fn begin_transaction(&self) -> StorageBackendResult<()>;
682
683 fn begin_read_transaction(&self) -> StorageBackendResult<()> {
687 self.begin_transaction()
688 }
689
690 fn begin_upgradeable_transaction(&self) -> StorageBackendResult<()> {
692 self.begin_transaction()
693 }
694
695 fn refresh_transaction_snapshot(
697 &self,
698 _cancellation: &uqa_core::CancellationToken,
699 ) -> StorageBackendResult<()> {
700 Err(StorageBackendError::Other(
701 "transaction snapshot refresh is not supported by this backend".into(),
702 ))
703 }
704
705 fn in_transaction(&self) -> bool;
707
708 fn transaction_has_written(&self) -> StorageBackendResult<bool>;
710
711 fn change_version(&self) -> StorageBackendResult<Option<u64>> {
714 Ok(None)
715 }
716
717 fn read_view_revision(
719 &self,
720 ) -> StorageBackendResult<Option<crate::key_value::KeyValueReadRevision>> {
721 Ok(None)
722 }
723
724 fn committed_data_revision(&self) -> StorageBackendResult<Option<CommittedDataRevision>> {
726 Ok(None)
727 }
728
729 fn commit_monitor_version(&self) -> StorageBackendResult<Option<u64>> {
731 Ok(None)
732 }
733
734 fn change_version_monitor_is_nonblocking(&self) -> StorageBackendResult<bool> {
736 Ok(true)
737 }
738
739 fn pin_transaction_snapshot(&self) -> StorageBackendResult<()> {
741 Ok(())
742 }
743
744 fn prepare_transaction_commit(&self) -> StorageBackendResult<()> {
746 Ok(())
747 }
748
749 fn commit_transaction(&self) -> StorageBackendResult<()>;
750
751 fn rollback_transaction(&self) -> StorageBackendResult<()>;
752
753 fn savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()>;
754
755 fn release_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()>;
756
757 fn rollback_to_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()>;
758}
759
760#[cfg(all(test, unix))]
761mod identity_tests {
762 use super::*;
763
764 #[test]
765 fn dangling_database_symlink_keeps_the_target_identity_after_creation() {
766 use std::os::unix::fs::symlink;
767
768 let directory = tempfile::tempdir().unwrap();
769 let target = directory.path().join("target.db");
770 let link = directory.path().join("database.db");
771 symlink("target.db", &link).unwrap();
772
773 let before = PersistentStorageIdentity::for_database_path(&link).unwrap();
774 std::fs::File::create(&target).unwrap();
775 let after = PersistentStorageIdentity::for_database_path(&link).unwrap();
776
777 assert_eq!(before, after);
778 assert_eq!(
779 after,
780 PersistentStorageIdentity::File(target.canonicalize().unwrap())
781 );
782 }
783}