1use std::collections::BTreeMap;
14use std::path::{Path, PathBuf};
15use std::sync::atomic::{AtomicU64, Ordering};
16use std::sync::Arc;
17
18use uqa_analysis::Analyzer;
19use uqa_core::{DocId, Value};
20
21use crate::document_store::DocumentStore;
22use crate::inverted_index::InvertedIndex;
23use crate::vector_index::{VectorIndex, VectorIndexOpenMode, VectorIndexSpec};
24use crate::CatalogFacade;
25
26#[derive(Debug, thiserror::Error)]
27pub enum StorageBackendError {
28 #[error(transparent)]
29 Memory(#[from] uqa_core::memory::MemoryError),
30 #[error(transparent)]
31 Cancelled(#[from] uqa_core::QueryCancelled),
32 #[error("text analysis failed: {0}")]
33 Analysis(#[from] uqa_analysis::AnalysisError),
34 #[error("payload serialization failed: {0}")]
35 Serde(#[from] serde_json::Error),
36 #[error("{backend} storage failed: {source}")]
37 Backend {
38 backend: &'static str,
39 #[source]
40 source: Box<dyn std::error::Error + Send + Sync>,
41 },
42 #[error("{0}")]
43 Other(String),
44}
45
46impl StorageBackendError {
47 pub fn backend(
48 backend: &'static str,
49 source: impl std::error::Error + Send + Sync + 'static,
50 ) -> Self {
51 Self::Backend {
52 backend,
53 source: Box::new(source),
54 }
55 }
56}
57
58pub type StorageBackendResult<T> = std::result::Result<T, StorageBackendError>;
59
60#[derive(Clone, Debug)]
62pub struct StorageSessionAffinity(Arc<()>);
63
64impl StorageSessionAffinity {
65 #[must_use]
66 pub fn new() -> Self {
67 Self(Arc::new(()))
68 }
69}
70
71impl Default for StorageSessionAffinity {
72 fn default() -> Self {
73 Self::new()
74 }
75}
76
77impl PartialEq for StorageSessionAffinity {
78 fn eq(&self, other: &Self) -> bool {
79 Arc::ptr_eq(&self.0, &other.0)
80 }
81}
82
83impl Eq for StorageSessionAffinity {}
84
85#[derive(Debug, thiserror::Error)]
86#[error("catalog and data backend must share one storage transaction context")]
87pub struct StorageSessionMismatch;
88
89#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
91pub struct StorageSavepointId(u64);
92
93impl StorageSavepointId {
94 #[must_use]
95 pub fn allocate() -> Self {
96 static NEXT_ID: AtomicU64 = AtomicU64::new(1);
97 let id = uqa_core::atomic::try_update_u64(
98 &NEXT_ID,
99 Ordering::Relaxed,
100 Ordering::Relaxed,
101 |current| current.checked_add(1),
102 )
103 .expect("storage savepoint identity space exhausted");
104 Self(id)
105 }
106
107 pub fn backend_name(self) -> String {
108 self.0.to_string()
109 }
110}
111
112pub struct PersistentStorageSession {
118 pub catalog: Arc<dyn CatalogFacade>,
119 pub backend: Arc<dyn PersistentStorageBackend>,
120}
121
122#[derive(Clone, Debug, PartialEq, Eq, Hash)]
124pub enum PersistentStorageIdentity {
125 File(PathBuf),
126 Opaque(String),
127}
128
129impl PersistentStorageIdentity {
130 pub fn for_database_path(path: &Path) -> StorageBackendResult<Self> {
132 let path = resolve_final_symlinks(path)?;
133 match std::fs::canonicalize(&path) {
134 Ok(canonical) => Ok(Self::File(canonical)),
135 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
136 let file_name = path.file_name().ok_or_else(|| {
137 StorageBackendError::Other(format!(
138 "database path `{}` has no file name",
139 path.display()
140 ))
141 })?;
142 let parent = path
143 .parent()
144 .filter(|parent| !parent.as_os_str().is_empty())
145 .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
146 let parent = std::fs::canonicalize(&parent).map_err(|error| {
147 StorageBackendError::Other(format!(
148 "canonicalize database directory `{}`: {error}",
149 parent.display()
150 ))
151 })?;
152 Ok(Self::File(parent.join(file_name)))
153 }
154 Err(error) => Err(StorageBackendError::Other(format!(
155 "canonicalize database `{}`: {error}",
156 path.display()
157 ))),
158 }
159 }
160}
161
162fn resolve_final_symlinks(path: &Path) -> StorageBackendResult<PathBuf> {
163 let mut current = path.to_path_buf();
164 let mut followed = 0usize;
165 loop {
166 match std::fs::symlink_metadata(¤t) {
167 Ok(metadata) if metadata.file_type().is_symlink() => {
168 followed += 1;
169 if followed > 40 {
170 return Err(StorageBackendError::Other(format!(
171 "database path `{}` has too many symbolic-link levels",
172 path.display()
173 )));
174 }
175 let target = std::fs::read_link(¤t).map_err(|error| {
176 StorageBackendError::Other(format!(
177 "read database symbolic link `{}`: {error}",
178 current.display()
179 ))
180 })?;
181 current = if target.is_absolute() {
182 target
183 } else {
184 current
185 .parent()
186 .filter(|parent| !parent.as_os_str().is_empty())
187 .unwrap_or_else(|| Path::new("."))
188 .join(target)
189 };
190 }
191 Ok(_) => return Ok(current),
192 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(current),
193 Err(error) => {
194 return Err(StorageBackendError::Other(format!(
195 "inspect database path `{}`: {error}",
196 current.display()
197 )))
198 }
199 }
200 }
201}
202
203#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
205pub enum StorageTransactionModel {
206 #[default]
207 ProviderSerialized,
208 VersionedConcurrent { database: crate::mvcc::DatabaseId },
210}
211
212impl StorageTransactionModel {
213 pub fn is_versioned(self) -> bool {
214 matches!(self, Self::VersionedConcurrent { .. })
215 }
216}
217
218impl PersistentStorageSession {
219 pub fn new(
220 catalog: Arc<dyn CatalogFacade>,
221 backend: Arc<dyn PersistentStorageBackend>,
222 ) -> Self {
223 Self { catalog, backend }
224 }
225
226 pub fn validate_transaction_affinity(&self) -> StorageBackendResult<()> {
228 let affinity = self.backend.transaction_affinity();
229 let model = self.backend.transaction_model();
230 if self.catalog.transaction_affinity() != affinity
231 || self.catalog.transaction_model() != model
232 || (model.is_versioned() && affinity.is_none())
233 {
234 return Err(StorageBackendError::backend(
235 "session",
236 StorageSessionMismatch,
237 ));
238 }
239 if model.is_versioned() && self.backend.write_cancellation().is_none() {
240 return Err(StorageBackendError::Other(
241 "versioned storage backend does not expose write cancellation".into(),
242 ));
243 }
244 Ok(())
245 }
246}
247
248pub trait PersistentStorageProvider: Send + Sync {
254 fn open_session(&self) -> StorageBackendResult<PersistentStorageSession>;
255
256 fn open_session_with_cancellation(
258 &self,
259 cancellation: &uqa_core::CancellationToken,
260 ) -> StorageBackendResult<PersistentStorageSession> {
261 cancellation.check()?;
262 let session = self.open_session()?;
263 if session.backend.transaction_model().is_versioned() {
264 return Err(StorageBackendError::Other(
265 "cancellable independent storage sessions are not implemented by this provider"
266 .into(),
267 ));
268 }
269 Ok(session)
270 }
271
272 fn open_initial_session(&self) -> StorageBackendResult<PersistentStorageSession> {
274 self.open_session()
275 }
276
277 fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
279 Ok(None)
280 }
281
282 fn auxiliary_encryption_key(&self) -> Option<crate::StorageEncryptionKey> {
286 None
287 }
288}
289
290pub trait PersistentStorageBackend: Send + Sync {
292 fn retention_control(&self) -> Option<crate::read_control::StorageReadControl> {
294 None
295 }
296
297 fn notification_publications(
299 &self,
300 ) -> Option<&dyn crate::notifications::NotificationPublicationStore> {
301 None
302 }
303
304 fn serializable_session(&self) -> Option<&dyn crate::mvcc::SerializableSession> {
306 None
307 }
308
309 fn write_cancellation(&self) -> Option<uqa_core::CancellationToken> {
311 None
312 }
313
314 fn open_session_with_cancellation(
316 &self,
317 cancellation: &uqa_core::CancellationToken,
318 ) -> StorageBackendResult<PersistentStorageSession> {
319 cancellation.check()?;
320 if self.transaction_model().is_versioned() {
321 return Err(StorageBackendError::Other(
322 "cancellable independent storage sessions are not implemented by this backend"
323 .into(),
324 ));
325 }
326 self.open_session()
327 }
328
329 fn open_controlled_session(
331 &self,
332 _control: &crate::read_control::StorageReadControl,
333 ) -> StorageBackendResult<PersistentStorageSession> {
334 Err(StorageBackendError::Other(
335 "controlled independent sessions are not implemented by this backend".into(),
336 ))
337 }
338
339 fn open_retained_read_session(
341 &self,
342 cancellation: &uqa_core::CancellationToken,
343 ) -> StorageBackendResult<PersistentStorageSession> {
344 cancellation.check()?;
345 Err(StorageBackendError::Other(
346 "retained read sessions are not implemented by this storage backend".into(),
347 ))
348 }
349
350 fn transaction_model(&self) -> StorageTransactionModel {
352 StorageTransactionModel::ProviderSerialized
353 }
354
355 fn identifier_allocator(&self) -> Option<&dyn crate::mvcc::IdentifierAllocator> {
357 None
358 }
359
360 fn transaction_affinity(&self) -> Option<StorageSessionAffinity> {
362 None
363 }
364
365 fn storage_identity(&self) -> StorageBackendResult<Option<PersistentStorageIdentity>> {
367 Ok(None)
368 }
369
370 fn auxiliary_encryption_key(&self) -> Option<crate::StorageEncryptionKey> {
373 None
374 }
375
376 fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
378 Err(StorageBackendError::Other(
379 "independent sessions are not implemented for this persistent backend".into(),
380 ))
381 }
382
383 fn supports_concurrent_pinned_read_and_write(&self) -> bool {
385 false
386 }
387
388 fn document_store(&self, table: &str) -> Box<dyn DocumentStore>;
389
390 fn migrate_document_storage(&self) -> StorageBackendResult<()> {
392 Ok(())
393 }
394
395 fn inverted_index(&self, table: &str, analyzer: Analyzer) -> Box<dyn InvertedIndex>;
396
397 fn migrate_inverted_index_storage(&self) -> StorageBackendResult<()> {
400 Ok(())
401 }
402
403 fn vector_index(
404 &self,
405 table: &str,
406 field: &str,
407 dimensions: u32,
408 spec: VectorIndexSpec,
409 mode: VectorIndexOpenMode,
410 ) -> StorageBackendResult<Box<dyn VectorIndex>>;
411
412 fn diskann_index(
414 &self,
415 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
416 _options: crate::diskann_index::DiskANNIndexOptions,
417 _temporary: &crate::diskann_index::build::DiskANNTemporaryBudget,
418 _mode: VectorIndexOpenMode,
419 ) -> StorageBackendResult<Box<dyn VectorIndex>> {
420 Err(StorageBackendError::Other(
421 "this backend does not support catalog-bound DiskANN indexes".into(),
422 ))
423 }
424
425 fn diskann_journal_pruner(
427 &self,
428 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
429 _max_record_bytes: usize,
430 ) -> StorageBackendResult<Box<dyn crate::diskann_index::changes::DiskANNJournalPruner>> {
431 Err(StorageBackendError::Other(
432 "this backend does not support DiskANN journal maintenance".into(),
433 ))
434 }
435
436 fn diskann_maintenance_source(
438 &self,
439 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
440 _max_record_bytes: usize,
441 ) -> StorageBackendResult<Box<dyn crate::diskann_index::maintenance::DiskANNMaintenanceSource>>
442 {
443 Err(StorageBackendError::Other(
444 "this backend does not support DiskANN rebuild maintenance".into(),
445 ))
446 }
447
448 fn retire_diskann_index(
449 &self,
450 _binding: crate::diskann_index::DiskANNIndexBinding<'_>,
451 ) -> StorageBackendResult<()> {
452 Err(StorageBackendError::Other(
453 "this backend does not support DiskANN retirement".into(),
454 ))
455 }
456
457 fn drop_vector_index_metadata(&self, _table: &str, _field: &str) -> StorageBackendResult<()> {
458 Ok(())
459 }
460
461 fn persists_btree_indexes(&self) -> bool {
463 false
464 }
465
466 fn load_btree_index(
469 &self,
470 _table: &str,
471 _field: &crate::ValueIndexKey,
472 ) -> StorageBackendResult<Option<Vec<(DocId, Value)>>> {
473 Ok(None)
474 }
475
476 fn btree_index_fields(&self, _table: &str) -> StorageBackendResult<Vec<crate::ValueIndexKey>> {
477 Ok(Vec::new())
478 }
479
480 fn read_btree_index_entry(
482 &self,
483 _table: &str,
484 _field: &crate::ValueIndexKey,
485 _doc_id: DocId,
486 ) -> StorageBackendResult<crate::ValueIndexEntry> {
487 if self.persists_btree_indexes() {
488 return Err(crate::StorageBackendError::Other(
489 "point reads of stored B-tree entries are not implemented for this backend".into(),
490 ));
491 }
492 Ok(crate::ValueIndexEntry::Unbuilt)
493 }
494
495 fn btree_index_repairs(&self) -> StorageBackendResult<Vec<(String, crate::ValueIndexKey)>> {
499 Ok(Vec::new())
500 }
501
502 fn clear_btree_index_repair(
503 &self,
504 _table: &str,
505 _field: &crate::ValueIndexKey,
506 ) -> StorageBackendResult<()> {
507 Ok(())
508 }
509
510 fn replace_btree_index(
511 &self,
512 _table: &str,
513 _field: &crate::ValueIndexKey,
514 _values: &[(DocId, Value)],
515 ) -> StorageBackendResult<()> {
516 Ok(())
517 }
518
519 fn repair_btree_index(
523 &self,
524 table: &str,
525 field: &crate::ValueIndexKey,
526 complete: &[(DocId, Value)],
527 _stale_doc_ids: &[DocId],
528 _missing: &[(DocId, Value)],
529 ) -> StorageBackendResult<()> {
530 self.replace_btree_index(table, field, complete)
531 }
532
533 fn replace_btree_indexes(
536 &self,
537 table: &str,
538 indexes: &[(&crate::ValueIndexKey, &[(DocId, Value)])],
539 ) -> StorageBackendResult<()> {
540 for (field, values) in indexes {
541 self.replace_btree_index(table, field, values)?;
542 }
543 Ok(())
544 }
545
546 fn apply_btree_index_write(
547 &self,
548 _table: &str,
549 _doc_id: DocId,
550 _values: Option<&BTreeMap<crate::ValueIndexKey, Value>>,
551 ) -> StorageBackendResult<()> {
552 Ok(())
553 }
554
555 fn apply_unused_btree_index_write(
557 &self,
558 table: &str,
559 doc_id: DocId,
560 values: &BTreeMap<crate::ValueIndexKey, Value>,
561 namespace: crate::document_store::identifiers::DocumentIdNamespace,
562 ) -> StorageBackendResult<()> {
563 let _ = namespace;
564 self.apply_btree_index_write(table, doc_id, Some(values))
565 }
566
567 fn drop_btree_index(
568 &self,
569 _table: &str,
570 _field: &crate::ValueIndexKey,
571 ) -> StorageBackendResult<()> {
572 Ok(())
573 }
574
575 fn clear_btree_indexes(&self, _table: &str) -> StorageBackendResult<()> {
576 Ok(())
577 }
578
579 fn reclaim_obsolete(&self) -> StorageBackendResult<()> {
581 if self.transaction_model().is_versioned() {
582 return Err(StorageBackendError::Other(
583 "versioned storage reclamation is not implemented by this backend".into(),
584 ));
585 }
586 Ok(())
587 }
588
589 fn vacuum(&self) -> StorageBackendResult<()> {
591 Ok(())
592 }
593
594 fn begin_transaction(&self) -> StorageBackendResult<()>;
595
596 fn begin_read_transaction(&self) -> StorageBackendResult<()> {
600 self.begin_transaction()
601 }
602
603 fn begin_upgradeable_transaction(&self) -> StorageBackendResult<()> {
605 self.begin_transaction()
606 }
607
608 fn refresh_transaction_snapshot(
610 &self,
611 _cancellation: &uqa_core::CancellationToken,
612 ) -> StorageBackendResult<()> {
613 Err(StorageBackendError::Other(
614 "transaction snapshot refresh is not supported by this backend".into(),
615 ))
616 }
617
618 fn in_transaction(&self) -> bool;
620
621 fn transaction_has_written(&self) -> StorageBackendResult<bool>;
623
624 fn change_version(&self) -> StorageBackendResult<Option<u64>> {
627 Ok(None)
628 }
629
630 fn read_view_revision(
632 &self,
633 ) -> StorageBackendResult<Option<crate::key_value::KeyValueReadRevision>> {
634 Ok(None)
635 }
636
637 fn commit_monitor_version(&self) -> StorageBackendResult<Option<u64>> {
639 Ok(None)
640 }
641
642 fn change_version_monitor_is_nonblocking(&self) -> StorageBackendResult<bool> {
644 Ok(true)
645 }
646
647 fn pin_transaction_snapshot(&self) -> StorageBackendResult<()> {
649 Ok(())
650 }
651
652 fn commit_transaction(&self) -> StorageBackendResult<()>;
653
654 fn rollback_transaction(&self) -> StorageBackendResult<()>;
655
656 fn savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()>;
657
658 fn release_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()>;
659
660 fn rollback_to_savepoint(&self, id: StorageSavepointId) -> StorageBackendResult<()>;
661}
662
663#[cfg(all(test, unix))]
664mod identity_tests {
665 use super::*;
666
667 #[test]
668 fn dangling_database_symlink_keeps_the_target_identity_after_creation() {
669 use std::os::unix::fs::symlink;
670
671 let directory = tempfile::tempdir().unwrap();
672 let target = directory.path().join("target.db");
673 let link = directory.path().join("database.db");
674 symlink("target.db", &link).unwrap();
675
676 let before = PersistentStorageIdentity::for_database_path(&link).unwrap();
677 std::fs::File::create(&target).unwrap();
678 let after = PersistentStorageIdentity::for_database_path(&link).unwrap();
679
680 assert_eq!(before, after);
681 assert_eq!(
682 after,
683 PersistentStorageIdentity::File(target.canonicalize().unwrap())
684 );
685 }
686}