1use super::{
10 Arc, DeepModel, Engine, ManagedConnection, Path, PersistentStorageBackend,
11 PersistentStorageProvider, PersistentStorageSession, SQLiteCompressedContainerAnchor,
12 SQLiteCompressionOptions, SQLiteError, SQLiteStorageProvider, StorageBackendError,
13 StorageBackendResult,
14};
15
16struct BackendSessionProvider {
17 backend: Arc<dyn PersistentStorageBackend>,
18}
19
20impl PersistentStorageProvider for BackendSessionProvider {
21 fn open_session_with_cancellation(
22 &self,
23 cancellation: &uqa_core::CancellationToken,
24 ) -> StorageBackendResult<PersistentStorageSession> {
25 self.backend.open_session_with_cancellation(cancellation)
26 }
27
28 fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
29 self.backend.auxiliary_encryption_key()
30 }
31
32 fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
33 self.backend.open_session()
34 }
35
36 fn storage_identity(
37 &self,
38 ) -> StorageBackendResult<Option<uqa_storage::PersistentStorageIdentity>> {
39 self.backend.storage_identity()
40 }
41}
42
43impl Default for Engine {
44 fn default() -> Self {
45 Self::new()
46 }
47}
48
49impl Engine {
50 pub fn new() -> Self {
52 Self::in_memory_with_retention(uqa_core::memory::MemoryBudget::new(
53 uqa_storage::mvcc::VersionedSessionOptions::default().retained_bytes,
54 ))
55 }
56
57 pub(crate) fn in_memory_with_retention(memory: uqa_core::memory::MemoryBudget) -> Self {
58 let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
59 let notification_hub = Arc::new(crate::NotificationHub::default());
60 let session_id = row_locks.allocate_session();
61 let session = Arc::new(super::SessionContext::with_retention(
62 super::initial_random_state(),
63 memory,
64 ));
65 let client_level = session.state.client_level();
66 let runtime = super::QueryRuntime::new(
67 super::SQL_FUNCTION_DEPTH_LIMIT,
68 client_level,
69 session.state.io_timing(),
70 );
71 session.state.attach_cancellation(&runtime.cancellation);
72 Self {
73 storage: super::StorageContext::memory(),
74 durable: Arc::new(super::DurableCatalogState::new()),
75 extensions: super::RuntimeExtensions::new(&session, &runtime.statement_gate),
76 session,
77 epochs: Arc::new(super::EpochCoordinator::new()),
78 runtime,
79 statistics: crate::statistics::shared_statistics(&row_locks),
80 row_locks,
81 notification_hub,
82 session_id,
83 owns_session_registration: true,
84 query_table_snapshots: None,
85 query_view_snapshots: None,
86 query_sql_function_snapshots: None,
87 query_catalog_snapshot: None,
88 query_transaction_overlay: None,
89 query_transaction_origin: None,
90 }
91 }
92
93 pub fn open(path: &Path) -> Result<Self, SQLiteError> {
94 let conn = ManagedConnection::open(path)?;
95 Self::open_with_connection(&conn)
96 }
97
98 pub fn detect_database_file(
103 path: &Path,
104 ) -> std::io::Result<uqa_storage_sqlite::DatabaseFileFormat> {
105 uqa_storage_sqlite::detect_database_file_format(path)
106 }
107
108 pub fn open_auto(path: &Path, key: Option<&str>) -> Result<Self, SQLiteError> {
129 let connection = ManagedConnection::open_auto(path, key)?;
130 Self::open_with_connection(&connection)
131 }
132
133 pub fn open_encrypted(path: &Path, key: &str) -> Result<Self, SQLiteError> {
137 let conn = ManagedConnection::open_encrypted(path, key)?;
138 Self::open_with_connection(&conn)
139 }
140
141 pub fn open_compressed(
145 path: &Path,
146 compression: SQLiteCompressionOptions,
147 ) -> Result<Self, SQLiteError> {
148 let conn = ManagedConnection::open_compressed(path, compression)?;
149 Self::open_with_connection(&conn)
150 }
151
152 pub fn open_compressed_encrypted(
160 path: &Path,
161 key: &str,
162 compression: SQLiteCompressionOptions,
163 ) -> Result<Self, SQLiteError> {
164 let conn = ManagedConnection::open_compressed_encrypted(path, key, compression)?;
165 Self::open_with_connection(&conn)
166 }
167
168 pub fn open_compressed_encrypted_with_anchor(
172 path: &Path,
173 key: &str,
174 compression: SQLiteCompressionOptions,
175 trusted_anchor: SQLiteCompressedContainerAnchor,
176 ) -> Result<Self, SQLiteError> {
177 let conn = ManagedConnection::open_compressed_encrypted_with_anchor(
178 path,
179 key,
180 compression,
181 trusted_anchor,
182 )?;
183 Self::open_with_connection(&conn)
184 }
185
186 pub fn compressed_container_anchor(
189 path: &Path,
190 key: &str,
191 ) -> Result<SQLiteCompressedContainerAnchor, SQLiteError> {
192 Ok(uqa_storage_sqlite::read_authenticated_anchor(path, key)?)
193 }
194
195 fn open_with_connection(conn: &ManagedConnection) -> Result<Self, SQLiteError> {
196 let provider: Arc<dyn PersistentStorageProvider> =
197 Arc::new(SQLiteStorageProvider::new(conn.clone()));
198 Self::from_persistent_provider(provider).map_err(Self::sqlite_open_error)
199 }
200
201 pub fn new_session(&self) -> StorageBackendResult<Self> {
211 let _statement = self.lock_statement_gate();
212 self.new_sibling_session(false, None)
213 }
214
215 #[must_use]
217 pub fn shares_database_with(&self, other: &Self) -> bool {
218 Arc::ptr_eq(&self.row_locks, &other.row_locks)
219 }
220
221 pub(crate) fn new_internal_read_session(&self) -> StorageBackendResult<Self> {
223 let mut session = self.new_sibling_session(true, None)?;
224 session.runtime.diagnostics = self.runtime.diagnostics.fork();
225 Ok(session)
226 }
227
228 pub(crate) fn new_internal_retained_read_session(&self) -> StorageBackendResult<Self> {
229 let backend = self.storage.backend.as_ref().ok_or_else(|| {
230 StorageBackendError::Other("retained reads require persistent storage".into())
231 })?;
232 let storage_session =
233 backend.open_retained_read_session(&uqa_core::CancellationToken::new())?;
234 let mut session = self.new_sibling_session(true, Some(storage_session))?;
235 session.runtime.diagnostics = self.runtime.diagnostics.fork();
236 Ok(session)
237 }
238
239 fn new_sibling_session(
240 &self,
241 internal_read: bool,
242 retained_session: Option<PersistentStorageSession>,
243 ) -> StorageBackendResult<Self> {
244 let provider = self.storage.provider.as_ref().ok_or_else(|| {
245 StorageBackendError::Other(
246 "independent sessions require a PersistentStorageProvider".into(),
247 )
248 })?;
249 let retained = retained_session.is_some();
251 let share_catalog = !internal_read
252 && !retained
253 && self
254 .session
255 .transactions
256 .try_lock()
257 .is_some_and(|stack| stack.is_empty())
258 && self.session.state.read().temporary_namespace.is_none();
259 if share_catalog {
260 self.synchronize_table_catalog()?;
261 self.synchronize_table_data()?;
262 self.synchronize_catalog_registries()?;
263 }
264 let observed_epochs = self.epochs.published_epochs();
265 let storage_session = match retained_session {
266 Some(session) => session,
267 None => provider.open_session()?,
268 };
269 storage_session.validate_transaction_affinity()?;
270 let storage_version_before_restore = storage_session.backend.change_version()?;
271 let shared = if share_catalog {
272 self.session_from_shared_catalog(&storage_session, provider)?
273 } else {
274 None
275 };
276 let mut session = match shared {
277 Some(session) => session,
278 None => Self::from_initialized_persistent_session(
279 storage_session,
280 Some(Arc::clone(provider)),
281 )?,
282 };
283 session
284 .session
285 .statistics_worker
286 .store(internal_read, std::sync::atomic::Ordering::Release);
287 session.session.notification_subscriptions_required.store(
288 self.notification_subscriptions_required(),
289 std::sync::atomic::Ordering::Release,
290 );
291 session.row_locks = Arc::clone(&self.row_locks);
292 session.statistics = Arc::clone(&self.statistics);
293 session.install_notification_hub(Arc::clone(&self.notification_hub))?;
294 session.session_id = self.row_locks.allocate_session();
295 let storage_version_after_restore = session
296 .storage
297 .backend
298 .as_ref()
299 .ok_or_else(|| {
300 StorageBackendError::Other(
301 "persistent session lost its storage backend during restore".into(),
302 )
303 })?
304 .change_version()?;
305 let epochs =
306 Arc::get_mut(&mut session.epochs).expect("new session owns its catalog epoch state");
307 if storage_version_before_restore == storage_version_after_restore {
308 epochs.share_published_from_at(&self.epochs, observed_epochs);
309 } else {
310 epochs.share_published_from(&self.epochs);
311 }
312 session.extensions = super::RuntimeExtensions::shared_from(&self.extensions);
318 session
319 .extensions
320 .functions
321 .attach(&session.session, &session.runtime.statement_gate);
322 if internal_read {
323 session.runtime.function_context = Arc::clone(&self.runtime.function_context);
324 }
325 if !retained {
326 session.synchronize_table_catalog()?;
327 session.synchronize_table_data()?;
328 session.synchronize_catalog_registries()?;
329 }
330 session.start_automatic_statistics();
331 Ok(session)
332 }
333
334 #[allow(clippy::needless_pass_by_value)]
337 pub fn from_persistent_provider(
338 provider: Arc<dyn PersistentStorageProvider>,
339 ) -> StorageBackendResult<Self> {
340 let identity = provider.storage_identity()?;
341 let session = provider.open_initial_session()?;
342 let mut engine = Self::from_persistent_session(session, Some(Arc::clone(&provider)))?;
343 let row_locks = crate::row_locks::shared_provider_manager(identity.clone(), &provider);
344 let notification_hub =
345 crate::notifications::shared_provider_notification_hub(identity, &provider);
346 engine.session_id = row_locks.allocate_session();
347 engine.statistics = crate::statistics::shared_statistics(&row_locks);
348 engine.row_locks = row_locks;
349 engine.install_notification_hub(notification_hub)?;
350 engine.start_automatic_statistics();
351 Ok(engine)
352 }
353
354 pub fn from_persistent_backends(
356 catalog: Arc<dyn uqa_storage::CatalogFacade>,
357 backend: Arc<dyn PersistentStorageBackend>,
358 ) -> StorageBackendResult<Self> {
359 let identity = backend.storage_identity()?;
360 let row_locks = crate::row_locks::shared_backend_manager(identity.clone(), &backend);
361 let notification_hub =
362 crate::notifications::shared_backend_notification_hub(identity, &backend);
363 let provider: Arc<dyn PersistentStorageProvider> = Arc::new(BackendSessionProvider {
364 backend: Arc::clone(&backend),
365 });
366 let mut engine = Self::from_persistent_session(
367 PersistentStorageSession::new(catalog, backend),
368 Some(provider),
369 )?;
370 engine.session_id = row_locks.allocate_session();
371 engine.statistics = crate::statistics::shared_statistics(&row_locks);
372 engine.row_locks = row_locks;
373 engine.install_notification_hub(notification_hub)?;
374 engine.start_automatic_statistics();
375 Ok(engine)
376 }
377
378 fn install_notification_hub(
379 &mut self,
380 notification_hub: Arc<crate::NotificationHub>,
381 ) -> StorageBackendResult<()> {
382 let process_id = notification_hub
383 .allocate_backend_process_id()
384 .map_err(|error| {
385 StorageBackendError::Other(format!(
386 "allocate database backend process identifier: {error}"
387 ))
388 })?;
389 self.notification_hub = notification_hub;
390 if let Some(process_id) = process_id {
391 self.session.install_database_backend_process_id(process_id);
392 }
393 Ok(())
394 }
395
396 fn from_persistent_session(
397 storage_session: PersistentStorageSession,
398 provider: Option<Arc<dyn PersistentStorageProvider>>,
399 ) -> StorageBackendResult<Self> {
400 Self::build_persistent_session(storage_session, provider, true)
401 }
402
403 pub(crate) fn from_initialized_persistent_session(
404 storage_session: PersistentStorageSession,
405 provider: Option<Arc<dyn PersistentStorageProvider>>,
406 ) -> StorageBackendResult<Self> {
407 Self::build_persistent_session(storage_session, provider, false)
408 }
409
410 pub(super) fn empty_persistent_session(
411 storage_session: PersistentStorageSession,
412 provider: Option<Arc<dyn PersistentStorageProvider>>,
413 ) -> Self {
414 let PersistentStorageSession { catalog, backend } = storage_session;
415 let cancellation = backend.write_cancellation().unwrap_or_default();
416 let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
417 let notification_hub = Arc::new(crate::NotificationHub::default());
418 let session_id = row_locks.allocate_session();
419 let session = Arc::new(super::SessionContext::new(super::initial_random_state()));
420 let client_level = session.state.client_level();
421 let runtime = super::QueryRuntime::with_cancellation(
422 super::SQL_FUNCTION_DEPTH_LIMIT,
423 cancellation,
424 client_level,
425 session.state.io_timing(),
426 );
427 session.state.attach_cancellation(&runtime.cancellation);
428 Self {
429 storage: super::StorageContext::persistent(catalog, backend, provider),
430 durable: Arc::new(super::DurableCatalogState::new()),
431 extensions: super::RuntimeExtensions::new(&session, &runtime.statement_gate),
432 session,
433 epochs: Arc::new(super::EpochCoordinator::new()),
434 runtime,
435 statistics: crate::statistics::shared_statistics(&row_locks),
436 row_locks,
437 notification_hub,
438 session_id,
439 owns_session_registration: true,
440 query_table_snapshots: None,
441 query_view_snapshots: None,
442 query_sql_function_snapshots: None,
443 query_catalog_snapshot: None,
444 query_transaction_overlay: None,
445 query_transaction_origin: None,
446 }
447 }
448
449 fn repair_catalog_on_open(&self) -> StorageBackendResult<()> {
451 self.repair_persistent_value_indexes_on_open()?;
452 self.repair_partition_foreign_keys_on_open()?;
453 self.repair_derived_constraints_on_open()
454 }
455
456 fn build_persistent_session(
457 storage_session: PersistentStorageSession,
458 provider: Option<Arc<dyn PersistentStorageProvider>>,
459 initialize_catalog: bool,
460 ) -> StorageBackendResult<Self> {
461 storage_session.validate_transaction_affinity()?;
462 let restore_catalog = Arc::clone(&storage_session.catalog);
463 let restore_backend = Arc::clone(&storage_session.backend);
464 let mut engine = Self::empty_persistent_session(storage_session, provider);
465 let own_read = !initialize_catalog && !restore_backend.in_transaction();
466 let version_before_read = if own_read {
467 let version = restore_backend.change_version()?;
468 restore_backend.begin_read_transaction()?;
469 restore_backend.pin_transaction_snapshot()?;
470 Some(version)
471 } else {
472 None
473 };
474 let read_view_before = restore_backend.read_view_revision()?;
478 let cache_revisions_before;
479 if initialize_catalog {
480 restore_backend.migrate_document_storage()?;
481 restore_backend.begin_upgradeable_transaction()?;
483 let restore_result = (|| {
484 restore_catalog.initialize_storage()?;
485 let cache_revisions = restore_catalog.cache_revisions()?;
486 restore_backend.migrate_inverted_index_storage()?;
487 Self::prepare_catalog_for_initial_restore(restore_catalog.as_ref())?;
488 engine
489 .restore_initial_catalog(restore_catalog.as_ref(), restore_backend.as_ref())?;
490 uqa_execution::catalog::value_restoration::normalize_legacy_vectors(
491 restore_catalog.as_ref(),
492 restore_backend.as_ref(),
493 &engine,
494 )?;
495 engine.repair_reset_fts_storage(restore_catalog.as_ref())?;
496 engine.initialize_catalog_constraint_tuples(&restore_catalog, &restore_backend)?;
497 Ok(cache_revisions)
498 })();
499 cache_revisions_before = match restore_result {
500 Ok(revisions) => revisions,
501 Err(error) => {
502 return match restore_backend.rollback_transaction() {
503 Ok(()) => Err(error),
504 Err(rollback_error) => Err(StorageBackendError::Other(format!(
505 "rollback initial catalog migration after `{error}` failed: {rollback_error}"
506 ))),
507 };
508 }
509 };
510 if let Err(error) = restore_backend.commit_transaction() {
511 return match restore_backend.rollback_transaction() {
512 Ok(()) => Err(error),
513 Err(rollback_error) => Err(StorageBackendError::Other(format!(
514 "rollback failed initial catalog migration commit `{error}`: {rollback_error}"
515 ))),
516 };
517 }
518 } else {
519 cache_revisions_before = restore_catalog.cache_revisions()?;
520 engine.restore_from_catalog(
521 restore_catalog.as_ref(),
522 restore_backend.as_ref(),
523 super::CatalogRestoreMode::LoadOnly,
524 )?;
525 }
526 if initialize_catalog {
527 engine.repair_catalog_on_open()?;
528 }
529 for (name, json) in restore_catalog.load_models()? {
532 let model = serde_json::from_str::<DeepModel>(&json)?;
533 engine.durable.models.write().insert(name, model);
534 }
535 for (name, json) in restore_catalog.load_all_scoring_params()? {
536 engine.durable.scoring_params.write().insert(name, json);
537 }
538 let cache_revisions_after = restore_catalog.cache_revisions()?;
542 let stable_restore = cache_revisions_before == cache_revisions_after;
543 if stable_restore {
544 *engine.epochs.storage_cache_revisions.lock() = cache_revisions_after;
545 let read_view_after = restore_backend.read_view_revision()?;
546 if read_view_before == read_view_after {
547 *engine.epochs.seen_storage_read_view.lock() = read_view_after;
548 }
549 }
550 if let Some(version) = restore_backend.change_version()?.filter(|_| stable_restore) {
551 engine
552 .epochs
553 .seen_storage_change_version
554 .store(version, std::sync::atomic::Ordering::Release);
555 }
556 if let Some(before) = version_before_read {
557 restore_backend.rollback_transaction()?;
558 if restore_backend.change_version()? != before {
559 engine.epochs.seen_storage_change_version.store(
562 before.unwrap_or_default(),
563 std::sync::atomic::Ordering::Release,
564 );
565 *engine.epochs.seen_storage_read_view.lock() = None;
566 }
567 }
568 Ok(engine)
569 }
570
571 fn sqlite_open_error(err: StorageBackendError) -> SQLiteError {
572 SQLiteError::from(err)
573 }
574}