1use super::{
10 Arc, DeepModel, Engine, ManagedConnection, Path, PersistentStorageBackend,
11 PersistentStorageProvider, PersistentStorageSession, SQLiteCompressedContainerAnchor,
12 SQLiteCompressionOptions, SQLiteError, SQLiteStorageProvider, StorageBackendError,
13 StorageBackendResult,
14};
15
16struct BackendSessionProvider {
17 backend: Arc<dyn PersistentStorageBackend>,
18}
19
20impl PersistentStorageProvider for BackendSessionProvider {
21 fn open_session_with_cancellation(
22 &self,
23 cancellation: &uqa_core::CancellationToken,
24 ) -> StorageBackendResult<PersistentStorageSession> {
25 self.backend.open_session_with_cancellation(cancellation)
26 }
27
28 fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
29 self.backend.auxiliary_encryption_key()
30 }
31
32 fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
33 self.backend.open_session()
34 }
35
36 fn storage_identity(
37 &self,
38 ) -> StorageBackendResult<Option<uqa_storage::PersistentStorageIdentity>> {
39 self.backend.storage_identity()
40 }
41}
42
43impl Default for Engine {
44 fn default() -> Self {
45 Self::new()
46 }
47}
48
49impl Engine {
50 pub fn new() -> Self {
52 let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
53 let notification_hub = Arc::new(crate::NotificationHub::default());
54 let session_id = row_locks.allocate_session();
55 let session = Arc::new(super::SessionContext::new(super::initial_random_state()));
56 let client_level = session.state.client_level();
57 let runtime = super::QueryRuntime::new(super::SQL_FUNCTION_DEPTH_LIMIT, client_level);
58 session.state.attach_cancellation(&runtime.cancellation);
59 Self {
60 storage: super::StorageContext::memory(),
61 durable: Arc::new(super::DurableCatalogState::new()),
62 session,
63 extensions: super::RuntimeExtensions::new(),
64 epochs: super::EpochCoordinator::new(),
65 runtime,
66 statistics: crate::statistics::shared_statistics(&row_locks),
67 row_locks,
68 notification_hub,
69 session_id,
70 owns_session_registration: true,
71 query_table_snapshots: None,
72 query_view_snapshots: None,
73 query_sql_function_snapshots: None,
74 query_catalog_snapshot: None,
75 query_transaction_overlay: None,
76 query_transaction_origin: None,
77 }
78 }
79
80 pub fn open(path: &Path) -> Result<Self, SQLiteError> {
81 let conn = ManagedConnection::open(path)?;
82 Self::open_with_connection(&conn)
83 }
84
85 pub fn detect_database_file(
90 path: &Path,
91 ) -> std::io::Result<uqa_storage_sqlite::DatabaseFileFormat> {
92 uqa_storage_sqlite::detect_database_file_format(path)
93 }
94
95 pub fn open_auto(path: &Path, key: Option<&str>) -> Result<Self, SQLiteError> {
116 use uqa_storage_sqlite::DatabaseFileFormat;
117 let key = match key {
118 Some("") => return Err(SQLiteError::EmptyEncryptionKey),
119 other => other,
120 };
121 match uqa_storage_sqlite::detect_database_file_format(path)? {
122 DatabaseFileFormat::Missing => match key {
123 Some(key) => Self::open_encrypted(path, key),
124 None => Self::open(path),
125 },
126 DatabaseFileFormat::PlainSQLite => match key {
127 Some(_) => Err(SQLiteError::NotEncrypted),
128 None => Self::open(path),
129 },
130 DatabaseFileFormat::CompressedContainer { encrypted: true } => match key {
131 Some(key) => {
132 Self::open_compressed_encrypted(path, key, SQLiteCompressionOptions::default())
133 }
134 None => Err(SQLiteError::EncryptionKeyRequired),
135 },
136 DatabaseFileFormat::CompressedContainer { encrypted: false } => match key {
137 Some(_) => Err(SQLiteError::NotEncrypted),
138 None => Self::open_compressed(path, SQLiteCompressionOptions::default()),
139 },
140 DatabaseFileFormat::Unrecognized => match key {
141 Some(key) => Self::open_encrypted(path, key),
142 None => Err(SQLiteError::EncryptionKeyRequired),
143 },
144 }
145 }
146
147 pub fn open_encrypted(path: &Path, key: &str) -> Result<Self, SQLiteError> {
151 let conn = ManagedConnection::open_encrypted(path, key)?;
152 Self::open_with_connection(&conn)
153 }
154
155 pub fn open_compressed(
159 path: &Path,
160 compression: SQLiteCompressionOptions,
161 ) -> Result<Self, SQLiteError> {
162 let conn = ManagedConnection::open_compressed(path, compression)?;
163 Self::open_with_connection(&conn)
164 }
165
166 pub fn open_compressed_encrypted(
174 path: &Path,
175 key: &str,
176 compression: SQLiteCompressionOptions,
177 ) -> Result<Self, SQLiteError> {
178 let conn = ManagedConnection::open_compressed_encrypted(path, key, compression)?;
179 Self::open_with_connection(&conn)
180 }
181
182 pub fn open_compressed_encrypted_with_anchor(
186 path: &Path,
187 key: &str,
188 compression: SQLiteCompressionOptions,
189 trusted_anchor: SQLiteCompressedContainerAnchor,
190 ) -> Result<Self, SQLiteError> {
191 let conn = ManagedConnection::open_compressed_encrypted_with_anchor(
192 path,
193 key,
194 compression,
195 trusted_anchor,
196 )?;
197 Self::open_with_connection(&conn)
198 }
199
200 pub fn compressed_container_anchor(
203 path: &Path,
204 key: &str,
205 ) -> Result<SQLiteCompressedContainerAnchor, SQLiteError> {
206 Ok(uqa_storage_sqlite::read_authenticated_anchor(path, key)?)
207 }
208
209 fn open_with_connection(conn: &ManagedConnection) -> Result<Self, SQLiteError> {
210 let provider: Arc<dyn PersistentStorageProvider> =
211 Arc::new(SQLiteStorageProvider::new(conn.clone()));
212 Self::from_persistent_provider(provider).map_err(Self::sqlite_open_error)
213 }
214
215 pub fn new_session(&self) -> StorageBackendResult<Self> {
225 let _statement = self.lock_statement_gate();
226 self.new_sibling_session(false, None)
227 }
228
229 pub(crate) fn new_internal_read_session(&self) -> StorageBackendResult<Self> {
231 let mut session = self.new_sibling_session(true, None)?;
232 session.runtime.diagnostics = self.runtime.diagnostics.fork();
233 Ok(session)
234 }
235
236 pub(crate) fn new_internal_retained_read_session(&self) -> StorageBackendResult<Self> {
237 let backend = self.storage.backend.as_ref().ok_or_else(|| {
238 StorageBackendError::Other("retained reads require persistent storage".into())
239 })?;
240 let storage_session =
241 backend.open_retained_read_session(&uqa_core::CancellationToken::new())?;
242 let mut session = self.new_sibling_session(true, Some(storage_session))?;
243 session.runtime.diagnostics = self.runtime.diagnostics.fork();
244 Ok(session)
245 }
246
247 fn new_sibling_session(
248 &self,
249 internal_read: bool,
250 retained_session: Option<PersistentStorageSession>,
251 ) -> StorageBackendResult<Self> {
252 let provider = self.storage.provider.as_ref().ok_or_else(|| {
253 StorageBackendError::Other(
254 "independent sessions require a PersistentStorageProvider".into(),
255 )
256 })?;
257 let retained = retained_session.is_some();
259 let share_catalog = !internal_read
260 && !retained
261 && self
262 .session
263 .transactions
264 .try_lock()
265 .is_some_and(|stack| stack.is_empty())
266 && self.session.state.read().temporary_namespace.is_none();
267 if share_catalog {
268 self.synchronize_table_catalog()?;
269 self.synchronize_table_data()?;
270 self.synchronize_catalog_registries()?;
271 }
272 let observed_epochs = self.epochs.published_epochs();
273 let storage_session = match retained_session {
274 Some(session) => session,
275 None => provider.open_session()?,
276 };
277 storage_session.validate_transaction_affinity()?;
278 let storage_version_before_restore = storage_session.backend.change_version()?;
279 let shared = if share_catalog {
280 self.session_from_shared_catalog(&storage_session, provider)?
281 } else {
282 None
283 };
284 let mut session = match shared {
285 Some(session) => session,
286 None => Self::from_initialized_persistent_session(
287 storage_session,
288 Some(Arc::clone(provider)),
289 )?,
290 };
291 session
292 .session
293 .statistics_worker
294 .store(internal_read, std::sync::atomic::Ordering::Release);
295 session.session.notification_subscriptions_required.store(
296 self.notification_subscriptions_required(),
297 std::sync::atomic::Ordering::Release,
298 );
299 session.row_locks = Arc::clone(&self.row_locks);
300 session.statistics = Arc::clone(&self.statistics);
301 session.install_notification_hub(Arc::clone(&self.notification_hub))?;
302 session.session_id = self.row_locks.allocate_session();
303 let storage_version_after_restore = session
304 .storage
305 .backend
306 .as_ref()
307 .ok_or_else(|| {
308 StorageBackendError::Other(
309 "persistent session lost its storage backend during restore".into(),
310 )
311 })?
312 .change_version()?;
313 if storage_version_before_restore == storage_version_after_restore {
314 session
315 .epochs
316 .share_published_from_at(&self.epochs, observed_epochs);
317 } else {
318 session.epochs.share_published_from(&self.epochs);
319 }
320 session.extensions = super::RuntimeExtensions::shared_from(&self.extensions);
326 if !retained {
327 session.synchronize_table_catalog()?;
328 session.synchronize_table_data()?;
329 session.synchronize_catalog_registries()?;
330 }
331 session.start_automatic_statistics();
332 Ok(session)
333 }
334
335 #[allow(clippy::needless_pass_by_value)]
338 pub fn from_persistent_provider(
339 provider: Arc<dyn PersistentStorageProvider>,
340 ) -> StorageBackendResult<Self> {
341 let identity = provider.storage_identity()?;
342 let session = provider.open_initial_session()?;
343 let mut engine = Self::from_persistent_session(session, Some(Arc::clone(&provider)))?;
344 let row_locks = crate::row_locks::shared_provider_manager(identity.clone(), &provider);
345 let notification_hub =
346 crate::notifications::shared_provider_notification_hub(identity, &provider);
347 engine.session_id = row_locks.allocate_session();
348 engine.statistics = crate::statistics::shared_statistics(&row_locks);
349 engine.row_locks = row_locks;
350 engine.install_notification_hub(notification_hub)?;
351 engine.start_automatic_statistics();
352 Ok(engine)
353 }
354
355 pub fn from_persistent_backends(
357 catalog: Arc<dyn uqa_storage::CatalogFacade>,
358 backend: Arc<dyn PersistentStorageBackend>,
359 ) -> StorageBackendResult<Self> {
360 let identity = backend.storage_identity()?;
361 let row_locks = crate::row_locks::shared_backend_manager(identity.clone(), &backend);
362 let notification_hub =
363 crate::notifications::shared_backend_notification_hub(identity, &backend);
364 let provider: Arc<dyn PersistentStorageProvider> = Arc::new(BackendSessionProvider {
365 backend: Arc::clone(&backend),
366 });
367 let mut engine = Self::from_persistent_session(
368 PersistentStorageSession::new(catalog, backend),
369 Some(provider),
370 )?;
371 engine.session_id = row_locks.allocate_session();
372 engine.statistics = crate::statistics::shared_statistics(&row_locks);
373 engine.row_locks = row_locks;
374 engine.install_notification_hub(notification_hub)?;
375 engine.start_automatic_statistics();
376 Ok(engine)
377 }
378
379 fn install_notification_hub(
380 &mut self,
381 notification_hub: Arc<crate::NotificationHub>,
382 ) -> StorageBackendResult<()> {
383 let process_id = notification_hub
384 .allocate_backend_process_id()
385 .map_err(|error| {
386 StorageBackendError::Other(format!(
387 "allocate database backend process identifier: {error}"
388 ))
389 })?;
390 self.notification_hub = notification_hub;
391 if let Some(process_id) = process_id {
392 self.session.install_database_backend_process_id(process_id);
393 }
394 Ok(())
395 }
396
397 fn from_persistent_session(
398 storage_session: PersistentStorageSession,
399 provider: Option<Arc<dyn PersistentStorageProvider>>,
400 ) -> StorageBackendResult<Self> {
401 Self::build_persistent_session(storage_session, provider, true)
402 }
403
404 pub(crate) fn from_initialized_persistent_session(
405 storage_session: PersistentStorageSession,
406 provider: Option<Arc<dyn PersistentStorageProvider>>,
407 ) -> StorageBackendResult<Self> {
408 Self::build_persistent_session(storage_session, provider, false)
409 }
410
411 pub(super) fn empty_persistent_session(
412 storage_session: PersistentStorageSession,
413 provider: Option<Arc<dyn PersistentStorageProvider>>,
414 ) -> Self {
415 let PersistentStorageSession { catalog, backend } = storage_session;
416 let cancellation = backend.write_cancellation().unwrap_or_default();
417 let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
418 let notification_hub = Arc::new(crate::NotificationHub::default());
419 let session_id = row_locks.allocate_session();
420 let session = Arc::new(super::SessionContext::new(super::initial_random_state()));
421 let client_level = session.state.client_level();
422 let runtime = super::QueryRuntime::with_cancellation(
423 super::SQL_FUNCTION_DEPTH_LIMIT,
424 cancellation,
425 client_level,
426 );
427 session.state.attach_cancellation(&runtime.cancellation);
428 Self {
429 storage: super::StorageContext::persistent(catalog, backend, provider),
430 durable: Arc::new(super::DurableCatalogState::new()),
431 session,
432 extensions: super::RuntimeExtensions::new(),
433 epochs: super::EpochCoordinator::new(),
434 runtime,
435 statistics: crate::statistics::shared_statistics(&row_locks),
436 row_locks,
437 notification_hub,
438 session_id,
439 owns_session_registration: true,
440 query_table_snapshots: None,
441 query_view_snapshots: None,
442 query_sql_function_snapshots: None,
443 query_catalog_snapshot: None,
444 query_transaction_overlay: None,
445 query_transaction_origin: None,
446 }
447 }
448
449 fn repair_catalog_on_open(&self) -> StorageBackendResult<()> {
451 self.repair_persistent_value_indexes_on_open()?;
452 self.repair_partition_foreign_keys_on_open()?;
453 self.repair_derived_constraints_on_open()
454 }
455
456 fn build_persistent_session(
457 storage_session: PersistentStorageSession,
458 provider: Option<Arc<dyn PersistentStorageProvider>>,
459 initialize_catalog: bool,
460 ) -> StorageBackendResult<Self> {
461 storage_session.validate_transaction_affinity()?;
462 let restore_catalog = Arc::clone(&storage_session.catalog);
463 let restore_backend = Arc::clone(&storage_session.backend);
464 let mut engine = Self::empty_persistent_session(storage_session, provider);
465 let own_read = !initialize_catalog && !restore_backend.in_transaction();
466 let version_before_read = if own_read {
467 let version = restore_backend.change_version()?;
468 restore_backend.begin_read_transaction()?;
469 restore_backend.pin_transaction_snapshot()?;
470 Some(version)
471 } else {
472 None
473 };
474 let read_view_before = restore_backend.read_view_revision()?;
478 let cache_revisions_before;
479 if initialize_catalog {
480 restore_backend.migrate_document_storage()?;
481 restore_backend.begin_upgradeable_transaction()?;
483 let restore_result = (|| {
484 restore_catalog.initialize_storage()?;
485 let cache_revisions = restore_catalog.cache_revisions()?;
486 restore_backend.migrate_inverted_index_storage()?;
487 Self::prepare_catalog_for_initial_restore(restore_catalog.as_ref())?;
488 engine
489 .restore_initial_catalog(restore_catalog.as_ref(), restore_backend.as_ref())?;
490 uqa_execution::catalog::value_restoration::normalize_legacy_vectors(
491 restore_catalog.as_ref(),
492 restore_backend.as_ref(),
493 &engine,
494 )?;
495 engine.repair_reset_fts_storage(restore_catalog.as_ref())?;
496 engine.initialize_catalog_constraint_tuples(&restore_catalog, &restore_backend)?;
497 Ok(cache_revisions)
498 })();
499 cache_revisions_before = match restore_result {
500 Ok(revisions) => revisions,
501 Err(error) => {
502 return match restore_backend.rollback_transaction() {
503 Ok(()) => Err(error),
504 Err(rollback_error) => Err(StorageBackendError::Other(format!(
505 "rollback initial catalog migration after `{error}` failed: {rollback_error}"
506 ))),
507 };
508 }
509 };
510 if let Err(error) = restore_backend.commit_transaction() {
511 return match restore_backend.rollback_transaction() {
512 Ok(()) => Err(error),
513 Err(rollback_error) => Err(StorageBackendError::Other(format!(
514 "rollback failed initial catalog migration commit `{error}`: {rollback_error}"
515 ))),
516 };
517 }
518 } else {
519 cache_revisions_before = restore_catalog.cache_revisions()?;
520 engine.restore_from_catalog(
521 restore_catalog.as_ref(),
522 restore_backend.as_ref(),
523 super::CatalogRestoreMode::LoadOnly,
524 )?;
525 }
526 if initialize_catalog {
527 engine.repair_catalog_on_open()?;
528 }
529 for (name, json) in restore_catalog.load_models()? {
532 let model = serde_json::from_str::<DeepModel>(&json)?;
533 engine.durable.models.write().insert(name, model);
534 }
535 for (name, json) in restore_catalog.load_all_scoring_params()? {
536 engine.durable.scoring_params.write().insert(name, json);
537 }
538 let cache_revisions_after = restore_catalog.cache_revisions()?;
542 let stable_restore = cache_revisions_before == cache_revisions_after;
543 if stable_restore {
544 *engine.epochs.storage_cache_revisions.lock() = cache_revisions_after;
545 let read_view_after = restore_backend.read_view_revision()?;
546 if read_view_before == read_view_after {
547 *engine.epochs.seen_storage_read_view.lock() = read_view_after;
548 }
549 }
550 if let Some(version) = restore_backend.change_version()?.filter(|_| stable_restore) {
551 engine
552 .epochs
553 .seen_storage_change_version
554 .store(version, std::sync::atomic::Ordering::Release);
555 }
556 if let Some(before) = version_before_read {
557 restore_backend.rollback_transaction()?;
558 if restore_backend.change_version()? != before {
559 engine.epochs.seen_storage_change_version.store(
562 before.unwrap_or_default(),
563 std::sync::atomic::Ordering::Release,
564 );
565 *engine.epochs.seen_storage_read_view.lock() = None;
566 }
567 }
568 Ok(engine)
569 }
570
571 fn sqlite_open_error(err: StorageBackendError) -> SQLiteError {
572 SQLiteError::from(err)
573 }
574}