1use super::{
10 Arc, DeepModel, Engine, ManagedConnection, Path, PersistentStorageBackend,
11 PersistentStorageProvider, PersistentStorageSession, SQLiteCompressedContainerAnchor,
12 SQLiteCompressionOptions, SQLiteError, SQLiteStorageProvider, StorageBackendError,
13 StorageBackendResult,
14};
15
16struct BackendSessionProvider {
17 backend: Arc<dyn PersistentStorageBackend>,
18}
19
20impl PersistentStorageProvider for BackendSessionProvider {
21 fn open_session_with_cancellation(
22 &self,
23 cancellation: &uqa_core::CancellationToken,
24 ) -> StorageBackendResult<PersistentStorageSession> {
25 self.backend.open_session_with_cancellation(cancellation)
26 }
27
28 fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
29 self.backend.auxiliary_encryption_key()
30 }
31
32 fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
33 self.backend.open_session()
34 }
35
36 fn storage_identity(
37 &self,
38 ) -> StorageBackendResult<Option<uqa_storage::PersistentStorageIdentity>> {
39 self.backend.storage_identity()
40 }
41}
42
43impl Default for Engine {
44 fn default() -> Self {
45 Self::new()
46 }
47}
48
49impl Engine {
50 pub fn new() -> Self {
52 let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
53 let notification_hub = Arc::new(crate::NotificationHub::default());
54 let session_id = row_locks.allocate_session();
55 let session = Arc::new(super::SessionContext::new(super::initial_random_state()));
56 let client_level = session.state.client_level();
57 let runtime = super::QueryRuntime::new(
58 super::SQL_FUNCTION_DEPTH_LIMIT,
59 client_level,
60 session.state.io_timing(),
61 );
62 session.state.attach_cancellation(&runtime.cancellation);
63 Self {
64 storage: super::StorageContext::memory(),
65 durable: Arc::new(super::DurableCatalogState::new()),
66 extensions: super::RuntimeExtensions::new(&session, &runtime.statement_gate),
67 session,
68 epochs: Arc::new(super::EpochCoordinator::new()),
69 runtime,
70 statistics: crate::statistics::shared_statistics(&row_locks),
71 row_locks,
72 notification_hub,
73 session_id,
74 owns_session_registration: true,
75 query_table_snapshots: None,
76 query_view_snapshots: None,
77 query_sql_function_snapshots: None,
78 query_catalog_snapshot: None,
79 query_transaction_overlay: None,
80 query_transaction_origin: None,
81 }
82 }
83
84 pub fn open(path: &Path) -> Result<Self, SQLiteError> {
85 let conn = ManagedConnection::open(path)?;
86 Self::open_with_connection(&conn)
87 }
88
89 pub fn detect_database_file(
94 path: &Path,
95 ) -> std::io::Result<uqa_storage_sqlite::DatabaseFileFormat> {
96 uqa_storage_sqlite::detect_database_file_format(path)
97 }
98
99 pub fn open_auto(path: &Path, key: Option<&str>) -> Result<Self, SQLiteError> {
120 let connection = ManagedConnection::open_auto(path, key)?;
121 Self::open_with_connection(&connection)
122 }
123
124 pub fn open_encrypted(path: &Path, key: &str) -> Result<Self, SQLiteError> {
128 let conn = ManagedConnection::open_encrypted(path, key)?;
129 Self::open_with_connection(&conn)
130 }
131
132 pub fn open_compressed(
136 path: &Path,
137 compression: SQLiteCompressionOptions,
138 ) -> Result<Self, SQLiteError> {
139 let conn = ManagedConnection::open_compressed(path, compression)?;
140 Self::open_with_connection(&conn)
141 }
142
143 pub fn open_compressed_encrypted(
151 path: &Path,
152 key: &str,
153 compression: SQLiteCompressionOptions,
154 ) -> Result<Self, SQLiteError> {
155 let conn = ManagedConnection::open_compressed_encrypted(path, key, compression)?;
156 Self::open_with_connection(&conn)
157 }
158
159 pub fn open_compressed_encrypted_with_anchor(
163 path: &Path,
164 key: &str,
165 compression: SQLiteCompressionOptions,
166 trusted_anchor: SQLiteCompressedContainerAnchor,
167 ) -> Result<Self, SQLiteError> {
168 let conn = ManagedConnection::open_compressed_encrypted_with_anchor(
169 path,
170 key,
171 compression,
172 trusted_anchor,
173 )?;
174 Self::open_with_connection(&conn)
175 }
176
177 pub fn compressed_container_anchor(
180 path: &Path,
181 key: &str,
182 ) -> Result<SQLiteCompressedContainerAnchor, SQLiteError> {
183 Ok(uqa_storage_sqlite::read_authenticated_anchor(path, key)?)
184 }
185
186 fn open_with_connection(conn: &ManagedConnection) -> Result<Self, SQLiteError> {
187 let provider: Arc<dyn PersistentStorageProvider> =
188 Arc::new(SQLiteStorageProvider::new(conn.clone()));
189 Self::from_persistent_provider(provider).map_err(Self::sqlite_open_error)
190 }
191
192 pub fn new_session(&self) -> StorageBackendResult<Self> {
202 let _statement = self.lock_statement_gate();
203 self.new_sibling_session(false, None)
204 }
205
206 #[must_use]
208 pub fn shares_database_with(&self, other: &Self) -> bool {
209 Arc::ptr_eq(&self.row_locks, &other.row_locks)
210 }
211
212 pub(crate) fn new_internal_read_session(&self) -> StorageBackendResult<Self> {
214 let mut session = self.new_sibling_session(true, None)?;
215 session.runtime.diagnostics = self.runtime.diagnostics.fork();
216 Ok(session)
217 }
218
219 pub(crate) fn new_internal_retained_read_session(&self) -> StorageBackendResult<Self> {
220 let backend = self.storage.backend.as_ref().ok_or_else(|| {
221 StorageBackendError::Other("retained reads require persistent storage".into())
222 })?;
223 let storage_session =
224 backend.open_retained_read_session(&uqa_core::CancellationToken::new())?;
225 let mut session = self.new_sibling_session(true, Some(storage_session))?;
226 session.runtime.diagnostics = self.runtime.diagnostics.fork();
227 Ok(session)
228 }
229
230 fn new_sibling_session(
231 &self,
232 internal_read: bool,
233 retained_session: Option<PersistentStorageSession>,
234 ) -> StorageBackendResult<Self> {
235 let provider = self.storage.provider.as_ref().ok_or_else(|| {
236 StorageBackendError::Other(
237 "independent sessions require a PersistentStorageProvider".into(),
238 )
239 })?;
240 let retained = retained_session.is_some();
242 let share_catalog = !internal_read
243 && !retained
244 && self
245 .session
246 .transactions
247 .try_lock()
248 .is_some_and(|stack| stack.is_empty())
249 && self.session.state.read().temporary_namespace.is_none();
250 if share_catalog {
251 self.synchronize_table_catalog()?;
252 self.synchronize_table_data()?;
253 self.synchronize_catalog_registries()?;
254 }
255 let observed_epochs = self.epochs.published_epochs();
256 let storage_session = match retained_session {
257 Some(session) => session,
258 None => provider.open_session()?,
259 };
260 storage_session.validate_transaction_affinity()?;
261 let storage_version_before_restore = storage_session.backend.change_version()?;
262 let shared = if share_catalog {
263 self.session_from_shared_catalog(&storage_session, provider)?
264 } else {
265 None
266 };
267 let mut session = match shared {
268 Some(session) => session,
269 None => Self::from_initialized_persistent_session(
270 storage_session,
271 Some(Arc::clone(provider)),
272 )?,
273 };
274 session
275 .session
276 .statistics_worker
277 .store(internal_read, std::sync::atomic::Ordering::Release);
278 session.session.notification_subscriptions_required.store(
279 self.notification_subscriptions_required(),
280 std::sync::atomic::Ordering::Release,
281 );
282 session.row_locks = Arc::clone(&self.row_locks);
283 session.statistics = Arc::clone(&self.statistics);
284 session.install_notification_hub(Arc::clone(&self.notification_hub))?;
285 session.session_id = self.row_locks.allocate_session();
286 let storage_version_after_restore = session
287 .storage
288 .backend
289 .as_ref()
290 .ok_or_else(|| {
291 StorageBackendError::Other(
292 "persistent session lost its storage backend during restore".into(),
293 )
294 })?
295 .change_version()?;
296 let epochs =
297 Arc::get_mut(&mut session.epochs).expect("new session owns its catalog epoch state");
298 if storage_version_before_restore == storage_version_after_restore {
299 epochs.share_published_from_at(&self.epochs, observed_epochs);
300 } else {
301 epochs.share_published_from(&self.epochs);
302 }
303 session.extensions = super::RuntimeExtensions::shared_from(&self.extensions);
309 session
310 .extensions
311 .functions
312 .attach(&session.session, &session.runtime.statement_gate);
313 if internal_read {
314 session.runtime.function_context = Arc::clone(&self.runtime.function_context);
315 }
316 if !retained {
317 session.synchronize_table_catalog()?;
318 session.synchronize_table_data()?;
319 session.synchronize_catalog_registries()?;
320 }
321 session.start_automatic_statistics();
322 Ok(session)
323 }
324
325 #[allow(clippy::needless_pass_by_value)]
328 pub fn from_persistent_provider(
329 provider: Arc<dyn PersistentStorageProvider>,
330 ) -> StorageBackendResult<Self> {
331 let identity = provider.storage_identity()?;
332 let session = provider.open_initial_session()?;
333 let mut engine = Self::from_persistent_session(session, Some(Arc::clone(&provider)))?;
334 let row_locks = crate::row_locks::shared_provider_manager(identity.clone(), &provider);
335 let notification_hub =
336 crate::notifications::shared_provider_notification_hub(identity, &provider);
337 engine.session_id = row_locks.allocate_session();
338 engine.statistics = crate::statistics::shared_statistics(&row_locks);
339 engine.row_locks = row_locks;
340 engine.install_notification_hub(notification_hub)?;
341 engine.start_automatic_statistics();
342 Ok(engine)
343 }
344
345 pub fn from_persistent_backends(
347 catalog: Arc<dyn uqa_storage::CatalogFacade>,
348 backend: Arc<dyn PersistentStorageBackend>,
349 ) -> StorageBackendResult<Self> {
350 let identity = backend.storage_identity()?;
351 let row_locks = crate::row_locks::shared_backend_manager(identity.clone(), &backend);
352 let notification_hub =
353 crate::notifications::shared_backend_notification_hub(identity, &backend);
354 let provider: Arc<dyn PersistentStorageProvider> = Arc::new(BackendSessionProvider {
355 backend: Arc::clone(&backend),
356 });
357 let mut engine = Self::from_persistent_session(
358 PersistentStorageSession::new(catalog, backend),
359 Some(provider),
360 )?;
361 engine.session_id = row_locks.allocate_session();
362 engine.statistics = crate::statistics::shared_statistics(&row_locks);
363 engine.row_locks = row_locks;
364 engine.install_notification_hub(notification_hub)?;
365 engine.start_automatic_statistics();
366 Ok(engine)
367 }
368
369 fn install_notification_hub(
370 &mut self,
371 notification_hub: Arc<crate::NotificationHub>,
372 ) -> StorageBackendResult<()> {
373 let process_id = notification_hub
374 .allocate_backend_process_id()
375 .map_err(|error| {
376 StorageBackendError::Other(format!(
377 "allocate database backend process identifier: {error}"
378 ))
379 })?;
380 self.notification_hub = notification_hub;
381 if let Some(process_id) = process_id {
382 self.session.install_database_backend_process_id(process_id);
383 }
384 Ok(())
385 }
386
387 fn from_persistent_session(
388 storage_session: PersistentStorageSession,
389 provider: Option<Arc<dyn PersistentStorageProvider>>,
390 ) -> StorageBackendResult<Self> {
391 Self::build_persistent_session(storage_session, provider, true)
392 }
393
394 pub(crate) fn from_initialized_persistent_session(
395 storage_session: PersistentStorageSession,
396 provider: Option<Arc<dyn PersistentStorageProvider>>,
397 ) -> StorageBackendResult<Self> {
398 Self::build_persistent_session(storage_session, provider, false)
399 }
400
401 pub(super) fn empty_persistent_session(
402 storage_session: PersistentStorageSession,
403 provider: Option<Arc<dyn PersistentStorageProvider>>,
404 ) -> Self {
405 let PersistentStorageSession { catalog, backend } = storage_session;
406 let cancellation = backend.write_cancellation().unwrap_or_default();
407 let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
408 let notification_hub = Arc::new(crate::NotificationHub::default());
409 let session_id = row_locks.allocate_session();
410 let session = Arc::new(super::SessionContext::new(super::initial_random_state()));
411 let client_level = session.state.client_level();
412 let runtime = super::QueryRuntime::with_cancellation(
413 super::SQL_FUNCTION_DEPTH_LIMIT,
414 cancellation,
415 client_level,
416 session.state.io_timing(),
417 );
418 session.state.attach_cancellation(&runtime.cancellation);
419 Self {
420 storage: super::StorageContext::persistent(catalog, backend, provider),
421 durable: Arc::new(super::DurableCatalogState::new()),
422 extensions: super::RuntimeExtensions::new(&session, &runtime.statement_gate),
423 session,
424 epochs: Arc::new(super::EpochCoordinator::new()),
425 runtime,
426 statistics: crate::statistics::shared_statistics(&row_locks),
427 row_locks,
428 notification_hub,
429 session_id,
430 owns_session_registration: true,
431 query_table_snapshots: None,
432 query_view_snapshots: None,
433 query_sql_function_snapshots: None,
434 query_catalog_snapshot: None,
435 query_transaction_overlay: None,
436 query_transaction_origin: None,
437 }
438 }
439
440 fn repair_catalog_on_open(&self) -> StorageBackendResult<()> {
442 self.repair_persistent_value_indexes_on_open()?;
443 self.repair_partition_foreign_keys_on_open()?;
444 self.repair_derived_constraints_on_open()
445 }
446
447 fn build_persistent_session(
448 storage_session: PersistentStorageSession,
449 provider: Option<Arc<dyn PersistentStorageProvider>>,
450 initialize_catalog: bool,
451 ) -> StorageBackendResult<Self> {
452 storage_session.validate_transaction_affinity()?;
453 let restore_catalog = Arc::clone(&storage_session.catalog);
454 let restore_backend = Arc::clone(&storage_session.backend);
455 let mut engine = Self::empty_persistent_session(storage_session, provider);
456 let own_read = !initialize_catalog && !restore_backend.in_transaction();
457 let version_before_read = if own_read {
458 let version = restore_backend.change_version()?;
459 restore_backend.begin_read_transaction()?;
460 restore_backend.pin_transaction_snapshot()?;
461 Some(version)
462 } else {
463 None
464 };
465 let read_view_before = restore_backend.read_view_revision()?;
469 let cache_revisions_before;
470 if initialize_catalog {
471 restore_backend.migrate_document_storage()?;
472 restore_backend.begin_upgradeable_transaction()?;
474 let restore_result = (|| {
475 restore_catalog.initialize_storage()?;
476 let cache_revisions = restore_catalog.cache_revisions()?;
477 restore_backend.migrate_inverted_index_storage()?;
478 Self::prepare_catalog_for_initial_restore(restore_catalog.as_ref())?;
479 engine
480 .restore_initial_catalog(restore_catalog.as_ref(), restore_backend.as_ref())?;
481 uqa_execution::catalog::value_restoration::normalize_legacy_vectors(
482 restore_catalog.as_ref(),
483 restore_backend.as_ref(),
484 &engine,
485 )?;
486 engine.repair_reset_fts_storage(restore_catalog.as_ref())?;
487 engine.initialize_catalog_constraint_tuples(&restore_catalog, &restore_backend)?;
488 Ok(cache_revisions)
489 })();
490 cache_revisions_before = match restore_result {
491 Ok(revisions) => revisions,
492 Err(error) => {
493 return match restore_backend.rollback_transaction() {
494 Ok(()) => Err(error),
495 Err(rollback_error) => Err(StorageBackendError::Other(format!(
496 "rollback initial catalog migration after `{error}` failed: {rollback_error}"
497 ))),
498 };
499 }
500 };
501 if let Err(error) = restore_backend.commit_transaction() {
502 return match restore_backend.rollback_transaction() {
503 Ok(()) => Err(error),
504 Err(rollback_error) => Err(StorageBackendError::Other(format!(
505 "rollback failed initial catalog migration commit `{error}`: {rollback_error}"
506 ))),
507 };
508 }
509 } else {
510 cache_revisions_before = restore_catalog.cache_revisions()?;
511 engine.restore_from_catalog(
512 restore_catalog.as_ref(),
513 restore_backend.as_ref(),
514 super::CatalogRestoreMode::LoadOnly,
515 )?;
516 }
517 if initialize_catalog {
518 engine.repair_catalog_on_open()?;
519 }
520 for (name, json) in restore_catalog.load_models()? {
523 let model = serde_json::from_str::<DeepModel>(&json)?;
524 engine.durable.models.write().insert(name, model);
525 }
526 for (name, json) in restore_catalog.load_all_scoring_params()? {
527 engine.durable.scoring_params.write().insert(name, json);
528 }
529 let cache_revisions_after = restore_catalog.cache_revisions()?;
533 let stable_restore = cache_revisions_before == cache_revisions_after;
534 if stable_restore {
535 *engine.epochs.storage_cache_revisions.lock() = cache_revisions_after;
536 let read_view_after = restore_backend.read_view_revision()?;
537 if read_view_before == read_view_after {
538 *engine.epochs.seen_storage_read_view.lock() = read_view_after;
539 }
540 }
541 if let Some(version) = restore_backend.change_version()?.filter(|_| stable_restore) {
542 engine
543 .epochs
544 .seen_storage_change_version
545 .store(version, std::sync::atomic::Ordering::Release);
546 }
547 if let Some(before) = version_before_read {
548 restore_backend.rollback_transaction()?;
549 if restore_backend.change_version()? != before {
550 engine.epochs.seen_storage_change_version.store(
553 before.unwrap_or_default(),
554 std::sync::atomic::Ordering::Release,
555 );
556 *engine.epochs.seen_storage_read_view.lock() = None;
557 }
558 }
559 Ok(engine)
560 }
561
562 fn sqlite_open_error(err: StorageBackendError) -> SQLiteError {
563 SQLiteError::from(err)
564 }
565}