Skip to main content

uqa_engine/open/
lifecycle.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! File-format opening, connection binding, and independent session creation.
8
9use super::{
10    Arc, DeepModel, Engine, ManagedConnection, Path, PersistentStorageBackend,
11    PersistentStorageProvider, PersistentStorageSession, SQLiteCompressedContainerAnchor,
12    SQLiteCompressionOptions, SQLiteError, SQLiteStorageProvider, StorageBackendError,
13    StorageBackendResult,
14};
15
16struct BackendSessionProvider {
17    backend: Arc<dyn PersistentStorageBackend>,
18}
19
20impl PersistentStorageProvider for BackendSessionProvider {
21    fn open_session_with_cancellation(
22        &self,
23        cancellation: &uqa_core::CancellationToken,
24    ) -> StorageBackendResult<PersistentStorageSession> {
25        self.backend.open_session_with_cancellation(cancellation)
26    }
27
28    fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
29        self.backend.auxiliary_encryption_key()
30    }
31
32    fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
33        self.backend.open_session()
34    }
35
36    fn storage_identity(
37        &self,
38    ) -> StorageBackendResult<Option<uqa_storage::PersistentStorageIdentity>> {
39        self.backend.storage_identity()
40    }
41}
42
43impl Default for Engine {
44    fn default() -> Self {
45        Self::new()
46    }
47}
48
49impl Engine {
50    /// In-memory engine. State lives only as long as this `Engine`.
51    pub fn new() -> Self {
52        Self::in_memory_with_retention(uqa_core::memory::MemoryBudget::new(
53            uqa_storage::mvcc::VersionedSessionOptions::default().retained_bytes,
54        ))
55    }
56
57    pub(crate) fn in_memory_with_retention(memory: uqa_core::memory::MemoryBudget) -> Self {
58        let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
59        let notification_hub = Arc::new(crate::NotificationHub::default());
60        let session_id = row_locks.allocate_session();
61        let session = Arc::new(super::SessionContext::with_retention(
62            super::initial_random_state(),
63            memory,
64        ));
65        let client_level = session.state.client_level();
66        let runtime = super::QueryRuntime::new(
67            super::SQL_FUNCTION_DEPTH_LIMIT,
68            client_level,
69            session.state.io_timing(),
70        );
71        session.state.attach_cancellation(&runtime.cancellation);
72        Self {
73            storage: super::StorageContext::memory(),
74            durable: Arc::new(super::DurableCatalogState::new()),
75            extensions: super::RuntimeExtensions::new(&session, &runtime.statement_gate),
76            session,
77            epochs: Arc::new(super::EpochCoordinator::new()),
78            runtime,
79            statistics: crate::statistics::shared_statistics(&row_locks),
80            row_locks,
81            notification_hub,
82            session_id,
83            owns_session_registration: true,
84            query_table_snapshots: None,
85            query_view_snapshots: None,
86            query_sql_function_snapshots: None,
87            query_catalog_snapshot: None,
88            query_transaction_overlay: None,
89            query_transaction_origin: None,
90        }
91    }
92
93    pub fn open(path: &Path) -> Result<Self, SQLiteError> {
94        let conn = ManagedConnection::open(path)?;
95        Self::open_with_connection(&conn)
96    }
97
98    /// Classify the on-disk format of `path` without opening it: plain
99    /// `SQLite`, UQA compressed container (with its encryption flag), a
100    /// missing/empty file, or an unrecognized header (`SQLCipher`
101    /// databases fall here because `SQLCipher` encrypts the whole file).
102    pub fn detect_database_file(
103        path: &Path,
104    ) -> std::io::Result<uqa_storage_sqlite::DatabaseFileFormat> {
105        uqa_storage_sqlite::detect_database_file_format(path)
106    }
107
108    /// Open `path` with the variant its on-disk format calls for.
109    ///
110    /// - Missing/empty file: creates a new database, `SQLCipher`
111    ///   encrypted when `key` is provided, plaintext otherwise.
112    /// - Plain `SQLite`: opens plaintext; providing a key is an error
113    ///   ([`SQLiteError::NotEncrypted`]) rather than a silent no-op so
114    ///   callers never believe an unencrypted database is protected.
115    /// - Compressed container: opens with the codec recorded in the
116    ///   container header; the encryption flag decides whether `key`
117    ///   is required ([`SQLiteError::EncryptionKeyRequired`]) or
118    ///   rejected ([`SQLiteError::NotEncrypted`]).
119    /// - Unrecognized header: treated as `SQLCipher` when `key` is
120    ///   provided; without a key this fails with
121    ///   [`SQLiteError::EncryptionKeyRequired`] because an encrypted
122    ///   database cannot be told apart from a foreign file.
123    ///
124    /// New compressed containers are not created through this entry
125    /// point; use [`Engine::open_compressed`] or
126    /// [`Engine::open_compressed_encrypted`] to choose compression for
127    /// a new database.
128    pub fn open_auto(path: &Path, key: Option<&str>) -> Result<Self, SQLiteError> {
129        let connection = ManagedConnection::open_auto(path, key)?;
130        Self::open_with_connection(&connection)
131    }
132
133    /// SQLCipher-backed engine. Applies `key` before any catalog
134    /// access, runs migrations, and rebuilds the in-memory table
135    /// registry from the encrypted catalog.
136    pub fn open_encrypted(path: &Path, key: &str) -> Result<Self, SQLiteError> {
137        let conn = ManagedConnection::open_encrypted(path, key)?;
138        Self::open_with_connection(&conn)
139    }
140
141    /// Compressed SQLite-backed engine. The compression VFS is
142    /// schema-neutral: it compresses `SQLite` byte ranges in chunks
143    /// without knowledge of UQA catalog tables or columns.
144    pub fn open_compressed(
145        path: &Path,
146        compression: SQLiteCompressionOptions,
147    ) -> Result<Self, SQLiteError> {
148        let conn = ManagedConnection::open_compressed(path, compression)?;
149        Self::open_with_connection(&conn)
150    }
151
152    /// Compressed and encrypted SQLite-backed engine. Chunk payloads
153    /// are compressed first, then encrypted by the compressed VFS. The v2
154    /// format authenticates container metadata, chunk placement, and commit
155    /// records, but cannot distinguish replacement by an internally valid
156    /// snapshot or fork without an external trusted state anchor.
157    /// Security-sensitive deployments that do not require compression should
158    /// prefer [`Engine::open_encrypted`] and `SQLCipher`.
159    pub fn open_compressed_encrypted(
160        path: &Path,
161        key: &str,
162        compression: SQLiteCompressionOptions,
163    ) -> Result<Self, SQLiteError> {
164        let conn = ManagedConnection::open_compressed_encrypted(path, key, compression)?;
165        Self::open_with_connection(&conn)
166    }
167
168    /// Open an encrypted compressed database and reject a different file or
169    /// any state other than `trusted_anchor` before `SQLite` reads the main
170    /// database. Refresh the trusted anchor after every committed write.
171    pub fn open_compressed_encrypted_with_anchor(
172        path: &Path,
173        key: &str,
174        compression: SQLiteCompressionOptions,
175        trusted_anchor: SQLiteCompressedContainerAnchor,
176    ) -> Result<Self, SQLiteError> {
177        let conn = ManagedConnection::open_compressed_encrypted_with_anchor(
178            path,
179            key,
180            compression,
181            trusted_anchor,
182        )?;
183        Self::open_with_connection(&conn)
184    }
185
186    /// Authenticate and return the anchor to persist in a trusted store after
187    /// committed writes to an encrypted compressed database.
188    pub fn compressed_container_anchor(
189        path: &Path,
190        key: &str,
191    ) -> Result<SQLiteCompressedContainerAnchor, SQLiteError> {
192        Ok(uqa_storage_sqlite::read_authenticated_anchor(path, key)?)
193    }
194
195    fn open_with_connection(conn: &ManagedConnection) -> Result<Self, SQLiteError> {
196        let provider: Arc<dyn PersistentStorageProvider> =
197            Arc::new(SQLiteStorageProvider::new(conn.clone()));
198        Self::from_persistent_provider(provider).map_err(Self::sqlite_open_error)
199    }
200
201    /// Create an independent SQL session over this engine's durable database.
202    ///
203    /// The new session gets its own catalog/backend pair, transaction stack,
204    /// runtime variables, prepared statements, statement cache, and
205    /// cancellation token. Committed catalog definitions share immutable
206    /// allocations; mutations detach the session's copy and synchronize through
207    /// shared epochs. Runtime-only Rust extensions are shared. The provider
208    /// must return catalog and data handles bound to one session transaction
209    /// so every durable mutation commits atomically.
210    pub fn new_session(&self) -> StorageBackendResult<Self> {
211        let _statement = self.lock_statement_gate();
212        self.new_sibling_session(false, None)
213    }
214
215    /// Whether these sessions use the same database identity. This reuses the provider-identified coordinator shared by independent opens and sibling sessions; memory engines have independent identities. Providers without stable storage identity can share only through their retained provider or backend owner.
216    #[must_use]
217    pub fn shares_database_with(&self, other: &Self) -> bool {
218        Arc::ptr_eq(&self.row_locks, &other.row_locks)
219    }
220
221    /// Restore an independent internal read view without entering the source statement or acquiring another automatic-maintenance client lease.
222    pub(crate) fn new_internal_read_session(&self) -> StorageBackendResult<Self> {
223        let mut session = self.new_sibling_session(true, None)?;
224        session.runtime.diagnostics = self.runtime.diagnostics.fork();
225        Ok(session)
226    }
227
228    pub(crate) fn new_internal_retained_read_session(&self) -> StorageBackendResult<Self> {
229        let backend = self.storage.backend.as_ref().ok_or_else(|| {
230            StorageBackendError::Other("retained reads require persistent storage".into())
231        })?;
232        let storage_session =
233            backend.open_retained_read_session(&uqa_core::CancellationToken::new())?;
234        let mut session = self.new_sibling_session(true, Some(storage_session))?;
235        session.runtime.diagnostics = self.runtime.diagnostics.fork();
236        Ok(session)
237    }
238
239    fn new_sibling_session(
240        &self,
241        internal_read: bool,
242        retained_session: Option<PersistentStorageSession>,
243    ) -> StorageBackendResult<Self> {
244        let provider = self.storage.provider.as_ref().ok_or_else(|| {
245            StorageBackendError::Other(
246                "independent sessions require a PersistentStorageProvider".into(),
247            )
248        })?;
249        // Only the public factory owns the source statement gate. Internal readers can run on a worker of that statement and must restore their own catalog without borrowing the source transaction or temporary namespace. A retained pair supplies its own fixed catalog view.
250        let retained = retained_session.is_some();
251        let share_catalog = !internal_read
252            && !retained
253            && self
254                .session
255                .transactions
256                .try_lock()
257                .is_some_and(|stack| stack.is_empty())
258            && self.session.state.read().temporary_namespace.is_none();
259        if share_catalog {
260            self.synchronize_table_catalog()?;
261            self.synchronize_table_data()?;
262            self.synchronize_catalog_registries()?;
263        }
264        let observed_epochs = self.epochs.published_epochs();
265        let storage_session = match retained_session {
266            Some(session) => session,
267            None => provider.open_session()?,
268        };
269        storage_session.validate_transaction_affinity()?;
270        let storage_version_before_restore = storage_session.backend.change_version()?;
271        let shared = if share_catalog {
272            self.session_from_shared_catalog(&storage_session, provider)?
273        } else {
274            None
275        };
276        let mut session = match shared {
277            Some(session) => session,
278            None => Self::from_initialized_persistent_session(
279                storage_session,
280                Some(Arc::clone(provider)),
281            )?,
282        };
283        session
284            .session
285            .statistics_worker
286            .store(internal_read, std::sync::atomic::Ordering::Release);
287        session.session.notification_subscriptions_required.store(
288            self.notification_subscriptions_required(),
289            std::sync::atomic::Ordering::Release,
290        );
291        session.row_locks = Arc::clone(&self.row_locks);
292        session.statistics = Arc::clone(&self.statistics);
293        session.install_notification_hub(Arc::clone(&self.notification_hub))?;
294        session.session_id = self.row_locks.allocate_session();
295        let storage_version_after_restore = session
296            .storage
297            .backend
298            .as_ref()
299            .ok_or_else(|| {
300                StorageBackendError::Other(
301                    "persistent session lost its storage backend during restore".into(),
302                )
303            })?
304            .change_version()?;
305        let epochs =
306            Arc::get_mut(&mut session.epochs).expect("new session owns its catalog epoch state");
307        if storage_version_before_restore == storage_version_after_restore {
308            epochs.share_published_from_at(&self.epochs, observed_epochs);
309        } else {
310            epochs.share_published_from(&self.epochs);
311        }
312        // A commit that raced the load advanced a shared publication beyond
313        // the generation or storage version captured before the session
314        // restored. Only that case needs a second catalog or data refresh.
315        // Catalog cells keep independent mutable owners over shared immutable
316        // values, so a writer cannot expose uncommitted definitions to siblings.
317        session.extensions = super::RuntimeExtensions::shared_from(&self.extensions);
318        session
319            .extensions
320            .functions
321            .attach(&session.session, &session.runtime.statement_gate);
322        if internal_read {
323            session.runtime.function_context = Arc::clone(&self.runtime.function_context);
324        }
325        if !retained {
326            session.synchronize_table_catalog()?;
327            session.synchronize_table_data()?;
328            session.synchronize_catalog_registries()?;
329        }
330        session.start_automatic_statistics();
331        Ok(session)
332    }
333
334    /// Build an engine and retain the provider used to create future
335    /// independent sessions.
336    #[allow(clippy::needless_pass_by_value)]
337    pub fn from_persistent_provider(
338        provider: Arc<dyn PersistentStorageProvider>,
339    ) -> StorageBackendResult<Self> {
340        let identity = provider.storage_identity()?;
341        let session = provider.open_initial_session()?;
342        let mut engine = Self::from_persistent_session(session, Some(Arc::clone(&provider)))?;
343        let row_locks = crate::row_locks::shared_provider_manager(identity.clone(), &provider);
344        let notification_hub =
345            crate::notifications::shared_provider_notification_hub(identity, &provider);
346        engine.session_id = row_locks.allocate_session();
347        engine.statistics = crate::statistics::shared_statistics(&row_locks);
348        engine.row_locks = row_locks;
349        engine.install_notification_hub(notification_hub)?;
350        engine.start_automatic_statistics();
351        Ok(engine)
352    }
353
354    /// Build an engine from already-open persistent metadata and data backends. The backend's session factory is retained for independent SQL sessions and latest-committed row-lock rechecks. Prefer [`Self::from_persistent_provider`] when a database-level owner is already available.
355    pub fn from_persistent_backends(
356        catalog: Arc<dyn uqa_storage::CatalogFacade>,
357        backend: Arc<dyn PersistentStorageBackend>,
358    ) -> StorageBackendResult<Self> {
359        let identity = backend.storage_identity()?;
360        let row_locks = crate::row_locks::shared_backend_manager(identity.clone(), &backend);
361        let notification_hub =
362            crate::notifications::shared_backend_notification_hub(identity, &backend);
363        let provider: Arc<dyn PersistentStorageProvider> = Arc::new(BackendSessionProvider {
364            backend: Arc::clone(&backend),
365        });
366        let mut engine = Self::from_persistent_session(
367            PersistentStorageSession::new(catalog, backend),
368            Some(provider),
369        )?;
370        engine.session_id = row_locks.allocate_session();
371        engine.statistics = crate::statistics::shared_statistics(&row_locks);
372        engine.row_locks = row_locks;
373        engine.install_notification_hub(notification_hub)?;
374        engine.start_automatic_statistics();
375        Ok(engine)
376    }
377
378    fn install_notification_hub(
379        &mut self,
380        notification_hub: Arc<crate::NotificationHub>,
381    ) -> StorageBackendResult<()> {
382        let process_id = notification_hub
383            .allocate_backend_process_id()
384            .map_err(|error| {
385                StorageBackendError::Other(format!(
386                    "allocate database backend process identifier: {error}"
387                ))
388            })?;
389        self.notification_hub = notification_hub;
390        if let Some(process_id) = process_id {
391            self.session.install_database_backend_process_id(process_id);
392        }
393        Ok(())
394    }
395
396    fn from_persistent_session(
397        storage_session: PersistentStorageSession,
398        provider: Option<Arc<dyn PersistentStorageProvider>>,
399    ) -> StorageBackendResult<Self> {
400        Self::build_persistent_session(storage_session, provider, true)
401    }
402
403    pub(crate) fn from_initialized_persistent_session(
404        storage_session: PersistentStorageSession,
405        provider: Option<Arc<dyn PersistentStorageProvider>>,
406    ) -> StorageBackendResult<Self> {
407        Self::build_persistent_session(storage_session, provider, false)
408    }
409
410    pub(super) fn empty_persistent_session(
411        storage_session: PersistentStorageSession,
412        provider: Option<Arc<dyn PersistentStorageProvider>>,
413    ) -> Self {
414        let PersistentStorageSession { catalog, backend } = storage_session;
415        let cancellation = backend.write_cancellation().unwrap_or_default();
416        let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
417        let notification_hub = Arc::new(crate::NotificationHub::default());
418        let session_id = row_locks.allocate_session();
419        let session = Arc::new(super::SessionContext::new(super::initial_random_state()));
420        let client_level = session.state.client_level();
421        let runtime = super::QueryRuntime::with_cancellation(
422            super::SQL_FUNCTION_DEPTH_LIMIT,
423            cancellation,
424            client_level,
425            session.state.io_timing(),
426        );
427        session.state.attach_cancellation(&runtime.cancellation);
428        Self {
429            storage: super::StorageContext::persistent(catalog, backend, provider),
430            durable: Arc::new(super::DurableCatalogState::new()),
431            extensions: super::RuntimeExtensions::new(&session, &runtime.statement_gate),
432            session,
433            epochs: Arc::new(super::EpochCoordinator::new()),
434            runtime,
435            statistics: crate::statistics::shared_statistics(&row_locks),
436            row_locks,
437            notification_hub,
438            session_id,
439            owns_session_registration: true,
440            query_table_snapshots: None,
441            query_view_snapshots: None,
442            query_sql_function_snapshots: None,
443            query_catalog_snapshot: None,
444            query_transaction_overlay: None,
445            query_transaction_origin: None,
446        }
447    }
448
449    /// Repair what earlier releases left inconsistent once a writer has restored the catalog: physical value indexes, partitions without the copies of their parent's foreign keys, and foreign keys without the constraints they derive on referenced partitions.
450    fn repair_catalog_on_open(&self) -> StorageBackendResult<()> {
451        self.repair_persistent_value_indexes_on_open()?;
452        self.repair_partition_foreign_keys_on_open()?;
453        self.repair_derived_constraints_on_open()
454    }
455
456    fn build_persistent_session(
457        storage_session: PersistentStorageSession,
458        provider: Option<Arc<dyn PersistentStorageProvider>>,
459        initialize_catalog: bool,
460    ) -> StorageBackendResult<Self> {
461        storage_session.validate_transaction_affinity()?;
462        let restore_catalog = Arc::clone(&storage_session.catalog);
463        let restore_backend = Arc::clone(&storage_session.backend);
464        let mut engine = Self::empty_persistent_session(storage_session, provider);
465        let own_read = !initialize_catalog && !restore_backend.in_transaction();
466        let version_before_read = if own_read {
467            let version = restore_backend.change_version()?;
468            restore_backend.begin_read_transaction()?;
469            restore_backend.pin_transaction_snapshot()?;
470            Some(version)
471        } else {
472            None
473        };
474        // A load-only restore owns one read boundary. Registry validators may
475        // resolve tables without recursively restoring a newer catalog. The
476        // unpublished Engine releases this transaction on errors and unwinds.
477        let read_view_before = restore_backend.read_view_revision()?;
478        let cache_revisions_before;
479        if initialize_catalog {
480            restore_backend.migrate_document_storage()?;
481            // A clean restore remains read-only on backends that can promote a transaction, while backends without promotion reserve their writer before the atomic migration scan.
482            restore_backend.begin_upgradeable_transaction()?;
483            let restore_result = (|| {
484                restore_catalog.initialize_storage()?;
485                let cache_revisions = restore_catalog.cache_revisions()?;
486                restore_backend.migrate_inverted_index_storage()?;
487                Self::prepare_catalog_for_initial_restore(restore_catalog.as_ref())?;
488                engine
489                    .restore_initial_catalog(restore_catalog.as_ref(), restore_backend.as_ref())?;
490                uqa_execution::catalog::value_restoration::normalize_legacy_vectors(
491                    restore_catalog.as_ref(),
492                    restore_backend.as_ref(),
493                    &engine,
494                )?;
495                engine.repair_reset_fts_storage(restore_catalog.as_ref())?;
496                engine.initialize_catalog_constraint_tuples(&restore_catalog, &restore_backend)?;
497                Ok(cache_revisions)
498            })();
499            cache_revisions_before = match restore_result {
500                Ok(revisions) => revisions,
501                Err(error) => {
502                    return match restore_backend.rollback_transaction() {
503                        Ok(()) => Err(error),
504                        Err(rollback_error) => Err(StorageBackendError::Other(format!(
505                            "rollback initial catalog migration after `{error}` failed: {rollback_error}"
506                        ))),
507                    };
508                }
509            };
510            if let Err(error) = restore_backend.commit_transaction() {
511                return match restore_backend.rollback_transaction() {
512                    Ok(()) => Err(error),
513                    Err(rollback_error) => Err(StorageBackendError::Other(format!(
514                        "rollback failed initial catalog migration commit `{error}`: {rollback_error}"
515                    ))),
516                };
517            }
518        } else {
519            cache_revisions_before = restore_catalog.cache_revisions()?;
520            engine.restore_from_catalog(
521                restore_catalog.as_ref(),
522                restore_backend.as_ref(),
523                super::CatalogRestoreMode::LoadOnly,
524            )?;
525        }
526        if initialize_catalog {
527            engine.repair_catalog_on_open()?;
528        }
529        // Eagerly and fallibly populate read caches. Once open succeeds,
530        // cache misses mean absence rather than a swallowed catalog error.
531        for (name, json) in restore_catalog.load_models()? {
532            let model = serde_json::from_str::<DeepModel>(&json)?;
533            engine.durable.models.write().insert(name, model);
534        }
535        for (name, json) in restore_catalog.load_all_scoring_params()? {
536            engine.durable.scoring_params.write().insert(name, json);
537        }
538        // Initial catalog migrations and physical-index repairs above may
539        // commit. Establish the backend monitor baseline only after every
540        // one-time write has completed.
541        let cache_revisions_after = restore_catalog.cache_revisions()?;
542        let stable_restore = cache_revisions_before == cache_revisions_after;
543        if stable_restore {
544            *engine.epochs.storage_cache_revisions.lock() = cache_revisions_after;
545            let read_view_after = restore_backend.read_view_revision()?;
546            if read_view_before == read_view_after {
547                *engine.epochs.seen_storage_read_view.lock() = read_view_after;
548            }
549        }
550        if let Some(version) = restore_backend.change_version()?.filter(|_| stable_restore) {
551            engine
552                .epochs
553                .seen_storage_change_version
554                .store(version, std::sync::atomic::Ordering::Release);
555        }
556        if let Some(before) = version_before_read {
557            restore_backend.rollback_transaction()?;
558            if restore_backend.change_version()? != before {
559                // The restored definitions belong to the pinned snapshot,
560                // even when the independent monitor saw a later commit.
561                engine.epochs.seen_storage_change_version.store(
562                    before.unwrap_or_default(),
563                    std::sync::atomic::Ordering::Release,
564                );
565                *engine.epochs.seen_storage_read_view.lock() = None;
566            }
567        }
568        Ok(engine)
569    }
570
571    fn sqlite_open_error(err: StorageBackendError) -> SQLiteError {
572        SQLiteError::from(err)
573    }
574}