Skip to main content

uqa_engine/open/
lifecycle.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! File-format opening, connection binding, and independent session creation.
8
9use super::{
10    Arc, DeepModel, Engine, ManagedConnection, Path, PersistentStorageBackend,
11    PersistentStorageProvider, PersistentStorageSession, SQLiteCompressedContainerAnchor,
12    SQLiteCompressionOptions, SQLiteError, SQLiteStorageProvider, StorageBackendError,
13    StorageBackendResult,
14};
15
16struct BackendSessionProvider {
17    backend: Arc<dyn PersistentStorageBackend>,
18}
19
20impl PersistentStorageProvider for BackendSessionProvider {
21    fn open_session_with_cancellation(
22        &self,
23        cancellation: &uqa_core::CancellationToken,
24    ) -> StorageBackendResult<PersistentStorageSession> {
25        self.backend.open_session_with_cancellation(cancellation)
26    }
27
28    fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
29        self.backend.auxiliary_encryption_key()
30    }
31
32    fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
33        self.backend.open_session()
34    }
35
36    fn storage_identity(
37        &self,
38    ) -> StorageBackendResult<Option<uqa_storage::PersistentStorageIdentity>> {
39        self.backend.storage_identity()
40    }
41}
42
43impl Default for Engine {
44    fn default() -> Self {
45        Self::new()
46    }
47}
48
49impl Engine {
50    /// In-memory engine. State lives only as long as this `Engine`.
51    pub fn new() -> Self {
52        let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
53        let notification_hub = Arc::new(crate::NotificationHub::default());
54        let session_id = row_locks.allocate_session();
55        let session = Arc::new(super::SessionContext::new(super::initial_random_state()));
56        let client_level = session.state.client_level();
57        let runtime = super::QueryRuntime::new(super::SQL_FUNCTION_DEPTH_LIMIT, client_level);
58        session.state.attach_cancellation(&runtime.cancellation);
59        Self {
60            storage: super::StorageContext::memory(),
61            durable: Arc::new(super::DurableCatalogState::new()),
62            session,
63            extensions: super::RuntimeExtensions::new(),
64            epochs: super::EpochCoordinator::new(),
65            runtime,
66            statistics: crate::statistics::shared_statistics(&row_locks),
67            row_locks,
68            notification_hub,
69            session_id,
70            owns_session_registration: true,
71            query_table_snapshots: None,
72            query_view_snapshots: None,
73            query_sql_function_snapshots: None,
74            query_catalog_snapshot: None,
75            query_transaction_overlay: None,
76            query_transaction_origin: None,
77        }
78    }
79
80    pub fn open(path: &Path) -> Result<Self, SQLiteError> {
81        let conn = ManagedConnection::open(path)?;
82        Self::open_with_connection(&conn)
83    }
84
85    /// Classify the on-disk format of `path` without opening it: plain
86    /// `SQLite`, UQA compressed container (with its encryption flag), a
87    /// missing/empty file, or an unrecognized header (`SQLCipher`
88    /// databases fall here because `SQLCipher` encrypts the whole file).
89    pub fn detect_database_file(
90        path: &Path,
91    ) -> std::io::Result<uqa_storage_sqlite::DatabaseFileFormat> {
92        uqa_storage_sqlite::detect_database_file_format(path)
93    }
94
95    /// Open `path` with the variant its on-disk format calls for.
96    ///
97    /// - Missing/empty file: creates a new database, `SQLCipher`
98    ///   encrypted when `key` is provided, plaintext otherwise.
99    /// - Plain `SQLite`: opens plaintext; providing a key is an error
100    ///   ([`SQLiteError::NotEncrypted`]) rather than a silent no-op so
101    ///   callers never believe an unencrypted database is protected.
102    /// - Compressed container: opens with the codec recorded in the
103    ///   container header; the encryption flag decides whether `key`
104    ///   is required ([`SQLiteError::EncryptionKeyRequired`]) or
105    ///   rejected ([`SQLiteError::NotEncrypted`]).
106    /// - Unrecognized header: treated as `SQLCipher` when `key` is
107    ///   provided; without a key this fails with
108    ///   [`SQLiteError::EncryptionKeyRequired`] because an encrypted
109    ///   database cannot be told apart from a foreign file.
110    ///
111    /// New compressed containers are not created through this entry
112    /// point; use [`Engine::open_compressed`] or
113    /// [`Engine::open_compressed_encrypted`] to choose compression for
114    /// a new database.
115    pub fn open_auto(path: &Path, key: Option<&str>) -> Result<Self, SQLiteError> {
116        use uqa_storage_sqlite::DatabaseFileFormat;
117        let key = match key {
118            Some("") => return Err(SQLiteError::EmptyEncryptionKey),
119            other => other,
120        };
121        match uqa_storage_sqlite::detect_database_file_format(path)? {
122            DatabaseFileFormat::Missing => match key {
123                Some(key) => Self::open_encrypted(path, key),
124                None => Self::open(path),
125            },
126            DatabaseFileFormat::PlainSQLite => match key {
127                Some(_) => Err(SQLiteError::NotEncrypted),
128                None => Self::open(path),
129            },
130            DatabaseFileFormat::CompressedContainer { encrypted: true } => match key {
131                Some(key) => {
132                    Self::open_compressed_encrypted(path, key, SQLiteCompressionOptions::default())
133                }
134                None => Err(SQLiteError::EncryptionKeyRequired),
135            },
136            DatabaseFileFormat::CompressedContainer { encrypted: false } => match key {
137                Some(_) => Err(SQLiteError::NotEncrypted),
138                None => Self::open_compressed(path, SQLiteCompressionOptions::default()),
139            },
140            DatabaseFileFormat::Unrecognized => match key {
141                Some(key) => Self::open_encrypted(path, key),
142                None => Err(SQLiteError::EncryptionKeyRequired),
143            },
144        }
145    }
146
147    /// SQLCipher-backed engine. Applies `key` before any catalog
148    /// access, runs migrations, and rebuilds the in-memory table
149    /// registry from the encrypted catalog.
150    pub fn open_encrypted(path: &Path, key: &str) -> Result<Self, SQLiteError> {
151        let conn = ManagedConnection::open_encrypted(path, key)?;
152        Self::open_with_connection(&conn)
153    }
154
155    /// Compressed SQLite-backed engine. The compression VFS is
156    /// schema-neutral: it compresses `SQLite` byte ranges in chunks
157    /// without knowledge of UQA catalog tables or columns.
158    pub fn open_compressed(
159        path: &Path,
160        compression: SQLiteCompressionOptions,
161    ) -> Result<Self, SQLiteError> {
162        let conn = ManagedConnection::open_compressed(path, compression)?;
163        Self::open_with_connection(&conn)
164    }
165
166    /// Compressed and encrypted SQLite-backed engine. Chunk payloads
167    /// are compressed first, then encrypted by the compressed VFS. The v2
168    /// format authenticates container metadata, chunk placement, and commit
169    /// records, but cannot distinguish replacement by an internally valid
170    /// snapshot or fork without an external trusted state anchor.
171    /// Security-sensitive deployments that do not require compression should
172    /// prefer [`Engine::open_encrypted`] and `SQLCipher`.
173    pub fn open_compressed_encrypted(
174        path: &Path,
175        key: &str,
176        compression: SQLiteCompressionOptions,
177    ) -> Result<Self, SQLiteError> {
178        let conn = ManagedConnection::open_compressed_encrypted(path, key, compression)?;
179        Self::open_with_connection(&conn)
180    }
181
182    /// Open an encrypted compressed database and reject a different file or
183    /// any state other than `trusted_anchor` before `SQLite` reads the main
184    /// database. Refresh the trusted anchor after every committed write.
185    pub fn open_compressed_encrypted_with_anchor(
186        path: &Path,
187        key: &str,
188        compression: SQLiteCompressionOptions,
189        trusted_anchor: SQLiteCompressedContainerAnchor,
190    ) -> Result<Self, SQLiteError> {
191        let conn = ManagedConnection::open_compressed_encrypted_with_anchor(
192            path,
193            key,
194            compression,
195            trusted_anchor,
196        )?;
197        Self::open_with_connection(&conn)
198    }
199
200    /// Authenticate and return the anchor to persist in a trusted store after
201    /// committed writes to an encrypted compressed database.
202    pub fn compressed_container_anchor(
203        path: &Path,
204        key: &str,
205    ) -> Result<SQLiteCompressedContainerAnchor, SQLiteError> {
206        Ok(uqa_storage_sqlite::read_authenticated_anchor(path, key)?)
207    }
208
209    fn open_with_connection(conn: &ManagedConnection) -> Result<Self, SQLiteError> {
210        let provider: Arc<dyn PersistentStorageProvider> =
211            Arc::new(SQLiteStorageProvider::new(conn.clone()));
212        Self::from_persistent_provider(provider).map_err(Self::sqlite_open_error)
213    }
214
215    /// Create an independent SQL session over this engine's durable database.
216    ///
217    /// The new session gets its own catalog/backend pair, transaction stack,
218    /// runtime variables, prepared statements, statement cache, and
219    /// cancellation token. Committed catalog definitions share immutable
220    /// allocations; mutations detach the session's copy and synchronize through
221    /// shared epochs. Runtime-only Rust extensions are shared. The provider
222    /// must return catalog and data handles bound to one session transaction
223    /// so every durable mutation commits atomically.
224    pub fn new_session(&self) -> StorageBackendResult<Self> {
225        let _statement = self.lock_statement_gate();
226        self.new_sibling_session(false, None)
227    }
228
229    /// Restore an independent internal read view without entering the source statement or acquiring another automatic-maintenance client lease.
230    pub(crate) fn new_internal_read_session(&self) -> StorageBackendResult<Self> {
231        let mut session = self.new_sibling_session(true, None)?;
232        session.runtime.diagnostics = self.runtime.diagnostics.fork();
233        Ok(session)
234    }
235
236    pub(crate) fn new_internal_retained_read_session(&self) -> StorageBackendResult<Self> {
237        let backend = self.storage.backend.as_ref().ok_or_else(|| {
238            StorageBackendError::Other("retained reads require persistent storage".into())
239        })?;
240        let storage_session =
241            backend.open_retained_read_session(&uqa_core::CancellationToken::new())?;
242        let mut session = self.new_sibling_session(true, Some(storage_session))?;
243        session.runtime.diagnostics = self.runtime.diagnostics.fork();
244        Ok(session)
245    }
246
247    fn new_sibling_session(
248        &self,
249        internal_read: bool,
250        retained_session: Option<PersistentStorageSession>,
251    ) -> StorageBackendResult<Self> {
252        let provider = self.storage.provider.as_ref().ok_or_else(|| {
253            StorageBackendError::Other(
254                "independent sessions require a PersistentStorageProvider".into(),
255            )
256        })?;
257        // Only the public factory owns the source statement gate. Internal readers can run on a worker of that statement and must restore their own catalog without borrowing the source transaction or temporary namespace. A retained pair supplies its own fixed catalog view.
258        let retained = retained_session.is_some();
259        let share_catalog = !internal_read
260            && !retained
261            && self
262                .session
263                .transactions
264                .try_lock()
265                .is_some_and(|stack| stack.is_empty())
266            && self.session.state.read().temporary_namespace.is_none();
267        if share_catalog {
268            self.synchronize_table_catalog()?;
269            self.synchronize_table_data()?;
270            self.synchronize_catalog_registries()?;
271        }
272        let observed_epochs = self.epochs.published_epochs();
273        let storage_session = match retained_session {
274            Some(session) => session,
275            None => provider.open_session()?,
276        };
277        storage_session.validate_transaction_affinity()?;
278        let storage_version_before_restore = storage_session.backend.change_version()?;
279        let shared = if share_catalog {
280            self.session_from_shared_catalog(&storage_session, provider)?
281        } else {
282            None
283        };
284        let mut session = match shared {
285            Some(session) => session,
286            None => Self::from_initialized_persistent_session(
287                storage_session,
288                Some(Arc::clone(provider)),
289            )?,
290        };
291        session
292            .session
293            .statistics_worker
294            .store(internal_read, std::sync::atomic::Ordering::Release);
295        session.session.notification_subscriptions_required.store(
296            self.notification_subscriptions_required(),
297            std::sync::atomic::Ordering::Release,
298        );
299        session.row_locks = Arc::clone(&self.row_locks);
300        session.statistics = Arc::clone(&self.statistics);
301        session.install_notification_hub(Arc::clone(&self.notification_hub))?;
302        session.session_id = self.row_locks.allocate_session();
303        let storage_version_after_restore = session
304            .storage
305            .backend
306            .as_ref()
307            .ok_or_else(|| {
308                StorageBackendError::Other(
309                    "persistent session lost its storage backend during restore".into(),
310                )
311            })?
312            .change_version()?;
313        if storage_version_before_restore == storage_version_after_restore {
314            session
315                .epochs
316                .share_published_from_at(&self.epochs, observed_epochs);
317        } else {
318            session.epochs.share_published_from(&self.epochs);
319        }
320        // A commit that raced the load advanced a shared publication beyond
321        // the generation or storage version captured before the session
322        // restored. Only that case needs a second catalog or data refresh.
323        // Catalog cells keep independent mutable owners over shared immutable
324        // values, so a writer cannot expose uncommitted definitions to siblings.
325        session.extensions = super::RuntimeExtensions::shared_from(&self.extensions);
326        if !retained {
327            session.synchronize_table_catalog()?;
328            session.synchronize_table_data()?;
329            session.synchronize_catalog_registries()?;
330        }
331        session.start_automatic_statistics();
332        Ok(session)
333    }
334
335    /// Build an engine and retain the provider used to create future
336    /// independent sessions.
337    #[allow(clippy::needless_pass_by_value)]
338    pub fn from_persistent_provider(
339        provider: Arc<dyn PersistentStorageProvider>,
340    ) -> StorageBackendResult<Self> {
341        let identity = provider.storage_identity()?;
342        let session = provider.open_initial_session()?;
343        let mut engine = Self::from_persistent_session(session, Some(Arc::clone(&provider)))?;
344        let row_locks = crate::row_locks::shared_provider_manager(identity.clone(), &provider);
345        let notification_hub =
346            crate::notifications::shared_provider_notification_hub(identity, &provider);
347        engine.session_id = row_locks.allocate_session();
348        engine.statistics = crate::statistics::shared_statistics(&row_locks);
349        engine.row_locks = row_locks;
350        engine.install_notification_hub(notification_hub)?;
351        engine.start_automatic_statistics();
352        Ok(engine)
353    }
354
355    /// Build an engine from already-open persistent metadata and data backends. The backend's session factory is retained for independent SQL sessions and latest-committed row-lock rechecks. Prefer [`Self::from_persistent_provider`] when a database-level owner is already available.
356    pub fn from_persistent_backends(
357        catalog: Arc<dyn uqa_storage::CatalogFacade>,
358        backend: Arc<dyn PersistentStorageBackend>,
359    ) -> StorageBackendResult<Self> {
360        let identity = backend.storage_identity()?;
361        let row_locks = crate::row_locks::shared_backend_manager(identity.clone(), &backend);
362        let notification_hub =
363            crate::notifications::shared_backend_notification_hub(identity, &backend);
364        let provider: Arc<dyn PersistentStorageProvider> = Arc::new(BackendSessionProvider {
365            backend: Arc::clone(&backend),
366        });
367        let mut engine = Self::from_persistent_session(
368            PersistentStorageSession::new(catalog, backend),
369            Some(provider),
370        )?;
371        engine.session_id = row_locks.allocate_session();
372        engine.statistics = crate::statistics::shared_statistics(&row_locks);
373        engine.row_locks = row_locks;
374        engine.install_notification_hub(notification_hub)?;
375        engine.start_automatic_statistics();
376        Ok(engine)
377    }
378
379    fn install_notification_hub(
380        &mut self,
381        notification_hub: Arc<crate::NotificationHub>,
382    ) -> StorageBackendResult<()> {
383        let process_id = notification_hub
384            .allocate_backend_process_id()
385            .map_err(|error| {
386                StorageBackendError::Other(format!(
387                    "allocate database backend process identifier: {error}"
388                ))
389            })?;
390        self.notification_hub = notification_hub;
391        if let Some(process_id) = process_id {
392            self.session.install_database_backend_process_id(process_id);
393        }
394        Ok(())
395    }
396
397    fn from_persistent_session(
398        storage_session: PersistentStorageSession,
399        provider: Option<Arc<dyn PersistentStorageProvider>>,
400    ) -> StorageBackendResult<Self> {
401        Self::build_persistent_session(storage_session, provider, true)
402    }
403
404    pub(crate) fn from_initialized_persistent_session(
405        storage_session: PersistentStorageSession,
406        provider: Option<Arc<dyn PersistentStorageProvider>>,
407    ) -> StorageBackendResult<Self> {
408        Self::build_persistent_session(storage_session, provider, false)
409    }
410
411    pub(super) fn empty_persistent_session(
412        storage_session: PersistentStorageSession,
413        provider: Option<Arc<dyn PersistentStorageProvider>>,
414    ) -> Self {
415        let PersistentStorageSession { catalog, backend } = storage_session;
416        let cancellation = backend.write_cancellation().unwrap_or_default();
417        let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
418        let notification_hub = Arc::new(crate::NotificationHub::default());
419        let session_id = row_locks.allocate_session();
420        let session = Arc::new(super::SessionContext::new(super::initial_random_state()));
421        let client_level = session.state.client_level();
422        let runtime = super::QueryRuntime::with_cancellation(
423            super::SQL_FUNCTION_DEPTH_LIMIT,
424            cancellation,
425            client_level,
426        );
427        session.state.attach_cancellation(&runtime.cancellation);
428        Self {
429            storage: super::StorageContext::persistent(catalog, backend, provider),
430            durable: Arc::new(super::DurableCatalogState::new()),
431            session,
432            extensions: super::RuntimeExtensions::new(),
433            epochs: super::EpochCoordinator::new(),
434            runtime,
435            statistics: crate::statistics::shared_statistics(&row_locks),
436            row_locks,
437            notification_hub,
438            session_id,
439            owns_session_registration: true,
440            query_table_snapshots: None,
441            query_view_snapshots: None,
442            query_sql_function_snapshots: None,
443            query_catalog_snapshot: None,
444            query_transaction_overlay: None,
445            query_transaction_origin: None,
446        }
447    }
448
449    /// Repair what earlier releases left inconsistent once a writer has restored the catalog: physical value indexes, partitions without the copies of their parent's foreign keys, and foreign keys without the constraints they derive on referenced partitions.
450    fn repair_catalog_on_open(&self) -> StorageBackendResult<()> {
451        self.repair_persistent_value_indexes_on_open()?;
452        self.repair_partition_foreign_keys_on_open()?;
453        self.repair_derived_constraints_on_open()
454    }
455
456    fn build_persistent_session(
457        storage_session: PersistentStorageSession,
458        provider: Option<Arc<dyn PersistentStorageProvider>>,
459        initialize_catalog: bool,
460    ) -> StorageBackendResult<Self> {
461        storage_session.validate_transaction_affinity()?;
462        let restore_catalog = Arc::clone(&storage_session.catalog);
463        let restore_backend = Arc::clone(&storage_session.backend);
464        let mut engine = Self::empty_persistent_session(storage_session, provider);
465        let own_read = !initialize_catalog && !restore_backend.in_transaction();
466        let version_before_read = if own_read {
467            let version = restore_backend.change_version()?;
468            restore_backend.begin_read_transaction()?;
469            restore_backend.pin_transaction_snapshot()?;
470            Some(version)
471        } else {
472            None
473        };
474        // A load-only restore owns one read boundary. Registry validators may
475        // resolve tables without recursively restoring a newer catalog. The
476        // unpublished Engine releases this transaction on errors and unwinds.
477        let read_view_before = restore_backend.read_view_revision()?;
478        let cache_revisions_before;
479        if initialize_catalog {
480            restore_backend.migrate_document_storage()?;
481            // A clean restore remains read-only on backends that can promote a transaction, while backends without promotion reserve their writer before the atomic migration scan.
482            restore_backend.begin_upgradeable_transaction()?;
483            let restore_result = (|| {
484                restore_catalog.initialize_storage()?;
485                let cache_revisions = restore_catalog.cache_revisions()?;
486                restore_backend.migrate_inverted_index_storage()?;
487                Self::prepare_catalog_for_initial_restore(restore_catalog.as_ref())?;
488                engine
489                    .restore_initial_catalog(restore_catalog.as_ref(), restore_backend.as_ref())?;
490                uqa_execution::catalog::value_restoration::normalize_legacy_vectors(
491                    restore_catalog.as_ref(),
492                    restore_backend.as_ref(),
493                    &engine,
494                )?;
495                engine.repair_reset_fts_storage(restore_catalog.as_ref())?;
496                engine.initialize_catalog_constraint_tuples(&restore_catalog, &restore_backend)?;
497                Ok(cache_revisions)
498            })();
499            cache_revisions_before = match restore_result {
500                Ok(revisions) => revisions,
501                Err(error) => {
502                    return match restore_backend.rollback_transaction() {
503                        Ok(()) => Err(error),
504                        Err(rollback_error) => Err(StorageBackendError::Other(format!(
505                            "rollback initial catalog migration after `{error}` failed: {rollback_error}"
506                        ))),
507                    };
508                }
509            };
510            if let Err(error) = restore_backend.commit_transaction() {
511                return match restore_backend.rollback_transaction() {
512                    Ok(()) => Err(error),
513                    Err(rollback_error) => Err(StorageBackendError::Other(format!(
514                        "rollback failed initial catalog migration commit `{error}`: {rollback_error}"
515                    ))),
516                };
517            }
518        } else {
519            cache_revisions_before = restore_catalog.cache_revisions()?;
520            engine.restore_from_catalog(
521                restore_catalog.as_ref(),
522                restore_backend.as_ref(),
523                super::CatalogRestoreMode::LoadOnly,
524            )?;
525        }
526        if initialize_catalog {
527            engine.repair_catalog_on_open()?;
528        }
529        // Eagerly and fallibly populate read caches. Once open succeeds,
530        // cache misses mean absence rather than a swallowed catalog error.
531        for (name, json) in restore_catalog.load_models()? {
532            let model = serde_json::from_str::<DeepModel>(&json)?;
533            engine.durable.models.write().insert(name, model);
534        }
535        for (name, json) in restore_catalog.load_all_scoring_params()? {
536            engine.durable.scoring_params.write().insert(name, json);
537        }
538        // Initial catalog migrations and physical-index repairs above may
539        // commit. Establish the backend monitor baseline only after every
540        // one-time write has completed.
541        let cache_revisions_after = restore_catalog.cache_revisions()?;
542        let stable_restore = cache_revisions_before == cache_revisions_after;
543        if stable_restore {
544            *engine.epochs.storage_cache_revisions.lock() = cache_revisions_after;
545            let read_view_after = restore_backend.read_view_revision()?;
546            if read_view_before == read_view_after {
547                *engine.epochs.seen_storage_read_view.lock() = read_view_after;
548            }
549        }
550        if let Some(version) = restore_backend.change_version()?.filter(|_| stable_restore) {
551            engine
552                .epochs
553                .seen_storage_change_version
554                .store(version, std::sync::atomic::Ordering::Release);
555        }
556        if let Some(before) = version_before_read {
557            restore_backend.rollback_transaction()?;
558            if restore_backend.change_version()? != before {
559                // The restored definitions belong to the pinned snapshot,
560                // even when the independent monitor saw a later commit.
561                engine.epochs.seen_storage_change_version.store(
562                    before.unwrap_or_default(),
563                    std::sync::atomic::Ordering::Release,
564                );
565                *engine.epochs.seen_storage_read_view.lock() = None;
566            }
567        }
568        Ok(engine)
569    }
570
571    fn sqlite_open_error(err: StorageBackendError) -> SQLiteError {
572        SQLiteError::from(err)
573    }
574}