Skip to main content

uqa_engine/open/
lifecycle.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! File-format opening, connection binding, and independent session creation.
8
9use super::{
10    Arc, DeepModel, Engine, ManagedConnection, Path, PersistentStorageBackend,
11    PersistentStorageProvider, PersistentStorageSession, SQLiteCompressedContainerAnchor,
12    SQLiteCompressionOptions, SQLiteError, SQLiteStorageProvider, StorageBackendError,
13    StorageBackendResult,
14};
15
16struct BackendSessionProvider {
17    backend: Arc<dyn PersistentStorageBackend>,
18}
19
20impl PersistentStorageProvider for BackendSessionProvider {
21    fn open_session_with_cancellation(
22        &self,
23        cancellation: &uqa_core::CancellationToken,
24    ) -> StorageBackendResult<PersistentStorageSession> {
25        self.backend.open_session_with_cancellation(cancellation)
26    }
27
28    fn auxiliary_encryption_key(&self) -> Option<uqa_storage::StorageEncryptionKey> {
29        self.backend.auxiliary_encryption_key()
30    }
31
32    fn open_session(&self) -> StorageBackendResult<PersistentStorageSession> {
33        self.backend.open_session()
34    }
35
36    fn storage_identity(
37        &self,
38    ) -> StorageBackendResult<Option<uqa_storage::PersistentStorageIdentity>> {
39        self.backend.storage_identity()
40    }
41}
42
43impl Default for Engine {
44    fn default() -> Self {
45        Self::new()
46    }
47}
48
49impl Engine {
50    /// In-memory engine. State lives only as long as this `Engine`.
51    pub fn new() -> Self {
52        let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
53        let notification_hub = Arc::new(crate::NotificationHub::default());
54        let session_id = row_locks.allocate_session();
55        let session = Arc::new(super::SessionContext::new(super::initial_random_state()));
56        let client_level = session.state.client_level();
57        let runtime = super::QueryRuntime::new(
58            super::SQL_FUNCTION_DEPTH_LIMIT,
59            client_level,
60            session.state.io_timing(),
61        );
62        session.state.attach_cancellation(&runtime.cancellation);
63        Self {
64            storage: super::StorageContext::memory(),
65            durable: Arc::new(super::DurableCatalogState::new()),
66            extensions: super::RuntimeExtensions::new(&session, &runtime.statement_gate),
67            session,
68            epochs: Arc::new(super::EpochCoordinator::new()),
69            runtime,
70            statistics: crate::statistics::shared_statistics(&row_locks),
71            row_locks,
72            notification_hub,
73            session_id,
74            owns_session_registration: true,
75            query_table_snapshots: None,
76            query_view_snapshots: None,
77            query_sql_function_snapshots: None,
78            query_catalog_snapshot: None,
79            query_transaction_overlay: None,
80            query_transaction_origin: None,
81        }
82    }
83
84    pub fn open(path: &Path) -> Result<Self, SQLiteError> {
85        let conn = ManagedConnection::open(path)?;
86        Self::open_with_connection(&conn)
87    }
88
89    /// Classify the on-disk format of `path` without opening it: plain
90    /// `SQLite`, UQA compressed container (with its encryption flag), a
91    /// missing/empty file, or an unrecognized header (`SQLCipher`
92    /// databases fall here because `SQLCipher` encrypts the whole file).
93    pub fn detect_database_file(
94        path: &Path,
95    ) -> std::io::Result<uqa_storage_sqlite::DatabaseFileFormat> {
96        uqa_storage_sqlite::detect_database_file_format(path)
97    }
98
99    /// Open `path` with the variant its on-disk format calls for.
100    ///
101    /// - Missing/empty file: creates a new database, `SQLCipher`
102    ///   encrypted when `key` is provided, plaintext otherwise.
103    /// - Plain `SQLite`: opens plaintext; providing a key is an error
104    ///   ([`SQLiteError::NotEncrypted`]) rather than a silent no-op so
105    ///   callers never believe an unencrypted database is protected.
106    /// - Compressed container: opens with the codec recorded in the
107    ///   container header; the encryption flag decides whether `key`
108    ///   is required ([`SQLiteError::EncryptionKeyRequired`]) or
109    ///   rejected ([`SQLiteError::NotEncrypted`]).
110    /// - Unrecognized header: treated as `SQLCipher` when `key` is
111    ///   provided; without a key this fails with
112    ///   [`SQLiteError::EncryptionKeyRequired`] because an encrypted
113    ///   database cannot be told apart from a foreign file.
114    ///
115    /// New compressed containers are not created through this entry
116    /// point; use [`Engine::open_compressed`] or
117    /// [`Engine::open_compressed_encrypted`] to choose compression for
118    /// a new database.
119    pub fn open_auto(path: &Path, key: Option<&str>) -> Result<Self, SQLiteError> {
120        let connection = ManagedConnection::open_auto(path, key)?;
121        Self::open_with_connection(&connection)
122    }
123
124    /// SQLCipher-backed engine. Applies `key` before any catalog
125    /// access, runs migrations, and rebuilds the in-memory table
126    /// registry from the encrypted catalog.
127    pub fn open_encrypted(path: &Path, key: &str) -> Result<Self, SQLiteError> {
128        let conn = ManagedConnection::open_encrypted(path, key)?;
129        Self::open_with_connection(&conn)
130    }
131
132    /// Compressed SQLite-backed engine. The compression VFS is
133    /// schema-neutral: it compresses `SQLite` byte ranges in chunks
134    /// without knowledge of UQA catalog tables or columns.
135    pub fn open_compressed(
136        path: &Path,
137        compression: SQLiteCompressionOptions,
138    ) -> Result<Self, SQLiteError> {
139        let conn = ManagedConnection::open_compressed(path, compression)?;
140        Self::open_with_connection(&conn)
141    }
142
143    /// Compressed and encrypted SQLite-backed engine. Chunk payloads
144    /// are compressed first, then encrypted by the compressed VFS. The v2
145    /// format authenticates container metadata, chunk placement, and commit
146    /// records, but cannot distinguish replacement by an internally valid
147    /// snapshot or fork without an external trusted state anchor.
148    /// Security-sensitive deployments that do not require compression should
149    /// prefer [`Engine::open_encrypted`] and `SQLCipher`.
150    pub fn open_compressed_encrypted(
151        path: &Path,
152        key: &str,
153        compression: SQLiteCompressionOptions,
154    ) -> Result<Self, SQLiteError> {
155        let conn = ManagedConnection::open_compressed_encrypted(path, key, compression)?;
156        Self::open_with_connection(&conn)
157    }
158
159    /// Open an encrypted compressed database and reject a different file or
160    /// any state other than `trusted_anchor` before `SQLite` reads the main
161    /// database. Refresh the trusted anchor after every committed write.
162    pub fn open_compressed_encrypted_with_anchor(
163        path: &Path,
164        key: &str,
165        compression: SQLiteCompressionOptions,
166        trusted_anchor: SQLiteCompressedContainerAnchor,
167    ) -> Result<Self, SQLiteError> {
168        let conn = ManagedConnection::open_compressed_encrypted_with_anchor(
169            path,
170            key,
171            compression,
172            trusted_anchor,
173        )?;
174        Self::open_with_connection(&conn)
175    }
176
177    /// Authenticate and return the anchor to persist in a trusted store after
178    /// committed writes to an encrypted compressed database.
179    pub fn compressed_container_anchor(
180        path: &Path,
181        key: &str,
182    ) -> Result<SQLiteCompressedContainerAnchor, SQLiteError> {
183        Ok(uqa_storage_sqlite::read_authenticated_anchor(path, key)?)
184    }
185
186    fn open_with_connection(conn: &ManagedConnection) -> Result<Self, SQLiteError> {
187        let provider: Arc<dyn PersistentStorageProvider> =
188            Arc::new(SQLiteStorageProvider::new(conn.clone()));
189        Self::from_persistent_provider(provider).map_err(Self::sqlite_open_error)
190    }
191
192    /// Create an independent SQL session over this engine's durable database.
193    ///
194    /// The new session gets its own catalog/backend pair, transaction stack,
195    /// runtime variables, prepared statements, statement cache, and
196    /// cancellation token. Committed catalog definitions share immutable
197    /// allocations; mutations detach the session's copy and synchronize through
198    /// shared epochs. Runtime-only Rust extensions are shared. The provider
199    /// must return catalog and data handles bound to one session transaction
200    /// so every durable mutation commits atomically.
201    pub fn new_session(&self) -> StorageBackendResult<Self> {
202        let _statement = self.lock_statement_gate();
203        self.new_sibling_session(false, None)
204    }
205
206    /// Whether these sessions use the same database identity. This reuses the provider-identified coordinator shared by independent opens and sibling sessions; memory engines have independent identities. Providers without stable storage identity can share only through their retained provider or backend owner.
207    #[must_use]
208    pub fn shares_database_with(&self, other: &Self) -> bool {
209        Arc::ptr_eq(&self.row_locks, &other.row_locks)
210    }
211
212    /// Restore an independent internal read view without entering the source statement or acquiring another automatic-maintenance client lease.
213    pub(crate) fn new_internal_read_session(&self) -> StorageBackendResult<Self> {
214        let mut session = self.new_sibling_session(true, None)?;
215        session.runtime.diagnostics = self.runtime.diagnostics.fork();
216        Ok(session)
217    }
218
219    pub(crate) fn new_internal_retained_read_session(&self) -> StorageBackendResult<Self> {
220        let backend = self.storage.backend.as_ref().ok_or_else(|| {
221            StorageBackendError::Other("retained reads require persistent storage".into())
222        })?;
223        let storage_session =
224            backend.open_retained_read_session(&uqa_core::CancellationToken::new())?;
225        let mut session = self.new_sibling_session(true, Some(storage_session))?;
226        session.runtime.diagnostics = self.runtime.diagnostics.fork();
227        Ok(session)
228    }
229
230    fn new_sibling_session(
231        &self,
232        internal_read: bool,
233        retained_session: Option<PersistentStorageSession>,
234    ) -> StorageBackendResult<Self> {
235        let provider = self.storage.provider.as_ref().ok_or_else(|| {
236            StorageBackendError::Other(
237                "independent sessions require a PersistentStorageProvider".into(),
238            )
239        })?;
240        // Only the public factory owns the source statement gate. Internal readers can run on a worker of that statement and must restore their own catalog without borrowing the source transaction or temporary namespace. A retained pair supplies its own fixed catalog view.
241        let retained = retained_session.is_some();
242        let share_catalog = !internal_read
243            && !retained
244            && self
245                .session
246                .transactions
247                .try_lock()
248                .is_some_and(|stack| stack.is_empty())
249            && self.session.state.read().temporary_namespace.is_none();
250        if share_catalog {
251            self.synchronize_table_catalog()?;
252            self.synchronize_table_data()?;
253            self.synchronize_catalog_registries()?;
254        }
255        let observed_epochs = self.epochs.published_epochs();
256        let storage_session = match retained_session {
257            Some(session) => session,
258            None => provider.open_session()?,
259        };
260        storage_session.validate_transaction_affinity()?;
261        let storage_version_before_restore = storage_session.backend.change_version()?;
262        let shared = if share_catalog {
263            self.session_from_shared_catalog(&storage_session, provider)?
264        } else {
265            None
266        };
267        let mut session = match shared {
268            Some(session) => session,
269            None => Self::from_initialized_persistent_session(
270                storage_session,
271                Some(Arc::clone(provider)),
272            )?,
273        };
274        session
275            .session
276            .statistics_worker
277            .store(internal_read, std::sync::atomic::Ordering::Release);
278        session.session.notification_subscriptions_required.store(
279            self.notification_subscriptions_required(),
280            std::sync::atomic::Ordering::Release,
281        );
282        session.row_locks = Arc::clone(&self.row_locks);
283        session.statistics = Arc::clone(&self.statistics);
284        session.install_notification_hub(Arc::clone(&self.notification_hub))?;
285        session.session_id = self.row_locks.allocate_session();
286        let storage_version_after_restore = session
287            .storage
288            .backend
289            .as_ref()
290            .ok_or_else(|| {
291                StorageBackendError::Other(
292                    "persistent session lost its storage backend during restore".into(),
293                )
294            })?
295            .change_version()?;
296        let epochs =
297            Arc::get_mut(&mut session.epochs).expect("new session owns its catalog epoch state");
298        if storage_version_before_restore == storage_version_after_restore {
299            epochs.share_published_from_at(&self.epochs, observed_epochs);
300        } else {
301            epochs.share_published_from(&self.epochs);
302        }
303        // A commit that raced the load advanced a shared publication beyond
304        // the generation or storage version captured before the session
305        // restored. Only that case needs a second catalog or data refresh.
306        // Catalog cells keep independent mutable owners over shared immutable
307        // values, so a writer cannot expose uncommitted definitions to siblings.
308        session.extensions = super::RuntimeExtensions::shared_from(&self.extensions);
309        session
310            .extensions
311            .functions
312            .attach(&session.session, &session.runtime.statement_gate);
313        if internal_read {
314            session.runtime.function_context = Arc::clone(&self.runtime.function_context);
315        }
316        if !retained {
317            session.synchronize_table_catalog()?;
318            session.synchronize_table_data()?;
319            session.synchronize_catalog_registries()?;
320        }
321        session.start_automatic_statistics();
322        Ok(session)
323    }
324
325    /// Build an engine and retain the provider used to create future
326    /// independent sessions.
327    #[allow(clippy::needless_pass_by_value)]
328    pub fn from_persistent_provider(
329        provider: Arc<dyn PersistentStorageProvider>,
330    ) -> StorageBackendResult<Self> {
331        let identity = provider.storage_identity()?;
332        let session = provider.open_initial_session()?;
333        let mut engine = Self::from_persistent_session(session, Some(Arc::clone(&provider)))?;
334        let row_locks = crate::row_locks::shared_provider_manager(identity.clone(), &provider);
335        let notification_hub =
336            crate::notifications::shared_provider_notification_hub(identity, &provider);
337        engine.session_id = row_locks.allocate_session();
338        engine.statistics = crate::statistics::shared_statistics(&row_locks);
339        engine.row_locks = row_locks;
340        engine.install_notification_hub(notification_hub)?;
341        engine.start_automatic_statistics();
342        Ok(engine)
343    }
344
345    /// Build an engine from already-open persistent metadata and data backends. The backend's session factory is retained for independent SQL sessions and latest-committed row-lock rechecks. Prefer [`Self::from_persistent_provider`] when a database-level owner is already available.
346    pub fn from_persistent_backends(
347        catalog: Arc<dyn uqa_storage::CatalogFacade>,
348        backend: Arc<dyn PersistentStorageBackend>,
349    ) -> StorageBackendResult<Self> {
350        let identity = backend.storage_identity()?;
351        let row_locks = crate::row_locks::shared_backend_manager(identity.clone(), &backend);
352        let notification_hub =
353            crate::notifications::shared_backend_notification_hub(identity, &backend);
354        let provider: Arc<dyn PersistentStorageProvider> = Arc::new(BackendSessionProvider {
355            backend: Arc::clone(&backend),
356        });
357        let mut engine = Self::from_persistent_session(
358            PersistentStorageSession::new(catalog, backend),
359            Some(provider),
360        )?;
361        engine.session_id = row_locks.allocate_session();
362        engine.statistics = crate::statistics::shared_statistics(&row_locks);
363        engine.row_locks = row_locks;
364        engine.install_notification_hub(notification_hub)?;
365        engine.start_automatic_statistics();
366        Ok(engine)
367    }
368
369    fn install_notification_hub(
370        &mut self,
371        notification_hub: Arc<crate::NotificationHub>,
372    ) -> StorageBackendResult<()> {
373        let process_id = notification_hub
374            .allocate_backend_process_id()
375            .map_err(|error| {
376                StorageBackendError::Other(format!(
377                    "allocate database backend process identifier: {error}"
378                ))
379            })?;
380        self.notification_hub = notification_hub;
381        if let Some(process_id) = process_id {
382            self.session.install_database_backend_process_id(process_id);
383        }
384        Ok(())
385    }
386
387    fn from_persistent_session(
388        storage_session: PersistentStorageSession,
389        provider: Option<Arc<dyn PersistentStorageProvider>>,
390    ) -> StorageBackendResult<Self> {
391        Self::build_persistent_session(storage_session, provider, true)
392    }
393
394    pub(crate) fn from_initialized_persistent_session(
395        storage_session: PersistentStorageSession,
396        provider: Option<Arc<dyn PersistentStorageProvider>>,
397    ) -> StorageBackendResult<Self> {
398        Self::build_persistent_session(storage_session, provider, false)
399    }
400
401    pub(super) fn empty_persistent_session(
402        storage_session: PersistentStorageSession,
403        provider: Option<Arc<dyn PersistentStorageProvider>>,
404    ) -> Self {
405        let PersistentStorageSession { catalog, backend } = storage_session;
406        let cancellation = backend.write_cancellation().unwrap_or_default();
407        let row_locks = Arc::new(crate::row_locks::RowLockManager::new());
408        let notification_hub = Arc::new(crate::NotificationHub::default());
409        let session_id = row_locks.allocate_session();
410        let session = Arc::new(super::SessionContext::new(super::initial_random_state()));
411        let client_level = session.state.client_level();
412        let runtime = super::QueryRuntime::with_cancellation(
413            super::SQL_FUNCTION_DEPTH_LIMIT,
414            cancellation,
415            client_level,
416            session.state.io_timing(),
417        );
418        session.state.attach_cancellation(&runtime.cancellation);
419        Self {
420            storage: super::StorageContext::persistent(catalog, backend, provider),
421            durable: Arc::new(super::DurableCatalogState::new()),
422            extensions: super::RuntimeExtensions::new(&session, &runtime.statement_gate),
423            session,
424            epochs: Arc::new(super::EpochCoordinator::new()),
425            runtime,
426            statistics: crate::statistics::shared_statistics(&row_locks),
427            row_locks,
428            notification_hub,
429            session_id,
430            owns_session_registration: true,
431            query_table_snapshots: None,
432            query_view_snapshots: None,
433            query_sql_function_snapshots: None,
434            query_catalog_snapshot: None,
435            query_transaction_overlay: None,
436            query_transaction_origin: None,
437        }
438    }
439
440    /// Repair what earlier releases left inconsistent once a writer has restored the catalog: physical value indexes, partitions without the copies of their parent's foreign keys, and foreign keys without the constraints they derive on referenced partitions.
441    fn repair_catalog_on_open(&self) -> StorageBackendResult<()> {
442        self.repair_persistent_value_indexes_on_open()?;
443        self.repair_partition_foreign_keys_on_open()?;
444        self.repair_derived_constraints_on_open()
445    }
446
447    fn build_persistent_session(
448        storage_session: PersistentStorageSession,
449        provider: Option<Arc<dyn PersistentStorageProvider>>,
450        initialize_catalog: bool,
451    ) -> StorageBackendResult<Self> {
452        storage_session.validate_transaction_affinity()?;
453        let restore_catalog = Arc::clone(&storage_session.catalog);
454        let restore_backend = Arc::clone(&storage_session.backend);
455        let mut engine = Self::empty_persistent_session(storage_session, provider);
456        let own_read = !initialize_catalog && !restore_backend.in_transaction();
457        let version_before_read = if own_read {
458            let version = restore_backend.change_version()?;
459            restore_backend.begin_read_transaction()?;
460            restore_backend.pin_transaction_snapshot()?;
461            Some(version)
462        } else {
463            None
464        };
465        // A load-only restore owns one read boundary. Registry validators may
466        // resolve tables without recursively restoring a newer catalog. The
467        // unpublished Engine releases this transaction on errors and unwinds.
468        let read_view_before = restore_backend.read_view_revision()?;
469        let cache_revisions_before;
470        if initialize_catalog {
471            restore_backend.migrate_document_storage()?;
472            // A clean restore remains read-only on backends that can promote a transaction, while backends without promotion reserve their writer before the atomic migration scan.
473            restore_backend.begin_upgradeable_transaction()?;
474            let restore_result = (|| {
475                restore_catalog.initialize_storage()?;
476                let cache_revisions = restore_catalog.cache_revisions()?;
477                restore_backend.migrate_inverted_index_storage()?;
478                Self::prepare_catalog_for_initial_restore(restore_catalog.as_ref())?;
479                engine
480                    .restore_initial_catalog(restore_catalog.as_ref(), restore_backend.as_ref())?;
481                uqa_execution::catalog::value_restoration::normalize_legacy_vectors(
482                    restore_catalog.as_ref(),
483                    restore_backend.as_ref(),
484                    &engine,
485                )?;
486                engine.repair_reset_fts_storage(restore_catalog.as_ref())?;
487                engine.initialize_catalog_constraint_tuples(&restore_catalog, &restore_backend)?;
488                Ok(cache_revisions)
489            })();
490            cache_revisions_before = match restore_result {
491                Ok(revisions) => revisions,
492                Err(error) => {
493                    return match restore_backend.rollback_transaction() {
494                        Ok(()) => Err(error),
495                        Err(rollback_error) => Err(StorageBackendError::Other(format!(
496                            "rollback initial catalog migration after `{error}` failed: {rollback_error}"
497                        ))),
498                    };
499                }
500            };
501            if let Err(error) = restore_backend.commit_transaction() {
502                return match restore_backend.rollback_transaction() {
503                    Ok(()) => Err(error),
504                    Err(rollback_error) => Err(StorageBackendError::Other(format!(
505                        "rollback failed initial catalog migration commit `{error}`: {rollback_error}"
506                    ))),
507                };
508            }
509        } else {
510            cache_revisions_before = restore_catalog.cache_revisions()?;
511            engine.restore_from_catalog(
512                restore_catalog.as_ref(),
513                restore_backend.as_ref(),
514                super::CatalogRestoreMode::LoadOnly,
515            )?;
516        }
517        if initialize_catalog {
518            engine.repair_catalog_on_open()?;
519        }
520        // Eagerly and fallibly populate read caches. Once open succeeds,
521        // cache misses mean absence rather than a swallowed catalog error.
522        for (name, json) in restore_catalog.load_models()? {
523            let model = serde_json::from_str::<DeepModel>(&json)?;
524            engine.durable.models.write().insert(name, model);
525        }
526        for (name, json) in restore_catalog.load_all_scoring_params()? {
527            engine.durable.scoring_params.write().insert(name, json);
528        }
529        // Initial catalog migrations and physical-index repairs above may
530        // commit. Establish the backend monitor baseline only after every
531        // one-time write has completed.
532        let cache_revisions_after = restore_catalog.cache_revisions()?;
533        let stable_restore = cache_revisions_before == cache_revisions_after;
534        if stable_restore {
535            *engine.epochs.storage_cache_revisions.lock() = cache_revisions_after;
536            let read_view_after = restore_backend.read_view_revision()?;
537            if read_view_before == read_view_after {
538                *engine.epochs.seen_storage_read_view.lock() = read_view_after;
539            }
540        }
541        if let Some(version) = restore_backend.change_version()?.filter(|_| stable_restore) {
542            engine
543                .epochs
544                .seen_storage_change_version
545                .store(version, std::sync::atomic::Ordering::Release);
546        }
547        if let Some(before) = version_before_read {
548            restore_backend.rollback_transaction()?;
549            if restore_backend.change_version()? != before {
550                // The restored definitions belong to the pinned snapshot,
551                // even when the independent monitor saw a later commit.
552                engine.epochs.seen_storage_change_version.store(
553                    before.unwrap_or_default(),
554                    std::sync::atomic::Ordering::Release,
555                );
556                *engine.epochs.seen_storage_read_view.lock() = None;
557            }
558        }
559        Ok(engine)
560    }
561
562    fn sqlite_open_error(err: StorageBackendError) -> SQLiteError {
563        SQLiteError::from(err)
564    }
565}