Skip to main content

uqa_engine/
lib.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7// libtest generates a single descriptor array for the crate's unit tests.
8#![cfg_attr(
9    test,
10    expect(
11        clippy::large_stack_arrays,
12        reason = "the compiler-generated test descriptor array exceeds the stack-array lint threshold"
13    )
14)]
15
16//! Top-level engine: a per-table [`DocumentStore`] + [`InvertedIndex`]
17//! pair, document mutation entry points, and a minimal `search` API for
18//! text-only round trips. Backed either by in-memory stores
19//! ([`Engine::new`]), the `SQLite`/`SQLCipher` constructors, or a swappable
20//! [`uqa_storage::PersistentStorageProvider`]; the operator pipeline is
21//! identical across backends.
22//!
23//! # Public API surface
24//!
25//! Construction:
26//! - [`Engine::new`] - purely in-memory; great for tests and the REPL.
27//! - [`Engine::open`] - `SQLite`-backed catalog at the given path; reopens
28//!   restore tables, models, and graphs from disk.
29//! - [`Engine::open_encrypted`] - same catalog restore path, with a
30//!   `SQLCipher` key applied before any schema access.
31//! - [`Engine::open_compressed`] - schema-neutral compressed `SQLite` VFS.
32//! - [`Engine::open_compressed_encrypted`] - compressed chunks encrypted
33//!   after compression.
34//! - [`Engine::from_persistent_provider`] - storage-neutral construction for
35//!   redb and application-defined providers, with backend-neutral sessions.
36//!
37//! Schema and table lifecycle:
38//! - [`Engine::create_table`] - register a table with declared columns.
39//! - [`Engine::create_default_table`] - convenience for FTS-only tables.
40//! - [`Engine::create_vector_field`] - attach a vector field to an
41//!   existing table.
42//!
43//! Document mutation:
44//! - [`Engine::add_document`], [`Engine::add_document_with_vectors`]
45//! - [`Engine::add_vector`] - set or replace a vector for an existing doc.
46//! - [`Engine::get_document`], [`Engine::delete_document`]
47//! - [`Engine::document_count`]
48//! - [`Engine::transaction`], [`Engine::sql_batch`] - group writes under one
49//!   engine transaction.
50//!
51//! Querying:
52//! - `Engine::sql` (defined in [`sql`]) - full SQL surface (select /
53//!   insert / update / delete / create-table, plus the registered
54//!   functions: `text_match`, `knn_match`, `fuse_bayesian_evidence` (plus
55//!   exact alias `fuse_log_odds`), `pool_positive_evidence`,
56//!   `multi_field_match`, `staged_retrieval`, `graph_*`, `deep_predict`).
57//! - [`Engine::sql_cursor`] / [`Engine::sql_columnar`] - bounded, schema-ordered
58//!   column batches for result sets that should not be retained in memory.
59//! - [`Engine::poll_sql_notifications`], [`Engine::wait_for_sql_notifications`],
60//!   and [`Engine::take_sql_notifications`] - receive committed
61//!   `LISTEN`/`NOTIFY` messages for the current session.
62//! - [`Engine::search`] - direct text-only retrieval returning a posting
63//!   list.
64//! - [`Engine::knn_search`], [`Engine::vector_similarity_search`] - k-NN
65//!   over a vector field.
66//! - [`Engine::hybrid_search`] - exact signed single-prior fusion of text and
67//!   vector posting lists (no SQL parsing in the hot path).
68//! - [`Engine::robust_hybrid_search`] - explicitly requested gated,
69//!   confidence-scaled positive-evidence pooling.
70//!
71//! Deep-model persistence:
72//! - [`Engine::save_model`], [`Engine::load_model`], [`Engine::drop_model`]
73//! - [`Engine::deep_predict`] - runs a stored model against the cached
74//!   feature row and returns ranked `(doc_id, score)` pairs.
75//!
76//! Graph workspaces (used by the Cypher front-end and the `graph_*`
77//! SQL functions):
78//! - [`Engine::create_graph`], [`Engine::drop_graph`]
79//! - [`Engine::graph_with`] - read-only access by name.
80//! - [`Engine::graph_with_mut`] - exclusive mutable access.
81//!
82//! Result types ([`SQLResult`], [`SQLParam`]) are re-exported from
83//! `uqa-sql`. Errors flow through [`EngineError`], which wraps SQL and
84//! storage errors so callers only need to match one enum.
85
86pub mod functions;
87pub mod migration;
88pub mod operator_tree_bridge;
89mod queries;
90/// SQL result formatting, type metadata and bounded cursors.
91pub use uqa_execution::result as sql;
92
93mod analyzers;
94mod async_sql_engine;
95mod cancellation;
96mod capabilities;
97mod catalog_indexes;
98mod domains;
99mod fdw;
100mod fts;
101mod generated;
102mod graphs;
103mod hierarchy;
104mod hook;
105mod models;
106mod notifications;
107mod open;
108mod prepared;
109mod relations;
110mod roles;
111mod schema_security;
112mod search;
113mod sequence_catalog;
114mod sequence_session;
115mod sequence_snapshot;
116mod sequences;
117mod session;
118mod sql_registry;
119mod state;
120mod statistics;
121pub use statistics::AutomaticStatisticsStatus;
122pub use uqa_execution::maintenance::diskann::{
123    DiskANNMaintenanceCensus, DiskANNMaintenancePhase, DiskANNMaintenanceStatus,
124    DiskANNRebuildPolicy,
125};
126mod row_locks;
127mod statement_cache;
128mod table_storage;
129mod tables;
130mod transactions;
131mod user_functions;
132mod value_index;
133
134pub(crate) use uqa_execution::mutation::{
135    deferred::DeferredForeignKeyCheck, overlay::CommandMutationOverlay,
136};
137pub(crate) use uqa_execution::row_locks::publication::TransactionRowChange;
138
139use std::collections::{BTreeMap, BTreeSet};
140use std::path::Path;
141use std::sync::atomic::{AtomicBool, Ordering};
142use std::sync::Arc;
143
144use parking_lot::RwLock;
145use uqa_analysis::{analyzer::standard_analyzer, registry as analyzer_registry, Analyzer};
146use uqa_core::{DocId, FieldName, Value};
147use uqa_ml::{DeepLearnOutput, DeepModel, LearnOptions, TrainingSet};
148use uqa_operators::ExecutionContext;
149use uqa_scoring::{
150    BM25Params, BayesianBM25Params, BayesianBM25Scorer, CalibrationMetrics, CalibrationReport,
151    ParameterLearner, RawBm25Score, UnsupervisedBm25ScoreEstimator,
152};
153use uqa_sql::SQLError;
154use uqa_storage::{
155    document_store::Document, AnalyzerPhase, CatalogFacade, CatalogIndexRow, ColumnStatsInput,
156    ColumnStatsRow, DocumentStore, HNSWIndex, HNSWIndexParams, IVFIndex, IVFIndexParams,
157    InvertedIndex, MemoryDocumentStore, MemoryInvertedIndex, MemoryVectorIndex,
158    PersistentStorageBackend, PersistentStorageProvider, PersistentStorageSession,
159    RelationIdentity, SequenceRow, StorageBackendError, StorageBackendResult, StorageSavepointId,
160    TableSchema, VectorFieldSchema, VectorIndex, VectorIndexOpenMode, VectorIndexSpec,
161};
162use uqa_storage_sqlite::{
163    ManagedConnection, SQLiteCompressedContainerAnchor, SQLiteStorageProvider,
164};
165
166pub use notifications::{
167    NotificationSubscription, NotificationSubscriptionError, NotificationSubscriptionOptions,
168    NotificationSubscriptionPermit, NotificationWait, SQLNotification, SQLNotificationOptions,
169};
170pub use sql::{SQLCursor, SQLCursorSummary};
171pub use uqa_execution::{ColumnVector, ColumnarBatch};
172pub use uqa_sql::{
173    ast::{SequenceBound, SequenceDataType, SequenceRestart},
174    AsyncSQLEngine, NoticeLevel, SQLBatchError, SQLNotice, SQLParam, SQLResult,
175};
176pub use uqa_storage_sqlite::{DatabaseFileFormat, SQLiteCompressionOptions, SQLiteError};
177
178use functions::RegisteredSQLFunction;
179pub use functions::{
180    SQLAggregateFunction, SQLAggregateState, SQLFunctionOptions, SQLFunctionVolatility,
181    SQLScalarFunction, SQLTableFunction, SQLTableFunctionResult, SQLTableFunctionStream,
182};
183use notifications::{
184    NotificationHubOwner as NotificationHub, PendingListenAction, PendingNotification,
185};
186use state::{
187    DurableCatalogSnapshot, DurableCatalogState, EpochCoordinator, QueryRuntime, RuntimeExtensions,
188    SessionContext, StorageContext, StoredView, StoredViewKind,
189};
190use statement_cache::{PreparedStatementPlan, SQLStatementCache};
191
192#[cfg(test)]
193use uqa_execution::catalog::sequence::restoration::SEQUENCES_METADATA_KEY;
194/// Default nesting cap for user-defined function calls. Exceeding it
195/// raises `stack depth limit exceeded`, mirroring the `PostgreSQL`
196/// `max_stack_depth` guard.
197const SQL_FUNCTION_DEPTH_LIMIT: usize = 128;
198
199#[derive(Debug, thiserror::Error)]
200pub enum EngineError {
201    #[error("SQL error: {0}")]
202    SQL(#[from] SQLError),
203    #[error("storage error: {0}")]
204    Storage(#[from] SQLiteError),
205}
206
207pub type EngineResult<T> = std::result::Result<T, EngineError>;
208
209pub use uqa_core::ScoredEntry;
210
211pub use uqa_scoring::{TextSearchAlgorithm, TextSearchProfile};
212
213pub use uqa_storage::FtsIndexStat;
214
215pub use uqa_scoring::ScoringMode;
216
217type TableFieldAnalyzerRegistry = BTreeMap<(String, String), uqa_storage::FieldAnalyzerBinding>;
218type SessionPortalTableSnapshots = Arc<BTreeMap<RelationIdentity, Arc<TableState>>>;
219type SessionPortalViewSnapshots = Arc<BTreeMap<RelationIdentity, StoredView>>;
220type SessionPortalSQLFunctionSnapshots =
221    Arc<BTreeMap<String, Vec<Arc<user_functions::SQLUserFunction>>>>;
222type SessionPortalCatalogSnapshot = Arc<session::RetainedCatalogSnapshot>;
223type SessionPortalTransactionOverlay =
224    Arc<BTreeMap<String, uqa_execution::query::document_changes::DocumentChanges>>;
225type ColumnStatsMap = BTreeMap<String, uqa_planner::ColumnStats>;
226type FixedTransactionCatalogBaseline = BTreeMap<[u8; 16], (RelationIdentity, Vec<u8>)>;
227type NontransactionalSequenceValues = BTreeMap<[u8; 16], NontransactionalSequenceHistory>;
228
229#[derive(Clone, Default)]
230struct NontransactionalSequenceHistory {
231    values_by_definition: BTreeMap<[u8; 16], NontransactionalSequenceValue>,
232    object_id: [u8; 16],
233    session_currval: Option<SessionSequenceValue>,
234    defines_lastval: bool,
235}
236
237/// Unified query engine composed from explicit storage, durable-catalog,
238/// session, extension, epoch, and query-runtime ownership domains.
239pub struct Engine {
240    storage: StorageContext,
241    durable: Arc<DurableCatalogState>,
242    session: Arc<SessionContext>,
243    extensions: RuntimeExtensions,
244    epochs: EpochCoordinator,
245    runtime: QueryRuntime,
246    row_locks: Arc<row_locks::RowLockManager>,
247    statistics: Arc<statistics::StatisticsCoordinator>,
248    notification_hub: Arc<NotificationHub>,
249    session_id: u64,
250    owns_session_registration: bool,
251    query_table_snapshots: Option<SessionPortalTableSnapshots>,
252    query_view_snapshots: Option<SessionPortalViewSnapshots>,
253    query_sql_function_snapshots: Option<SessionPortalSQLFunctionSnapshots>,
254    query_catalog_snapshot: Option<SessionPortalCatalogSnapshot>,
255    query_transaction_overlay: Option<SessionPortalTransactionOverlay>,
256    query_transaction_origin: Option<u64>,
257}
258
259pub use uqa_execution::catalog::sequence::SequenceState;
260
261#[derive(Clone, Copy, Default)]
262struct TransactionDirtyState {
263    table_data: bool,
264    table_catalog: bool,
265    catalog_registry: bool,
266}
267
268/// Physical write admission for the lifetime of a transaction. Savepoint undo restores SQL access characteristics but cannot turn an already promoted physical writer back into an unwritten reader.
269#[derive(Clone, Copy, PartialEq, Eq)]
270enum TransactionIntent {
271    ReadOnly,
272    ReadWrite,
273}
274
275#[derive(Clone, Copy, PartialEq, Eq)]
276enum BackendTransactionMode {
277    Deferred,
278    Writer,
279    Versioned,
280}
281
282#[derive(Clone, Copy, PartialEq, Eq)]
283enum TransactionStatus {
284    Active,
285    Failed,
286    FailedBackendAborted,
287    CommitPending(uqa_storage::mvcc::TransactionOutcomeId),
288    RollbackPending(uqa_storage::mvcc::TransactionOutcomeId),
289}
290
291#[derive(Clone, Copy, PartialEq, Eq)]
292enum TransactionFrameKind {
293    ExplicitBlock,
294    ImplicitStatement,
295    SimpleQuery,
296}
297
298#[derive(Debug, Clone, Copy, PartialEq, Eq)]
299struct TransactionCharacteristicsState {
300    isolation: uqa_sql::ast::TransactionIsolationLevel,
301    read_only: bool,
302    deferrable: bool,
303    /// The characteristics the transaction assigned itself with `BEGIN`, `SET TRANSACTION` or `SET transaction_*`, which `pg_settings` reports with the source `session` rather than `override`.
304    assigned: u8,
305}
306
307impl TransactionCharacteristicsState {
308    const ISOLATION_ASSIGNED: u8 = 1;
309    const READ_ONLY_ASSIGNED: u8 = 2;
310    const DEFERRABLE_ASSIGNED: u8 = 4;
311}
312
313impl Default for TransactionCharacteristicsState {
314    fn default() -> Self {
315        Self {
316            isolation: uqa_sql::ast::TransactionIsolationLevel::ReadCommitted,
317            read_only: false,
318            deferrable: false,
319            assigned: 0,
320        }
321    }
322}
323
324pub(crate) use uqa_sql::catalog::constraints::ConstraintIdentity;
325
326#[derive(Debug, Clone, Default)]
327pub(crate) struct ConstraintModeState {
328    all: Option<bool>,
329    named: BTreeMap<ConstraintIdentity, bool>,
330}
331
332struct TransactionFrame {
333    started_at_micros: i64,
334    /// Whether this outer frame is an implicit SQL-driver boundary rather than a user-visible `BEGIN` block. A simple-query batch promotes it when the batch reaches `BEGIN`.
335    implicit_statement: bool,
336    /// Whether the user has entered an explicit transaction block. A multi-statement simple-query message still owns one atomic frame, but `PostgreSQL` permits `DISCARD` there until `BEGIN` promotes it.
337    explicit_transaction_block: bool,
338    storage_savepoint: Option<StorageSavepointId>,
339    intent: TransactionIntent,
340    backend_mode: BackendTransactionMode,
341    status: TransactionStatus,
342    characteristics: TransactionCharacteristicsState,
343    first_snapshot_set: bool,
344    fixed_snapshot: Option<FixedTransactionSnapshot>,
345    fixed_catalog_records: Option<Arc<uqa_storage::mvcc::CommittedReadBoundary>>,
346    /// Last committed table catalog installed while a fixed row snapshot is active. Immutable fingerprints distinguish this transaction's own DDL overlay from definitions refreshed from sibling commits.
347    fixed_catalog_baseline: Option<FixedTransactionCatalogBaseline>,
348    /// `PostgreSQL` transaction/subtransaction IDs along the active frame path. The first slot belongs to this frame and each following slot to the correspondingly indexed SQL savepoint.
349    xid_levels: Vec<Option<u32>>,
350    savepoints: Vec<TransactionSavepoint>,
351    session_snapshot: SessionStateSnapshot,
352    data_snapshot: Option<EngineDataSnapshot>,
353    dirty_at_begin: TransactionDirtyState,
354    prepared_changes: uqa_execution::statement::prepared::invalidation::PreparedInvalidationLog,
355    /// Lock mark this frame started with. Rolling the whole frame back releases every acquisition at or above it, independent of the savepoint marks the frame allocated later.
356    begin_lock_mark: u32,
357    lock_mark: u32,
358    next_lock_mark: u32,
359    snapshot_change_baseline: row_locks::RowChangeBaseline,
360    row_changes: Vec<TransactionRowChange>,
361    /// The index of the rows the transaction changed, which reads at a fixed snapshot take views of; only the outer frame keeps one.
362    fixed_identities: Option<uqa_execution::query::document_changes::ChangedIdentities>,
363    statistics_changes: statistics::StatisticsChanges,
364    /// What this frame's commit did with its statistics changes, for the session to take over when the commit has succeeded.
365    statistics_settlement: statistics::StatisticsSettlement,
366    deferred_foreign_key_checks: Vec<DeferredForeignKeyCheck>,
367    deferred_constraint_trigger_events:
368        Vec<uqa_execution::mutation::triggers::DeferredConstraintTriggerEvent>,
369    pending_listen_actions: Vec<PendingListenAction>,
370    pending_notifications: Vec<PendingNotification>,
371    pending_notification_commit:
372        parking_lot::Mutex<Option<Box<notifications::CrossNotificationCommit>>>,
373    constraint_modes: ConstraintModeState,
374    /// Values allocated by `nextval` or installed by `setval` are not rolled back in `PostgreSQL`, except that allocations made against a transactionally changed sequence definition roll back with that definition. Every active frame records values by definition generation so transaction, savepoint, and PL/pgSQL exception rollback can reapply exactly the generation owned by the rollback target while preserving the latest session `currval` and `lastval` effects.
375    nontransactional_sequence_values: NontransactionalSequenceValues,
376}
377
378enum FixedTransactionSnapshot {
379    Pinned(Arc<Engine>),
380    Detached(SessionPortalTableSnapshots),
381}
382
383impl FixedTransactionSnapshot {
384    fn table(&self, relation: &RelationIdentity) -> Option<Arc<TableState>> {
385        match self {
386            Self::Pinned(snapshot) => snapshot.storage.tables.read().get(relation).cloned(),
387            Self::Detached(tables) => tables.get(relation).cloned(),
388        }
389    }
390
391    fn table_for_live_relation(
392        &self,
393        relation: &RelationIdentity,
394        live: &TableState,
395    ) -> Option<Arc<TableState>> {
396        let storage_generation = live.storage_generation();
397        let exact = self.table(relation);
398        if exact
399            .as_ref()
400            .is_some_and(|table| table.storage_generation() == storage_generation)
401        {
402            return exact;
403        }
404        match self {
405            Self::Pinned(snapshot) => snapshot
406                .storage
407                .tables
408                .read()
409                .values()
410                .find(|table| table.storage_generation() == storage_generation)
411                .cloned(),
412            Self::Detached(tables) => tables
413                .values()
414                .find(|table| table.storage_generation() == storage_generation)
415                .cloned(),
416        }
417    }
418}
419
420struct TransactionSavepoint {
421    name: String,
422    storage_savepoint: StorageSavepointId,
423    characteristics: TransactionCharacteristicsState,
424    session_snapshot: SessionStateSnapshot,
425    data_snapshot: Option<EngineDataSnapshot>,
426    dirty: TransactionDirtyState,
427    prepared_change_mark:
428        uqa_execution::statement::prepared::invalidation::PreparedInvalidationMark,
429    lock_mark: u32,
430    /// The length of the frame's row changes at the savepoint. The changes only grow after it, so a rollback truncates them to this length.
431    row_changes: usize,
432    statistics_changes: statistics::StatisticsChanges,
433    deferred_foreign_key_checks: Vec<DeferredForeignKeyCheck>,
434    deferred_constraint_trigger_events:
435        Vec<uqa_execution::mutation::triggers::DeferredConstraintTriggerEvent>,
436    pending_listen_actions: Vec<PendingListenAction>,
437    pending_notifications: Vec<PendingNotification>,
438    constraint_modes: ConstraintModeState,
439}
440
441/// Lightweight SQL-session state that follows transaction/savepoint rollback for every backend. It is intentionally separate from the database-sized memory-engine snapshot so persistent sessions receive identical SET, search-path, and statement-cache semantics. Sequence `currval` and last-used entries produced after the snapshot are reapplied because sequence functions are nontransactional in `PostgreSQL`.
442#[derive(Clone, Default)]
443struct SessionStateSnapshot {
444    command_overlays: Vec<uqa_execution::mutation::overlay::CommandOverlayCheckpoint>,
445    /// A pinned physical graph view plus this transaction's changed identities. Savepoints retain only handles and changed-id checkpoints, never graph payload replicas.
446    graph_overlay: Option<GraphTransactionOverlay>,
447    search_path: Vec<String>,
448    /// The OIDs of the session's temporary namespace and its TOAST namespace once its first temporary object created them; a rollback past that creation forgets them.
449    temporary_namespace: Option<uqa_sql::catalog::temporary_namespace::TemporaryNamespaceOids>,
450    session_vars: BTreeMap<String, String>,
451    parameter_scopes: uqa_sql::semantics::parameters::ParameterScopes<state::RuntimeParameterValue>,
452    sequence_currvals: BTreeMap<RelationIdentity, SessionSequenceValue>,
453    last_sequence: Option<SessionLastSequenceReference>,
454    sequence_discard_generation: u64,
455    sql_statement_cache: SQLStatementCache,
456    /// Names of portals that existed at this transaction or savepoint boundary. Rollback removes portals created later without rewinding cursor positions or resurrecting closed portals.
457    portal_names: BTreeSet<String>,
458    listened_channels: Vec<String>,
459    authorization: uqa_sql::catalog::roles::session::SessionAuthorization,
460}
461
462#[derive(Clone)]
463struct GraphTransactionOverlay {
464    store: Arc<uqa_graph::PersistentGraphStore>,
465    names: Arc<BTreeSet<String>>,
466}
467
468#[derive(Clone, Copy, PartialEq, Eq)]
469enum PinnedPortalTransactionControl {
470    MakeHoldable,
471    Reject,
472}
473
474struct SessionPortalState {
475    data: SessionPortalData,
476    schema: uqa_sql::RowSchema,
477    transaction_origin: u64,
478    position: SessionPortalPosition,
479    scrollable: bool,
480    holdable: bool,
481    /// Action to take when procedural transaction control encounters this pinned PL/pgSQL loop portal.
482    pinned_transaction_control: PinnedPortalTransactionControl,
483    /// A PL/pgSQL row loop pins its portal while user statements run so the loop body cannot close the executor that owns its current tuple batch.
484    pin_count: usize,
485    /// Retain live catalog visibility while the executor is temporarily outside the session map.
486    _registration: uqa_execution::statement::portal::PortalRegistration,
487}
488
489use uqa_execution::statement::portal::{SessionPortalCommandDeclaration, SessionPortalDeclaration};
490
491enum SessionPortalData {
492    Pending {
493        query: uqa_planner::QueryPlan,
494        params: Vec<SQLParam>,
495        table_snapshots: SessionPortalTableSnapshots,
496        view_snapshots: SessionPortalViewSnapshots,
497        sql_function_snapshots: SessionPortalSQLFunctionSnapshots,
498        catalog_snapshot: SessionPortalCatalogSnapshot,
499        restart: Option<SessionPortalRestart>,
500    },
501    PendingCommand {
502        command: Box<uqa_planner::CommandPlan>,
503        params: Vec<SQLParam>,
504        /// Preserve `PostgreSQL` 18's explicit-`SCROLL` DML tuple image while retaining the command's returned row count.
505        null_returning_values: bool,
506    },
507    Result(SQLResult),
508    Indexed(SessionPortalMaterialization),
509    Streaming {
510        worker: SessionPortalWorker,
511        materialized: Option<SessionPortalMaterialization>,
512        eof: bool,
513        restart: Option<SessionPortalRestart>,
514    },
515}
516
517struct SessionPortalRestart {
518    query: uqa_planner::QueryPlan,
519    params: Vec<SQLParam>,
520    table_snapshots: SessionPortalTableSnapshots,
521    view_snapshots: SessionPortalViewSnapshots,
522    sql_function_snapshots: SessionPortalSQLFunctionSnapshots,
523    catalog_snapshot: SessionPortalCatalogSnapshot,
524}
525
526struct SessionPortalMaterialization {
527    rows: uqa_execution::IndexedSpill,
528}
529
530use uqa_execution::statement::portal::worker::{
531    SessionPortalWorker, SessionPortalWorkerRequest, SessionPortalWorkerResponse,
532};
533
534/// `PostgreSQL` distinguishes the positions before the first row and after the last row from a position on a row.
535#[derive(Debug, Clone, Copy, PartialEq, Eq)]
536enum SessionPortalPosition {
537    BeforeFirst,
538    /// The one-based position of the current row.
539    OnRow(usize),
540    /// Number of rows processed while moving forward before the executor reported end of scan.
541    AfterLast(usize),
542}
543
544#[derive(Clone, Copy)]
545struct SessionRandomState {
546    s0: u64,
547    s1: u64,
548}
549
550impl Default for SessionRandomState {
551    fn default() -> Self {
552        Self {
553            s0: 0x5851_f42d_4c95_7f2d,
554            s1: 0x1405_7b7e_f767_814f,
555        }
556    }
557}
558
559#[derive(Clone)]
560struct EngineDataSnapshot {
561    tables: BTreeMap<RelationIdentity, TableDataSnapshot>,
562    durable: DurableCatalogSnapshot,
563    foreign_memory_tables: BTreeMap<RelationIdentity, Vec<uqa_sql::ResultRow>>,
564}
565
566#[derive(Clone)]
567#[expect(clippy::struct_excessive_bools, reason = "independent snapshot flags")]
568struct TableDataSnapshot {
569    state: Arc<TableState>,
570    security: state::BoundTableSecurity,
571    storage_generation: [u8; 16],
572    document_store: Arc<dyn DocumentStore>,
573    inverted_index: Arc<dyn InvertedIndex>,
574    vector_indexes: BTreeMap<FieldName, Arc<dyn VectorIndex>>,
575    value_indexes: BTreeMap<uqa_storage::ValueIndexKey, value_index::ColumnValueIndex>,
576    fts_fields: Vec<FieldName>,
577    columns: Vec<uqa_sql::ast::ColumnDef>,
578    columns_declared: bool,
579    /// One-past-the-last allocated document id. `u128` is intentional: it
580    /// represents `u64::MAX + 1`, so exhaustion is distinguishable from an
581    /// available final id and can never wrap or issue a duplicate.
582    next_id: u128,
583    analyzer: Analyzer,
584    column_stats: BTreeMap<String, uqa_planner::ColumnStats>,
585    column_stats_loaded: bool,
586    column_stats_dirty: bool,
587    statistics_maintenance: statistics::MaintenanceState,
588    table_checks: Vec<uqa_sql::ast::TableCheck>,
589    foreign_keys: Vec<uqa_sql::ast::ForeignKey>,
590    key_constraints: Vec<uqa_sql::ast::TableKeyConstraint>,
591    hierarchy: uqa_sql::ast::TableHierarchy,
592    row_type_array_name: Option<String>,
593    dropped_attributes: Vec<uqa_sql::catalog::relation_attributes::DroppedAttribute>,
594    doc_count_cache: u64,
595    doc_count_dirty: bool,
596}
597
598pub(crate) struct TableState {
599    /// Session-local relation generation. Reloads of the same catalog object preserve this value, while CREATE allocates a new value even when a dropped relation's name is reused.
600    lifecycle_id: std::sync::atomic::AtomicU64,
601    /// Durable logical relation identity used by `PostgreSQL` catalogs. Renames, schema changes, `TRUNCATE`, and reopen preserve it.
602    object_id: [u8; 16],
603    /// Durable SQL role ownership and ACL. Mutations publish this value atomically and preserve the relation's logical and physical identities.
604    security: state::CatalogCell<state::BoundTableSecurity>,
605    /// Durable physical-storage generation shared by every session. Schema-only changes preserve it; CREATE and TRUNCATE replace it so a fixed transaction snapshot never aliases a different physical relation lifetime.
606    storage_generation: RwLock<[u8; 16]>,
607    pub(crate) document_store: RwLock<Box<dyn DocumentStore>>,
608    inverted_index: RwLock<Box<dyn InvertedIndex>>,
609    vector_indexes: RwLock<uqa_storage::vector_index::VectorIndexes>,
610    fts_fields: state::CatalogCell<Vec<FieldName>>,
611    /// Column schema captured at CREATE TABLE / ALTER TABLE time, driving auto-id allocation and ALTER COLUMN bookkeeping.
612    columns: state::CatalogCell<Vec<uqa_sql::ast::ColumnDef>>,
613    columns_declared: state::CatalogCell<bool>,
614    /// Retained document-ID floor. Capable persistent backends reserve from the storage-owned durable namespace; temporary, memory and serialized backends use this local state. `u128` preserves the exhausted `u64::MAX + 1` value.
615    next_id: parking_lot::Mutex<u128>,
616    /// Whether the table's single integer primary key, if it has one, names its rows' identities (see `MutationIdentifiers::maps_integer_keys`). A table an earlier version wrote with a row at an identity its key does not name resolves its keys through the key's index instead.
617    maps_integer_keys: AtomicBool,
618    analyzer: state::CatalogCell<Analyzer>,
619    /// Per-column statistics refreshed by `ANALYZE table_name` or lazily
620    /// by `column_stats` after writes mark the table dirty. Keyed by column
621    /// name.
622    column_stats: state::CatalogCell<BTreeMap<String, uqa_planner::ColumnStats>>,
623    column_stats_loaded: AtomicBool,
624    column_stats_dirty: AtomicBool,
625    /// Changes since the last analysis of a table that no catalog tracks: tables of a memory-only engine and temporary tables. Their lazy analysis follows the maintenance policy of durable tables instead of re-analyzing after every write.
626    statistics_maintenance: parking_lot::Mutex<statistics::MaintenanceState>,
627    /// Table-level `CHECK` constraints, evaluated against every row
628    /// at INSERT / UPDATE time.
629    table_checks: state::CatalogCell<Vec<uqa_sql::ast::TableCheck>>,
630    /// Table-level `FOREIGN KEY` constraints. Each entry binds local
631    /// columns to a `(ref_table, ref_columns)` lookup target.
632    foreign_keys: state::CatalogCell<Vec<uqa_sql::ast::ForeignKey>>,
633    /// Typed PRIMARY KEY / UNIQUE tuples, including composite keys and
634    /// their SQL NULL-equality policy.
635    key_constraints: state::CatalogCell<Vec<uqa_sql::ast::TableKeyConstraint>>,
636    /// Direct parents, an optional partition key, and an optional child bound.
637    /// The complete object is persisted with the table's constraint envelope.
638    hierarchy: state::CatalogCell<uqa_sql::ast::TableHierarchy>,
639    /// Lazily built per-column value indexes for PRIMARY KEY / UNIQUE
640    /// / `CREATE INDEX` btree columns. Maintained incrementally by the
641    /// document write paths; cleared on bulk reloads.
642    value_indexes: RwLock<BTreeMap<uqa_storage::ValueIndexKey, value_index::ColumnValueIndex>>,
643    /// Cached `document_store.len()`. Persistent stores answer `len`
644    /// with a `COUNT(*)` query, which used to run once per SQL
645    /// statement for planner row estimates; the cache is invalidated
646    /// by every write and recomputed on demand.
647    doc_count_cache: std::sync::atomic::AtomicU64,
648    doc_count_dirty: AtomicBool,
649    /// Immutable relation lifecycle attributes captured at creation.
650    persistence: uqa_sql::ast::RelationPersistence,
651    on_commit: uqa_sql::ast::OnCommitAction,
652    /// The public OIDs allocated when the table was created; `None` for a table created before OIDs were recorded, which derives them from its identity.
653    catalog_oids: Option<uqa_sql::catalog::relation_oids::RelationCatalogOids>,
654    row_type_array_name: state::CatalogCell<Option<String>>,
655    dropped_attributes:
656        state::CatalogCell<Vec<uqa_sql::catalog::relation_attributes::DroppedAttribute>>,
657}
658
659impl TableState {
660    fn lifecycle_id(&self) -> u64 {
661        self.lifecycle_id.load(Ordering::Acquire)
662    }
663
664    fn storage_generation(&self) -> [u8; 16] {
665        *self.storage_generation.read()
666    }
667
668    fn object_id(&self) -> [u8; 16] {
669        self.object_id
670    }
671
672    fn recorded_catalog_oids(
673        &self,
674    ) -> Option<uqa_sql::catalog::relation_oids::RelationCatalogOids> {
675        self.catalog_oids
676    }
677
678    /// The table's public OIDs: the recorded ones, or those its identity derives.
679    fn relation_oids(&self) -> uqa_sql::catalog::relation_oids::RelationCatalogOids {
680        self.catalog_oids.unwrap_or_else(|| {
681            uqa_sql::catalog::relation_oids::RelationCatalogOids::legacy(
682                uqa_sql::catalog::relation_oids::RelationOidKind::Table,
683                &self.object_id,
684            )
685        })
686    }
687
688    /// The namespace this table's document identities are allocated and observed in.
689    fn document_id_namespace(
690        &self,
691    ) -> uqa_storage::document_store::identifiers::DocumentIdNamespace {
692        uqa_storage::document_store::identifiers::DocumentIdNamespace {
693            object: self.object_id(),
694            generation: self.storage_generation(),
695        }
696    }
697
698    fn role_owner(&self) -> uqa_sql::catalog::roles::RoleIdentity {
699        self.security.read().role_owner
700    }
701
702    fn security(&self) -> state::BoundTableSecurity {
703        self.security.read().clone()
704    }
705
706    fn fts_fields(&self) -> Vec<FieldName> {
707        self.fts_fields.read().clone()
708    }
709}
710
711fn next_table_lifecycle_id() -> u64 {
712    static NEXT_TABLE_LIFECYCLE_ID: std::sync::atomic::AtomicU64 =
713        std::sync::atomic::AtomicU64::new(1);
714    uqa_core::atomic::try_update_u64(
715        &NEXT_TABLE_LIFECYCLE_ID,
716        Ordering::Relaxed,
717        Ordering::Relaxed,
718        |current| current.checked_add(1),
719    )
720    .expect("table lifecycle id space exhausted")
721}
722
723pub(crate) use uqa_execution::catalog::identity::new_nonzero_catalog_identity;
724
725fn new_table_object_id() -> StorageBackendResult<[u8; 16]> {
726    new_nonzero_catalog_identity("table", "object identity")
727}
728
729fn new_view_object_id() -> StorageBackendResult<[u8; 16]> {
730    new_nonzero_catalog_identity("view", "object identity")
731}
732
733fn new_table_storage_generation() -> StorageBackendResult<[u8; 16]> {
734    new_nonzero_catalog_identity("table", "storage generation")
735}
736
737fn new_sequence_object_id() -> StorageBackendResult<[u8; 16]> {
738    new_nonzero_catalog_identity("sequence", "object identity")
739}
740
741fn new_sequence_definition_generation() -> StorageBackendResult<[u8; 16]> {
742    new_nonzero_catalog_identity("sequence", "definition generation")
743}
744
745fn normalize_analyzer_config_value(value: &mut serde_json::Value) {
746    if let Some(tokenizer) = value.get_mut("tokenizer") {
747        if let Some(name) = tokenizer.as_str() {
748            *tokenizer = serde_json::json!({
749                "type": name.to_ascii_lowercase().replace('-', "_")
750            });
751        }
752    }
753    if let Some(filters) = value
754        .get_mut("token_filters")
755        .and_then(|v| v.as_array_mut())
756    {
757        for filter in filters {
758            if let Some(name) = filter.as_str() {
759                *filter = serde_json::json!({
760                    "type": name.to_ascii_lowercase().replace('-', "_")
761                });
762            }
763        }
764    }
765}
766
767fn parse_analyzer_config(name: &str, config_json: &str) -> std::result::Result<Analyzer, String> {
768    let mut value: serde_json::Value = serde_json::from_str(config_json)
769        .map_err(|e| format!("analyzer `{name}` config is not valid JSON: {e}"))?;
770    normalize_analyzer_config_value(&mut value);
771    let analyzer: Analyzer = serde_json::from_value(value)
772        .map_err(|e| format!("analyzer `{name}` config is not a valid analyzer: {e}"))?;
773    analyzer
774        .validate()
775        .map_err(|e| format!("analyzer `{name}` config is invalid: {e}"))?;
776    Ok(analyzer)
777}
778
779fn normalize_analyzer_phase(phase: &str) -> std::result::Result<(String, AnalyzerPhase), String> {
780    let phase = AnalyzerPhase::parse(&phase.to_ascii_lowercase())?;
781    let normalized = match phase {
782        AnalyzerPhase::Index => "index",
783        AnalyzerPhase::Search => "search",
784        AnalyzerPhase::Both => "both",
785    };
786    Ok((normalized.to_string(), phase))
787}
788
789impl Drop for Engine {
790    fn drop(&mut self) {
791        if self.owns_session_registration {
792            self.session.portals.lock().clear();
793            if let Some(backend) = self.storage.backend.as_ref() {
794                if backend.in_transaction() {
795                    let _ = backend.rollback_transaction();
796                }
797            }
798            for frame in self.session.transactions.lock().iter() {
799                drop(frame.pending_notification_commit.lock().take());
800            }
801            self.row_locks.close_temporary_roles(self.session_id);
802            self.row_locks.release_session(self.session_id);
803            self.notification_hub.unregister(self.session_id);
804            self.release_automatic_statistics_client();
805        }
806    }
807}
808
809fn initial_random_state() -> SessionRandomState {
810    use std::sync::atomic::{AtomicU64, Ordering};
811
812    static NEXT_STATE: AtomicU64 = AtomicU64::new(0x4d59_5df4_d0f3_3173);
813    random_state_from_seed(NEXT_STATE.fetch_add(0x9e37_79b9_7f4a_7c15, Ordering::Relaxed))
814}
815
816fn random_state_from_seed(mut seed: u64) -> SessionRandomState {
817    let mut splitmix64 = || {
818        seed = seed.wrapping_add(0x9e37_79b9_7f4a_7c15);
819        let mut value = seed;
820        value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
821        value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
822        value ^ (value >> 31)
823    };
824    let state = SessionRandomState {
825        s0: splitmix64(),
826        s1: splitmix64(),
827    };
828    if state.s0 == 0 && state.s1 == 0 {
829        SessionRandomState::default()
830    } else {
831        state
832    }
833}
834
835/// Exact signed single-prior hybrid-search arguments. Keeps
836/// [`Engine::hybrid_search`] borrowing-friendly without an explosion of
837/// positional parameters.
838#[derive(Debug, Clone)]
839pub struct HybridSearchParams<'a> {
840    pub table: &'a str,
841    pub text_field: &'a str,
842    pub text_query: &'a str,
843    pub vector_field: &'a str,
844    pub query_vector: Vec<f32>,
845    /// How many KNN candidates to pull from the vector index before
846    /// fusion. Tune above `top_k` to widen the recall pool.
847    pub knn_pool: usize,
848    pub top_k: usize,
849}
850
851/// Explicit robust-ranking variant of [`HybridSearchParams`]. This contract
852/// applies positive-evidence gating and confidence scaling rather than exact
853/// single-prior Bayesian evidence fusion.
854#[derive(Debug, Clone)]
855pub struct RobustHybridSearchParams<'a> {
856    pub table: &'a str,
857    pub text_field: &'a str,
858    pub text_query: &'a str,
859    pub vector_field: &'a str,
860    pub query_vector: Vec<f32>,
861    /// How many KNN candidates to pull from the vector index before
862    /// fusion. Tune above `top_k` to widen the recall pool.
863    pub knn_pool: usize,
864    /// Confidence-scaling exponent for robust positive-evidence pooling.
865    /// Must be finite and in `[0, 1]`.
866    pub alpha: f64,
867    pub top_k: usize,
868}
869
870#[cfg(test)]
871mod tests;
872
873mod copy;
874
875use uqa_execution::catalog::sequence::session::{
876    NontransactionalSequenceValue, SessionLastSequenceReference, SessionSequenceCache,
877    SessionSequenceValue,
878};