Skip to main content

uqa_engine/
fts.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7use super::{
8    analyzer_registry, Arc, BTreeMap, DocId, Document, Engine, FieldName, FtsIndexStat, SQLError,
9    TableState, Value,
10};
11use uqa_storage::inverted_index::DocumentTextSource;
12use uqa_storage::{InvertedIndex, StorageBackendResult};
13
14impl Engine {
15    pub(crate) fn fts_fields_for_table(&self, name: &str) -> Result<Vec<FieldName>, SQLError> {
16        Ok(self
17            .try_query_table(name)
18            .map_err(|err| SQLError::Internal(format!("resolve table `{name}`: {err}")))?
19            .map_or_else(Vec::new, |table| table.fts_fields()))
20    }
21
22    /// Validate the physical text-search contract for one concrete field.
23    /// A declared TEXT column is not searchable until it has been registered
24    /// in a GIN/FTS index; treating that state as an empty posting list hides a
25    /// schema/configuration error from both the public search API and the
26    /// operator-tree executor.
27    pub(crate) fn validate_text_search_field(
28        &self,
29        table: &str,
30        field: &str,
31    ) -> Result<(), SQLError> {
32        let Some(table_state) = self
33            .try_query_table(table)
34            .map_err(|error| SQLError::Internal(format!("resolve text-search table: {error}")))?
35        else {
36            return Err(SQLError::UnknownTable(table.to_string()));
37        };
38        uqa_sql::semantics::text_indexes::require_physical_text_index(
39            table,
40            field,
41            &table_state.fts_fields(),
42            || table_state.columns.read().clone(),
43        )
44    }
45
46    pub fn fts_index_stats(
47        &self,
48        table_filter: Option<&str>,
49    ) -> Result<Vec<FtsIndexStat>, SQLError> {
50        self.with_direct_read_snapshot(|engine| {
51            engine.fts_index_stats_with_tables(table_filter, |name| {
52                engine
53                    .bind_query_table_read(name)
54                    .map(|binding| binding.value)
55            })
56        })
57    }
58
59    pub(crate) fn fts_index_stats_in_execution(
60        &self,
61        table_filter: Option<&str>,
62    ) -> Result<Vec<FtsIndexStat>, SQLError> {
63        self.fts_index_stats_with_tables(table_filter, |name| self.require_query_table(name))
64    }
65
66    fn fts_index_stats_with_tables(
67        &self,
68        table_filter: Option<&str>,
69        bind: impl Fn(&str) -> Result<Arc<TableState>, SQLError>,
70    ) -> Result<Vec<FtsIndexStat>, SQLError> {
71        let mut out = Vec::new();
72        for table_name in self.fts_stats_table_names(table_filter)? {
73            let table = bind(&table_name)?;
74            let mut fields = table.fts_fields();
75            fields.sort();
76            let index = table.inverted_index.read();
77            let index = uqa_execution::serializable::text::ObservedTextIndex::new(
78                index.as_ref(),
79                self.serializable_table_state_read(&table)?,
80                table.columns.snapshot(),
81            );
82            for field in fields {
83                let analyzer = self
84                    .table_field_analyzer_in_execution(&table_name, &field)
85                    .map_err(SQLError::Internal)?
86                    .map_or_else(
87                        || analyzer_registry::DEFAULT_ANALYZER_NAME.to_string(),
88                        |(name, _)| name,
89                    );
90                let doc_length_count = index.doc_length_count(Some(&field)).map_err(|error| {
91                    uqa_execution::storage_errors::storage_error(
92                        "read FTS document-length count",
93                        &error,
94                    )
95                })?;
96                out.push(FtsIndexStat {
97                    table_name: table_name.clone(),
98                    field: field.clone(),
99                    analyzer,
100                    posting_count: index.posting_count(Some(&field)).map_err(|error| {
101                        uqa_execution::storage_errors::storage_error(
102                            "read FTS posting count",
103                            &error,
104                        )
105                    })?,
106                    doc_length_count,
107                    indexed_doc_count: doc_length_count,
108                    term_count: index.term_count(Some(&field)).map_err(|error| {
109                        uqa_execution::storage_errors::storage_error("read FTS term count", &error)
110                    })?,
111                    total_field_length: index.total_field_length(&field).map_err(|error| {
112                        uqa_execution::storage_errors::storage_error(
113                            "read FTS field length",
114                            &error,
115                        )
116                    })?,
117                });
118            }
119        }
120        Ok(out)
121    }
122
123    fn fts_stats_table_names(&self, table_filter: Option<&str>) -> Result<Vec<String>, SQLError> {
124        self.synchronize_table_catalog()
125            .map_err(|err| SQLError::Internal(format!("refresh table catalog: {err}")))?;
126        let mut names = if let Some(name) = table_filter {
127            vec![self
128                .try_resolve_query_table_name(name)
129                .map_err(|err| SQLError::Internal(format!("resolve table filter: {err}")))?
130                .ok_or_else(|| SQLError::UnknownTable(name.to_string()))?]
131        } else if let Some(tables) = self.query_table_snapshots.as_ref() {
132            tables
133                .keys()
134                .map(uqa_core::RelationIdentity::qualified_name)
135                .collect()
136        } else {
137            self.storage
138                .tables
139                .read()
140                .keys()
141                .map(uqa_core::RelationIdentity::qualified_name)
142                .collect()
143        };
144        names.sort_unstable();
145        Ok(names)
146    }
147
148    /// The text of the table's indexed fields, which a rebuild reads a page at a time from a snapshot of its documents instead of collecting it first.
149    pub(crate) fn fts_source(
150        t: &Arc<TableState>,
151        cancellation: Option<&uqa_core::CancellationToken>,
152    ) -> StorageBackendResult<DocumentTextSource> {
153        if let Some(cancellation) = cancellation {
154            cancellation.check()?;
155        }
156        let documents = t.document_store.read().snapshot()?;
157        Ok(DocumentTextSource::new(
158            documents,
159            t.fts_fields(),
160            cancellation.cloned(),
161        ))
162    }
163
164    pub(crate) fn rebuild_fts_index(t: &Arc<TableState>) -> StorageBackendResult<()> {
165        let mut source = Self::fts_source(t, None)?;
166        t.inverted_index.write().try_rebuild_documents(&mut source)
167    }
168
169    pub(crate) fn rebuild_fts_index_cancellable(
170        t: &Arc<TableState>,
171        cancellation: &uqa_core::CancellationToken,
172    ) -> StorageBackendResult<()> {
173        let mut source = Self::fts_source(t, Some(cancellation))?;
174        t.inverted_index
175            .write()
176            .try_rebuild_documents_cancellable(&mut source, cancellation)
177    }
178
179    pub fn add_document(
180        &self,
181        table: &str,
182        doc_id: DocId,
183        document: Document,
184    ) -> Result<(), SQLError> {
185        self.with_evaluating_row_write_transaction(
186            table,
187            doc_id,
188            uqa_sql::ast::LockStrength::ForUpdate,
189            |engine| {
190                engine.add_document_impl(
191                    table,
192                    doc_id,
193                    document,
194                    uqa_execution::mutation::publication::InsertedIdentity::Unknown,
195                )
196            },
197        )
198    }
199
200    pub(crate) fn add_document_impl(
201        &self,
202        table: &str,
203        doc_id: DocId,
204        mut document: Document,
205        inserted: uqa_execution::mutation::publication::InsertedIdentity,
206    ) -> Result<(), SQLError> {
207        uqa_execution::mutation::assignment::refresh_stored_generated_columns(
208            self.mutation_assignment_context(),
209            table,
210            &mut document,
211        )?;
212        uqa_execution::serializable::observe_row_write(self, table, doc_id)?;
213        self.add_prepared_document_impl(table, doc_id, document, inserted)
214    }
215
216    pub(crate) fn add_prepared_document_impl(
217        &self,
218        table: &str,
219        doc_id: DocId,
220        document: Document,
221        inserted: uqa_execution::mutation::publication::InsertedIdentity,
222    ) -> Result<(), SQLError> {
223        self.add_prepared_document_impl_with_fts(table, doc_id, document, inserted, true, None)
224    }
225
226    pub(crate) fn add_prepared_document_without_fts_impl(
227        &self,
228        table: &str,
229        doc_id: DocId,
230        document: Document,
231        inserted: uqa_execution::mutation::publication::InsertedIdentity,
232    ) -> Result<(), SQLError> {
233        self.add_prepared_document_impl_with_fts(table, doc_id, document, inserted, false, None)
234    }
235
236    pub(crate) fn add_prepared_stored_document_impl(
237        &self,
238        table: &str,
239        doc_id: DocId,
240        document: uqa_storage::StoredDocument,
241        inserted: uqa_execution::mutation::publication::InsertedIdentity,
242    ) -> Result<(), SQLError> {
243        let (fields, metadata) = document.into_parts();
244        self.add_prepared_document_impl_with_fts(
245            table,
246            doc_id,
247            fields,
248            inserted,
249            true,
250            Some(metadata),
251        )
252    }
253
254    pub(crate) fn prepared_document_text_fields(
255        &self,
256        table: &str,
257        document: &Document,
258    ) -> Result<BTreeMap<FieldName, String>, SQLError> {
259        let Some(t) = self
260            .try_table(table)
261            .map_err(|err| SQLError::Internal(format!("resolve table `{table}`: {err}")))?
262        else {
263            return Err(SQLError::UnknownTable(table.to_string()));
264        };
265        let mut text_fields = BTreeMap::new();
266        for name in &t.fts_fields() {
267            if let Some(Value::Str(value)) = document.get(name) {
268                text_fields.insert(name.clone(), value.clone());
269            }
270        }
271        Ok(text_fields)
272    }
273
274    pub(crate) fn add_prepared_fts_documents(
275        &self,
276        table: &str,
277        documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
278    ) -> Result<(), SQLError> {
279        let Some(t) = self
280            .try_table(table)
281            .map_err(|err| SQLError::Internal(format!("resolve table `{table}`: {err}")))?
282        else {
283            return Err(SQLError::UnknownTable(table.to_string()));
284        };
285        let result = uqa_execution::serializable::text::add_documents(
286            self,
287            table,
288            t.columns.snapshot(),
289            t.inverted_index.write().as_mut(),
290            documents,
291        );
292        result
293    }
294
295    /// Replacement is one atomic inverted-index operation even when the new document has no indexed text. Skipping an empty field map would leave stale postings from the previous version; remove-then-add would expose a destructive failure window when analysis fails. Only a document known to be new has no previous version, and without indexed text it leaves the index alone.
296    fn publish_prepared_document_text(
297        &self,
298        table_name: &str,
299        table: &TableState,
300        doc_id: DocId,
301        text_fields: BTreeMap<FieldName, String>,
302        known_new: bool,
303    ) -> Result<(), SQLError> {
304        if !uqa_execution::mutation::publication::document_changes_text_index(
305            known_new,
306            &text_fields,
307        ) {
308            return Ok(());
309        }
310        uqa_execution::serializable::text::add_document(
311            self,
312            table_name,
313            table.columns.snapshot(),
314            table.inverted_index.write().as_mut(),
315            doc_id,
316            text_fields,
317        )
318    }
319
320    fn add_prepared_document_impl_with_fts(
321        &self,
322        table: &str,
323        doc_id: DocId,
324        mut document: Document,
325        inserted: uqa_execution::mutation::publication::InsertedIdentity,
326        index_fts: bool,
327        metadata: Option<uqa_storage::DocumentMetadata>,
328    ) -> Result<(), SQLError> {
329        let known_new = inserted.is_vacant();
330        let Some(table_name) = self
331            .try_resolve_table_name(table)
332            .map_err(|err| SQLError::Internal(format!("resolve table `{table}`: {err}")))?
333        else {
334            return Err(SQLError::UnknownTable(table.to_string()));
335        };
336        let Some(t) = self
337            .try_table(table)
338            .map_err(|err| SQLError::Internal(format!("resolve table `{table}`: {err}")))?
339        else {
340            return Err(SQLError::UnknownTable(table.to_string()));
341        };
342        let existed = if known_new {
343            false
344        } else {
345            t.document_store
346                .read()
347                .contains_doc_id(doc_id)
348                .map_err(|error| SQLError::Internal(format!("read existing document: {error}")))?
349        };
350        // Value-index maintenance: unindex the previous field values
351        // (put may replace an existing document), index the new ones.
352        // `old_indexed` is `None` exactly when no index is built, so
353        // the common path costs one read-lock check. A failed put must
354        // leave the value indexes untouched. A known-new document has
355        // no previous values to unindex, so its writes skip the per-row
356        // storage lookup entirely and only insert the new values.
357        let (old_indexed, indexed_fields) = if known_new {
358            (None, Self::value_indexes_built_fields(&t))
359        } else {
360            let old = self.value_indexes_old_values(&t, doc_id)?;
361            let fields = old.as_ref().map(|old| {
362                old.keys()
363                    .cloned()
364                    .collect::<Vec<uqa_storage::ValueIndexKey>>()
365            });
366            (old, fields)
367        };
368        let persistent_indexed =
369            self.persistent_value_index_document_values(&table_name, &document)?;
370        let new_indexed = indexed_fields
371            .map(|fields| {
372                if let Some(values) = &persistent_indexed {
373                    return Ok(values.clone());
374                }
375                self.value_index_document_values(&table_name, &fields, &document)
376            })
377            .transpose()?;
378        self.observe_value_index_write(
379            &table_name,
380            &t,
381            doc_id,
382            existed,
383            old_indexed.as_ref(),
384            persistent_indexed.as_ref().or(new_indexed.as_ref()),
385        )?;
386        if index_fts {
387            let text_fields = self.prepared_document_text_fields(table, &document)?;
388            self.publish_prepared_document_text(&table_name, &t, doc_id, text_fields, known_new)?;
389        }
390        let columns = t.columns.read().clone();
391        crate::generated::strip_virtual_generated_columns(&columns, &mut document);
392        let metadata = match metadata {
393            Some(metadata) => metadata,
394            None => uqa_storage::DocumentMetadata::with_tuple_xmin(self.tuple_version_xid()?),
395        };
396        self.advance_next_id(&table_name, doc_id).map_err(|error| {
397            uqa_execution::mutation::errors::identifier_storage_error(
398                "observe inserted document identity",
399                &error,
400            )
401        })?;
402        // An identity no document ever had has no earlier records in the namespace its watermark was read in, so a store that keeps the table under that namespace writes without reading what it replaces.
403        let unused = inserted.is_unused().then(|| t.document_id_namespace());
404        let mut store = t.document_store.write();
405        let stored = uqa_storage::StoredDocument::with_metadata(document, metadata);
406        match unused {
407            Some(namespace) => store.put_stored_unused(doc_id, stored, namespace),
408            None => store.put_stored(doc_id, stored),
409        }
410        .map_err(|err| crate::table_storage::document_store_write_error(&err))?;
411        if let Some(new) = persistent_indexed.as_ref() {
412            self.persist_value_indexes_apply_write(&table_name, doc_id, Some(new), unused)?;
413        }
414        if let Some(new) = new_indexed.as_ref() {
415            self.value_indexes_apply_write(&t, doc_id, Some(new))?;
416        }
417        drop(store);
418        let documents = if existed {
419            crate::table_storage::DocumentCountChange::Unchanged
420        } else {
421            crate::table_storage::DocumentCountChange::Added
422        };
423        self.mark_row_write(&table_name, &t, documents)
424            .map_err(|err| SQLError::Internal(format!("invalidate column stats: {err}")))?;
425        if existed {
426            self.note_row_changed(&table_name, doc_id)?;
427        } else {
428            self.note_row_inserted(&table_name, doc_id)?;
429        }
430        self.publish_command_document(&table_name, doc_id)?;
431        Ok(())
432    }
433}