1use super::{
8 analyzer_registry, Arc, BTreeMap, DocId, Document, Engine, FieldName, FtsIndexStat, SQLError,
9 TableState, Value,
10};
11use uqa_storage::inverted_index::DocumentTextSource;
12use uqa_storage::{InvertedIndex, StorageBackendResult};
13
14impl Engine {
15 pub(crate) fn fts_fields_for_table(&self, name: &str) -> Result<Vec<FieldName>, SQLError> {
16 Ok(self
17 .try_query_table(name)
18 .map_err(|err| SQLError::Internal(format!("resolve table `{name}`: {err}")))?
19 .map_or_else(Vec::new, |table| table.fts_fields()))
20 }
21
22 pub(crate) fn validate_text_search_field(
28 &self,
29 table: &str,
30 field: &str,
31 ) -> Result<(), SQLError> {
32 let Some(table_state) = self
33 .try_query_table(table)
34 .map_err(|error| SQLError::Internal(format!("resolve text-search table: {error}")))?
35 else {
36 return Err(SQLError::UnknownTable(table.to_string()));
37 };
38 uqa_sql::semantics::text_indexes::require_physical_text_index(
39 table,
40 field,
41 &table_state.fts_fields(),
42 || table_state.columns.read().clone(),
43 )
44 }
45
46 pub fn fts_index_stats(
47 &self,
48 table_filter: Option<&str>,
49 ) -> Result<Vec<FtsIndexStat>, SQLError> {
50 self.with_direct_read_snapshot(|engine| {
51 engine.fts_index_stats_with_tables(table_filter, |name| {
52 engine
53 .bind_query_table_read(name)
54 .map(|binding| binding.value)
55 })
56 })
57 }
58
59 pub(crate) fn fts_index_stats_in_execution(
60 &self,
61 table_filter: Option<&str>,
62 ) -> Result<Vec<FtsIndexStat>, SQLError> {
63 self.fts_index_stats_with_tables(table_filter, |name| self.require_query_table(name))
64 }
65
66 fn fts_index_stats_with_tables(
67 &self,
68 table_filter: Option<&str>,
69 bind: impl Fn(&str) -> Result<Arc<TableState>, SQLError>,
70 ) -> Result<Vec<FtsIndexStat>, SQLError> {
71 let mut out = Vec::new();
72 for table_name in self.fts_stats_table_names(table_filter)? {
73 let table = bind(&table_name)?;
74 let mut fields = table.fts_fields();
75 fields.sort();
76 let index = table.inverted_index.read();
77 let index = uqa_execution::serializable::text::ObservedTextIndex::new(
78 index.as_ref(),
79 self.serializable_table_state_read(&table)?,
80 table.columns.snapshot(),
81 );
82 for field in fields {
83 let analyzer = self
84 .table_field_analyzer_in_execution(&table_name, &field)
85 .map_err(SQLError::Internal)?
86 .map_or_else(
87 || analyzer_registry::DEFAULT_ANALYZER_NAME.to_string(),
88 |(name, _)| name,
89 );
90 let doc_length_count = index.doc_length_count(Some(&field)).map_err(|error| {
91 uqa_execution::storage_errors::storage_error(
92 "read FTS document-length count",
93 &error,
94 )
95 })?;
96 out.push(FtsIndexStat {
97 table_name: table_name.clone(),
98 field: field.clone(),
99 analyzer,
100 posting_count: index.posting_count(Some(&field)).map_err(|error| {
101 uqa_execution::storage_errors::storage_error(
102 "read FTS posting count",
103 &error,
104 )
105 })?,
106 doc_length_count,
107 indexed_doc_count: doc_length_count,
108 term_count: index.term_count(Some(&field)).map_err(|error| {
109 uqa_execution::storage_errors::storage_error("read FTS term count", &error)
110 })?,
111 total_field_length: index.total_field_length(&field).map_err(|error| {
112 uqa_execution::storage_errors::storage_error(
113 "read FTS field length",
114 &error,
115 )
116 })?,
117 });
118 }
119 }
120 Ok(out)
121 }
122
123 fn fts_stats_table_names(&self, table_filter: Option<&str>) -> Result<Vec<String>, SQLError> {
124 self.synchronize_table_catalog()
125 .map_err(|err| SQLError::Internal(format!("refresh table catalog: {err}")))?;
126 let mut names = if let Some(name) = table_filter {
127 vec![self
128 .try_resolve_query_table_name(name)
129 .map_err(|err| SQLError::Internal(format!("resolve table filter: {err}")))?
130 .ok_or_else(|| SQLError::UnknownTable(name.to_string()))?]
131 } else if let Some(tables) = self.query_table_snapshots.as_ref() {
132 tables
133 .keys()
134 .map(uqa_core::RelationIdentity::qualified_name)
135 .collect()
136 } else {
137 self.storage
138 .tables
139 .read()
140 .keys()
141 .map(uqa_core::RelationIdentity::qualified_name)
142 .collect()
143 };
144 names.sort_unstable();
145 Ok(names)
146 }
147
148 pub(crate) fn fts_source(
150 t: &Arc<TableState>,
151 cancellation: Option<&uqa_core::CancellationToken>,
152 ) -> StorageBackendResult<DocumentTextSource> {
153 if let Some(cancellation) = cancellation {
154 cancellation.check()?;
155 }
156 let documents = t.document_store.read().snapshot()?;
157 Ok(DocumentTextSource::new(
158 documents,
159 t.fts_fields(),
160 cancellation.cloned(),
161 ))
162 }
163
164 pub(crate) fn rebuild_fts_index(t: &Arc<TableState>) -> StorageBackendResult<()> {
165 let mut source = Self::fts_source(t, None)?;
166 t.inverted_index.write().try_rebuild_documents(&mut source)
167 }
168
169 pub(crate) fn rebuild_fts_index_cancellable(
170 t: &Arc<TableState>,
171 cancellation: &uqa_core::CancellationToken,
172 ) -> StorageBackendResult<()> {
173 let mut source = Self::fts_source(t, Some(cancellation))?;
174 t.inverted_index
175 .write()
176 .try_rebuild_documents_cancellable(&mut source, cancellation)
177 }
178
179 pub fn add_document(
180 &self,
181 table: &str,
182 doc_id: DocId,
183 document: Document,
184 ) -> Result<(), SQLError> {
185 self.with_evaluating_row_write_transaction(
186 table,
187 doc_id,
188 uqa_sql::ast::LockStrength::ForUpdate,
189 |engine| {
190 engine.add_document_impl(
191 table,
192 doc_id,
193 document,
194 uqa_execution::mutation::publication::InsertedIdentity::Unknown,
195 )
196 },
197 )
198 }
199
200 pub(crate) fn add_document_impl(
201 &self,
202 table: &str,
203 doc_id: DocId,
204 mut document: Document,
205 inserted: uqa_execution::mutation::publication::InsertedIdentity,
206 ) -> Result<(), SQLError> {
207 uqa_execution::mutation::assignment::refresh_stored_generated_columns(
208 self.mutation_assignment_context(),
209 table,
210 &mut document,
211 )?;
212 uqa_execution::serializable::observe_row_write(self, table, doc_id)?;
213 self.add_prepared_document_impl(table, doc_id, document, inserted)
214 }
215
216 pub(crate) fn add_prepared_document_impl(
217 &self,
218 table: &str,
219 doc_id: DocId,
220 document: Document,
221 inserted: uqa_execution::mutation::publication::InsertedIdentity,
222 ) -> Result<(), SQLError> {
223 self.add_prepared_document_impl_with_fts(table, doc_id, document, inserted, true, None)
224 }
225
226 pub(crate) fn add_prepared_document_without_fts_impl(
227 &self,
228 table: &str,
229 doc_id: DocId,
230 document: Document,
231 inserted: uqa_execution::mutation::publication::InsertedIdentity,
232 ) -> Result<(), SQLError> {
233 self.add_prepared_document_impl_with_fts(table, doc_id, document, inserted, false, None)
234 }
235
236 pub(crate) fn add_prepared_stored_document_impl(
237 &self,
238 table: &str,
239 doc_id: DocId,
240 document: uqa_storage::StoredDocument,
241 inserted: uqa_execution::mutation::publication::InsertedIdentity,
242 ) -> Result<(), SQLError> {
243 let (fields, metadata) = document.into_parts();
244 self.add_prepared_document_impl_with_fts(
245 table,
246 doc_id,
247 fields,
248 inserted,
249 true,
250 Some(metadata),
251 )
252 }
253
254 pub(crate) fn prepared_document_text_fields(
255 &self,
256 table: &str,
257 document: &Document,
258 ) -> Result<BTreeMap<FieldName, String>, SQLError> {
259 let Some(t) = self
260 .try_table(table)
261 .map_err(|err| SQLError::Internal(format!("resolve table `{table}`: {err}")))?
262 else {
263 return Err(SQLError::UnknownTable(table.to_string()));
264 };
265 let mut text_fields = BTreeMap::new();
266 for name in &t.fts_fields() {
267 if let Some(Value::Str(value)) = document.get(name) {
268 text_fields.insert(name.clone(), value.clone());
269 }
270 }
271 Ok(text_fields)
272 }
273
274 pub(crate) fn add_prepared_fts_documents(
275 &self,
276 table: &str,
277 documents: Vec<(DocId, BTreeMap<FieldName, String>)>,
278 ) -> Result<(), SQLError> {
279 let Some(t) = self
280 .try_table(table)
281 .map_err(|err| SQLError::Internal(format!("resolve table `{table}`: {err}")))?
282 else {
283 return Err(SQLError::UnknownTable(table.to_string()));
284 };
285 let result = uqa_execution::serializable::text::add_documents(
286 self,
287 table,
288 t.columns.snapshot(),
289 t.inverted_index.write().as_mut(),
290 documents,
291 );
292 result
293 }
294
295 fn publish_prepared_document_text(
297 &self,
298 table_name: &str,
299 table: &TableState,
300 doc_id: DocId,
301 text_fields: BTreeMap<FieldName, String>,
302 known_new: bool,
303 ) -> Result<(), SQLError> {
304 if !uqa_execution::mutation::publication::document_changes_text_index(
305 known_new,
306 &text_fields,
307 ) {
308 return Ok(());
309 }
310 uqa_execution::serializable::text::add_document(
311 self,
312 table_name,
313 table.columns.snapshot(),
314 table.inverted_index.write().as_mut(),
315 doc_id,
316 text_fields,
317 )
318 }
319
320 fn add_prepared_document_impl_with_fts(
321 &self,
322 table: &str,
323 doc_id: DocId,
324 mut document: Document,
325 inserted: uqa_execution::mutation::publication::InsertedIdentity,
326 index_fts: bool,
327 metadata: Option<uqa_storage::DocumentMetadata>,
328 ) -> Result<(), SQLError> {
329 let known_new = inserted.is_vacant();
330 let Some(table_name) = self
331 .try_resolve_table_name(table)
332 .map_err(|err| SQLError::Internal(format!("resolve table `{table}`: {err}")))?
333 else {
334 return Err(SQLError::UnknownTable(table.to_string()));
335 };
336 let Some(t) = self
337 .try_table(table)
338 .map_err(|err| SQLError::Internal(format!("resolve table `{table}`: {err}")))?
339 else {
340 return Err(SQLError::UnknownTable(table.to_string()));
341 };
342 let existed = if known_new {
343 false
344 } else {
345 t.document_store
346 .read()
347 .contains_doc_id(doc_id)
348 .map_err(|error| SQLError::Internal(format!("read existing document: {error}")))?
349 };
350 let (old_indexed, indexed_fields) = if known_new {
358 (None, Self::value_indexes_built_fields(&t))
359 } else {
360 let old = self.value_indexes_old_values(&t, doc_id)?;
361 let fields = old.as_ref().map(|old| {
362 old.keys()
363 .cloned()
364 .collect::<Vec<uqa_storage::ValueIndexKey>>()
365 });
366 (old, fields)
367 };
368 let persistent_indexed =
369 self.persistent_value_index_document_values(&table_name, &document)?;
370 let new_indexed = indexed_fields
371 .map(|fields| {
372 if let Some(values) = &persistent_indexed {
373 return Ok(values.clone());
374 }
375 self.value_index_document_values(&table_name, &fields, &document)
376 })
377 .transpose()?;
378 self.observe_value_index_write(
379 &table_name,
380 &t,
381 doc_id,
382 existed,
383 old_indexed.as_ref(),
384 persistent_indexed.as_ref().or(new_indexed.as_ref()),
385 )?;
386 if index_fts {
387 let text_fields = self.prepared_document_text_fields(table, &document)?;
388 self.publish_prepared_document_text(&table_name, &t, doc_id, text_fields, known_new)?;
389 }
390 let columns = t.columns.read().clone();
391 crate::generated::strip_virtual_generated_columns(&columns, &mut document);
392 let metadata = match metadata {
393 Some(metadata) => metadata,
394 None => uqa_storage::DocumentMetadata::with_tuple_xmin(self.tuple_version_xid()?),
395 };
396 self.advance_next_id(&table_name, doc_id).map_err(|error| {
397 uqa_execution::mutation::errors::identifier_storage_error(
398 "observe inserted document identity",
399 &error,
400 )
401 })?;
402 let unused = inserted.is_unused().then(|| t.document_id_namespace());
404 let mut store = t.document_store.write();
405 let stored = uqa_storage::StoredDocument::with_metadata(document, metadata);
406 match unused {
407 Some(namespace) => store.put_stored_unused(doc_id, stored, namespace),
408 None => store.put_stored(doc_id, stored),
409 }
410 .map_err(|err| crate::table_storage::document_store_write_error(&err))?;
411 if let Some(new) = persistent_indexed.as_ref() {
412 self.persist_value_indexes_apply_write(&table_name, doc_id, Some(new), unused)?;
413 }
414 if let Some(new) = new_indexed.as_ref() {
415 self.value_indexes_apply_write(&t, doc_id, Some(new))?;
416 }
417 drop(store);
418 let documents = if existed {
419 crate::table_storage::DocumentCountChange::Unchanged
420 } else {
421 crate::table_storage::DocumentCountChange::Added
422 };
423 self.mark_row_write(&table_name, &t, documents)
424 .map_err(|err| SQLError::Internal(format!("invalidate column stats: {err}")))?;
425 if existed {
426 self.note_row_changed(&table_name, doc_id)?;
427 } else {
428 self.note_row_inserted(&table_name, doc_id)?;
429 }
430 self.publish_command_document(&table_name, doc_id)?;
431 Ok(())
432 }
433}