Skip to main content

uqa_engine/
analyzers.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7use super::{
8    analyzer_registry, normalize_analyzer_phase, parse_analyzer_config, AnalyzerPhase, Arc, Engine,
9    TableState,
10};
11use uqa_execution::storage_errors::storage_error;
12use uqa_sql::ast::ColumnType;
13use uqa_sql::SQLError;
14use uqa_storage::{
15    AnalyzerBindingOwner, FieldAnalyzerBinding, StorageBackendError, StorageBackendResult,
16};
17
18mod persistence;
19
20impl Engine {
21    pub(crate) fn resolve_analyzer_revision(
22        &self,
23        name: &str,
24    ) -> StorageBackendResult<Arc<uqa_analysis::CompiledAnalyzer>> {
25        let name = name.trim();
26        if name.is_empty() {
27            return Err(StorageBackendError::Other(
28                "analyzer name cannot be empty".into(),
29            ));
30        }
31        if let Ok(analyzer) = analyzer_registry::get_analyzer(name) {
32            return Ok(analyzer.compile()?);
33        }
34        self.durable
35            .named_analyzers
36            .read()
37            .get(name)
38            .cloned()
39            .ok_or_else(|| {
40                StorageBackendError::Other(format!("analyzer `{name}` is not registered"))
41            })
42    }
43
44    pub fn register_named_analyzer(
45        &self,
46        name: &str,
47        config_json: &str,
48    ) -> std::result::Result<(), String> {
49        self.with_implicit_string_transaction(|engine| {
50            engine.register_named_analyzer_inner(name, config_json)
51        })
52    }
53
54    fn register_named_analyzer_inner(
55        &self,
56        name: &str,
57        config_json: &str,
58    ) -> std::result::Result<(), String> {
59        self.synchronize_catalog_registries()
60            .map_err(|err| format!("refresh analyzer catalog: {err}"))?;
61        let name = name.trim();
62        if analyzer_registry::is_builtin_analyzer(name) {
63            return Err(format!("cannot overwrite built-in analyzer `{name}`"));
64        }
65        let compiled = parse_analyzer_config(name, config_json)?
66            .compile()
67            .map_err(|error| error.to_string())?;
68        let configuration = serde_json::to_string(
69            &compiled
70                .descriptor()
71                .configuration()
72                .map_err(|error| error.to_string())?,
73        )
74        .map_err(|error| error.to_string())?;
75        let mut analyzers = self.durable.named_analyzers.write();
76        if let Some(catalog) = self.storage.catalog.as_ref() {
77            catalog
78                .save_analyzer_revision(
79                    name,
80                    &configuration,
81                    compiled.descriptor().canonical_json(),
82                )
83                .map_err(|err| format!("persist analyzer `{name}`: {err}"))?;
84        }
85        analyzers.insert(name.to_owned(), compiled);
86        drop(analyzers);
87        self.note_catalog_registry_changed();
88        Ok(())
89    }
90
91    /// Analyze one input with a resolved revision and return the SQL diagnostic object.
92    pub fn analyze_text(&self, name: &str, input: &str) -> Result<uqa_core::Value, String> {
93        self.with_direct_query_snapshot(
94            true,
95            |engine| {
96                engine
97                    .synchronize_catalog_registries()
98                    .map_err(|err| format!("refresh analyzer catalog: {err}"))?;
99                uqa_execution::query::table_functions::analyze_text(
100                    engine.query_runtime_view(),
101                    engine,
102                    name,
103                    input,
104                )
105                .map_err(|error| error.to_string())
106            },
107            |error| error.to_string(),
108        )
109    }
110
111    pub fn drop_named_analyzer(&self, name: &str) -> Result<bool, String> {
112        self.with_implicit_string_transaction(|engine| engine.drop_named_analyzer_inner(name))
113    }
114
115    fn drop_named_analyzer_inner(&self, name: &str) -> Result<bool, String> {
116        let name = name.trim();
117        self.synchronize_catalog_registries()
118            .map_err(|err| format!("refresh analyzer catalog: {err}"))?;
119        if self
120            .durable
121            .table_field_analyzers
122            .read()
123            .values()
124            .any(|binding| binding.uses_name(name))
125        {
126            return Err(format!(
127                "analyzer `{name}` is still assigned to a table field"
128            ));
129        }
130        let mut analyzers = self.durable.named_analyzers.write();
131        if !analyzers.contains_key(name) {
132            return Ok(false);
133        }
134        if let Some(catalog) = self.storage.catalog.as_ref() {
135            catalog
136                .drop_analyzer(name)
137                .map_err(|err| format!("drop analyzer `{name}`: {err}"))?;
138        }
139        let removed = analyzers.remove(name).is_some();
140        drop(analyzers);
141        if removed {
142            self.note_catalog_registry_changed();
143        }
144        Ok(removed)
145    }
146
147    pub fn list_named_analyzers(&self) -> Result<Vec<String>, String> {
148        self.with_direct_query_snapshot(true, Self::named_analyzers_in_execution, |error| {
149            error.to_string()
150        })
151    }
152
153    pub(crate) fn named_analyzers_in_execution(&self) -> Result<Vec<String>, String> {
154        self.synchronize_catalog_registries()
155            .map_err(|err| format!("refresh analyzer catalog: {err}"))?;
156        let mut names: Vec<String> = self
157            .durable
158            .named_analyzers
159            .read()
160            .keys()
161            .cloned()
162            .collect();
163        names.sort();
164        Ok(names)
165    }
166
167    pub fn set_table_field_analyzer(
168        &self,
169        table: &str,
170        field: &str,
171        analyzer_name: &str,
172        phase: &str,
173    ) -> std::result::Result<(), String> {
174        self.with_implicit_string_transaction(|engine| {
175            engine
176                .set_table_field_analyzer_inner(table, field, analyzer_name, phase)
177                .map_err(|error| error.to_string())
178        })
179    }
180
181    pub(crate) fn set_table_field_analyzer_inner(
182        &self,
183        table: &str,
184        field: &str,
185        analyzer_name: &str,
186        phase: &str,
187    ) -> Result<(), SQLError> {
188        self.synchronize_catalog_registries()
189            .map_err(|error| storage_error("refresh analyzer catalog", &error))?;
190        let table_name = self
191            .try_resolve_table_name(table)
192            .map_err(|error| storage_error("resolve analyzer table", &error))?
193            .ok_or_else(|| SQLError::UnknownTable(table.into()))?;
194        let Some(t) = self
195            .try_table(&table_name)
196            .map_err(|error| storage_error("resolve analyzer table", &error))?
197        else {
198            return Err(SQLError::UnknownTable(table.into()));
199        };
200        Self::validate_table_analyzer_field(&table_name, &t, field)
201            .map_err(SQLError::Unsupported)?;
202        let analyzer_name = analyzer_name.trim();
203        let analyzer =
204            self.resolve_analyzer_revision(analyzer_name)
205                .map_err(|error| match error {
206                    StorageBackendError::Other(message) => SQLError::Unsupported(message),
207                    error => storage_error("resolve analyzer revision", &error),
208                })?;
209        let (_, phase) = normalize_analyzer_phase(phase).map_err(SQLError::Unsupported)?;
210        let (old_index, old_search) = {
211            let index = t.inverted_index.read();
212            (
213                index
214                    .index_analyzer_revision(field)
215                    .map_err(|error| storage_error("resolve prior index analyzer", &error))?,
216                index
217                    .search_analyzer_revision(field)
218                    .map_err(|error| storage_error("resolve prior search analyzer", &error))?,
219            )
220        };
221        let previous = self
222            .durable
223            .table_field_analyzers
224            .read()
225            .get(&(table_name.clone(), field.to_owned()))
226            .cloned()
227            .unwrap_or_else(|| {
228                FieldAnalyzerBinding::unassigned(old_index.clone(), old_search.clone())
229            });
230        if previous.owner == AnalyzerBindingOwner::Gin {
231            return Err(SQLError::Unsupported(format!("field `{table_name}`.`{field}` has a GIN-owned analyzer; recreate its owning index before assigning a field analyzer")));
232        }
233        let candidate = previous.assigned(
234            analyzer_name,
235            analyzer.clone(),
236            phase,
237            AnalyzerBindingOwner::Field,
238        );
239        let rebuild = matches!(phase, AnalyzerPhase::Index | AnalyzerPhase::Both)
240            && t.fts_fields().iter().any(|f| f == field);
241        if rebuild {
242            let mut source = Self::fts_source(&t, Some(&self.runtime.cancellation))
243                .map_err(|error| storage_error("read FTS rebuild source", &error))?;
244            t.inverted_index
245                .write()
246                .rebuild_with_analyzer_revision_cancellable(
247                    field,
248                    analyzer,
249                    phase,
250                    &mut source,
251                    &self.runtime.cancellation,
252                )
253                .map_err(|error| storage_error("set_table_analyzer", &error))?;
254        } else {
255            t.inverted_index
256                .write()
257                .set_field_analyzer_revision(field, analyzer, phase)
258                .map_err(|error| SQLError::Unsupported(format!("set_table_analyzer: {error}")))?;
259        }
260        if let Err(error) = self.persist_field_analyzer_binding(&table_name, field, &candidate) {
261            return Err(Self::restore_analyzer_error(
262                &t,
263                field,
264                old_index,
265                old_search,
266                rebuild,
267                storage_error("persist table analyzer", &error),
268            ));
269        }
270        self.durable
271            .table_field_analyzers
272            .write()
273            .insert((table_name, field.to_owned()), candidate);
274        if self.is_persistent() {
275            self.note_table_catalog_changed();
276            self.note_catalog_registry_changed();
277        }
278        Ok(())
279    }
280
281    /// A per-field analyzer is meaningful only for a real text column that is
282    /// already part of the table's physical FTS index.  Enforce the same
283    /// invariant both when accepting a new assignment and while hydrating
284    /// persisted assignments on reopen.
285    pub(crate) fn validate_table_analyzer_field(
286        table_name: &str,
287        table: &Arc<TableState>,
288        field: &str,
289    ) -> Result<(), String> {
290        let column_type = table
291            .columns
292            .read()
293            .iter()
294            .find(|column| column.name == field)
295            .map(|column| column.ty.clone())
296            .ok_or_else(|| {
297                format!("set_table_analyzer: column `{table_name}`.`{field}` does not exist")
298            })?;
299        if column_type != ColumnType::Text {
300            return Err(format!(
301                "set_table_analyzer: column `{table_name}`.`{field}` must be TEXT, got {column_type:?}"
302            ));
303        }
304        if !table
305            .fts_fields()
306            .iter()
307            .any(|candidate| candidate == field)
308        {
309            return Err(format!(
310                "set_table_analyzer: field `{table_name}`.`{field}` is not registered in the physical FTS index"
311            ));
312        }
313        Ok(())
314    }
315
316    fn restore_analyzer_error(
317        table: &std::sync::Arc<super::TableState>,
318        field: &str,
319        index_analyzer: Arc<uqa_analysis::CompiledAnalyzer>,
320        search_analyzer: Arc<uqa_analysis::CompiledAnalyzer>,
321        rebuild: bool,
322        original: SQLError,
323    ) -> SQLError {
324        match Self::restore_field_analyzers(table, field, index_analyzer, search_analyzer, rebuild)
325        {
326            Ok(()) => original,
327            Err(cleanup) => SQLError::Routine {
328                sqlstate: original.sqlstate().unwrap_or("XX000").into(),
329                message: format!(
330                    "{original}; restoring the prior field analyzer also failed: {cleanup}"
331                ),
332            },
333        }
334    }
335
336    fn restore_field_analyzers(
337        table: &std::sync::Arc<super::TableState>,
338        field: &str,
339        index_analyzer: Arc<uqa_analysis::CompiledAnalyzer>,
340        search_analyzer: Arc<uqa_analysis::CompiledAnalyzer>,
341        rebuild: bool,
342    ) -> StorageBackendResult<()> {
343        let source = if rebuild {
344            Some(Self::fts_source(table, None)?)
345        } else {
346            None
347        };
348        let mut index = table.inverted_index.write();
349        if let Some(mut source) = source {
350            index.rebuild_with_analyzer_revision(
351                field,
352                index_analyzer,
353                AnalyzerPhase::Index,
354                &mut source,
355            )?;
356        } else {
357            index
358                .set_field_analyzer_revision(field, index_analyzer, AnalyzerPhase::Index)
359                .map_err(StorageBackendError::Other)?;
360        }
361        index
362            .set_field_analyzer_revision(field, search_analyzer, AnalyzerPhase::Search)
363            .map_err(StorageBackendError::Other)?;
364        Ok(())
365    }
366
367    pub fn table_field_analyzer(
368        &self,
369        table: &str,
370        field: &str,
371    ) -> Result<Option<(String, String)>, String> {
372        self.with_direct_query_snapshot(
373            true,
374            |engine| engine.table_field_analyzer_in_execution(table, field),
375            |error| error.to_string(),
376        )
377    }
378
379    pub(crate) fn table_field_analyzer_in_execution(
380        &self,
381        table: &str,
382        field: &str,
383    ) -> Result<Option<(String, String)>, String> {
384        self.synchronize_catalog_registries()
385            .map_err(|err| format!("refresh analyzer catalog: {err}"))?;
386        let Some(table) = self
387            .try_resolve_table_name(table)
388            .map_err(|err| format!("resolve table `{table}`: {err}"))?
389        else {
390            return Ok(None);
391        };
392        Ok(self
393            .durable
394            .table_field_analyzers
395            .read()
396            .get(&(table, field.to_string()))
397            .and_then(FieldAnalyzerBinding::last_assignment))
398    }
399
400    /// compatibility alias for [`Engine::register_named_analyzer`].
401    pub fn create_analyzer(
402        &self,
403        name: &str,
404        config_json: &str,
405    ) -> std::result::Result<(), String> {
406        self.register_named_analyzer(name, config_json)
407    }
408
409    /// compatibility alias for [`Engine::drop_named_analyzer`].
410    pub fn drop_analyzer(&self, name: &str) -> Result<bool, String> {
411        self.drop_named_analyzer(name)
412    }
413
414    /// compatibility alias for [`Engine::set_table_field_analyzer`].
415    pub fn set_table_analyzer(
416        &self,
417        table: &str,
418        field: &str,
419        analyzer_name: &str,
420        phase: &str,
421    ) -> std::result::Result<(), String> {
422        self.set_table_field_analyzer(table, field, analyzer_name, phase)
423    }
424
425    /// Resolve the analyzer assigned to `(table, field)` for the given
426    /// phase. `phase` is `"index"`, `"search"`, or `"both"`. Returns the
427    /// exact bound configuration JSON. `both` requires the same named revision on both sides; an unnamed default has no explicit assignment to return.
428    pub fn get_table_analyzer(
429        &self,
430        table: &str,
431        field: &str,
432        phase: &str,
433    ) -> Result<Option<String>, String> {
434        self.with_direct_query_snapshot(
435            true,
436            |engine| {
437                engine
438                    .synchronize_catalog_registries()
439                    .map_err(|error| error.to_string())?;
440                let (_, phase) = normalize_analyzer_phase(phase)?;
441                let Some(table) = engine
442                    .try_resolve_table_name(table)
443                    .map_err(|error| error.to_string())?
444                else {
445                    return Ok(None);
446                };
447                let Some(binding) = engine
448                    .durable
449                    .table_field_analyzers
450                    .read()
451                    .get(&(table, field.to_owned()))
452                    .cloned()
453                else {
454                    return Ok(None);
455                };
456                let side = match phase {
457                    AnalyzerPhase::Index => &binding.index,
458                    AnalyzerPhase::Search => &binding.search,
459                    AnalyzerPhase::Both
460                        if binding.index.name == binding.search.name
461                            && binding.index.compiled.descriptor().fingerprint()
462                                == binding.search.compiled.descriptor().fingerprint() =>
463                    {
464                        &binding.index
465                    }
466                    AnalyzerPhase::Both => return Ok(None),
467                };
468                if side.name.is_none() {
469                    return Ok(None);
470                }
471                let configuration = side
472                    .compiled
473                    .descriptor()
474                    .configuration()
475                    .map_err(|error| error.to_string())?;
476                serde_json::to_string(&configuration)
477                    .map(Some)
478                    .map_err(|error| error.to_string())
479            },
480            |error| error.to_string(),
481        )
482    }
483}