uqa_storage/inverted_index/
analysis.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Shared staging of immutable analysis results before any provider mutation.
8
9use std::collections::BTreeMap;
10
11use uqa_analysis::{
12    AnalysisError, AnalyzerFingerprint, CompiledAnalyzer, SourceOffsets, TokenLengthPolicy,
13};
14use uqa_core::{TokenOccurrence, TokenOffsets};
15
16use crate::{StorageBackendError, StorageBackendResult, TokenTermKey};
17
18/// Metadata published with a complete document field's occurrences. The field's retained index revision owns the matching descriptor and resources.
19#[derive(Debug, Clone, Copy, PartialEq, Eq)]
20pub struct IndexedFieldMetadata {
21    pub analyzer_fingerprint: AnalyzerFingerprint,
22    pub occurrence_format_version: u8,
23    pub length_policy: TokenLengthPolicy,
24    pub length: u64,
25    pub final_offsets: TokenOffsets,
26    pub final_position_increment: u32,
27}
28
29impl IndexedFieldMetadata {
30    pub fn new(analyzer: &CompiledAnalyzer, field: &AnalyzedField) -> Self {
31        Self {
32            analyzer_fingerprint: analyzer.descriptor().fingerprint(),
33            occurrence_format_version: crate::clustered_postings::OCCURRENCE_FORMAT_VERSION,
34            length_policy: analyzer.descriptor().length_policy(),
35            length: field.length,
36            final_offsets: field.final_offsets,
37            final_position_increment: field.final_position_increment,
38        }
39    }
40}
41
42#[derive(Debug, Clone, PartialEq, Eq)]
43pub struct AnalyzedField {
44    pub length: u64,
45    pub terms: BTreeMap<TokenTermKey, Vec<TokenOccurrence>>,
46    pub final_offsets: TokenOffsets,
47    pub final_position_increment: u32,
48}
49
50/// Analyze one complete source field with a resolved revision, retaining every emitted graph edge.
51///
52/// ```
53/// use uqa_analysis::{Analyzer, AnalyzerLimits, AnalyzerResources, TokenLengthPolicy};
54/// use uqa_storage::{inverted_index::analyze_index_field, TokenTermKey};
55/// let config: Analyzer = serde_json::from_str(r#"{"tokenizer":{"type":"whitespace"},"token_filters":[{"type":"synonym","synonyms":{"a":["a","a"]}}]}"#)?;
56/// let compiled = AnalyzerResources::new(AnalyzerLimits::default())
57///     .compile_with_length_policy(&config, TokenLengthPolicy::DiscountOverlaps)?;
58/// let field = analyze_index_field(&compiled, "a")?;
59/// assert_eq!(field.length, 1);
60/// assert_eq!(field.terms[&TokenTermKey::from_text("a")].len(), 3);
61/// # Ok::<(), Box<dyn std::error::Error>>(())
62/// ```
63pub fn analyze_index_field(
64    analyzer: &CompiledAnalyzer,
65    text: &str,
66) -> StorageBackendResult<AnalyzedField> {
67    analyze_index_field_with_poll(analyzer, text, || Ok(()))
68}
69
70/// Analyze and project one source field while observing the caller's cancellation token.
71pub fn analyze_index_field_cancellable(
72    analyzer: &CompiledAnalyzer,
73    text: &str,
74    cancellation: &uqa_core::CancellationToken,
75) -> StorageBackendResult<AnalyzedField> {
76    analyze_index_field_with_poll(analyzer, text, || {
77        cancellation.check().map_err(|_| AnalysisError::Cancelled)
78    })
79    .map_err(|error| match error {
80        StorageBackendError::Analysis(AnalysisError::Cancelled) => {
81            StorageBackendError::Cancelled(uqa_core::QueryCancelled)
82        }
83        other => other,
84    })
85}
86
87fn analyze_index_field_with_poll(
88    analyzer: &CompiledAnalyzer,
89    text: &str,
90    mut poll: impl FnMut() -> Result<(), AnalysisError>,
91) -> StorageBackendResult<AnalyzedField> {
92    let output = analyzer
93        .analyze_tokens_budgeted(
94            text,
95            &uqa_core::memory::MemoryBudget::new(usize::MAX),
96            &mut poll,
97        )?
98        .into_parts()
99        .0;
100    let mut staged = AnalyzedField {
101        length: 0,
102        terms: BTreeMap::new(),
103        final_offsets: source_offsets(output.final_offsets())?,
104        final_position_increment: output.final_position_increment(),
105    };
106    let mut position = -1_i64;
107    for token in output.tokens() {
108        poll()?;
109        position = position
110            .checked_add(i64::from(token.position_increment()))
111            .ok_or(AnalysisError::TokenPositionOverflow)?;
112        let position = u32::try_from(position).map_err(|_| AnalysisError::TokenPositionOverflow)?;
113        let offsets = token.offsets().map(source_offsets).transpose()?;
114        if offsets.is_some_and(|offsets| {
115            offsets.end_utf8 > staged.final_offsets.end_utf8
116                || offsets.end_utf16 > staged.final_offsets.end_utf16
117        }) {
118            return Err(StorageBackendError::Other(
119                "token occurrence exceeds original source".into(),
120            ));
121        }
122        let occurrence = TokenOccurrence {
123            position,
124            position_length: token.position_length(),
125            offsets,
126        };
127        occurrence
128            .validate()
129            .map_err(|error| StorageBackendError::Other(error.to_string()))?;
130        staged
131            .terms
132            .entry(TokenTermKey::from_term(token.term()))
133            .or_default()
134            .push(occurrence);
135        if analyzer.descriptor().length_policy() == TokenLengthPolicy::EmittedTokens
136            || token.position_increment() > 0
137        {
138            staged.length = staged
139                .length
140                .checked_add(1)
141                .ok_or_else(|| super::counter_error("document token count"))?;
142        }
143    }
144    poll()?;
145    Ok(staged)
146}
147
148mod query;
149pub use query::{
150    analyze_query_graph, analyze_query_graph_budgeted, analyze_query_terms,
151    analyze_query_terms_budgeted,
152};
153
154fn source_offsets(offsets: &SourceOffsets) -> StorageBackendResult<TokenOffsets> {
155    fn offset(value: usize) -> StorageBackendResult<u64> {
156        u64::try_from(value).map_err(|_| super::counter_error("source offset"))
157    }
158    Ok(TokenOffsets {
159        start_utf8: offset(offsets.utf8.start)?,
160        end_utf8: offset(offsets.utf8.end)?,
161        start_utf16: offset(offsets.utf16.start)?,
162        end_utf16: offset(offsets.utf16.end)?,
163    })
164}
165
166#[cfg(test)]
167#[path = "analysis/index_tests.rs"]
168mod index_tests;
uqa_storage/inverted_index/analysis.rs

uqa_storage/inverted_index/
analysis.rs
