uqa_analysis/
token.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Structured analysis tokens and end-of-stream position state.
8
9use std::ops::Range;
10#[cfg(feature = "nori")]
11use std::sync::Arc;
12
13use serde::Serialize;
14#[cfg(feature = "nori")]
15use uqa_core::memory::Budgeted;
16
17#[cfg(test)]
18use crate::FilteredText;
19use crate::{AnalysisError, AnalysisResult, SourceOffsets, TokenTerm};
20
21pub(crate) mod allocation;
22
23#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
24pub struct AnalysisToken {
25    pub(crate) term: TokenTerm,
26    pub(crate) offsets: Option<SourceOffsets>,
27    pub(crate) position_increment: u32,
28    pub(crate) position_length: u32,
29    pub(crate) keyword: bool,
30    #[serde(skip_serializing_if = "Option::is_none")]
31    filtered_utf16: Option<Range<usize>>,
32    #[cfg(feature = "nori")]
33    #[serde(skip_serializing_if = "Option::is_none")]
34    korean_morphology: Option<crate::nori::KoreanMorphology>,
35    #[serde(skip)]
36    verbatim: bool,
37}
38
39impl AnalysisToken {
40    pub fn term(&self) -> &TokenTerm {
41        &self.term
42    }
43
44    pub fn offsets(&self) -> Option<&SourceOffsets> {
45        self.offsets.as_ref()
46    }
47
48    pub fn position_increment(&self) -> u32 {
49        self.position_increment
50    }
51
52    pub fn position_length(&self) -> u32 {
53        self.position_length
54    }
55
56    pub fn is_keyword(&self) -> bool {
57        self.keyword
58    }
59
60    /// Exact tokenizer coordinates before character-filter source correction, when supplied.
61    pub fn filtered_utf16(&self) -> Option<&Range<usize>> {
62        self.filtered_utf16.as_ref()
63    }
64
65    #[cfg(feature = "nori")]
66    pub fn korean_morphology(&self) -> Option<&crate::nori::KoreanMorphology> {
67        self.korean_morphology.as_ref()
68    }
69
70    #[cfg(test)]
71    pub(crate) fn from_source(
72        input: &FilteredText<'_>,
73        range: Range<usize>,
74    ) -> AnalysisResult<Self> {
75        let budget = uqa_core::memory::MemoryBudget::new(usize::MAX);
76        input.prepare_coordinates(&budget, &mut || Ok(()))?;
77        Ok(
78            Self::from_source_budgeted(input, range, &budget, &mut || Ok(()))?
79                .into_parts()
80                .0,
81        )
82    }
83
84    fn term_only(term: String) -> Self {
85        Self {
86            term: term.into(),
87            offsets: None,
88            position_increment: 1,
89            position_length: 1,
90            keyword: false,
91            filtered_utf16: None,
92            #[cfg(feature = "nori")]
93            korean_morphology: None,
94            verbatim: false,
95        }
96    }
97
98    #[cfg(test)]
99    pub(crate) fn replace_term(&mut self, term: TokenTerm) {
100        if term != self.term {
101            self.verbatim = false;
102            self.term = term;
103        }
104    }
105
106    #[cfg(test)]
107    pub(crate) fn substring(&self, range: Range<usize>) -> Self {
108        let mut token = Self {
109            term: self.term.substring(range.clone()),
110            offsets: self.offsets.clone(),
111            position_increment: self.position_increment,
112            position_length: self.position_length,
113            keyword: self.keyword,
114            filtered_utf16: self.filtered_utf16.clone(),
115            #[cfg(feature = "nori")]
116            korean_morphology: self.korean_morphology.clone(),
117            verbatim: self.verbatim,
118        };
119        if self.verbatim {
120            if let Some(offsets) = &self.offsets {
121                let original = self.term.as_str().expect("verbatim Unicode input");
122                let start_utf16 = original[..range.start].encode_utf16().count();
123                let length_utf16 = token.term.utf16_len();
124                token.offsets = Some(SourceOffsets {
125                    utf8: offsets.utf8.start + range.start..offsets.utf8.start + range.end,
126                    utf16: offsets.utf16.start + start_utf16
127                        ..offsets.utf16.start + start_utf16 + length_utf16,
128                });
129                if let Some(filtered) = &self.filtered_utf16 {
130                    if filtered.len() == self.term.utf16_len() {
131                        token.filtered_utf16 = Some(
132                            filtered.start + start_utf16
133                                ..filtered.start + start_utf16 + length_utf16,
134                        );
135                    }
136                }
137            }
138        }
139        token
140    }
141}
142
143/// A complete analyzed input with explicit token graph and source end state.
144#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
145pub struct AnalyzedText {
146    #[serde(flatten)]
147    pub(crate) batch: TokenBatch,
148    pub(crate) final_offsets: SourceOffsets,
149    #[cfg(feature = "nori")]
150    #[serde(skip)]
151    pub(crate) projection: Arc<Budgeted<crate::source::SourceProjection>>,
152}
153
154impl AnalyzedText {
155    pub fn tokens(&self) -> &[AnalysisToken] {
156        &self.batch.tokens
157    }
158
159    pub fn into_tokens(self) -> Vec<AnalysisToken> {
160        self.batch.tokens
161    }
162
163    pub fn into_terms(self) -> AnalysisResult<Vec<String>> {
164        self.batch.into_terms()
165    }
166
167    pub fn final_offsets(&self) -> &SourceOffsets {
168        &self.final_offsets
169    }
170
171    pub fn final_position_increment(&self) -> u32 {
172        self.batch.final_position_increment
173    }
174
175    #[cfg(test)]
176    pub(crate) fn from_source(
177        tokens: Vec<AnalysisToken>,
178        input: &FilteredText<'_>,
179    ) -> AnalysisResult<Self> {
180        let batch = TokenBatch {
181            tokens,
182            final_position_increment: 0,
183            terminal: None,
184        };
185        batch.validate_positions()?;
186        Ok(Self {
187            batch,
188            final_offsets: input.final_offsets(),
189            #[cfg(feature = "nori")]
190            projection: input.projection(),
191        })
192    }
193}
194
195#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
196pub(crate) struct TokenBatch<T = AnalysisToken> {
197    pub tokens: Vec<T>,
198    pub final_position_increment: u32,
199    #[serde(skip)]
200    pub terminal: Option<Box<T>>,
201}
202
203impl TokenBatch {
204    pub fn from_terms(terms: Vec<String>) -> Self {
205        Self {
206            tokens: terms.into_iter().map(AnalysisToken::term_only).collect(),
207            final_position_increment: 0,
208            terminal: None,
209        }
210    }
211
212    pub fn into_terms(self) -> AnalysisResult<Vec<String>> {
213        self.tokens
214            .into_iter()
215            .map(|token| token.term.into_string())
216            .collect()
217    }
218
219    #[cfg(any(test, feature = "nori"))]
220    pub fn validate_positions(&self) -> AnalysisResult<()> {
221        self.validate_positions_with_control(&mut || Ok(()))
222    }
223
224    pub(crate) fn validate_positions_with_control(
225        &self,
226        poll: &mut dyn FnMut() -> AnalysisResult<()>,
227    ) -> AnalysisResult<()> {
228        let mut position = -1_i64;
229        for (index, token) in self.tokens.iter().enumerate() {
230            if index % 1024 == 0 {
231                poll()?;
232            }
233            if token.position_length == 0 || (position < 0 && token.position_increment == 0) {
234                return Err(AnalysisError::InvalidTokenPosition);
235            }
236            position = position
237                .checked_add(i64::from(token.position_increment))
238                .ok_or(AnalysisError::TokenPositionOverflow)?;
239            let position =
240                u32::try_from(position).map_err(|_| AnalysisError::TokenPositionOverflow)?;
241            position
242                .checked_add(token.position_length)
243                .ok_or(AnalysisError::TokenPositionOverflow)?;
244        }
245        let final_position = position
246            .checked_add(i64::from(self.final_position_increment))
247            .ok_or(AnalysisError::TokenPositionOverflow)?;
248        if final_position > i64::from(u32::MAX) {
249            return Err(AnalysisError::TokenPositionOverflow);
250        }
251        Ok(())
252    }
253}
254
255#[cfg(test)]
256mod tests;
257
258#[cfg(feature = "nori")]
259mod korean;
uqa_analysis/token.rs

uqa_analysis/
token.rs
