uqa_analysis/
token.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Structured analysis tokens and end-of-stream position state.
8
9use std::ops::Range;
10#[cfg(any(feature = "nori", feature = "kuromoji"))]
11use std::sync::Arc;
12
13use serde::Serialize;
14#[cfg(any(feature = "nori", feature = "kuromoji"))]
15use uqa_core::memory::Budgeted;
16
17#[cfg(test)]
18use crate::FilteredText;
19use crate::{AnalysisError, AnalysisResult, SourceOffsets, TokenTerm};
20
21pub(crate) mod allocation;
22
23#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
24pub struct AnalysisToken {
25    pub(crate) term: TokenTerm,
26    pub(crate) offsets: Option<SourceOffsets>,
27    pub(crate) position_increment: u32,
28    pub(crate) position_length: u32,
29    pub(crate) keyword: bool,
30    #[serde(skip_serializing_if = "Option::is_none")]
31    filtered_utf16: Option<Range<usize>>,
32    #[cfg(any(feature = "nori", feature = "kuromoji"))]
33    #[serde(flatten, skip_serializing_if = "Option::is_none")]
34    morphology: Option<Morphology>,
35    #[serde(skip)]
36    verbatim: bool,
37}
38
39impl AnalysisToken {
40    pub fn term(&self) -> &TokenTerm {
41        &self.term
42    }
43
44    pub fn offsets(&self) -> Option<&SourceOffsets> {
45        self.offsets.as_ref()
46    }
47
48    pub fn position_increment(&self) -> u32 {
49        self.position_increment
50    }
51
52    pub fn position_length(&self) -> u32 {
53        self.position_length
54    }
55
56    pub fn is_keyword(&self) -> bool {
57        self.keyword
58    }
59
60    /// Exact tokenizer coordinates before character-filter source correction, when supplied.
61    pub fn filtered_utf16(&self) -> Option<&Range<usize>> {
62        self.filtered_utf16.as_ref()
63    }
64
65    #[cfg(feature = "nori")]
66    pub fn korean_morphology(&self) -> Option<&crate::nori::KoreanMorphology> {
67        match self.morphology.as_ref() {
68            Some(Morphology::Korean(value)) => Some(value),
69            _ => None,
70        }
71    }
72
73    #[cfg(feature = "kuromoji")]
74    pub fn japanese_morphology(&self) -> Option<&crate::kuromoji::JapaneseMorphology> {
75        match self.morphology.as_ref() {
76            Some(Morphology::Japanese(value)) => Some(value),
77            _ => None,
78        }
79    }
80
81    #[cfg(test)]
82    pub(crate) fn from_source(
83        input: &FilteredText<'_>,
84        range: Range<usize>,
85    ) -> AnalysisResult<Self> {
86        let budget = uqa_core::memory::MemoryBudget::new(usize::MAX);
87        input.prepare_coordinates(&budget, &mut || Ok(()))?;
88        Ok(
89            Self::from_source_budgeted(input, range, &budget, &mut || Ok(()))?
90                .into_parts()
91                .0,
92        )
93    }
94
95    fn term_only(term: String) -> Self {
96        Self {
97            term: term.into(),
98            offsets: None,
99            position_increment: 1,
100            position_length: 1,
101            keyword: false,
102            filtered_utf16: None,
103            #[cfg(any(feature = "nori", feature = "kuromoji"))]
104            morphology: None,
105            verbatim: false,
106        }
107    }
108
109    #[cfg(test)]
110    pub(crate) fn replace_term(&mut self, term: TokenTerm) {
111        if term != self.term {
112            self.verbatim = false;
113            self.term = term;
114        }
115    }
116
117    #[cfg(test)]
118    pub(crate) fn substring(&self, range: Range<usize>) -> Self {
119        let mut token = Self {
120            term: self.term.substring(range.clone()),
121            offsets: self.offsets.clone(),
122            position_increment: self.position_increment,
123            position_length: self.position_length,
124            keyword: self.keyword,
125            filtered_utf16: self.filtered_utf16.clone(),
126            #[cfg(any(feature = "nori", feature = "kuromoji"))]
127            morphology: self.morphology.clone(),
128            verbatim: self.verbatim,
129        };
130        if self.verbatim {
131            if let Some(offsets) = &self.offsets {
132                let original = self.term.as_str().expect("verbatim Unicode input");
133                let start_utf16 = original[..range.start].encode_utf16().count();
134                let length_utf16 = token.term.utf16_len();
135                token.offsets = Some(SourceOffsets {
136                    utf8: offsets.utf8.start + range.start..offsets.utf8.start + range.end,
137                    utf16: offsets.utf16.start + start_utf16
138                        ..offsets.utf16.start + start_utf16 + length_utf16,
139                });
140                if let Some(filtered) = &self.filtered_utf16 {
141                    if filtered.len() == self.term.utf16_len() {
142                        token.filtered_utf16 = Some(
143                            filtered.start + start_utf16
144                                ..filtered.start + start_utf16 + length_utf16,
145                        );
146                    }
147                }
148            }
149        }
150        token
151    }
152}
153
154/// A complete analyzed input with explicit token graph and source end state.
155#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
156pub struct AnalyzedText {
157    #[serde(flatten)]
158    pub(crate) batch: TokenBatch,
159    pub(crate) final_offsets: SourceOffsets,
160    #[cfg(any(feature = "nori", feature = "kuromoji"))]
161    #[serde(skip)]
162    pub(crate) projection: Arc<Budgeted<crate::source::SourceProjection>>,
163}
164
165impl AnalyzedText {
166    pub fn tokens(&self) -> &[AnalysisToken] {
167        &self.batch.tokens
168    }
169
170    pub fn into_tokens(self) -> Vec<AnalysisToken> {
171        self.batch.tokens
172    }
173
174    pub fn into_terms(self) -> AnalysisResult<Vec<String>> {
175        self.batch.into_terms()
176    }
177
178    pub fn final_offsets(&self) -> &SourceOffsets {
179        &self.final_offsets
180    }
181
182    pub fn final_position_increment(&self) -> u32 {
183        self.batch.final_position_increment
184    }
185
186    #[cfg(test)]
187    pub(crate) fn from_source(
188        tokens: Vec<AnalysisToken>,
189        input: &FilteredText<'_>,
190    ) -> AnalysisResult<Self> {
191        let batch = TokenBatch {
192            tokens,
193            final_position_increment: 0,
194            terminal: None,
195        };
196        batch.validate_positions()?;
197        Ok(Self {
198            batch,
199            final_offsets: input.final_offsets(),
200            #[cfg(any(feature = "nori", feature = "kuromoji"))]
201            projection: input.projection(),
202        })
203    }
204}
205
206#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
207pub(crate) struct TokenBatch<T = AnalysisToken> {
208    pub tokens: Vec<T>,
209    pub final_position_increment: u32,
210    #[serde(skip)]
211    pub terminal: Option<Box<T>>,
212}
213
214impl TokenBatch {
215    pub fn from_terms(terms: Vec<String>) -> Self {
216        Self {
217            tokens: terms.into_iter().map(AnalysisToken::term_only).collect(),
218            final_position_increment: 0,
219            terminal: None,
220        }
221    }
222
223    pub fn into_terms(self) -> AnalysisResult<Vec<String>> {
224        self.tokens
225            .into_iter()
226            .map(|token| token.term.into_string())
227            .collect()
228    }
229
230    #[cfg(any(test, feature = "nori", feature = "kuromoji"))]
231    pub fn validate_positions(&self) -> AnalysisResult<()> {
232        self.validate_positions_with_control(&mut || Ok(()))
233    }
234
235    pub(crate) fn validate_positions_with_control(
236        &self,
237        poll: &mut dyn FnMut() -> AnalysisResult<()>,
238    ) -> AnalysisResult<()> {
239        let mut position = -1_i64;
240        for (index, token) in self.tokens.iter().enumerate() {
241            if index % 1024 == 0 {
242                poll()?;
243            }
244            if token.position_length == 0 || (position < 0 && token.position_increment == 0) {
245                return Err(AnalysisError::InvalidTokenPosition);
246            }
247            position = position
248                .checked_add(i64::from(token.position_increment))
249                .ok_or(AnalysisError::TokenPositionOverflow)?;
250            let position =
251                u32::try_from(position).map_err(|_| AnalysisError::TokenPositionOverflow)?;
252            position
253                .checked_add(token.position_length)
254                .ok_or(AnalysisError::TokenPositionOverflow)?;
255        }
256        let final_position = position
257            .checked_add(i64::from(self.final_position_increment))
258            .ok_or(AnalysisError::TokenPositionOverflow)?;
259        if final_position > i64::from(u32::MAX) {
260            return Err(AnalysisError::TokenPositionOverflow);
261        }
262        Ok(())
263    }
264}
265
266#[cfg(test)]
267mod tests;
268
269#[cfg(feature = "nori")]
270mod korean;
271
272#[cfg(any(feature = "nori", feature = "kuromoji"))]
273mod morphology;
274#[cfg(any(feature = "nori", feature = "kuromoji"))]
275use morphology::Morphology;
276
277#[cfg(any(feature = "nori", feature = "kuromoji"))]
278mod native;
279
280#[cfg(feature = "kuromoji")]
281mod japanese;
282
283#[cfg(any(feature = "nori", feature = "kuromoji"))]
284mod filter;
uqa_analysis/token.rs

uqa_analysis/
token.rs
