use std::ops::Range;
#[cfg(any(feature = "nori", feature = "kuromoji"))]
use std::sync::Arc;
use serde::Serialize;
#[cfg(any(feature = "nori", feature = "kuromoji"))]
use uqa_core::memory::Budgeted;
#[cfg(test)]
use crate::FilteredText;
use crate::{AnalysisError, AnalysisResult, SourceOffsets, TokenTerm};
pub(crate) mod allocation;
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct AnalysisToken {
pub(crate) term: TokenTerm,
pub(crate) offsets: Option<SourceOffsets>,
pub(crate) position_increment: u32,
pub(crate) position_length: u32,
pub(crate) keyword: bool,
#[serde(skip_serializing_if = "Option::is_none")]
filtered_utf16: Option<Range<usize>>,
#[cfg(any(feature = "nori", feature = "kuromoji"))]
#[serde(flatten, skip_serializing_if = "Option::is_none")]
morphology: Option<Morphology>,
#[serde(skip)]
verbatim: bool,
}
impl AnalysisToken {
pub fn term(&self) -> &TokenTerm {
&self.term
}
pub fn offsets(&self) -> Option<&SourceOffsets> {
self.offsets.as_ref()
}
pub fn position_increment(&self) -> u32 {
self.position_increment
}
pub fn position_length(&self) -> u32 {
self.position_length
}
pub fn is_keyword(&self) -> bool {
self.keyword
}
pub fn filtered_utf16(&self) -> Option<&Range<usize>> {
self.filtered_utf16.as_ref()
}
#[cfg(feature = "nori")]
pub fn korean_morphology(&self) -> Option<&crate::nori::KoreanMorphology> {
match self.morphology.as_ref() {
Some(Morphology::Korean(value)) => Some(value),
_ => None,
}
}
#[cfg(feature = "kuromoji")]
pub fn japanese_morphology(&self) -> Option<&crate::kuromoji::JapaneseMorphology> {
match self.morphology.as_ref() {
Some(Morphology::Japanese(value)) => Some(value),
_ => None,
}
}
#[cfg(test)]
pub(crate) fn from_source(
input: &FilteredText<'_>,
range: Range<usize>,
) -> AnalysisResult<Self> {
let budget = uqa_core::memory::MemoryBudget::new(usize::MAX);
input.prepare_coordinates(&budget, &mut || Ok(()))?;
Ok(
Self::from_source_budgeted(input, range, &budget, &mut || Ok(()))?
.into_parts()
.0,
)
}
fn term_only(term: String) -> Self {
Self {
term: term.into(),
offsets: None,
position_increment: 1,
position_length: 1,
keyword: false,
filtered_utf16: None,
#[cfg(any(feature = "nori", feature = "kuromoji"))]
morphology: None,
verbatim: false,
}
}
#[cfg(test)]
pub(crate) fn replace_term(&mut self, term: TokenTerm) {
if term != self.term {
self.verbatim = false;
self.term = term;
}
}
#[cfg(test)]
pub(crate) fn substring(&self, range: Range<usize>) -> Self {
let mut token = Self {
term: self.term.substring(range.clone()),
offsets: self.offsets.clone(),
position_increment: self.position_increment,
position_length: self.position_length,
keyword: self.keyword,
filtered_utf16: self.filtered_utf16.clone(),
#[cfg(any(feature = "nori", feature = "kuromoji"))]
morphology: self.morphology.clone(),
verbatim: self.verbatim,
};
if self.verbatim {
if let Some(offsets) = &self.offsets {
let original = self.term.as_str().expect("verbatim Unicode input");
let start_utf16 = original[..range.start].encode_utf16().count();
let length_utf16 = token.term.utf16_len();
token.offsets = Some(SourceOffsets {
utf8: offsets.utf8.start + range.start..offsets.utf8.start + range.end,
utf16: offsets.utf16.start + start_utf16
..offsets.utf16.start + start_utf16 + length_utf16,
});
if let Some(filtered) = &self.filtered_utf16 {
if filtered.len() == self.term.utf16_len() {
token.filtered_utf16 = Some(
filtered.start + start_utf16
..filtered.start + start_utf16 + length_utf16,
);
}
}
}
}
token
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct AnalyzedText {
#[serde(flatten)]
pub(crate) batch: TokenBatch,
pub(crate) final_offsets: SourceOffsets,
#[cfg(any(feature = "nori", feature = "kuromoji"))]
#[serde(skip)]
pub(crate) projection: Arc<Budgeted<crate::source::SourceProjection>>,
}
impl AnalyzedText {
pub fn tokens(&self) -> &[AnalysisToken] {
&self.batch.tokens
}
pub fn into_tokens(self) -> Vec<AnalysisToken> {
self.batch.tokens
}
pub fn into_terms(self) -> AnalysisResult<Vec<String>> {
self.batch.into_terms()
}
pub fn final_offsets(&self) -> &SourceOffsets {
&self.final_offsets
}
pub fn final_position_increment(&self) -> u32 {
self.batch.final_position_increment
}
#[cfg(test)]
pub(crate) fn from_source(
tokens: Vec<AnalysisToken>,
input: &FilteredText<'_>,
) -> AnalysisResult<Self> {
let batch = TokenBatch {
tokens,
final_position_increment: 0,
terminal: None,
};
batch.validate_positions()?;
Ok(Self {
batch,
final_offsets: input.final_offsets(),
#[cfg(any(feature = "nori", feature = "kuromoji"))]
projection: input.projection(),
})
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct TokenBatch<T = AnalysisToken> {
pub tokens: Vec<T>,
pub final_position_increment: u32,
#[serde(skip)]
pub terminal: Option<Box<T>>,
}
impl TokenBatch {
pub fn from_terms(terms: Vec<String>) -> Self {
Self {
tokens: terms.into_iter().map(AnalysisToken::term_only).collect(),
final_position_increment: 0,
terminal: None,
}
}
pub fn into_terms(self) -> AnalysisResult<Vec<String>> {
self.tokens
.into_iter()
.map(|token| token.term.into_string())
.collect()
}
#[cfg(any(test, feature = "nori", feature = "kuromoji"))]
pub fn validate_positions(&self) -> AnalysisResult<()> {
self.validate_positions_with_control(&mut || Ok(()))
}
pub(crate) fn validate_positions_with_control(
&self,
poll: &mut dyn FnMut() -> AnalysisResult<()>,
) -> AnalysisResult<()> {
let mut position = -1_i64;
for (index, token) in self.tokens.iter().enumerate() {
if index % 1024 == 0 {
poll()?;
}
if token.position_length == 0 || (position < 0 && token.position_increment == 0) {
return Err(AnalysisError::InvalidTokenPosition);
}
position = position
.checked_add(i64::from(token.position_increment))
.ok_or(AnalysisError::TokenPositionOverflow)?;
let position =
u32::try_from(position).map_err(|_| AnalysisError::TokenPositionOverflow)?;
position
.checked_add(token.position_length)
.ok_or(AnalysisError::TokenPositionOverflow)?;
}
let final_position = position
.checked_add(i64::from(self.final_position_increment))
.ok_or(AnalysisError::TokenPositionOverflow)?;
if final_position > i64::from(u32::MAX) {
return Err(AnalysisError::TokenPositionOverflow);
}
Ok(())
}
}
#[cfg(test)]
mod tests;
#[cfg(feature = "nori")]
mod korean;
#[cfg(any(feature = "nori", feature = "kuromoji"))]
mod morphology;
#[cfg(any(feature = "nori", feature = "kuromoji"))]
use morphology::Morphology;
#[cfg(any(feature = "nori", feature = "kuromoji"))]
mod native;
#[cfg(feature = "kuromoji")]
mod japanese;
#[cfg(any(feature = "nori", feature = "kuromoji"))]
mod filter;