uqa-analysis 0.3.5

//
// Unified Query Algebra
//
// Copyright (c) 2023-2026 Cognica, Inc.
//

//! Composable text analysis pipeline.
//!
//! ```text
//! text -> CharFilter* -> Tokenizer -> TokenFilter* -> tokens
//! ```

use serde::{Deserialize, Serialize};

use crate::char_filter::CharFilter;
use crate::error::AnalysisResult;
use crate::token_filter::TokenFilter;
use crate::tokenizer::Tokenizer;

mod compiled;
mod diagnostics;
pub use compiled::CompiledAnalyzer;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Analyzer {
    #[serde(default = "default_tokenizer")]
    pub tokenizer: Tokenizer,
    #[serde(default)]
    pub token_filters: Vec<TokenFilter>,
    #[serde(default)]
    pub char_filters: Vec<CharFilter>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub normalization: Option<crate::NormalizationConfig>,
}

fn default_tokenizer() -> Tokenizer {
    Tokenizer::Whitespace
}

impl Default for Analyzer {
    fn default() -> Self {
        Self {
            tokenizer: default_tokenizer(),
            token_filters: Vec::new(),
            char_filters: Vec::new(),
            normalization: None,
        }
    }
}

impl Analyzer {
    /// Identify components requiring immutable morphology and token-graph revisions.
    pub fn uses_morphology_stages(&self) -> bool {
        self.uses_korean_stages() || self.uses_japanese_stages()
    }

    /// Identify Japanese token stages independently of optional normalization profiles.
    pub fn uses_japanese_stages(&self) -> bool {
        #[cfg(feature = "kuromoji")]
        {
            matches!(self.tokenizer, Tokenizer::Kuromoji(_))
                || self
                    .token_filters
                    .iter()
                    .any(crate::kuromoji::pipeline::is_japanese_filter)
        }
        #[cfg(not(feature = "kuromoji"))]
        {
            false
        }
    }

    /// Identify Korean components for storage-format and catalog capability preflight.
    pub fn uses_korean_stages(&self) -> bool {
        #[cfg(feature = "nori")]
        {
            matches!(self.tokenizer, Tokenizer::Nori(_))
                || self.token_filters.iter().any(|filter| match filter {
                    TokenFilter::NoriPartOfSpeech(_)
                    | TokenFilter::NoriReadingForm(_)
                    | TokenFilter::NoriNumber(_) => true,
                    TokenFilter::UnicodeSimpleLowercase(config) => {
                        config.unicode_profile.nori_dictionary().is_some()
                    }
                    _ => false,
                })
        }
        #[cfg(not(feature = "nori"))]
        {
            false
        }
    }

    pub fn new(
        tokenizer: Tokenizer,
        token_filters: Vec<TokenFilter>,
        char_filters: Vec<CharFilter>,
    ) -> Self {
        Self {
            tokenizer,
            token_filters,
            char_filters,
            normalization: None,
        }
    }

    /// Select normalization independently of tokenization and analysis filters.
    pub fn with_normalization(mut self, normalization: crate::NormalizationConfig) -> Self {
        self.normalization = Some(normalization);
        self
    }

    pub fn analyze(&self, text: &str) -> AnalysisResult<Vec<String>> {
        self.analyze_tokens(text)?.into_terms()
    }

    /// Analyze complete input without discarding token graph or original source metadata.
    ///
    /// ```
    /// use uqa_analysis::standard_analyzer;
    ///
    /// let analyzed = standard_analyzer("english").analyze_tokens("The cats and")?;
    /// let token = &analyzed.tokens()[0];
    /// assert_eq!(token.term(), "cat");
    /// assert_eq!(token.offsets().unwrap().utf8, 4..8);
    /// assert_eq!(token.position_increment(), 2);
    /// assert_eq!(analyzed.final_position_increment(), 1);
    /// assert_eq!(analyzed.final_offsets().utf8, 12..12);
    /// # Ok::<(), uqa_analysis::AnalysisError>(())
    /// ```
    pub fn analyze_tokens(&self, text: &str) -> AnalysisResult<crate::AnalyzedText> {
        Ok(self
            .analyze_tokens_budgeted(
                text,
                &uqa_core::memory::MemoryBudget::new(usize::MAX),
                || Ok(()),
            )?
            .into_parts()
            .0)
    }

    /// Analyze with one runtime allowance while retaining the uncompiled API's resource reload behavior.
    ///
    /// Filter preparation and caller configuration have separate ownership. Use a compiled analyzer to resolve immutable resources before execution and reuse them across calls.
    pub fn analyze_tokens_budgeted(
        &self,
        text: &str,
        budget: &uqa_core::memory::MemoryBudget,
        mut poll: impl FnMut() -> AnalysisResult<()>,
    ) -> AnalysisResult<uqa_core::memory::Budgeted<crate::AnalyzedText>> {
        poll()?;
        let mut filtered = crate::FilteredText::new(text);
        for filter in &self.char_filters {
            filtered = filter
                .prepare()?
                .filter_mapped_budgeted(filtered, budget, &mut poll)?;
        }
        let mut tokens = self
            .tokenizer
            .prepare()?
            .tokenize_mapped_for_filters_budgeted(&filtered, budget, &mut poll)?;
        drop(filtered);
        for filter in &self.token_filters {
            poll()?;
            tokens = filter
                .prepare()?
                .filter_analyzed_budgeted(tokens, &mut poll)?;
        }
        #[cfg(feature = "kuromoji")]
        if self.uses_japanese_stages() {
            tokens.validate_japanese_attributes(&mut poll)?;
        }
        poll()?;
        Ok(tokens)
    }

    /// Validate every configured stage without consuming input.
    ///
    /// Registration paths should call this to fail before persistence. Each
    /// execution method remains independently fallible because analyzer values
    /// can come from legacy persisted data and external synonym files can
    /// become unreadable after validation.
    pub fn validate(&self) -> AnalysisResult<()> {
        #[cfg(any(feature = "nori", feature = "kuromoji"))]
        if let Some(normalization) = &self.normalization {
            normalization.validate()?;
        }
        for char_filter in &self.char_filters {
            char_filter.validate()?;
        }
        self.tokenizer.validate()?;
        for token_filter in &self.token_filters {
            token_filter.validate()?;
        }
        Ok(())
    }
}

/// `WhitespaceTokenizer` + `Lowercase`.
pub fn whitespace_analyzer() -> Analyzer {
    Analyzer::new(
        Tokenizer::Whitespace,
        vec![TokenFilter::Lowercase],
        Vec::new(),
    )
}

/// `Standard` + `Lowercase` + `ASCIIFolding` + `Stop` + `PorterStem`.
pub fn standard_analyzer(language: &str) -> Analyzer {
    Analyzer::new(
        Tokenizer::Standard,
        vec![
            TokenFilter::Lowercase,
            TokenFilter::ASCIIFolding,
            TokenFilter::Stop {
                language: language.to_string(),
                custom_words: Vec::new(),
            },
            TokenFilter::PorterStem,
        ],
        Vec::new(),
    )
}

/// `standard_analyzer` extended with character-level n-grams (2..=3) for
/// CJK-style text where words are not whitespace-delimited.
pub fn standard_cjk_analyzer(language: &str) -> Analyzer {
    Analyzer::new(
        Tokenizer::Standard,
        vec![
            TokenFilter::Lowercase,
            TokenFilter::ASCIIFolding,
            TokenFilter::Stop {
                language: language.to_string(),
                custom_words: Vec::new(),
            },
            TokenFilter::PorterStem,
            TokenFilter::Ngram {
                min_gram: 2,
                max_gram: 3,
                keep_short: true,
            },
        ],
        Vec::new(),
    )
}

/// `Keyword` tokenizer with no filters.
pub fn keyword_analyzer() -> Analyzer {
    Analyzer::new(Tokenizer::Keyword, Vec::new(), Vec::new())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn standard_pipeline_lowers_stops_and_stems() {
        let a = standard_analyzer("english");
        // "The Running" -> standard tokens ["The", "Running"]
        // -> lowercase ["the", "running"]
        // -> ascii_fold same
        // -> stop ["running"]
        // -> porter_stem ["run"]
        assert_eq!(a.analyze("The Running").unwrap(), vec!["run"]);
    }

    #[test]
    fn whitespace_pipeline_just_lowers() {
        let a = whitespace_analyzer();
        assert_eq!(a.analyze("Hello WORLD").unwrap(), vec!["hello", "world"]);
    }

    #[test]
    fn keyword_pipeline_emits_whole_input() {
        let a = keyword_analyzer();
        assert_eq!(
            a.analyze("the quick brown").unwrap(),
            vec!["the quick brown"]
        );
    }

    #[test]
    fn round_trips_via_serde_json() {
        let a = standard_analyzer("english");
        let s = serde_json::to_string(&a).unwrap();
        let back: Analyzer = serde_json::from_str(&s).unwrap();
        assert_eq!(
            back.analyze("The Running").unwrap(),
            a.analyze("The Running").unwrap()
        );
    }
}