luci/analysis/

mod.rs

// Obsidian [[wikilinks]] in doc comments are intentional — they link to
// design and reference docs in docs/. Rustdoc doesn't understand them.
#![allow(rustdoc::broken_intra_doc_links)]

//! `luci-analysis` — text analysis pipeline for Luci.
//!
//! Transforms raw text into indexed terms via a three-stage pipeline:
//!
//! ```text
//! Raw Text → Tokenizer → Token Filters → Indexed Terms
//! ```
//!
//! Provides the `Tokenizer` and `TokenFilter` traits with built-in
//! implementations matching Elasticsearch's analyzer model: `standard`,
//! `simple`, `whitespace`, and `keyword` analyzers.
//!
//! See [[analyzers]] for the full specification.

mod char_filter;
pub mod config;
mod filter;
mod token;
mod tokenizer;

pub use char_filter::{
    CharFilter, HtmlStripCharFilter, MappingCharFilter, OffsetCorrection, PatternReplaceCharFilter,
    correct_offset,
};
pub use filter::{
    AsciiFoldingFilter, EdgeNGramTokenFilter, LowercaseFilter, NGramTokenFilter, ShingleFilter,
    StemmerAlgorithm, StemmerFilter, StopFilter, SynonymFilter, TokenFilter,
};
pub use token::Token;
pub use tokenizer::{
    EdgeNGramTokenizer, KeywordTokenizer, LetterTokenizer, NGramTokenizer, PathHierarchyTokenizer,
    PatternTokenizer, StandardTokenizer, Tokenizer, WhitespaceTokenizer,
};

use std::collections::HashMap;

/// A complete text analysis pipeline: char filters + tokenizer + token filters.
///
/// Combines zero or more [`CharFilter`]s, a single [`Tokenizer`], and zero or
/// more [`TokenFilter`]s. Character filters preprocess the raw text, the
/// tokenizer breaks it into tokens, then each token filter transforms the
/// stream in order.
///
/// See [[analyzers#Pipeline Stages]].
pub struct Analyzer {
    name: String,
    char_filters: Vec<Box<dyn CharFilter>>,
    tokenizer: Box<dyn Tokenizer>,
    filters: Vec<Box<dyn TokenFilter>>,
}

impl Analyzer {
    /// Create a new analyzer with the given name, tokenizer, and filters.
    pub fn new(
        name: impl Into<String>,
        tokenizer: impl Tokenizer + 'static,
        filters: Vec<Box<dyn TokenFilter>>,
    ) -> Self {
        Self {
            name: name.into(),
            char_filters: Vec::new(),
            tokenizer: Box::new(tokenizer),
            filters,
        }
    }

    /// Create a new analyzer with char filters, tokenizer, and token filters.
    pub fn with_char_filters(
        name: impl Into<String>,
        char_filters: Vec<Box<dyn CharFilter>>,
        tokenizer: impl Tokenizer + 'static,
        filters: Vec<Box<dyn TokenFilter>>,
    ) -> Self {
        Self {
            name: name.into(),
            char_filters,
            tokenizer: Box::new(tokenizer),
            filters,
        }
    }

    /// Create from already-boxed components (used by config builder).
    pub fn from_boxed(
        name: impl Into<String>,
        char_filters: Vec<Box<dyn CharFilter>>,
        tokenizer: Box<dyn Tokenizer>,
        filters: Vec<Box<dyn TokenFilter>>,
    ) -> Self {
        Self {
            name: name.into(),
            char_filters,
            tokenizer,
            filters,
        }
    }

    /// Run the full analysis pipeline on the input text.
    ///
    /// Applies char filters, tokenizes, corrects offsets, then applies
    /// token filters in order.
    pub fn analyze(&self, text: &str) -> Vec<Token> {
        // Phase 1: apply char filters
        let (filtered_text, corrections) = self.apply_char_filters(text);
        let tokenize_input = if corrections.is_empty() {
            text
        } else {
            &filtered_text
        };

        // Phase 2: tokenize
        let mut tokens = Vec::new();
        self.tokenizer.tokenize(tokenize_input, &mut tokens);

        // Phase 3: correct offsets back to original text
        if !corrections.is_empty() {
            for token in &mut tokens {
                token.offset_from = correct_offset(token.offset_from, &corrections);
                token.offset_to = correct_offset(token.offset_to, &corrections);
            }
        }

        // Phase 4: token filters
        for filter in &self.filters {
            filter.apply(&mut tokens);
        }
        tokens
    }

    /// The analyzer's name (e.g., `"standard"`, `"simple"`).
    pub fn name(&self) -> &str {
        &self.name
    }

    /// Apply all char filters in sequence, accumulating corrections.
    fn apply_char_filters(&self, text: &str) -> (String, Vec<OffsetCorrection>) {
        if self.char_filters.is_empty() {
            return (String::new(), Vec::new());
        }

        let mut current = text.to_string();
        let mut all_corrections = Vec::new();

        for cf in &self.char_filters {
            let (filtered, corrections) = cf.filter(&current);
            all_corrections.extend(corrections);
            current = filtered;
        }

        (current, all_corrections)
    }
}

/// Registry of named analyzers with fallback resolution.
///
/// Implements the analyzer resolution chain from [[analyzers#Analyzer Resolution]]:
/// 1. Look up by exact name
/// 2. Fall back to the `standard` analyzer
///
/// All built-in analyzers are registered on construction.
pub struct AnalyzerRegistry {
    analyzers: HashMap<String, Analyzer>,
}

impl std::fmt::Debug for AnalyzerRegistry {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("AnalyzerRegistry")
            .field("analyzers", &self.analyzers.keys().collect::<Vec<_>>())
            .finish()
    }
}

impl AnalyzerRegistry {
    /// Create a new registry with all built-in analyzers pre-registered.
    pub fn new() -> Self {
        let mut registry = Self {
            analyzers: HashMap::new(),
        };
        registry.register(standard_analyzer());
        registry.register(simple_analyzer());
        registry.register(whitespace_analyzer());
        registry.register(keyword_analyzer());
        registry.register(stop_analyzer());
        registry
    }

    /// Register a custom analyzer. Overwrites any existing analyzer with
    /// the same name.
    pub fn register(&mut self, analyzer: Analyzer) {
        self.analyzers.insert(analyzer.name.clone(), analyzer);
    }

    /// Look up an analyzer by name, falling back to `standard`.
    pub fn get(&self, name: &str) -> &Analyzer {
        self.analyzers
            .get(name)
            .unwrap_or_else(|| self.analyzers.get("standard").unwrap())
    }

    /// Look up an analyzer by name. Returns `None` if not found.
    pub fn try_get(&self, name: &str) -> Option<&Analyzer> {
        self.analyzers.get(name)
    }

    /// List all registered analyzer names.
    pub fn names(&self) -> Vec<&str> {
        self.analyzers.keys().map(String::as_str).collect()
    }
}

impl Default for AnalyzerRegistry {
    fn default() -> Self {
        Self::new()
    }
}

// --- Built-in analyzer constructors ---

/// `standard` analyzer: UAX#29 tokenizer + lowercase filter.
///
/// The default analyzer for `text` fields.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn standard_analyzer() -> Analyzer {
    Analyzer::new(
        "standard",
        StandardTokenizer,
        vec![Box::new(LowercaseFilter)],
    )
}

/// `simple` analyzer: letter tokenizer + lowercase filter.
///
/// Splits on non-letter characters and lowercases.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn simple_analyzer() -> Analyzer {
    Analyzer::new("simple", LetterTokenizer, vec![Box::new(LowercaseFilter)])
}

/// `whitespace` analyzer: whitespace tokenizer, no filters.
///
/// Splits on whitespace only, preserving case and punctuation.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn whitespace_analyzer() -> Analyzer {
    Analyzer::new("whitespace", WhitespaceTokenizer, vec![])
}

/// `keyword` analyzer: keyword tokenizer, no filters.
///
/// Emits the entire input as a single token. Used for exact-match fields.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn keyword_analyzer() -> Analyzer {
    Analyzer::new("keyword", KeywordTokenizer, vec![])
}

/// `stop` analyzer: UAX#29 tokenizer + lowercase + English stop words.
///
/// Like `standard` but removes common English stop words.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn stop_analyzer() -> Analyzer {
    Analyzer::new(
        "stop",
        StandardTokenizer,
        vec![Box::new(LowercaseFilter), Box::new(StopFilter::english())],
    )
}

/// `language` analyzer: UAX#29 tokenizer + lowercase + stop words + stemmer.
///
/// The most aggressive built-in analyzer — normalizes, removes stop words,
/// and stems. Best recall for free-text search.
///
/// See [[analyzers#Built-in Analyzers]].
pub fn language_analyzer(algorithm: StemmerAlgorithm) -> Analyzer {
    Analyzer::new(
        "language",
        StandardTokenizer,
        vec![
            Box::new(LowercaseFilter),
            Box::new(StopFilter::english()),
            Box::new(StemmerFilter::new(algorithm)),
        ],
    )
}

#[cfg(test)]
mod tests {
    use super::*;

    // --- Analyzer ---

    #[test]
    fn standard_analyzer_basic() {
        let analyzer = standard_analyzer();
        let tokens = analyzer.analyze("The Quick Brown Fox");
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();
        assert_eq!(texts, vec!["the", "quick", "brown", "fox"]);
    }

    #[test]
    fn standard_analyzer_name() {
        let analyzer = standard_analyzer();
        assert_eq!(analyzer.name(), "standard");
    }

    #[test]
    fn simple_analyzer_strips_numbers() {
        let analyzer = simple_analyzer();
        let tokens = analyzer.analyze("Hello123World");
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();
        assert_eq!(texts, vec!["hello", "world"]);
    }

    #[test]
    fn whitespace_analyzer_preserves_everything() {
        let analyzer = whitespace_analyzer();
        let tokens = analyzer.analyze("Hello, World!");
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();
        assert_eq!(texts, vec!["Hello,", "World!"]);
    }

    #[test]
    fn keyword_analyzer_single_token() {
        let analyzer = keyword_analyzer();
        let tokens = analyzer.analyze("Hello, World!");
        assert_eq!(tokens.len(), 1);
        assert_eq!(tokens[0].text, "Hello, World!");
    }

    #[test]
    fn stop_analyzer_removes_stop_words() {
        let analyzer = stop_analyzer();
        let tokens = analyzer.analyze("The quick brown fox is a test");
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();
        assert_eq!(texts, vec!["quick", "brown", "fox", "test"]);
    }

    #[test]
    fn language_analyzer_stems() {
        let analyzer = language_analyzer(StemmerAlgorithm::English);
        let tokens = analyzer.analyze("The cats are running quickly");
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();
        assert_eq!(texts, vec!["cat", "run", "quick"]);
    }

    #[test]
    fn analyzer_preserves_positions() {
        let analyzer = stop_analyzer();
        let tokens = analyzer.analyze("the quick brown fox");
        // "the" removed, but positions preserved from tokenization.
        assert_eq!(tokens[0].text, "quick");
        assert_eq!(tokens[0].position, 1); // position 0 was "the"
    }

    #[test]
    fn analyzer_empty_input() {
        let analyzer = standard_analyzer();
        let tokens = analyzer.analyze("");
        assert!(tokens.is_empty());
    }

    // --- AnalyzerRegistry ---

    #[test]
    fn registry_has_builtins() {
        let registry = AnalyzerRegistry::new();
        let names = registry.names();
        assert!(names.contains(&"standard"));
        assert!(names.contains(&"simple"));
        assert!(names.contains(&"whitespace"));
        assert!(names.contains(&"keyword"));
        assert!(names.contains(&"stop"));
    }

    #[test]
    fn registry_get_standard() {
        let registry = AnalyzerRegistry::new();
        let analyzer = registry.get("standard");
        assert_eq!(analyzer.name(), "standard");
    }

    #[test]
    fn registry_fallback_to_standard() {
        let registry = AnalyzerRegistry::new();
        let analyzer = registry.get("nonexistent");
        assert_eq!(analyzer.name(), "standard");
    }

    #[test]
    fn registry_try_get_returns_none() {
        let registry = AnalyzerRegistry::new();
        assert!(registry.try_get("nonexistent").is_none());
        assert!(registry.try_get("standard").is_some());
    }

    #[test]
    fn registry_custom_analyzer() {
        let mut registry = AnalyzerRegistry::new();
        registry.register(Analyzer::new(
            "custom",
            WhitespaceTokenizer,
            vec![Box::new(LowercaseFilter)],
        ));

        let analyzer = registry.get("custom");
        assert_eq!(analyzer.name(), "custom");
        let tokens = analyzer.analyze("Hello World");
        assert_eq!(tokens[0].text, "hello");
    }

    // --- End-to-end ---

    #[test]
    fn analyze_realistic_document() {
        let analyzer = standard_analyzer();
        let text = "Elasticsearch is a distributed, RESTful search and \
                    analytics engine. It centrally stores your data for \
                    lightning fast search.";
        let tokens = analyzer.analyze(text);

        // Should produce lowercased terms without punctuation.
        assert!(tokens.len() > 10);
        assert!(tokens.iter().all(|t| t.text == t.text.to_lowercase()));

        // Offsets should point back to the original text.
        for token in &tokens {
            assert_eq!(
                text[token.offset_from..token.offset_to].to_lowercase(),
                token.text
            );
        }
    }

    #[test]
    fn stop_analyzer_realistic() {
        let analyzer = stop_analyzer();
        let text = "The quick brown fox jumps over the lazy dog";
        let tokens = analyzer.analyze(text);
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();

        // "the" (appears twice) and "over" (not a stop word) should be handled correctly.
        assert!(!texts.contains(&"the"));
        assert!(texts.contains(&"quick"));
        assert!(texts.contains(&"over")); // "over" is not in default stop words
    }

    #[test]
    fn language_analyzer_realistic() {
        let analyzer = language_analyzer(StemmerAlgorithm::English);
        let text = "The users were searching for documents containing these keywords";
        let tokens = analyzer.analyze(text);
        let texts: Vec<&str> = tokens.iter().map(|t| t.text.as_str()).collect();

        // Stop words removed, remaining words stemmed.
        assert!(!texts.contains(&"the"));
        assert!(!texts.contains(&"these")); // stop word
        assert!(texts.contains(&"user")); // "users" → "user"
        assert!(texts.contains(&"search")); // "searching" → "search"
        assert!(texts.contains(&"document")); // "documents" → "document"
        assert!(texts.contains(&"keyword")); // "keywords" → "keyword"
    }
}