readsight 1.0.0

//! Splits text into words, sentences, and letters using language regexes.

use regex::Regex;

use crate::error::{Error, Result};
use crate::language::Language;

/// Language-aware tokenizer.
pub struct TextSplitter {
    word_re: Regex,
    sentence_re: Regex,
    letter_re: Regex,
}

fn compile(body: &str) -> Result<Regex> {
    Regex::new(body).map_err(|e| Error::InvalidPattern(format!("{body}: {e}")))
}

impl TextSplitter {
    /// Build a splitter from a language's regex bodies.
    pub fn new(language: &Language) -> Result<Self> {
        Ok(TextSplitter {
            word_re: compile(&language.word_split_pattern)?,
            sentence_re: compile(&language.sentence_boundary_pattern)?,
            letter_re: compile(&language.letter_pattern)?,
        })
    }

    /// Split into words, dropping empty fragments.
    pub fn split_words(&self, text: &str) -> Vec<String> {
        let text = text.trim();
        if text.is_empty() {
            return Vec::new();
        }
        self.word_re
            .split(text)
            .filter(|w| !w.is_empty())
            .map(String::from)
            .collect()
    }

    /// Split into sentences (trimmed, no empties).
    pub fn split_sentences(&self, text: &str) -> Vec<String> {
        let text = text.trim();
        if text.is_empty() {
            return Vec::new();
        }
        self.sentence_re
            .split(text)
            .filter(|p| !p.is_empty())
            .map(|p| p.trim().to_string())
            .collect()
    }

    /// Count letters (matches of the letter regex).
    pub fn count_letters(&self, text: &str) -> i64 {
        let text = text.trim();
        if text.is_empty() {
            return 0;
        }
        self.letter_re.find_iter(text).count() as i64
    }

    /// Count words.
    pub fn count_words(&self, text: &str) -> i64 {
        self.split_words(text).len() as i64
    }

    /// Count sentences (at least 1 for non-empty text).
    pub fn count_sentences(&self, text: &str) -> i64 {
        let text = text.trim();
        if text.is_empty() {
            return 0;
        }
        let count = self.sentence_re.find_iter(text).count() as i64;
        if count == 0 {
            1
        } else {
            count
        }
    }

    /// Count words whose letter count exceeds `threshold`.
    pub fn count_long_words(&self, text: &str, threshold: i64) -> i64 {
        self.split_words(text)
            .iter()
            .filter(|w| self.count_letters(w) > threshold)
            .count() as i64
    }
}