sentencex 0.1.31

mod am;
mod ar;
mod bg;
mod bn;
mod ca;
mod da;
mod de;
mod el;
mod en;
mod es;
mod fallbacks;
mod fi;
mod fr;
mod fronting;
mod gu;
mod hi;
mod hy;
mod it;
mod ja;
mod kk;
mod kn;
mod language;
mod list_markers;
mod ml;
mod mr;
mod my;
mod nl;
mod pa;
mod pl;
mod pt;
mod quotes;
mod ru;
mod sk;
mod ta;
mod te;
mod trailing_markers;
mod uk;

pub use am::Amharic;
pub use ar::Arabic;
pub use bg::Bulgarian;
pub use bn::Bengali;
pub use ca::Catalan;
pub use da::Danish;
pub use de::German;
pub use el::Greek;
pub use en::English;
pub use es::Spanish;
pub use fallbacks::get_fallbacks;
pub use fi::Finnish;
pub use fr::French;
pub use gu::Gujarati;
pub use hi::Hindi;
pub use hy::Armenian;
pub use it::Italian;
pub use ja::Japanese;
pub use kk::Kazakh;
pub use kn::Kannada;
pub use language::Language;
pub(crate) use language::paragraph_breaks;
pub use ml::Malayalam;
pub use mr::Marathi;
pub use my::Burmese;
pub use nl::Dutch;
pub use pa::Punjabi;
pub use pl::Polish;
pub use pt::Portuguese;
pub use ru::Russian;
pub use sk::Slovak;
pub use ta::Tamil;
pub use te::Telugu;
pub use uk::Ukrainian;

use rustc_hash::FxHashSet;

/// Parse one or more bundled word-list files into a deduplicated set.
/// Lines are trimmed; blank lines and `//` line comments are dropped.
pub(crate) fn parse_word_list<'a>(sources: impl IntoIterator<Item = &'a str>) -> FxHashSet<String> {
    sources
        .into_iter()
        .flat_map(str::lines)
        .map(str::trim)
        .filter(|line| !line.is_empty() && !line.starts_with("//"))
        .map(String::from)
        .collect()
}

pub(crate) fn parse_lowercase_word_list<'a>(
    sources: impl IntoIterator<Item = &'a str>,
) -> FxHashSet<String> {
    parse_word_list(sources)
        .iter()
        .map(|s| s.to_lowercase())
        .collect()
}

pub(crate) fn parse_markers_list(source: &'static str) -> Vec<trailing_markers::MarkerDef> {
    source.lines().filter_map(parse_marker_line).collect()
}

fn parse_marker_line(raw: &'static str) -> Option<trailing_markers::MarkerDef> {
    let line = raw.trim();
    if line.is_empty() || line.starts_with("//") {
        return None;
    }

    let mut parts = line.split('|').map(str::trim);
    let suffix = parts.next().expect("split yields at least one element");

    let case = parts
        .next()
        .unwrap_or_else(|| panic!("marker line missing `case` field: {raw:?}"));

    let digit_field = parts
        .next()
        .unwrap_or_else(|| panic!("marker line missing `digit_breaks` field: {raw:?}"));

    let uppercase_field = parts.next();
    let flag_field = parts.next();
    if parts.next().is_some() {
        panic!("marker line has trailing fields after flag: {raw:?}");
    }

    let ignore_case = match case {
        "i" => true,
        "s" => false,
        other => panic!("marker line has unknown case `{other}`: {raw:?}"),
    };

    let digit_only = match flag_field {
        None => false,
        Some("digit-only") => true,
        Some(other) => panic!("marker line has unknown flag `{other}`: {raw:?}"),
    };

    Some(trailing_markers::MarkerDef {
        matcher: trailing_markers::SuffixMatcher {
            suffix,
            ignore_case,
            digit_only,
        },
        policy: trailing_markers::MarkerPolicy {
            digit_breaks: parse_break_flag(digit_field, "digit_breaks", raw),
            uppercase_breaks: uppercase_field
                .map(|f| parse_break_flag(f, "uppercase_breaks", raw))
                .unwrap_or(true),
        },
    })
}

fn parse_break_flag(field: &str, axis: &str, raw: &str) -> bool {
    match field {
        "break" => true,
        "cont" => false,
        other => panic!("marker line has unknown {axis} `{other}`: {raw:?}"),
    }
}

#[cfg(test)]
mod tests {
    use std::fs;

    use super::*;

    pub fn run_language_tests<T: Language>(language: T, test_file: &str) {
        let raw = fs::read_to_string(test_file).expect("Failed to read test file");
        let content = raw.replace("\r\n", "\n");
        let test_cases: Vec<&str> = content.split("===").collect();

        for case in test_cases {
            // Explicit '#' comment handling. This lets fixture files include
            // section headers and structural comments anywhere — not just at
            // the very top of the file — without silently skipping cases that
            // happen to share a chunk with leading comments.
            let cleaned: String = case
                .lines()
                .filter(|line| !line.trim_start().starts_with('#'))
                .collect::<Vec<_>>()
                .join("\n");

            let cleaned = cleaned.trim();
            if cleaned.is_empty() {
                continue;
            }

            let parts: Vec<&str> = cleaned
                .split("---")
                .map(|part| part.trim())
                .filter(|part| !part.is_empty())
                .collect();

            if parts.len() != 2 {
                continue; // Skip malformed test cases
            }

            // Comparison normalises whitespace on both sides — any run of
            // whitespace (spaces, tabs, newlines) collapses to a single
            // space.
            let normalise = |s: &str| s.split_whitespace().collect::<Vec<_>>().join(" ");
            let input = parts[0];

            let expected: Vec<String> = parts[1]
                .lines()
                .map(&normalise)
                .filter(|s| !s.is_empty())
                .collect();

            let result = language.segment(input);

            let actual: Vec<String> = result
                .iter()
                .map(|item| normalise(item))
                .filter(|s| !s.is_empty())
                .collect();

            assert_eq!(actual, expected, "Failed for input: \n{}", input);
        }
    }
}