mlmorph 1.4.3

Malayalam morphological analyzer
Documentation
use crate::{
    AnalysisResult, Morpheme, ParsedAnalysis, foreign_word_detector::check_foreign_word,
    normalizer::normalize,
};
use regex::Regex;
use sfst::Sfst;
use std::collections::HashMap;

lazy_static::lazy_static! {
    // One morpheme: a root (non-tag chars) followed by one or more tags.
    // Unlike Python's `regex` module, the Rust `regex` crate cannot return every
    // repetition of a capture group, so we match each morpheme separately with
    // `captures_iter` instead of relying on an outer `+`.
    static ref MORPHEME_REGEX: Regex = {
        Regex::new(r"(?P<root>[^<]+)(?P<tags>(?:<[^>]+>)+)").unwrap()
    };

    // A single tag. `captures_iter` yields one capture per tag.
    static ref POS_REGEX: Regex = {
        Regex::new(r"<(?P<tag>[^>]+)>").unwrap()
    };
}

pub struct Analyser {
    sfst: Sfst,
}

impl Analyser {
    pub fn new() -> Result<Self, Box<dyn std::error::Error>> {
        let sfst = crate::create_sfst()?;
        Ok(Analyser { sfst })
    }

    pub fn analyse(
        &self,
        word: &str,
        weighted: bool,
        foreign_word_check: bool,
    ) -> Result<Vec<AnalysisResult>, Box<dyn std::error::Error>> {
        let word = normalize(word);
        let mut analysis_results = self.sfst.analyse(&word)?;

        if analysis_results.is_empty() {
            if foreign_word_check && check_foreign_word(&word) == 1 {
                analysis_results = vec![format!("{}<fw>", word)];
            }
        }

        if !weighted {
            return Ok(analysis_results.into_iter().map(|a| (a, 0)).collect());
        }

        let mut processed_result = Vec::new();
        for analysis in analysis_results {
            let weight = if weighted {
                Self::parse_analysis(&analysis)?.weight
            } else {
                0
            };
            processed_result.push((analysis, weight));
        }

        processed_result.sort_by_key(|(_, weight)| *weight);
        Ok(processed_result)
    }

    pub fn parse_analysis(analysis: &str) -> Result<ParsedAnalysis, Box<dyn std::error::Error>> {
        if analysis.is_empty() {
            return Ok(ParsedAnalysis {
                morphemes: Vec::new(),
                weight: 0,
            });
        }

        let analysis = if analysis.starts_with('<') {
            format!(" {}", analysis)
        } else {
            analysis.to_string()
        };

        let mut morphemes = Vec::new();
        for cap in MORPHEME_REGEX.captures_iter(&analysis) {
            let root = cap.name("root").unwrap().as_str().to_string();
            let tags = cap.name("tags").unwrap().as_str();
            let pos: Vec<String> = POS_REGEX
                .captures_iter(tags)
                .filter_map(|cap| cap.name("tag"))
                .map(|m| m.as_str().to_string())
                .collect();

            morphemes.push(Morpheme { root, pos });
        }

        if morphemes.is_empty() {
            return Err("Could not parse the analysis".into());
        }

        let weight = Self::get_weight(&morphemes);
        Ok(ParsedAnalysis { morphemes, weight })
    }

    fn get_weight(morphemes: &[Morpheme]) -> i32 {
        let morpheme_length = morphemes.len() as i32;
        let mut weight = morpheme_length * 100;

        for morpheme in morphemes {
            for pos in &morpheme.pos {
                weight += morpheme.pos.len() as i32 * 5
                    + morpheme.root.chars().count() as i32 * 2
                    + Self::get_pos_weight(pos) * 3;
            }
        }

        weight
    }

    fn get_pos_weight(pos: &str) -> i32 {
        let weights: HashMap<&str, i32> = [
            ("v", 1),
            ("n", 2),
            ("adv", 3),
            ("adj", 4),
            ("coordinative", 4),
            ("v-n-compound", 4),
            ("prn", 5),
            ("past", 4),
            ("cvb-adv-part-past", 5),
            ("np", 5),
            ("fw", 1),
        ]
        .iter()
        .cloned()
        .collect();

        weights.get(pos).copied().unwrap_or(pos.len() as i32)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_parse_analysis_multi_morpheme() {
        let parsed = Analyser::parse_analysis("കരുത്ത്<n><sociative>എ<indeclinable>").unwrap();

        assert_eq!(parsed.morphemes.len(), 2);

        assert_eq!(parsed.morphemes[0].root, "കരുത്ത്");
        assert_eq!(parsed.morphemes[0].pos, vec!["n", "sociative"]);

        assert_eq!(parsed.morphemes[1].root, "");
        assert_eq!(parsed.morphemes[1].pos, vec!["indeclinable"]);

        // Weight matches the Python reference implementation.
        assert_eq!(parsed.weight, 324);
    }
}