use crate::{
AnalysisResult, Morpheme, ParsedAnalysis, foreign_word_detector::check_foreign_word,
normalizer::normalize,
};
use regex::Regex;
use sfst::Sfst;
use std::collections::HashMap;
lazy_static::lazy_static! {
static ref MORPHEME_REGEX: Regex = {
Regex::new(r"(?P<root>[^<]+)(?P<tags>(?:<[^>]+>)+)").unwrap()
};
static ref POS_REGEX: Regex = {
Regex::new(r"<(?P<tag>[^>]+)>").unwrap()
};
}
pub struct Analyser {
sfst: Sfst,
}
impl Analyser {
pub fn new() -> Result<Self, Box<dyn std::error::Error>> {
let sfst = crate::create_sfst()?;
Ok(Analyser { sfst })
}
pub fn analyse(
&self,
word: &str,
weighted: bool,
foreign_word_check: bool,
) -> Result<Vec<AnalysisResult>, Box<dyn std::error::Error>> {
let word = normalize(word);
let mut analysis_results = self.sfst.analyse(&word)?;
if analysis_results.is_empty() {
if foreign_word_check && check_foreign_word(&word) == 1 {
analysis_results = vec![format!("{}<fw>", word)];
}
}
if !weighted {
return Ok(analysis_results.into_iter().map(|a| (a, 0)).collect());
}
let mut processed_result = Vec::new();
for analysis in analysis_results {
let weight = if weighted {
Self::parse_analysis(&analysis)?.weight
} else {
0
};
processed_result.push((analysis, weight));
}
processed_result.sort_by_key(|(_, weight)| *weight);
Ok(processed_result)
}
pub fn parse_analysis(analysis: &str) -> Result<ParsedAnalysis, Box<dyn std::error::Error>> {
if analysis.is_empty() {
return Ok(ParsedAnalysis {
morphemes: Vec::new(),
weight: 0,
});
}
let analysis = if analysis.starts_with('<') {
format!(" {}", analysis)
} else {
analysis.to_string()
};
let mut morphemes = Vec::new();
for cap in MORPHEME_REGEX.captures_iter(&analysis) {
let root = cap.name("root").unwrap().as_str().to_string();
let tags = cap.name("tags").unwrap().as_str();
let pos: Vec<String> = POS_REGEX
.captures_iter(tags)
.filter_map(|cap| cap.name("tag"))
.map(|m| m.as_str().to_string())
.collect();
morphemes.push(Morpheme { root, pos });
}
if morphemes.is_empty() {
return Err("Could not parse the analysis".into());
}
let weight = Self::get_weight(&morphemes);
Ok(ParsedAnalysis { morphemes, weight })
}
fn get_weight(morphemes: &[Morpheme]) -> i32 {
let morpheme_length = morphemes.len() as i32;
let mut weight = morpheme_length * 100;
for morpheme in morphemes {
for pos in &morpheme.pos {
weight += morpheme.pos.len() as i32 * 5
+ morpheme.root.chars().count() as i32 * 2
+ Self::get_pos_weight(pos) * 3;
}
}
weight
}
fn get_pos_weight(pos: &str) -> i32 {
let weights: HashMap<&str, i32> = [
("v", 1),
("n", 2),
("adv", 3),
("adj", 4),
("coordinative", 4),
("v-n-compound", 4),
("prn", 5),
("past", 4),
("cvb-adv-part-past", 5),
("np", 5),
("fw", 1),
]
.iter()
.cloned()
.collect();
weights.get(pos).copied().unwrap_or(pos.len() as i32)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_analysis_multi_morpheme() {
let parsed = Analyser::parse_analysis("കരുത്ത്<n><sociative>എ<indeclinable>").unwrap();
assert_eq!(parsed.morphemes.len(), 2);
assert_eq!(parsed.morphemes[0].root, "കരുത്ത്");
assert_eq!(parsed.morphemes[0].pos, vec!["n", "sociative"]);
assert_eq!(parsed.morphemes[1].root, "എ");
assert_eq!(parsed.morphemes[1].pos, vec!["indeclinable"]);
assert_eq!(parsed.weight, 324);
}
}