use crate::core::hash::{FastMap, FastSet};
#[derive(Clone, Debug)]
pub struct MeaningUnit {
pub subject: String,
pub relation: String,
pub object: String,
}
pub struct Decomposer {
prepositions: FastSet<&'static str>,
irregular_verbs: FastMap<&'static str, &'static str>,
auxiliaries: FastSet<&'static str>,
determiners: FastSet<&'static str>,
coordinating_conj: FastSet<&'static str>,
adverbs: FastSet<&'static str>,
}
impl Decomposer {
pub fn new() -> Self {
let prepositions: FastSet<&'static str> = [
"about",
"above",
"across",
"after",
"against",
"along",
"amid",
"among",
"around",
"as",
"at",
"before",
"behind",
"below",
"beneath",
"beside",
"between",
"beyond",
"by",
"concerning",
"despite",
"down",
"during",
"except",
"for",
"from",
"in",
"inside",
"into",
"like",
"near",
"of",
"off",
"on",
"onto",
"opposite",
"out",
"outside",
"over",
"past",
"per",
"regarding",
"since",
"than",
"through",
"throughout",
"till",
"to",
"toward",
"towards",
"under",
"unlike",
"until",
"up",
"upon",
"via",
"with",
"within",
"without",
]
.into_iter()
.collect();
let irregular_verbs: FastMap<&'static str, &'static str> = [
("is", "be"),
("are", "be"),
("was", "be"),
("were", "be"),
("am", "be"),
("been", "be"),
("being", "be"),
("has", "have"),
("had", "have"),
("having", "have"),
("does", "do"),
("did", "do"),
("done", "do"),
("says", "say"),
("said", "say"),
("makes", "make"),
("made", "make"),
("making", "make"),
("goes", "go"),
("went", "go"),
("gone", "go"),
("going", "go"),
("takes", "take"),
("took", "take"),
("taken", "take"),
("taking", "take"),
("comes", "come"),
("came", "come"),
("coming", "come"),
("sees", "see"),
("saw", "see"),
("seen", "see"),
("seeing", "see"),
("knows", "know"),
("knew", "know"),
("known", "know"),
("gets", "get"),
("got", "get"),
("gotten", "get"),
("getting", "get"),
("gives", "give"),
("gave", "give"),
("given", "give"),
("giving", "give"),
("finds", "find"),
("found", "find"),
("finding", "find"),
("thinks", "think"),
("thought", "think"),
("thinking", "think"),
("tells", "tell"),
("told", "tell"),
("telling", "tell"),
("becomes", "become"),
("became", "become"),
("leaves", "leave"),
("left", "leave"),
("leaving", "leave"),
("puts", "put"),
("putting", "put"),
("keeps", "keep"),
("kept", "keep"),
("keeping", "keep"),
("lets", "let"),
("letting", "let"),
("begins", "begin"),
("began", "begin"),
("begun", "begin"),
("shows", "show"),
("showed", "show"),
("shown", "show"),
("showing", "show"),
("hears", "hear"),
("heard", "hear"),
("runs", "run"),
("ran", "run"),
("running", "run"),
("moves", "move"),
("moved", "move"),
("moving", "move"),
("lives", "live"),
("lived", "live"),
("living", "live"),
("believes", "believe"),
("believed", "believe"),
("holds", "hold"),
("held", "hold"),
("holding", "hold"),
("brings", "bring"),
("brought", "bring"),
("writes", "write"),
("wrote", "write"),
("written", "write"),
("writing", "write"),
("sits", "sit"),
("sat", "sit"),
("sitting", "sit"),
("stands", "stand"),
("stood", "stand"),
("standing", "stand"),
("loses", "lose"),
("lost", "lose"),
("losing", "lose"),
("pays", "pay"),
("paid", "pay"),
("meets", "meet"),
("met", "meet"),
("sets", "set"),
("setting", "set"),
("learns", "learn"),
("learnt", "learn"),
("learned", "learn"),
("leads", "lead"),
("led", "lead"),
("leading", "lead"),
("grows", "grow"),
("grew", "grow"),
("grown", "grow"),
("builds", "build"),
("built", "build"),
("building", "build"),
("sends", "send"),
("sent", "send"),
("falls", "fall"),
("fell", "fall"),
("fallen", "fall"),
("buys", "buy"),
("bought", "buy"),
("feels", "feel"),
("felt", "feel"),
("speaks", "speak"),
("spoke", "speak"),
("spoken", "speak"),
("reads", "read"),
("spends", "spend"),
("spent", "spend"),
("wins", "win"),
("won", "win"),
("teaches", "teach"),
("taught", "teach"),
("eats", "eat"),
("ate", "eat"),
("eaten", "eat"),
("drinks", "drink"),
("drank", "drink"),
("drunk", "drink"),
("drives", "drive"),
("drove", "drive"),
("driven", "drive"),
("breaks", "break"),
("broke", "break"),
("broken", "break"),
("catches", "catch"),
("caught", "catch"),
("draws", "draw"),
("drew", "draw"),
("drawn", "draw"),
("flies", "fly"),
("flew", "fly"),
("flown", "fly"),
("means", "mean"),
("meant", "mean"),
("contains", "contain"),
("contained", "contain"),
("includes", "include"),
("included", "include"),
("requires", "require"),
("required", "require"),
("provides", "provide"),
("provided", "provide"),
("creates", "create"),
("created", "create"),
("creating", "create"),
("causes", "cause"),
("caused", "cause"),
("supports", "support"),
("supported", "support"),
("produces", "produce"),
("produced", "produce"),
("uses", "use"),
("used", "use"),
("using", "use"),
("needs", "need"),
("needed", "need"),
("wants", "want"),
("wanted", "want"),
("calls", "call"),
("called", "call"),
("tries", "try"),
("tried", "try"),
("asks", "ask"),
("asked", "ask"),
("works", "work"),
("worked", "work"),
("plays", "play"),
("played", "play"),
("kills", "kill"),
("killed", "kill"),
("turns", "turn"),
("turned", "turn"),
("helps", "help"),
("helped", "help"),
("starts", "start"),
("started", "start"),
("follows", "follow"),
("followed", "follow"),
("stops", "stop"),
("stopped", "stop"),
("opens", "open"),
("opened", "open"),
("closes", "close"),
("closed", "close"),
("carries", "carry"),
("carried", "carry"),
("offers", "offer"),
("offered", "offer"),
("remembers", "remember"),
("remembered", "remember"),
("loves", "love"),
("loved", "love"),
("considers", "consider"),
("considered", "consider"),
("appears", "appear"),
("appeared", "appear"),
("allows", "allow"),
("allowed", "allow"),
("serves", "serve"),
("served", "serve"),
("expects", "expect"),
("expected", "expect"),
("remains", "remain"),
("remained", "remain"),
("suggests", "suggest"),
("suggested", "suggest"),
("raises", "raise"),
("raised", "raise"),
("develops", "develop"),
("developed", "develop"),
("describes", "describe"),
("described", "describe"),
("owns", "own"),
("owned", "own"),
("defines", "define"),
("defined", "define"),
("connects", "connect"),
("connected", "connect"),
("represents", "represent"),
("represented", "represent"),
("involves", "involve"),
("involved", "involve"),
("belongs", "belong"),
("belonged", "belong"),
]
.into_iter()
.collect();
let auxiliaries: FastSet<&'static str> = [
"is", "are", "was", "were", "am", "has", "have", "had", "been", "being", "gets", "got",
"gotten",
]
.into_iter()
.collect();
let determiners: FastSet<&'static str> = [
"the", "a", "an", "this", "that", "these", "those", "my", "your", "his", "her", "its",
"our", "their", "some", "any", "each", "every", "no",
]
.into_iter()
.collect();
let coordinating_conj: FastSet<&'static str> =
["and", "or", "but", "nor"].into_iter().collect();
let adverbs: FastSet<&'static str> = [
"very",
"really",
"quite",
"always",
"never",
"often",
"sometimes",
"usually",
"also",
"just",
"still",
"already",
"only",
"even",
"quickly",
"slowly",
"carefully",
"easily",
"strongly",
"highly",
"deeply",
"widely",
"nearly",
"recently",
"currently",
"generally",
"simply",
"actually",
"probably",
"certainly",
"clearly",
"directly",
"exactly",
"finally",
"however",
"perhaps",
"possibly",
"rapidly",
"suddenly",
"truly",
"mostly",
]
.into_iter()
.collect();
Self {
prepositions,
irregular_verbs,
auxiliaries,
determiners,
coordinating_conj,
adverbs,
}
}
pub fn decompose(&self, text: &str) -> Vec<MeaningUnit> {
let mut results = Vec::new();
for sentence in text.split(['.', '!', '?', ';']) {
let sentence = sentence.trim();
if sentence.is_empty() {
continue;
}
let clauses = self.split_conjunctions(sentence);
for clause in &clauses {
let tokens = self.tokenize(clause);
if tokens.len() < 2 {
continue;
}
results.extend(self.extract_all(&tokens));
}
}
results
}
fn tokenize(&self, text: &str) -> Vec<String> {
text.split_whitespace()
.map(|w| {
w.trim_matches(|c: char| c.is_ascii_punctuation() && c != '-' && c != '\'')
.to_string()
})
.filter(|w| !w.is_empty())
.collect()
}
fn split_conjunctions(&self, sentence: &str) -> Vec<String> {
let words: Vec<&str> = sentence.split_whitespace().collect();
if words.len() < 5 {
return vec![sentence.to_string()];
}
for (i, &word) in words.iter().enumerate() {
let lower = word
.trim_matches(|c: char| c.is_ascii_punctuation())
.to_lowercase();
if !self.coordinating_conj.contains(lower.as_str()) || i == 0 || i >= words.len() - 1 {
continue;
}
let after_conj = &words[i + 1..];
if let Some(verb_offset) = self.find_verb_position(after_conj) {
if verb_offset > 0 {
let verb_and_rest: String = after_conj[verb_offset..].join(" ");
let subj1: String = words[..i].join(" ");
let subj2: String = after_conj[..verb_offset].join(" ");
if !subj1.is_empty() && !subj2.is_empty() {
return vec![
format!("{} {}", subj1, verb_and_rest),
format!("{} {}", subj2, verb_and_rest),
];
}
}
}
}
vec![sentence.to_string()]
}
fn find_verb_position(&self, words: &[&str]) -> Option<usize> {
for (i, &word) in words.iter().enumerate() {
let lower = word
.trim_matches(|c: char| c.is_ascii_punctuation())
.to_lowercase();
if self.irregular_verbs.contains_key(lower.as_str()) {
return Some(i);
}
}
for (i, &word) in words.iter().enumerate() {
let lower = word
.trim_matches(|c: char| c.is_ascii_punctuation())
.to_lowercase();
if self.lemmatize_verb(&lower).is_some() {
return Some(i);
}
}
None
}
fn extract_all(&self, tokens: &[String]) -> Vec<MeaningUnit> {
let mut results = Vec::new();
if let Some(unit) = self.try_passive(tokens) {
results.push(unit);
return results;
}
if let Some(unit) = self.try_copula(tokens) {
results.push(unit);
}
results.extend(self.try_verb_patterns(tokens));
results
}
fn try_passive(&self, tokens: &[String]) -> Option<MeaningUnit> {
for (i, token) in tokens.iter().enumerate() {
let lower = token.to_lowercase();
if !self.auxiliaries.contains(lower.as_str()) || i == 0 {
continue;
}
let mut pp_idx = i + 1;
while pp_idx < tokens.len()
&& self
.adverbs
.contains(tokens[pp_idx].to_lowercase().as_str())
{
pp_idx += 1;
}
if pp_idx >= tokens.len() {
continue;
}
let pp_lower = tokens[pp_idx].to_lowercase();
let lemma = self.lemmatize_verb(&pp_lower)?;
let by_idx =
(pp_idx + 1..tokens.len()).find(|&j| tokens[j].eq_ignore_ascii_case("by"))?;
if by_idx + 1 >= tokens.len() {
continue;
}
let patient = self.extract_np(&tokens[..i]);
let agent = self.extract_np(&tokens[by_idx + 1..]);
if !patient.is_empty() && !agent.is_empty() {
return Some(MeaningUnit {
subject: agent,
relation: lemma,
object: patient,
});
}
}
None
}
fn try_copula(&self, tokens: &[String]) -> Option<MeaningUnit> {
let copulas = ["is", "are", "was", "were", "am"];
for (i, token) in tokens.iter().enumerate() {
let lower = token.to_lowercase();
if !copulas.contains(&lower.as_str()) || i == 0 || i + 1 >= tokens.len() {
continue;
}
if i + 1 < tokens.len() {
let next_lower = tokens[i + 1].to_lowercase();
if self.lemmatize_verb(&next_lower).is_some()
&& (next_lower.ends_with("ed")
|| next_lower.ends_with("en")
|| self.is_irregular_participle(&next_lower))
{
continue;
}
}
let subject = self.extract_np(&tokens[..i]);
let object = self.extract_np(&tokens[i + 1..]);
if !subject.is_empty() && !object.is_empty() {
return Some(MeaningUnit {
subject,
relation: "is_a".to_string(),
object,
});
}
}
None
}
fn try_verb_patterns(&self, tokens: &[String]) -> Vec<MeaningUnit> {
let mut results = Vec::new();
for (i, token) in tokens.iter().enumerate() {
if i == 0 {
continue;
}
let lower = token.to_lowercase();
let Some(lemma) = self.lemmatize_verb(&lower) else {
continue;
};
if lemma == "be" {
continue;
}
if i + 2 < tokens.len() {
let next_lower = tokens[i + 1].to_lowercase();
if self.prepositions.contains(next_lower.as_str()) {
let subject = self.extract_np(&tokens[..i]);
let object = self.extract_np(&tokens[i + 2..]);
if !subject.is_empty() && !object.is_empty() {
results.push(MeaningUnit {
subject,
relation: format!("{}_{}", lemma, next_lower),
object,
});
break; }
}
}
if i + 1 < tokens.len() {
let subject = self.extract_np(&tokens[..i]);
let object = self.extract_np(&tokens[i + 1..]);
if !subject.is_empty() && !object.is_empty() {
results.push(MeaningUnit {
subject,
relation: lemma,
object,
});
break;
}
}
}
results
}
fn lemmatize_verb(&self, word: &str) -> Option<String> {
if let Some(&lemma) = self.irregular_verbs.get(word) {
return Some(lemma.to_string());
}
if self.irregular_verbs.values().any(|&v| v == word) {
return Some(word.to_string());
}
if word.len() < 3 {
return None;
}
if word.ends_with("ing") && word.len() > 4 {
let stem = &word[..word.len() - 3];
if stem.len() >= 2 && stem.as_bytes()[stem.len() - 1] == stem.as_bytes()[stem.len() - 2]
{
return Some(stem[..stem.len() - 1].to_string());
}
if stem.ends_with('t') {
let with_e = format!("{}e", stem);
return Some(with_e);
}
return Some(stem.to_string());
}
if word.ends_with("ed") && word.len() > 3 {
let stem = &word[..word.len() - 2];
if word.ends_with("ied") && word.len() > 4 {
return Some(format!("{}y", &word[..word.len() - 3]));
}
if stem.len() >= 2 && stem.as_bytes()[stem.len() - 1] == stem.as_bytes()[stem.len() - 2]
{
return Some(stem[..stem.len() - 1].to_string());
}
if stem.ends_with('t') || stem.ends_with('d') || stem.ends_with('k') {
return Some(stem.to_string());
}
return Some(format!("{}e", stem));
}
if word.ends_with("es") && word.len() > 3 {
if let Some(stem) = word.strip_suffix("ies") {
return Some(format!("{}y", stem));
}
if word.ends_with("ches")
|| word.ends_with("shes")
|| word.ends_with("sses")
|| word.ends_with("xes")
|| word.ends_with("zes")
{
return Some(word[..word.len() - 2].to_string());
}
return Some(word[..word.len() - 1].to_string());
}
if word.ends_with('s') && !word.ends_with("ss") && word.len() > 2 {
return Some(word[..word.len() - 1].to_string());
}
None
}
fn is_irregular_participle(&self, word: &str) -> bool {
matches!(
word,
"been"
| "done"
| "gone"
| "seen"
| "known"
| "given"
| "taken"
| "made"
| "found"
| "told"
| "left"
| "kept"
| "begun"
| "shown"
| "heard"
| "held"
| "brought"
| "written"
| "sat"
| "stood"
| "lost"
| "met"
| "led"
| "grown"
| "built"
| "sent"
| "fallen"
| "bought"
| "felt"
| "spoken"
| "spent"
| "won"
| "taught"
| "eaten"
| "drunk"
| "driven"
| "broken"
| "caught"
| "drawn"
| "flown"
| "meant"
| "thought"
| "paid"
| "run"
| "put"
| "set"
| "read"
)
}
fn extract_np(&self, tokens: &[String]) -> String {
let filtered: Vec<&str> = tokens
.iter()
.map(|t| t.as_str())
.filter(|w| {
let lower = w.to_lowercase();
!self.determiners.contains(lower.as_str()) && !self.adverbs.contains(lower.as_str())
})
.collect();
if filtered.is_empty() {
return String::new();
}
filtered.join(" ")
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn first_unit_extracts_subject_relation_and_object() {
let d = Decomposer::new();
for (sentence, subject, relation, object) in [
(
"Paris is the capital of France",
Some("Paris"),
Some("is_a"),
None,
),
(
"The cat goes to the park",
Some("cat"),
Some("go_to"),
Some("park"),
),
(
"The book was written by John",
Some("John"),
Some("write"),
Some("book"),
),
("Alice created the project", None, Some("create"), None),
("John runs the company", None, Some("run"), None),
("Birds fly over the ocean", None, Some("fly_over"), None),
("Paris is a city, indeed!", Some("Paris"), None, None),
] {
let units = d.decompose(sentence);
let Some(first) = units.first() else {
panic!("{sentence:?} produced no units");
};
if let Some(subject) = subject {
assert_eq!(first.subject, subject, "{sentence:?}");
}
if let Some(relation) = relation {
assert_eq!(first.relation, relation, "{sentence:?}");
}
if let Some(object) = object {
assert_eq!(first.object, object, "{sentence:?}");
}
}
}
#[test]
fn splits_sentences_and_skips_text_without_a_relation() {
let d = Decomposer::new();
for (text, units) in [
("", 0),
("hi", 0),
("Dogs bark", 0),
("Paris is a city. Berlin is a city.", 2),
("Paris is a city; Berlin is a city", 2),
] {
assert_eq!(d.decompose(text).len(), units, "{text:?}");
}
}
#[test]
fn test_conjunction_splitting() {
let d = Decomposer::new();
let units = d.decompose("Dogs and cats are animals");
assert_eq!(units.len(), 2, "Should split conjunction into 2 triples");
assert!(units.iter().any(|u| u.subject == "Dogs"));
assert!(units.iter().any(|u| u.subject == "cats"));
}
}