pub mod compression;
pub mod document;
pub mod lexical;
pub mod readability;
use crate::domain::Document;
pub type Metric = Box<dyn Fn(&mut Document)>;
pub fn default_suite() -> Vec<Metric> {
vec![
Box::new(readability::compute),
Box::new(lexical::compute),
Box::new(document::compute),
Box::new(compression::compute),
]
}
pub fn run_suite(analysis: &mut Document, suite: &[Metric]) {
for metric in suite {
metric(analysis);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::{Paragraph, Section, Sentence, Token};
fn make_token(text: &str, pos: &str, dep: &str, head: usize) -> Token {
Token {
id: 0,
text: text.to_string(),
lemma: text.to_lowercase(),
pos: pos.to_string(),
xpos: String::new(),
feats: String::new(),
dep: dep.to_string(),
head,
deps: String::new(),
misc: String::new(),
is_punct: pos == "PUNCT",
}
}
fn make_sentences(input: Vec<(&str, Vec<Token>)>) -> Vec<Sentence> {
input
.into_iter()
.map(|(text, tokens)| Sentence::new(text.to_string(), tokens))
.collect()
}
fn analysis_from_paragraphs(paragraphs: Vec<(Paragraph, Vec<Sentence>)>) -> Document {
let mut paras = Vec::new();
for (mut para, sents) in paragraphs {
if !para.in_blockquote {
para.sentences = sents;
}
paras.push(para);
}
let sections = vec![Section {
heading: None,
level: 0,
paragraphs: paras,
}];
Document::new(sections)
}
#[test]
fn passive_voice_propagates_through_suite() {
let para_text = "The system was built by the team";
let sents = make_sentences(vec![(
para_text,
vec![
make_token("The", "DET", "det", 2),
make_token("system", "NOUN", "nsubj:pass", 4),
make_token("was", "AUX", "aux:pass", 4),
make_token("built", "VERB", "root", 0),
make_token("by", "ADP", "case", 6),
make_token("the", "DET", "det", 6),
make_token("team", "NOUN", "obl", 4),
],
)]);
let mut analysis =
analysis_from_paragraphs(vec![(Paragraph::new(para_text.to_string(), false), sents)]);
let suite = default_suite();
run_suite(&mut analysis, &suite);
assert_eq!(analysis.total_sentences(), 1);
assert!(
analysis.sentences().next().unwrap().is_passive(),
"should detect passive voice"
);
}
#[test]
fn blockquote_paragraphs_have_no_sentences() {
let mut analysis = analysis_from_paragraphs(vec![(
Paragraph::new("Some text here".to_string(), true),
vec![],
)]);
let suite = default_suite();
run_suite(&mut analysis, &suite);
assert_eq!(
analysis.total_sentences(),
0,
"blockquote paragraphs should have no sentences"
);
}
#[test]
fn document_passive_ratio_via_suite() {
let s1 = make_sentences(vec![(
"The system was built",
vec![
make_token("The", "DET", "det", 2),
make_token("system", "NOUN", "nsubj:pass", 3),
make_token("was", "AUX", "aux:pass", 3),
make_token("built", "VERB", "root", 0),
],
)]);
let s2 = make_sentences(vec![(
"The team shipped the product",
vec![
make_token("The", "DET", "det", 2),
make_token("team", "NOUN", "nsubj", 3),
make_token("shipped", "VERB", "root", 0),
make_token("the", "DET", "det", 5),
make_token("product", "NOUN", "obj", 3),
],
)]);
let mut analysis = analysis_from_paragraphs(vec![
(
Paragraph::new("The system was built".to_string(), false),
s1,
),
(
Paragraph::new("The team shipped the product".to_string(), false),
s2,
),
]);
let suite = default_suite();
run_suite(&mut analysis, &suite);
assert_eq!(analysis.total_sentences(), 2);
assert!(
(analysis.passive_ratio() - 0.5).abs() < 0.01,
"expected ~50% passive, got {}",
analysis.passive_ratio()
);
}
}