pub(crate) mod compression;
pub(crate) mod document;
pub(crate) mod lexical;
pub(crate) mod readability;
use crate::domain::Document;
pub(crate) type Metric = Box<dyn Fn(&mut Document)>;
pub(crate) fn default_suite() -> Vec<Metric> {
vec![
Box::new(readability::compute),
Box::new(lexical::compute),
Box::new(document::compute),
Box::new(compression::compute),
]
}
pub(crate) fn run_suite(analysis: &mut Document, suite: &[Metric]) {
for metric in suite {
metric(analysis);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::{Paragraph, ParagraphKind, Section, Sentence, Span, Token};
fn make_token(text: &str, pos: &str, dep: &str, head: usize) -> Token {
Token {
id: 0,
span: Default::default(),
text: text.to_string(),
lemma: text.to_lowercase(),
pos: pos.to_string(),
xpos: String::new(),
features: std::collections::BTreeMap::new(),
dep: dep.to_string(),
head,
deps: String::new(),
misc: String::new(),
is_punct: pos == "PUNCT",
}
}
fn make_sentences(input: Vec<(&str, Vec<Token>)>) -> Vec<Sentence> {
input
.into_iter()
.map(|(text, tokens)| Sentence::new(text.to_string(), tokens))
.collect()
}
fn analysis_from_paragraphs(paragraphs: Vec<(Paragraph, Vec<Sentence>)>) -> Document {
let mut paras = Vec::new();
for (mut para, sents) in paragraphs {
if para.kind != ParagraphKind::Quote {
para.sentences = sents;
}
paras.push(para);
}
let sections = vec![Section {
heading: None,
level: 0,
span: None,
paragraphs: paras,
}];
Document::new(crate::domain::Provenance::for_test(), sections, Vec::new())
}
#[test]
fn passive_voice_propagates_through_suite() {
let para_text = "The system was built by the team";
let sents = make_sentences(vec![(
para_text,
vec![
make_token("The", "DET", "det", 2),
make_token("system", "NOUN", "nsubj:pass", 4),
make_token("was", "AUX", "aux:pass", 4),
make_token("built", "VERB", "root", 0),
make_token("by", "ADP", "case", 6),
make_token("the", "DET", "det", 6),
make_token("team", "NOUN", "obl", 4),
],
)]);
let mut analysis = analysis_from_paragraphs(vec![(
Paragraph::new(
para_text.to_string(),
ParagraphKind::Paragraph,
Span::default(),
),
sents,
)]);
let suite = default_suite();
run_suite(&mut analysis, &suite);
assert_eq!(analysis.metrics.total_sentences, 1);
assert!(
analysis.sentences().next().unwrap().passive,
"should detect passive voice"
);
}
#[test]
fn blockquote_paragraphs_have_no_sentences() {
let mut analysis = analysis_from_paragraphs(vec![(
Paragraph::new(
"Some text here".to_string(),
ParagraphKind::Quote,
Span::default(),
),
vec![],
)]);
let suite = default_suite();
run_suite(&mut analysis, &suite);
assert_eq!(
analysis.metrics.total_sentences, 0,
"blockquote paragraphs should have no sentences"
);
assert_eq!(analysis.metrics.paragraph_count, 1);
assert_eq!(analysis.metrics.mean_sentence_length, None);
assert_eq!(analysis.metrics.passive_ratio, None);
}
#[test]
fn document_passive_ratio_via_suite() {
let s1 = make_sentences(vec![(
"The system was built",
vec![
make_token("The", "DET", "det", 2),
make_token("system", "NOUN", "nsubj:pass", 3),
make_token("was", "AUX", "aux:pass", 3),
make_token("built", "VERB", "root", 0),
],
)]);
let s2 = make_sentences(vec![(
"The team shipped the product",
vec![
make_token("The", "DET", "det", 2),
make_token("team", "NOUN", "nsubj", 3),
make_token("shipped", "VERB", "root", 0),
make_token("the", "DET", "det", 5),
make_token("product", "NOUN", "obj", 3),
],
)]);
let mut analysis = analysis_from_paragraphs(vec![
(
Paragraph::new(
"The system was built".to_string(),
ParagraphKind::Paragraph,
Span::default(),
),
s1,
),
(
Paragraph::new(
"The team shipped the product".to_string(),
ParagraphKind::Paragraph,
Span::default(),
),
s2,
),
]);
let suite = default_suite();
run_suite(&mut analysis, &suite);
assert_eq!(analysis.metrics.total_sentences, 2);
assert_eq!(analysis.metrics.total_words, 9);
assert_eq!(analysis.metrics.paragraph_count, 2);
assert_eq!(analysis.metrics.mean_sentence_length, Some(4.5));
let passive = analysis.metrics.passive_ratio.expect("two sentences");
assert!(
(passive - 0.5).abs() < 0.01,
"expected ~50% passive, got {passive}"
);
}
}