use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
pub const MAX_INPUT_BYTES: usize = 8 * 1024 * 1024;
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct ProvisionNotice {
pub artifact: String,
pub bytes: u64,
pub destination: PathBuf,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Embedding(pub Vec<f32>);
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct SemanticClusters {
pub model_hash: String,
pub threshold: f32,
pub clusters: Vec<SemanticCluster>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct SemanticCluster {
pub members: Vec<usize>,
pub edges: Vec<SemanticEdge>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct SemanticEdge {
pub a: usize,
pub b: usize,
pub score: f32,
}
#[derive(thiserror::Error, Debug)]
#[non_exhaustive]
pub enum Error {
#[error("model not found: {}", .0.display())]
ModelNotFound(PathBuf),
#[error("invalid model: {0}")]
ModelInvalid(String),
#[error("parse failed: {0}")]
ParseFailed(String),
#[error("{what} input too large: {actual} > limit {limit}")]
InputTooLarge {
limit: usize,
actual: usize,
what: &'static str,
},
#[error("unsupported format: {0:?}")]
UnsupportedFormat(Format),
#[error("invalid input: {0}")]
InvalidInput(String),
#[error("io error: {0}")]
Io(#[from] std::io::Error),
}
impl Error {
pub fn kind(&self) -> &'static str {
match self {
Error::ModelNotFound(_) => "model_not_found",
Error::ModelInvalid(_) => "model_invalid",
Error::ParseFailed(_) => "parse_failed",
Error::InputTooLarge { .. } => "input_too_large",
Error::UnsupportedFormat(_) => "unsupported_format",
Error::InvalidInput(_) => "invalid_input",
Error::Io(_) => "io",
}
}
}
pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Token {
pub id: usize,
pub text: String,
pub lemma: String,
pub pos: String,
pub xpos: String,
pub feats: String,
pub head: usize,
pub dep: String,
pub deps: String,
pub misc: String,
pub is_punct: bool,
}
impl Token {
pub fn builder(
id: usize,
text: String,
lemma: String,
pos: String,
head: usize,
dep: String,
) -> TokenBuilder {
TokenBuilder {
id,
text,
lemma,
pos,
head,
dep,
xpos: String::new(),
feats: String::new(),
deps: String::new(),
misc: String::new(),
is_punct: false,
}
}
pub fn feat(&self, key: &str) -> Option<&str> {
self.feats
.split('|')
.find_map(|pair| match pair.split_once('=') {
Some((k, v)) if k == key => Some(v),
_ => None,
})
}
}
pub struct TokenBuilder {
id: usize,
text: String,
lemma: String,
pos: String,
head: usize,
dep: String,
xpos: String,
feats: String,
deps: String,
misc: String,
is_punct: bool,
}
impl TokenBuilder {
pub fn xpos(mut self, xpos: String) -> Self {
self.xpos = xpos;
self
}
pub fn feats(mut self, feats: String) -> Self {
self.feats = feats;
self
}
pub fn deps(mut self, deps: String) -> Self {
self.deps = deps;
self
}
pub fn misc(mut self, misc: String) -> Self {
self.misc = misc;
self
}
pub fn is_punct(mut self, is_punct: bool) -> Self {
self.is_punct = is_punct;
self
}
pub fn build(self) -> Token {
Token {
id: self.id,
text: self.text,
lemma: self.lemma,
pos: self.pos,
xpos: self.xpos,
feats: self.feats,
head: self.head,
dep: self.dep,
deps: self.deps,
misc: self.misc,
is_punct: self.is_punct,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Negation {
pub cue_id: usize,
pub cue_lemma: String,
pub head_id: usize,
}
fn detect_negations(tokens: &[Token]) -> Vec<Negation> {
const CUE_LEMMAS: [&str; 5] = ["not", "never", "no", "neither", "nor"];
const CUE_DEPS: [&str; 3] = ["advmod", "det", "cc"];
tokens
.iter()
.filter(|t| CUE_LEMMAS.contains(&t.lemma.as_str()) && CUE_DEPS.contains(&t.dep.as_str()))
.map(|t| Negation {
cue_id: t.id,
cue_lemma: t.lemma.clone(),
head_id: t.head,
})
.collect()
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Modal {
pub aux_id: usize,
pub aux_lemma: String,
pub head_id: usize,
}
fn detect_modals(tokens: &[Token]) -> Vec<Modal> {
const MODAL_LEMMAS: [&str; 10] = [
"can", "could", "may", "might", "must", "ought", "shall", "should", "will", "would",
];
tokens
.iter()
.filter(|t| (t.dep == "aux" || t.pos == "AUX") && MODAL_LEMMAS.contains(&t.lemma.as_str()))
.map(|t| Modal {
aux_id: t.id,
aux_lemma: t.lemma.clone(),
head_id: t.head,
})
.collect()
}
fn detect_bare_assertion(tokens: &[Token], modals: &[Modal]) -> bool {
const FINITE_CARRIER_DEPS: [&str; 3] = ["cop", "aux", "aux:pass"];
tokens.iter().find(|t| t.head == 0).is_some_and(|root| {
let unmodalized = !modals
.iter()
.any(|m| m.head_id == root.id || m.aux_id == root.id);
let indicative = root.feat("Mood") == Some("Ind")
|| tokens.iter().any(|t| {
t.head == root.id
&& FINITE_CARRIER_DEPS.contains(&t.dep.as_str())
&& t.feat("Mood") == Some("Ind")
});
unmodalized && indicative
})
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Reporting {
pub verb_id: usize,
pub verb_lemma: String,
pub ccomp_id: usize,
pub subject_id: Option<usize>,
pub subject_lemma: Option<String>,
}
fn detect_reportings(tokens: &[Token]) -> Vec<Reporting> {
const SUBJECT_DEPS: [&str; 3] = ["nsubj", "nsubj:pass", "nsubjpass"];
tokens
.iter()
.filter(|c| c.dep == "ccomp")
.filter_map(|c| {
let verb = tokens.iter().find(|t| t.id == c.head && t.pos == "VERB")?;
let subject = tokens
.iter()
.find(|t| t.head == verb.id && SUBJECT_DEPS.contains(&t.dep.as_str()));
Some(Reporting {
verb_id: verb.id,
verb_lemma: verb.lemma.clone(),
ccomp_id: c.id,
subject_id: subject.map(|s| s.id),
subject_lemma: subject.map(|s| s.lemma.clone()),
})
})
.collect()
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct RootAdverbial {
pub adv_id: usize,
pub adv_lemma: String,
}
fn detect_root_adverbials(tokens: &[Token]) -> Vec<RootAdverbial> {
let Some(root) = tokens.iter().find(|t| t.head == 0) else {
return Vec::new();
};
tokens
.iter()
.filter(|t| t.dep == "advmod" && t.head == root.id)
.map(|t| RootAdverbial {
adv_id: t.id,
adv_lemma: t.lemma.clone(),
})
.collect()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum HearstPattern {
SuchAs,
SuchNpAs,
Including,
Especially,
AndOther,
OrOther,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct HearstSpan {
pub head_id: usize,
pub head_lemma: String,
pub first_id: usize,
pub last_id: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct HearstPair {
pub pattern: HearstPattern,
pub hypernym: HearstSpan,
pub hyponym: HearstSpan,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Sentence {
pub text: String,
pub tokens: Vec<Token>,
#[serde(default)]
pub negations: Vec<Negation>,
#[serde(default)]
pub modals: Vec<Modal>,
#[serde(default)]
pub bare_assertion: bool,
#[serde(default)]
pub reportings: Vec<Reporting>,
#[serde(default)]
pub root_adverbials: Vec<RootAdverbial>,
#[serde(default)]
pub hearst_pairs: Vec<HearstPair>,
}
impl Sentence {
pub fn new(text: String, tokens: Vec<Token>) -> Self {
let negations = detect_negations(&tokens);
let modals = detect_modals(&tokens);
let bare_assertion = detect_bare_assertion(&tokens, &modals);
let reportings = detect_reportings(&tokens);
let root_adverbials = detect_root_adverbials(&tokens);
Self {
text,
tokens,
negations,
modals,
bare_assertion,
reportings,
root_adverbials,
hearst_pairs: Vec::new(),
}
}
pub fn reportings_in(&self, lexicon: &[&str]) -> Vec<&Reporting> {
self.reportings
.iter()
.filter(|r| lexicon.contains(&r.verb_lemma.as_str()))
.collect()
}
pub fn root_adverbials_in(&self, lexicon: &[&str]) -> Vec<&RootAdverbial> {
self.root_adverbials
.iter()
.filter(|a| lexicon.contains(&a.adv_lemma.as_str()))
.collect()
}
pub fn content_tokens(&self) -> Vec<&Token> {
self.tokens.iter().filter(|t| !t.is_punct).collect()
}
pub fn word_count(&self) -> usize {
self.tokens.iter().filter(|t| !t.is_punct).count()
}
pub fn is_passive(&self) -> bool {
self.tokens
.iter()
.any(|t| t.dep == "nsubj:pass" || t.dep == "nsubjpass" || t.dep == "aux:pass")
}
pub fn tree_depth(&self) -> usize {
use std::collections::{HashMap, HashSet};
let head_by_id: HashMap<usize, usize> =
self.tokens.iter().map(|t| (t.id, t.head)).collect();
let mut depth_by_id: HashMap<usize, usize> = HashMap::new();
fn depth_of(
id: usize,
head_by_id: &HashMap<usize, usize>,
depth_by_id: &mut HashMap<usize, usize>,
) -> usize {
if let Some(&d) = depth_by_id.get(&id) {
return d;
}
let mut visited: HashSet<usize> = HashSet::new();
let mut chain: Vec<usize> = Vec::new();
let mut cur = id;
let base_depth = loop {
if !visited.insert(cur) {
for &cid in &chain {
depth_by_id.insert(cid, usize::MAX);
}
return usize::MAX;
}
let head = head_by_id.get(&cur).copied().unwrap_or(0);
if head == 0 {
chain.push(cur);
break 0usize;
}
if let Some(&d) = depth_by_id.get(&head) {
chain.push(cur);
break d.saturating_add(1);
}
chain.push(cur);
cur = head;
};
let n = chain.len();
for (i, &cid) in chain.iter().enumerate() {
let d = base_depth.saturating_add(n - 1 - i);
depth_by_id.insert(cid, d);
}
depth_by_id[&id]
}
self.tokens
.iter()
.map(|t| depth_of(t.id, &head_by_id, &mut depth_by_id))
.max()
.unwrap_or(0)
}
pub fn root_token(&self) -> Option<&Token> {
self.tokens.iter().find(|t| t.head == 0)
}
pub fn children_of(&self, id: usize) -> Vec<&Token> {
self.tokens.iter().filter(|t| t.head == id).collect()
}
pub fn head_of(&self, id: usize) -> Option<&Token> {
self.tokens.iter().find(|t| t.id == id).and_then(|t| {
if t.head == 0 {
None
} else {
self.tokens.iter().find(|h| h.id == t.head)
}
})
}
pub fn subtree(&self, id: usize) -> Vec<&Token> {
let mut result = Vec::new();
let mut stack = vec![id];
let mut visited = std::collections::HashSet::new();
while let Some(current) = stack.pop() {
if !visited.insert(current) {
continue;
}
if let Some(token) = self.tokens.iter().find(|t| t.id == current) {
result.push(token);
for child in &self.tokens {
if child.head == current && child.id != current {
stack.push(child.id);
}
}
}
}
result.sort_by_key(|t| t.id);
result
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Paragraph {
pub text: String,
pub in_blockquote: bool,
pub sentences: Vec<Sentence>,
pub readability_grade: Option<f64>,
pub lexical_density: Option<f64>,
pub compression_ratio: Option<f64>,
}
impl Paragraph {
pub fn new(text: String, in_blockquote: bool) -> Self {
Self {
text,
in_blockquote,
sentences: Vec::new(),
readability_grade: None,
lexical_density: None,
compression_ratio: None,
}
}
pub fn word_count(&self) -> usize {
self.sentences.iter().map(|s| s.word_count()).sum()
}
pub fn sentence_count(&self) -> usize {
self.sentences.len()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Section {
pub heading: Option<String>,
pub level: usize,
pub paragraphs: Vec<Paragraph>,
}
impl Section {
pub fn new(heading: Option<String>, level: usize, paragraphs: Vec<Paragraph>) -> Self {
Self {
heading,
level,
paragraphs,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Document {
pub sections: Vec<Section>,
pub vocabulary_ttr: Option<f64>,
pub nominalization_ratio: Option<f64>,
#[serde(default)]
pub passive_ratio: Option<f64>,
}
impl Document {
pub fn new(sections: Vec<Section>) -> Self {
Self {
sections,
vocabulary_ttr: None,
nominalization_ratio: None,
passive_ratio: None,
}
}
pub fn paragraphs(&self) -> impl Iterator<Item = &Paragraph> {
self.sections.iter().flat_map(|s| s.paragraphs.iter())
}
pub fn paragraphs_mut(&mut self) -> impl Iterator<Item = &mut Paragraph> {
self.sections
.iter_mut()
.flat_map(|s| s.paragraphs.iter_mut())
}
pub fn paragraph_count(&self) -> usize {
self.sections.iter().map(|s| s.paragraphs.len()).sum()
}
pub fn sentences(&self) -> impl Iterator<Item = &Sentence> {
self.sections
.iter()
.flat_map(|s| s.paragraphs.iter())
.flat_map(|p| p.sentences.iter())
}
pub fn tokens(&self) -> impl Iterator<Item = &Token> {
self.sections
.iter()
.flat_map(|s| s.paragraphs.iter())
.flat_map(|p| p.sentences.iter())
.flat_map(|s| s.tokens.iter())
}
pub fn total_sentences(&self) -> usize {
self.paragraphs().map(|p| p.sentence_count()).sum()
}
pub fn total_words(&self) -> usize {
self.sentences().map(|s| s.word_count()).sum()
}
pub fn passive_ratio(&self) -> f64 {
let total = self.total_sentences();
if total == 0 {
return 0.0;
}
self.sentences().filter(|s| s.is_passive()).count() as f64 / total as f64
}
pub fn mean_sentence_length(&self) -> f64 {
let total = self.total_sentences();
if total == 0 {
return 0.0;
}
self.total_words() as f64 / total as f64
}
pub fn sentence_length_std(&self) -> f64 {
let lengths: Vec<f64> = self.sentences().map(|s| s.word_count() as f64).collect();
if lengths.len() <= 1 {
return 0.0;
}
let mean = lengths.iter().sum::<f64>() / lengths.len() as f64;
let var =
lengths.iter().map(|l| (l - mean).powi(2)).sum::<f64>() / (lengths.len() - 1) as f64;
var.sqrt()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct ScoredSentence {
pub text: String,
pub score: f64,
pub position: usize,
}
impl ScoredSentence {
pub fn new(text: String, score: f64, position: usize) -> Self {
Self {
text,
score,
position,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Keyphrase {
pub phrase: String,
pub score: f64,
}
impl Keyphrase {
pub fn new(phrase: String, score: f64) -> Self {
Self { phrase, score }
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub enum Format {
Markdown,
PlainText,
Pdf,
Docx,
}
impl Format {
pub fn from_path(path: impl AsRef<Path>) -> Format {
match path.as_ref().extension().and_then(|e| e.to_str()) {
Some("md" | "markdown") => Format::Markdown,
Some("pdf") => Format::Pdf,
Some("docx") => Format::Docx,
_ => Format::PlainText,
}
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct RawDocument {
pub text: String,
pub path: Option<PathBuf>,
pub format: Format,
}
impl RawDocument {
pub fn new(text: String, path: Option<PathBuf>, format: Format) -> Self {
Self { text, path, format }
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct CorpusEntry {
pub path: Option<PathBuf>,
pub analysis: Document,
}
impl CorpusEntry {
pub fn new(path: Option<PathBuf>, analysis: Document) -> Self {
Self { path, analysis }
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Corpus {
pub entries: Vec<CorpusEntry>,
}
impl Corpus {
pub fn new(entries: Vec<CorpusEntry>) -> Self {
Self { entries }
}
pub fn total_words(&self) -> usize {
self.entries.iter().map(|e| e.analysis.total_words()).sum()
}
pub fn passive_ratio(&self) -> f64 {
let total: usize = self
.entries
.iter()
.map(|e| e.analysis.total_sentences())
.sum();
if total == 0 {
return 0.0;
}
let passive: usize = self
.entries
.iter()
.map(|e| e.analysis.sentences().filter(|s| s.is_passive()).count())
.sum();
passive as f64 / total as f64
}
pub fn mean_readability(&self) -> f64 {
let grades: Vec<f64> = self
.entries
.iter()
.flat_map(|e| e.analysis.paragraphs())
.filter_map(|p| p.readability_grade)
.collect();
if grades.is_empty() {
return 0.0;
}
grades.iter().sum::<f64>() / grades.len() as f64
}
}
#[derive(Debug)]
#[non_exhaustive]
pub struct DocumentError {
pub path: Option<PathBuf>,
pub error: Error,
}
impl DocumentError {
pub fn new(path: Option<PathBuf>, error: Error) -> Self {
Self { path, error }
}
}
impl std::fmt::Display for DocumentError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match &self.path {
Some(p) => write!(f, "{}: {}", p.display(), self.error),
None => write!(f, "{}", self.error),
}
}
}
impl std::error::Error for DocumentError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.error)
}
}
#[derive(Debug)]
#[non_exhaustive]
pub struct CorpusResult {
pub corpus: Corpus,
pub errors: Vec<DocumentError>,
}
impl CorpusResult {
pub fn new(corpus: Corpus, errors: Vec<DocumentError>) -> Self {
Self { corpus, errors }
}
}
impl FromIterator<std::result::Result<CorpusEntry, DocumentError>> for CorpusResult {
fn from_iter<I: IntoIterator<Item = std::result::Result<CorpusEntry, DocumentError>>>(
iter: I,
) -> Self {
let mut entries = Vec::new();
let mut errors = Vec::new();
for item in iter {
match item {
Ok(entry) => entries.push(entry),
Err(err) => errors.push(err),
}
}
CorpusResult::new(Corpus::new(entries), errors)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn embedding_serializes_as_bare_array() {
let e = Embedding(vec![1.0, -0.5, 0.25]);
let json = serde_json::to_string(&e).expect("serialize");
assert_eq!(json, "[1.0,-0.5,0.25]");
let back: Embedding = serde_json::from_str(&json).expect("deserialize");
assert_eq!(back, e);
}
fn make_token(id: usize, text: &str, pos: &str, dep: &str, head: usize) -> Token {
Token {
id,
text: text.to_string(),
lemma: text.to_lowercase(),
pos: pos.to_string(),
xpos: String::new(),
feats: String::new(),
head,
dep: dep.to_string(),
deps: String::new(),
misc: String::new(),
is_punct: pos == "PUNCT",
}
}
fn make_token_feats(
id: usize,
text: &str,
pos: &str,
dep: &str,
head: usize,
feats: &str,
) -> Token {
let mut tok = make_token(id, text, pos, dep, head);
tok.feats = feats.to_string();
tok
}
fn passive_sentence() -> Sentence {
Sentence::new(
"The system was built by the team".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "system", "NOUN", "nsubj:pass", 4),
make_token(3, "was", "AUX", "aux:pass", 4),
make_token(4, "built", "VERB", "root", 0),
make_token(5, "by", "ADP", "case", 7),
make_token(6, "the", "DET", "det", 7),
make_token(7, "team", "NOUN", "obl", 4),
],
)
}
fn active_sentence() -> Sentence {
Sentence::new(
"The team shipped the product".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "team", "NOUN", "nsubj", 3),
make_token(3, "shipped", "VERB", "root", 0),
make_token(4, "the", "DET", "det", 5),
make_token(5, "product", "NOUN", "obj", 3),
],
)
}
#[test]
fn content_tokens_excludes_punct() {
let sent = Sentence::new(
"Hello, world!".to_string(),
vec![
make_token(1, "Hello", "INTJ", "root", 0),
make_token(2, ",", "PUNCT", "punct", 1),
make_token(3, "world", "NOUN", "vocative", 1),
make_token(4, "!", "PUNCT", "punct", 1),
],
);
assert_eq!(sent.content_tokens().len(), 2);
assert_eq!(sent.word_count(), 2);
}
#[test]
fn is_passive_detects_passive() {
assert!(passive_sentence().is_passive());
assert!(!active_sentence().is_passive());
}
fn token_with_feats(feats: &str) -> Token {
Token::builder(
1,
"x".to_string(),
"x".to_string(),
"NOUN".to_string(),
0,
"root".to_string(),
)
.feats(feats.to_string())
.build()
}
const HARVESTED_FEATS: &[&str] = &[
"",
"Case=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs",
"Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin",
"Tense=Past|VerbForm=Part|Voice=Pass",
"Definite=Def|PronType=Art",
"Number=Sing",
"Degree=Pos",
"Mood=Ind|Tense=Past|VerbForm=Fin",
"VerbForm=Fin",
"Case=Nom|Number=Plur|Person=1|PronType=Prs",
"VerbForm=Inf",
"Number=Sing|PronType=Dem",
"Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin",
"Number=Plur",
];
#[test]
fn feat_round_trips_every_harvested_pair() {
for feats in HARVESTED_FEATS {
let tok = token_with_feats(feats);
for pair in feats.split('|').filter(|p| !p.is_empty()) {
let (k, v) = pair.split_once('=').expect("harvested pair has =");
assert_eq!(tok.feat(k), Some(v), "feats {feats:?} key {k:?}");
}
assert_eq!(tok.feat("Absent"), None, "feats {feats:?}");
assert_eq!(tok.feat(""), None, "feats {feats:?}");
}
}
#[test]
fn feat_none_on_empty_and_underscore_placeholder() {
for feats in ["", "_"] {
let tok = token_with_feats(feats);
for key in ["Mood", "VerbForm", "_", ""] {
assert_eq!(tok.feat(key), None, "feats {feats:?} key {key:?}");
}
}
}
#[test]
fn feat_single_feature() {
let tok = token_with_feats("Number=Sing");
assert_eq!(tok.feat("Number"), Some("Sing"));
assert_eq!(tok.feat("number"), None, "keys match exactly");
assert_eq!(tok.feat("Num"), None, "no prefix match");
}
#[test]
fn feat_multi_value_returned_unsplit() {
let tok = token_with_feats("Case=Nom,Acc|Number=Sing");
assert_eq!(tok.feat("Case"), Some("Nom,Acc"));
}
#[test]
fn feat_malformed_input_does_not_panic() {
let tok = token_with_feats("junk|Mood=Ind=extra|Mood=Sub|=orphan|Tense=");
assert_eq!(tok.feat("junk"), None);
assert_eq!(tok.feat("Mood"), Some("Ind=extra"));
assert_eq!(tok.feat("Tense"), Some(""));
assert_eq!(tok.feat("orphan"), None);
}
#[test]
fn negation_detects_not_as_advmod() {
let sent = Sentence::new(
"The plan is not ready.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "plan", "NOUN", "nsubj", 5),
make_token(3, "is", "AUX", "cop", 5),
make_token(4, "not", "PART", "advmod", 5),
make_token(5, "ready", "ADJ", "root", 0),
make_token(6, ".", "PUNCT", "punct", 5),
],
);
assert_eq!(sent.negations.len(), 1);
assert_eq!(sent.negations[0].cue_id, 4);
assert_eq!(sent.negations[0].cue_lemma, "not");
assert_eq!(sent.negations[0].head_id, 5);
}
#[test]
fn negation_detects_never_as_advmod() {
let sent = Sentence::new(
"It was never reviewed.".to_string(),
vec![
make_token(1, "It", "PRON", "nsubj:pass", 4),
make_token(2, "was", "AUX", "aux:pass", 4),
make_token(3, "never", "ADV", "advmod", 4),
make_token(4, "reviewed", "VERB", "root", 0),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.negations.len(), 1);
assert_eq!(sent.negations[0].cue_lemma, "never");
assert_eq!(sent.negations[0].cue_id, 3);
assert_eq!(sent.negations[0].head_id, 4);
}
#[test]
fn negation_detects_no_as_det() {
let sent = Sentence::new(
"No changes were made.".to_string(),
vec![
make_token(1, "No", "DET", "det", 2),
make_token(2, "changes", "NOUN", "nsubj:pass", 4),
make_token(3, "were", "AUX", "aux:pass", 4),
make_token(4, "made", "VERB", "root", 0),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.negations.len(), 1);
assert_eq!(sent.negations[0].cue_lemma, "no");
assert_eq!(sent.negations[0].head_id, 2);
}
#[test]
fn negation_detects_neither_and_nor() {
let sent = Sentence::new(
"Neither option worked, nor did the fallback.".to_string(),
vec![
make_token(1, "Neither", "DET", "det", 2),
make_token(2, "option", "NOUN", "nsubj", 3),
make_token(3, "worked", "VERB", "root", 0),
make_token(4, ",", "PUNCT", "punct", 6),
make_token(5, "nor", "CCONJ", "cc", 6),
make_token(6, "did", "VERB", "conj", 3),
make_token(7, "the", "DET", "det", 8),
make_token(8, "fallback", "NOUN", "obj", 6),
make_token(9, ".", "PUNCT", "punct", 3),
],
);
let lemmas: Vec<&str> = sent
.negations
.iter()
.map(|n| n.cue_lemma.as_str())
.collect();
assert_eq!(lemmas, vec!["neither", "nor"]);
assert_eq!(sent.negations[0].head_id, 2);
assert_eq!(sent.negations[1].head_id, 6);
}
#[test]
fn negation_no_false_positive_on_nothing_as_subject() {
let sent = Sentence::new(
"Nothing happened.".to_string(),
vec![
make_token(1, "Nothing", "PRON", "nsubj", 2),
make_token(2, "happened", "VERB", "root", 0),
make_token(3, ".", "PUNCT", "punct", 2),
],
);
assert!(sent.negations.is_empty());
}
#[test]
fn negation_no_fire_on_pronominal_neither() {
let sent = Sentence::new(
"Neither of them came.".to_string(),
vec![
make_token(1, "Neither", "DET", "nsubj", 4),
make_token(2, "of", "ADP", "case", 3),
make_token(3, "them", "PRON", "nmod", 1),
make_token(4, "came", "VERB", "root", 0),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert!(sent.negations.is_empty());
}
#[test]
fn negation_on_tokenizer_split_cannot_fires_once_on_not() {
let sent = Sentence::new(
"We cannot merge this.".to_string(),
vec![
make_token(1, "We", "PRON", "nsubj", 4),
make_token(2, "can", "AUX", "aux", 4),
make_token(3, "not", "PART", "advmod", 4),
make_token(4, "merge", "VERB", "root", 0),
make_token(5, "this", "PRON", "obj", 4),
make_token(6, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.negations.len(), 1);
assert_eq!(sent.negations[0].cue_id, 3);
assert_eq!(sent.negations[0].cue_lemma, "not");
assert_eq!(sent.negations[0].head_id, 4);
}
#[test]
fn negation_empty_on_unnegated_sentences() {
assert!(active_sentence().negations.is_empty());
assert!(passive_sentence().negations.is_empty());
}
#[test]
fn negations_serialize_and_default_on_old_json() {
let sent = Sentence::new(
"It was never reviewed.".to_string(),
vec![make_token(1, "never", "ADV", "advmod", 0)],
);
let json = serde_json::to_value(&sent).unwrap();
assert_eq!(json["negations"][0]["cue_lemma"], "never");
let old = r#"{"text":"old","tokens":[]}"#;
let sent: Sentence = serde_json::from_str(old).unwrap();
assert!(sent.negations.is_empty());
}
fn modal_clause(lemma: &str) -> Sentence {
Sentence::new(
format!("They {lemma} leave."),
vec![
make_token(1, "They", "PRON", "nsubj", 3),
make_token_feats(2, lemma, "AUX", "aux", 3, "VerbForm=Fin"),
make_token_feats(3, "leave", "VERB", "root", 0, "VerbForm=Inf"),
make_token(4, ".", "PUNCT", "punct", 3),
],
)
}
#[test]
fn modal_every_closed_class_lemma_is_found() {
for lemma in [
"can", "could", "may", "might", "must", "ought", "shall", "should", "will", "would",
] {
let sent = modal_clause(lemma);
assert_eq!(sent.modals.len(), 1, "lemma {lemma:?}");
assert_eq!(sent.modals[0].aux_id, 2, "lemma {lemma:?}");
assert_eq!(sent.modals[0].aux_lemma, lemma);
assert_eq!(sent.modals[0].head_id, 3, "lemma {lemma:?}");
assert!(!sent.bare_assertion, "modalized clause, lemma {lemma:?}");
}
}
#[test]
fn modal_chain_reports_its_modal_with_the_content_verb_head() {
let sent = Sentence::new(
"It might have been done.".to_string(),
vec![
make_token(1, "It", "PRON", "nsubj:pass", 5),
make_token_feats(2, "might", "AUX", "aux", 5, "VerbForm=Fin"),
make_token_feats(3, "have", "AUX", "aux", 5, "VerbForm=Inf"),
make_token_feats(4, "been", "AUX", "aux:pass", 5, "Tense=Past|VerbForm=Part"),
make_token_feats(
5,
"done",
"VERB",
"root",
0,
"Tense=Past|VerbForm=Part|Voice=Pass",
),
make_token(6, ".", "PUNCT", "punct", 5),
],
);
assert_eq!(sent.modals.len(), 1);
assert_eq!(sent.modals[0].aux_id, 2);
assert_eq!(sent.modals[0].aux_lemma, "might");
assert_eq!(sent.modals[0].head_id, 5);
assert!(!sent.bare_assertion, "participle root carries no Mood");
}
#[test]
fn modal_coordinated_clauses_report_both() {
let sent = Sentence::new(
"He can stay, and she may leave.".to_string(),
vec![
make_token(1, "He", "PRON", "nsubj", 3),
make_token_feats(2, "can", "AUX", "aux", 3, "VerbForm=Fin"),
make_token_feats(3, "stay", "VERB", "root", 0, "VerbForm=Inf"),
make_token(4, ",", "PUNCT", "punct", 8),
make_token(5, "and", "CCONJ", "cc", 8),
make_token(6, "she", "PRON", "nsubj", 8),
make_token_feats(7, "may", "AUX", "aux", 8, "VerbForm=Fin"),
make_token_feats(8, "leave", "VERB", "conj", 3, "VerbForm=Inf"),
make_token(9, ".", "PUNCT", "punct", 3),
],
);
let found: Vec<(&str, usize)> = sent
.modals
.iter()
.map(|m| (m.aux_lemma.as_str(), m.head_id))
.collect();
assert_eq!(found, vec![("can", 3), ("may", 8)]);
}
#[test]
fn modal_promoted_to_root_under_vp_ellipsis_is_found() {
let sent = Sentence::new(
"Yes, you should.".to_string(),
vec![
make_token(1, "Yes", "INTJ", "discourse", 4),
make_token(2, ",", "PUNCT", "punct", 4),
make_token(3, "you", "PRON", "nsubj", 4),
make_token_feats(4, "should", "AUX", "root", 0, "VerbForm=Fin"),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.modals.len(), 1);
assert_eq!(sent.modals[0].aux_lemma, "should");
assert_eq!(sent.modals[0].head_id, 0);
assert!(!sent.bare_assertion, "root modal means a modalized clause");
}
#[test]
fn modal_coordinated_auxiliaries_report_both() {
let sent = Sentence::new(
"He can and will succeed.".to_string(),
vec![
make_token(1, "He", "PRON", "nsubj", 2),
make_token_feats(2, "can", "AUX", "root", 0, "VerbForm=Fin"),
make_token(3, "and", "CCONJ", "cc", 5),
make_token_feats(4, "will", "AUX", "aux", 5, "VerbForm=Fin"),
make_token_feats(5, "succeed", "VERB", "conj", 2, "VerbForm=Inf"),
make_token(6, ".", "PUNCT", "punct", 2),
],
);
let found: Vec<(&str, usize)> = sent
.modals
.iter()
.map(|m| (m.aux_lemma.as_str(), m.head_id))
.collect();
assert_eq!(found, vec![("can", 0), ("will", 5)]);
assert!(!sent.bare_assertion);
}
#[test]
fn modal_second_coordinated_auxiliary_as_conj_is_found() {
let sent = Sentence::new(
"He can, and he will, succeed.".to_string(),
vec![
make_token(1, "He", "PRON", "nsubj", 2),
make_token_feats(2, "can", "AUX", "root", 0, "VerbForm=Fin"),
make_token(3, ",", "PUNCT", "punct", 6),
make_token(4, "and", "CCONJ", "cc", 6),
make_token(5, "he", "PRON", "nsubj", 6),
make_token_feats(6, "will", "AUX", "conj", 2, "VerbForm=Fin"),
make_token(7, ",", "PUNCT", "punct", 8),
make_token_feats(8, "succeed", "VERB", "conj", 2, "Mood=Ind|VerbForm=Fin"),
make_token(9, ".", "PUNCT", "punct", 2),
],
);
let found: Vec<(&str, usize)> = sent
.modals
.iter()
.map(|m| (m.aux_lemma.as_str(), m.head_id))
.collect();
assert_eq!(found, vec![("can", 0), ("will", 2)]);
assert!(!sent.bare_assertion, "root modal means a modalized clause");
}
#[test]
fn copular_assertion_is_bare() {
let sent = Sentence::new(
"The sky is blue.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "sky", "NOUN", "nsubj", 4),
make_token_feats(
3,
"is",
"AUX",
"cop",
4,
"Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin",
),
make_token_feats(4, "blue", "ADJ", "root", 0, "Degree=Pos"),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert!(sent.modals.is_empty(), "the copula be is outside the class");
assert!(sent.bare_assertion);
}
#[test]
fn do_support_assertion_is_bare() {
let sent = Sentence::new(
"He did leave.".to_string(),
vec![
make_token(1, "He", "PRON", "nsubj", 3),
make_token_feats(2, "do", "AUX", "aux", 3, "Mood=Ind|Tense=Past|VerbForm=Fin"),
make_token_feats(3, "leave", "VERB", "root", 0, "VerbForm=Inf"),
make_token(4, ".", "PUNCT", "punct", 3),
],
);
assert!(sent.modals.is_empty());
assert!(sent.bare_assertion);
}
#[test]
fn modal_contraction_fires_on_its_lemma() {
let sent = Sentence::new(
"They won't stop.".to_string(),
vec![
make_token(1, "They", "PRON", "nsubj", 4),
make_token_feats(2, "will", "AUX", "aux", 4, "VerbForm=Fin"),
make_token(3, "not", "PART", "advmod", 4),
make_token_feats(4, "stop", "VERB", "root", 0, "VerbForm=Inf"),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.modals.len(), 1);
assert_eq!(sent.modals[0].aux_lemma, "will");
assert_eq!(sent.negations.len(), 1, "the split not still fires");
}
#[test]
fn modal_no_false_positive_on_lexical_will() {
let sent = Sentence::new(
"The will was signed yesterday.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "will", "NOUN", "nsubj:pass", 4),
make_token_feats(
3,
"was",
"AUX",
"aux:pass",
4,
"Mood=Ind|Tense=Past|VerbForm=Fin",
),
make_token_feats(
4,
"signed",
"VERB",
"root",
0,
"Tense=Past|VerbForm=Part|Voice=Pass",
),
make_token(5, "yesterday", "NOUN", "obl:tmod", 4),
make_token(6, ".", "PUNCT", "punct", 4),
],
);
assert!(sent.modals.is_empty());
assert!(sent.bare_assertion);
}
#[test]
fn directive_with_modal_is_not_a_bare_assertion() {
let sent = Sentence::new(
"You must complete the form by Friday.".to_string(),
vec![
make_token(1, "You", "PRON", "nsubj", 3),
make_token_feats(2, "must", "AUX", "aux", 3, "VerbForm=Fin"),
make_token_feats(3, "complete", "VERB", "root", 0, "VerbForm=Inf"),
make_token(4, "the", "DET", "det", 5),
make_token(5, "form", "NOUN", "obj", 3),
make_token(6, "by", "ADP", "case", 7),
make_token(7, "Friday", "PROPN", "obl", 3),
make_token(8, ".", "PUNCT", "punct", 3),
],
);
assert_eq!(sent.modals.len(), 1);
assert_eq!(sent.modals[0].aux_lemma, "must");
assert!(!sent.bare_assertion);
}
#[test]
fn bare_assertion_on_indicative_root_without_modal() {
let sent = Sentence::new(
"The committee approved it.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "committee", "NOUN", "nsubj", 3),
make_token_feats(
3,
"approved",
"VERB",
"root",
0,
"Mood=Ind|Tense=Past|VerbForm=Fin",
),
make_token(4, "it", "PRON", "obj", 3),
make_token(5, ".", "PUNCT", "punct", 3),
],
);
assert!(sent.modals.is_empty());
assert!(sent.bare_assertion);
}
#[test]
fn imperative_root_is_not_a_bare_assertion() {
let sent = Sentence::new(
"Complete the form by Friday.".to_string(),
vec![
make_token_feats(1, "Complete", "VERB", "root", 0, "Mood=Imp|VerbForm=Fin"),
make_token(2, "the", "DET", "det", 3),
make_token(3, "form", "NOUN", "obj", 1),
make_token(4, "by", "ADP", "case", 5),
make_token(5, "Friday", "PROPN", "obl", 1),
make_token(6, ".", "PUNCT", "punct", 1),
],
);
assert!(sent.modals.is_empty());
assert!(!sent.bare_assertion);
}
#[test]
fn subordinate_modal_is_reported_and_does_not_defeat_bare_assertion() {
let sent = Sentence::new(
"I know it might rain.".to_string(),
vec![
make_token(1, "I", "PRON", "nsubj", 2),
make_token_feats(
2,
"know",
"VERB",
"root",
0,
"Mood=Ind|Tense=Pres|VerbForm=Fin",
),
make_token(3, "it", "PRON", "nsubj", 5),
make_token_feats(4, "might", "AUX", "aux", 5, "VerbForm=Fin"),
make_token(5, "rain", "NOUN", "ccomp", 2),
make_token(6, ".", "PUNCT", "punct", 5),
],
);
assert_eq!(sent.modals.len(), 1);
assert_eq!(sent.modals[0].aux_lemma, "might");
assert_eq!(sent.modals[0].head_id, 5);
assert!(sent.bare_assertion);
}
#[test]
fn modals_serialize_and_default_on_old_json() {
let sent = modal_clause("might");
let json = serde_json::to_value(&sent).unwrap();
assert_eq!(json["modals"][0]["aux_lemma"], "might");
assert_eq!(json["bare_assertion"], false);
let old = r#"{"text":"old","tokens":[]}"#;
let sent: Sentence = serde_json::from_str(old).unwrap();
assert!(sent.modals.is_empty());
assert!(!sent.bare_assertion);
}
#[test]
fn reporting_self_attribution() {
let sent = Sentence::new(
"We show that the method works.".to_string(),
vec![
make_token(1, "We", "PRON", "nsubj", 2),
make_token(2, "show", "VERB", "root", 0),
make_token(3, "that", "SCONJ", "mark", 6),
make_token(4, "the", "DET", "det", 5),
make_token(5, "method", "NOUN", "nsubj", 6),
make_token(6, "works", "VERB", "ccomp", 2),
make_token(7, ".", "PUNCT", "punct", 2),
],
);
assert_eq!(sent.reportings.len(), 1);
let r = &sent.reportings[0];
assert_eq!(r.verb_id, 2);
assert_eq!(r.verb_lemma, "show");
assert_eq!(r.ccomp_id, 6);
assert_eq!(r.subject_id, Some(1));
assert_eq!(r.subject_lemma.as_deref(), Some("we"));
}
#[test]
fn reporting_other_attribution() {
let sent = Sentence::new(
"Smith reported that the effect vanished.".to_string(),
vec![
make_token(1, "Smith", "PROPN", "nsubj", 2),
make_token(2, "reported", "VERB", "root", 0),
make_token(3, "that", "SCONJ", "mark", 6),
make_token(4, "the", "DET", "det", 5),
make_token(5, "effect", "NOUN", "nsubj", 6),
make_token(6, "vanished", "VERB", "ccomp", 2),
make_token(7, ".", "PUNCT", "punct", 2),
],
);
assert_eq!(sent.reportings.len(), 1);
let r = &sent.reportings[0];
assert_eq!((r.verb_id, r.ccomp_id), (2, 6));
assert_eq!(r.verb_lemma, "reported");
assert_eq!(r.subject_id, Some(1));
}
#[test]
fn reporting_impersonal() {
let sent = Sentence::new(
"These results suggest that the mechanism is shared.".to_string(),
vec![
make_token(1, "These", "DET", "det", 2),
make_token(2, "results", "NOUN", "nsubj", 3),
make_token(3, "suggest", "VERB", "root", 0),
make_token(4, "that", "SCONJ", "mark", 8),
make_token(5, "the", "DET", "det", 6),
make_token(6, "mechanism", "NOUN", "nsubj:pass", 8),
make_token(7, "is", "AUX", "aux:pass", 8),
make_token(8, "shared", "VERB", "ccomp", 3),
make_token(9, ".", "PUNCT", "punct", 3),
],
);
assert_eq!(sent.reportings.len(), 1);
let r = &sent.reportings[0];
assert_eq!((r.verb_id, r.ccomp_id), (3, 8));
assert_eq!(r.subject_id, Some(2));
assert_eq!(r.subject_lemma.as_deref(), Some("results"));
}
#[test]
fn reporting_survives_upstream_split_with_no_subject() {
let sent = Sentence::new(
"reported that the effect vanished.".to_string(),
vec![
make_token(1, "reported", "VERB", "root", 0),
make_token(2, "that", "SCONJ", "mark", 5),
make_token(3, "the", "DET", "det", 4),
make_token(4, "effect", "NOUN", "nsubj", 5),
make_token(5, "vanished", "VERB", "ccomp", 1),
make_token(6, ".", "PUNCT", "punct", 1),
],
);
assert_eq!(sent.reportings.len(), 1);
let r = &sent.reportings[0];
assert_eq!((r.verb_id, r.ccomp_id), (1, 5));
assert_eq!(r.subject_id, None);
assert_eq!(r.subject_lemma, None);
}
#[test]
fn reporting_no_fire_without_ccomp() {
let sent = Sentence::new(
"Smith reported a similar finding.".to_string(),
vec![
make_token(1, "Smith", "PROPN", "nsubj", 2),
make_token(2, "reported", "VERB", "root", 0),
make_token(3, "a", "DET", "det", 5),
make_token(4, "similar", "ADJ", "amod", 5),
make_token(5, "finding", "NOUN", "obj", 2),
make_token(6, ".", "PUNCT", "punct", 2),
],
);
assert!(sent.reportings.is_empty());
assert!(active_sentence().reportings.is_empty());
}
#[test]
fn reporting_no_fire_on_non_verb_head() {
let sent = Sentence::new(
"I am sure that it works.".to_string(),
vec![
make_token(1, "I", "PRON", "nsubj", 3),
make_token(2, "am", "AUX", "cop", 3),
make_token(3, "sure", "ADJ", "root", 0),
make_token(4, "that", "SCONJ", "mark", 6),
make_token(5, "it", "PRON", "nsubj", 6),
make_token(6, "works", "VERB", "ccomp", 3),
make_token(7, ".", "PUNCT", "punct", 3),
],
);
assert!(sent.reportings.is_empty());
}
#[test]
fn root_adverbial_detects_sentence_scope_advmod() {
let sent = Sentence::new(
"Reportedly, the deal closed.".to_string(),
vec![
make_token(1, "Reportedly", "ADV", "advmod", 5),
make_token(2, ",", "PUNCT", "punct", 5),
make_token(3, "the", "DET", "det", 4),
make_token(4, "deal", "NOUN", "nsubj", 5),
make_token(5, "closed", "VERB", "root", 0),
make_token(6, ".", "PUNCT", "punct", 5),
],
);
assert_eq!(sent.root_adverbials.len(), 1);
assert_eq!(sent.root_adverbials[0].adv_id, 1);
assert_eq!(sent.root_adverbials[0].adv_lemma, "reportedly");
}
#[test]
fn root_adverbial_reports_manner_adverbs_too() {
let sent = Sentence::new(
"The team quickly shipped.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "team", "NOUN", "nsubj", 4),
make_token(3, "quickly", "ADV", "advmod", 4),
make_token(4, "shipped", "VERB", "root", 0),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.root_adverbials.len(), 1);
assert_eq!(sent.root_adverbials[0].adv_lemma, "quickly");
assert!(
sent.root_adverbials_in(&["reportedly", "allegedly"])
.is_empty()
);
}
#[test]
fn root_adverbial_ignores_non_root_attachment() {
let sent = Sentence::new(
"She says he ran quickly.".to_string(),
vec![
make_token(1, "She", "PRON", "nsubj", 2),
make_token(2, "says", "VERB", "root", 0),
make_token(3, "he", "PRON", "nsubj", 4),
make_token(4, "ran", "VERB", "ccomp", 2),
make_token(5, "quickly", "ADV", "advmod", 4),
make_token(6, ".", "PUNCT", "punct", 2),
],
);
assert!(sent.root_adverbials.is_empty());
assert_eq!(sent.reportings.len(), 1);
}
#[test]
fn root_adverbial_overlaps_negation_by_design() {
let sent = Sentence::new(
"The plan is not ready.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "plan", "NOUN", "nsubj", 5),
make_token(3, "is", "AUX", "cop", 5),
make_token(4, "not", "PART", "advmod", 5),
make_token(5, "ready", "ADJ", "root", 0),
make_token(6, ".", "PUNCT", "punct", 5),
],
);
assert_eq!(sent.negations.len(), 1);
assert_eq!(sent.root_adverbials.len(), 1);
assert_eq!(sent.root_adverbials[0].adv_id, 4);
}
#[test]
fn reportings_in_filters_by_caller_lexicon() {
let sent = Sentence::new(
"Smith reported that the effect vanished.".to_string(),
vec![
make_token(1, "Smith", "PROPN", "nsubj", 2),
make_token(2, "reported", "VERB", "root", 0),
make_token(3, "that", "SCONJ", "mark", 6),
make_token(4, "the", "DET", "det", 5),
make_token(5, "effect", "NOUN", "nsubj", 6),
make_token(6, "vanished", "VERB", "ccomp", 2),
make_token(7, ".", "PUNCT", "punct", 2),
],
);
assert_eq!(sent.reportings_in(&["reported", "suggest"]).len(), 1);
assert!(sent.reportings_in(&["say", "claim"]).is_empty());
}
#[test]
fn reportings_serialize_and_default_on_old_json() {
let sent = Sentence::new(
"Reportedly, it works.".to_string(),
vec![
make_token(1, "Reportedly", "ADV", "advmod", 3),
make_token(2, "it", "PRON", "nsubj", 3),
make_token(3, "works", "VERB", "root", 0),
make_token(4, ".", "PUNCT", "punct", 3),
],
);
let json = serde_json::to_value(&sent).unwrap();
assert_eq!(json["root_adverbials"][0]["adv_lemma"], "reportedly");
assert!(json["reportings"].as_array().unwrap().is_empty());
let old = r#"{"text":"old","tokens":[]}"#;
let sent: Sentence = serde_json::from_str(old).unwrap();
assert!(sent.reportings.is_empty());
assert!(sent.root_adverbials.is_empty());
}
#[test]
fn tree_depth_computes_max() {
let sent = active_sentence();
assert!(sent.tree_depth() > 0);
}
#[test]
fn root_token_finds_root() {
let sent = passive_sentence();
let root = sent.root_token().unwrap();
assert_eq!(root.text, "built");
}
#[test]
fn children_of_returns_direct_children() {
let sent = passive_sentence();
let children = sent.children_of(4);
let texts: Vec<&str> = children.iter().map(|t| t.text.as_str()).collect();
assert!(texts.contains(&"system"));
assert!(texts.contains(&"was"));
assert!(texts.contains(&"team"));
}
#[test]
fn head_of_returns_head() {
let sent = passive_sentence();
let head = sent.head_of(2).unwrap();
assert_eq!(head.text, "built");
}
#[test]
fn head_of_root_returns_none() {
let sent = passive_sentence();
assert!(sent.head_of(4).is_none());
}
#[test]
fn subtree_includes_all_descendants() {
let sent = passive_sentence();
let sub = sent.subtree(4);
assert_eq!(sub.len(), sent.tokens.len());
}
#[test]
fn subtree_leaf_returns_single() {
let sent = passive_sentence();
let sub = sent.subtree(1);
assert_eq!(sub.len(), 1);
assert_eq!(sub[0].text, "The");
}
#[test]
fn invalid_id_returns_empty() {
let sent = passive_sentence();
assert!(sent.children_of(99).is_empty());
assert!(sent.head_of(99).is_none());
assert!(sent.subtree(99).is_empty());
}
#[test]
fn tree_depth_with_non_contiguous_ids() {
let sent = Sentence::new(
"non contiguous ids".to_string(),
vec![
make_token(10, "A", "NOUN", "dep", 20),
make_token(20, "B", "NOUN", "dep", 30),
make_token(30, "C", "VERB", "root", 0),
],
);
assert_eq!(sent.tree_depth(), 2);
assert_eq!(sent.root_token().unwrap().text, "C");
assert_eq!(sent.children_of(30).len(), 1);
assert_eq!(sent.head_of(10).unwrap().text, "B");
}
#[test]
fn subtree_safe_on_cyclic_graph() {
let sent = Sentence::new(
"cyclic".to_string(),
vec![
make_token(1, "A", "NOUN", "dep", 2),
make_token(2, "B", "NOUN", "dep", 1),
],
);
let sub = sent.subtree(1);
assert_eq!(sub.len(), 2);
}
#[test]
fn empty_sentence_is_safe() {
let sent = Sentence::new(String::new(), vec![]);
assert_eq!(sent.word_count(), 0);
assert!(!sent.is_passive());
assert_eq!(sent.tree_depth(), 0);
assert!(sent.root_token().is_none());
assert!(sent.children_of(1).is_empty());
assert!(sent.head_of(1).is_none());
assert!(sent.subtree(1).is_empty());
}
fn straight_chain(n: usize) -> Sentence {
let tokens: Vec<Token> = (1..=n)
.map(|i| make_token(i, "x", "NOUN", "dep", if i == 1 { 0 } else { i - 1 }))
.collect();
Sentence::new("chain".to_string(), tokens)
}
#[test]
fn tree_depth_25_chain_returns_24() {
let sent = straight_chain(25);
assert_eq!(sent.tree_depth(), 24);
}
#[test]
fn tree_depth_1000_chain_returns_999_in_bounded_time() {
let sent = straight_chain(1000);
let start = std::time::Instant::now();
let depth = sent.tree_depth();
let elapsed = start.elapsed();
assert_eq!(depth, 999);
assert!(
elapsed < std::time::Duration::from_millis(50),
"tree_depth on 1000-chain took {elapsed:?} (expected < 50ms; suggests non-linear complexity)"
);
}
#[test]
fn corpus_result_partition_holds() {
let items: Vec<std::result::Result<CorpusEntry, DocumentError>> = vec![
Ok(CorpusEntry::new(
Some(PathBuf::from("a.md")),
Document::new(Vec::new()),
)),
Err(DocumentError::new(
Some(PathBuf::from("b.md")),
Error::ParseFailed("bad".to_string()),
)),
Ok(CorpusEntry::new(None, Document::new(Vec::new()))),
Err(DocumentError::new(
None,
Error::ParseFailed("worse".to_string()),
)),
];
let consumed = items.len();
let result: CorpusResult = items.into_iter().collect();
assert_eq!(result.corpus.entries.len() + result.errors.len(), consumed);
assert_eq!(result.corpus.entries.len(), 2);
assert_eq!(
result.corpus.entries[0].path,
Some(PathBuf::from("a.md")),
"success order preserved"
);
assert_eq!(
result.errors[0].path,
Some(PathBuf::from("b.md")),
"failure order preserved"
);
assert_eq!(result.errors[1].path, None, "path-less failure stays None");
}
#[test]
fn corpus_result_from_empty_iterator() {
let result: CorpusResult = std::iter::empty().collect();
assert!(result.corpus.entries.is_empty());
assert!(result.errors.is_empty());
}
#[test]
fn document_error_display_with_and_without_path() {
let with_path = DocumentError::new(
Some(PathBuf::from("essay.md")),
Error::ParseFailed("boom".to_string()),
);
assert_eq!(with_path.to_string(), "essay.md: parse failed: boom");
let without = DocumentError::new(None, Error::ParseFailed("boom".to_string()));
assert_eq!(without.to_string(), "parse failed: boom");
}
#[test]
fn document_error_source_is_the_domain_error() {
use std::error::Error as _;
let err = DocumentError::new(None, Error::ParseFailed("boom".to_string()));
assert!(err.source().is_some());
}
#[test]
fn format_equality() {
assert_eq!(Format::Markdown, Format::Markdown);
assert_ne!(Format::Markdown, Format::PlainText);
}
#[test]
fn format_from_path_reads_the_extension() {
for (path, expected) in [
("notes.md", Format::Markdown),
("notes.markdown", Format::Markdown),
("report.pdf", Format::Pdf),
("report.docx", Format::Docx),
("notes.txt", Format::PlainText),
("notes.rs", Format::PlainText),
("/tmp/deep/notes.md", Format::Markdown),
("archive.md.gz", Format::PlainText),
] {
assert_eq!(Format::from_path(path), expected, "{path}");
}
}
#[test]
fn format_from_path_falls_back_to_plain_text() {
assert_eq!(Format::from_path("README"), Format::PlainText);
assert_eq!(Format::from_path("<stdin>"), Format::PlainText);
assert_eq!(Format::from_path(".gitignore"), Format::PlainText);
assert_eq!(Format::from_path(""), Format::PlainText);
}
#[test]
fn tree_depth_returns_max_on_cycle() {
let sent = Sentence::new(
"cyclic".to_string(),
vec![
make_token(1, "A", "NOUN", "dep", 2),
make_token(2, "B", "NOUN", "dep", 1),
],
);
assert_eq!(sent.tree_depth(), usize::MAX);
}
#[test]
fn error_kind_names_every_variant_distinctly() {
let cases: [(Error, &str); 7] = [
(
Error::ModelNotFound(PathBuf::from("m.udpipe")),
"model_not_found",
),
(
Error::ModelInvalid("truncated".to_string()),
"model_invalid",
),
(Error::ParseFailed("no parse".to_string()), "parse_failed"),
(
Error::InputTooLarge {
limit: 1,
actual: 2,
what: "input",
},
"input_too_large",
),
(Error::UnsupportedFormat(Format::Pdf), "unsupported_format"),
(
Error::InvalidInput("bad argument".to_string()),
"invalid_input",
),
(
Error::Io(std::io::Error::from(std::io::ErrorKind::NotFound)),
"io",
),
];
let mut seen: Vec<&'static str> = Vec::new();
for (error, expected) in &cases {
assert_eq!(error.kind(), *expected, "{error}");
assert!(!seen.contains(expected), "duplicate kind {expected}");
seen.push(expected);
}
}
}