use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
pub mod limits {
pub const MAX_INPUT_BYTES: usize = 8 * 1024 * 1024;
pub const MAX_CONFIG_FILE_BYTES: usize = 64 * 1024;
pub const MAX_DOWNLOAD_BYTES: usize = 64 * 1024 * 1024;
pub const MAX_TFIDF_SENTENCES: usize = 2000;
pub const MAX_TEXTRANK_SENTENCES: usize = 2000;
pub const MAX_CLUSTER_SENTENCES: usize = MAX_TEXTRANK_SENTENCES;
pub const MAX_RAKE_TOKENS: usize = 200_000;
pub const MAX_YAKE_TOKENS: usize = 200_000;
pub const MAX_COMPRESSION_PARAGRAPH_BYTES: usize = 256 * 1024;
pub const MIN_READABILITY_WORDS: usize = 11;
pub const MIN_COMPRESSION_WORDS: usize = 51;
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct ProvisionNotice {
pub artifact: String,
pub bytes: u64,
pub destination: PathBuf,
pub license: String,
pub license_url: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Embedding(pub Vec<f32>);
struct GateSubject<'a>(&'a str);
impl std::fmt::Display for GateSubject<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self.0 {
"input" => f.write_str("input"),
label => write!(f, "{label} input"),
}
}
}
struct FormatSubject<'a>(Option<&'a Format>, Option<&'static str>);
impl std::fmt::Display for FormatSubject<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match (self.0, self.1) {
(Some(format), Some(feature)) => write!(
f,
"unsupported format: {}; enable the `{feature}` feature to read it",
format.wire_name()
),
(Some(Format::Other(name)), None) => write!(
f,
"unsupported format: {name}; no reader is registered for it"
),
(Some(format), None) => write!(
f,
"unsupported format: {}; no feature of this matra version reads it",
format.wire_name()
),
(None, _) => f.write_str(
"unsupported format: the input is not UTF-8 text and carries no signature matra recognises",
),
}
}
}
#[derive(thiserror::Error, Debug)]
#[non_exhaustive]
pub enum Error {
#[error("model not found: {}", .0.display())]
ModelNotFound(PathBuf),
#[error("invalid model: {0}")]
ModelInvalid(String),
#[error("parse failed: {cause}")]
ParseFailed {
reason: Option<&'static str>,
cause: String,
},
#[error("{} too large: {actual} > limit {limit}", GateSubject(.what))]
InputTooLarge {
limit: usize,
actual: usize,
what: &'static str,
},
#[error("{}", FormatSubject(.format.as_ref(), *.feature))]
UnsupportedFormat {
format: Option<Format>,
feature: Option<&'static str>,
},
#[error("invalid input: {message}")]
InvalidInput {
reason: &'static str,
message: String,
},
#[error("io error: {0}")]
Io(#[from] std::io::Error),
}
impl Error {
pub fn kind(&self) -> &'static str {
match self {
Error::ModelNotFound(_) => "model_not_found",
Error::ModelInvalid(_) => "model_invalid",
Error::ParseFailed { .. } => "parse_failed",
Error::InputTooLarge { .. } => "input_too_large",
Error::UnsupportedFormat { .. } => "unsupported_format",
Error::InvalidInput { .. } => "invalid_input",
Error::Io(_) => "io",
}
}
pub fn details(&self) -> ErrorDetails {
let details = ErrorDetails::default();
match self {
Error::ModelNotFound(path) => details.text("path", path.to_string_lossy()),
Error::ModelInvalid(cause) => details.text("cause", cause.as_str()),
Error::ParseFailed { reason, cause } => {
let details = match reason {
Some(reason) => details.text("reason", *reason),
None => details,
};
details.text("cause", cause.as_str())
}
Error::InvalidInput { reason, .. } => details.text("reason", *reason),
Error::InputTooLarge {
limit,
actual,
what,
} => details
.text("what", *what)
.integer("limit", *limit)
.integer("actual", *actual),
Error::UnsupportedFormat { format, feature } => {
let details = match format {
Some(format) => details.text("format", format.wire_name()),
None => details,
};
match feature {
Some(feature) => details.text("feature", *feature),
None => details,
}
}
Error::Io(error) => {
let details = details.text("io_kind", io_kind_name(error.kind()));
let context = error
.get_ref()
.and_then(|inner| inner.downcast_ref::<IoContext>());
let details = match context {
Some(context) => context.add_to(details),
None => details,
};
details.text("cause", error.to_string())
}
}
}
}
impl Serialize for Error {
fn serialize<S: serde::Serializer>(
&self,
serializer: S,
) -> std::result::Result<S::Ok, S::Error> {
use serde::ser::SerializeStruct;
let mut state = serializer.serialize_struct("Error", 3)?;
state.serialize_field("kind", self.kind())?;
state.serialize_field("message", &self.to_string())?;
state.serialize_field("details", &self.details())?;
state.end()
}
}
pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ErrorDetails {
entries: Vec<(&'static str, DetailValue)>,
}
impl ErrorDetails {
pub fn get(&self, key: &str) -> Option<&DetailValue> {
self.entries
.iter()
.find(|(name, _)| *name == key)
.map(|(_, value)| value)
}
pub fn iter(&self) -> impl Iterator<Item = (&'static str, &DetailValue)> + '_ {
self.entries.iter().map(|(name, value)| (*name, value))
}
pub fn len(&self) -> usize {
self.entries.len()
}
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
fn text(mut self, key: &'static str, value: impl Into<String>) -> Self {
self.entries.push((key, DetailValue::Text(value.into())));
self
}
fn integer(mut self, key: &'static str, value: usize) -> Self {
let value = u64::try_from(value).unwrap_or(u64::MAX);
self.entries.push((key, DetailValue::Integer(value)));
self
}
fn flag(mut self, key: &'static str, value: bool) -> Self {
self.entries.push((key, DetailValue::Bool(value)));
self
}
}
impl Serialize for ErrorDetails {
fn serialize<S: serde::Serializer>(
&self,
serializer: S,
) -> std::result::Result<S::Ok, S::Error> {
use serde::ser::SerializeMap;
let mut map = serializer.serialize_map(Some(self.entries.len()))?;
for (key, value) in &self.entries {
map.serialize_entry(key, value)?;
}
map.end()
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(untagged)]
#[non_exhaustive]
pub enum DetailValue {
Text(String),
Integer(u64),
Bool(bool),
}
#[derive(Debug)]
pub(crate) struct IoContext {
message: String,
path: Option<PathBuf>,
url: Option<String>,
retryable: bool,
}
impl IoContext {
pub(crate) fn at_path(
kind: std::io::ErrorKind,
path: &Path,
message: impl Into<String>,
) -> std::io::Error {
std::io::Error::new(
kind,
IoContext {
message: message.into(),
path: Some(path.to_path_buf()),
url: None,
retryable: false,
},
)
}
pub(crate) fn cannot(operation: &str, path: &Path, error: &std::io::Error) -> std::io::Error {
IoContext::at_path(
error.kind(),
path,
format!("cannot {operation} {}: {error}", path.display()),
)
}
#[cfg_attr(
not(any(
feature = "udpipe",
all(feature = "model2vec", not(target_arch = "wasm32"))
)),
allow(dead_code)
)]
pub(crate) fn download(
kind: std::io::ErrorKind,
url: &str,
message: impl Into<String>,
retryable: bool,
) -> std::io::Error {
std::io::Error::new(
kind,
IoContext {
message: message.into(),
path: None,
url: Some(url.to_string()),
retryable,
},
)
}
fn add_to(&self, details: ErrorDetails) -> ErrorDetails {
let details = match &self.path {
Some(path) => details.text("path", path.to_string_lossy()),
None => details,
};
match &self.url {
Some(url) => details
.text("url", url.as_str())
.flag("retryable", self.retryable),
None => details,
}
}
}
impl std::fmt::Display for IoContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.message)
}
}
impl std::error::Error for IoContext {}
fn io_kind_name(kind: std::io::ErrorKind) -> &'static str {
use std::io::ErrorKind;
match kind {
ErrorKind::NotFound => "not_found",
ErrorKind::PermissionDenied => "permission_denied",
ErrorKind::AlreadyExists => "already_exists",
ErrorKind::Unsupported => "unsupported",
ErrorKind::InvalidInput => "invalid_input",
ErrorKind::InvalidData => "invalid_data",
ErrorKind::TimedOut => "timed_out",
ErrorKind::NotConnected => "not_connected",
ErrorKind::ConnectionRefused => "connection_refused",
ErrorKind::ConnectionReset => "connection_reset",
ErrorKind::UnexpectedEof => "unexpected_eof",
ErrorKind::Interrupted => "interrupted",
ErrorKind::NotADirectory => "not_a_directory",
ErrorKind::IsADirectory => "is_a_directory",
ErrorKind::StorageFull => "storage_full",
_ => "other",
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Token {
pub id: usize,
pub span: Span,
pub text: String,
pub lemma: String,
pub pos: String,
pub xpos: String,
pub features: std::collections::BTreeMap<String, String>,
pub head: usize,
pub dep: String,
pub deps: String,
pub misc: String,
pub is_punct: bool,
}
impl Token {
pub fn builder(
id: usize,
text: String,
lemma: String,
pos: String,
head: usize,
dep: String,
) -> TokenBuilder {
TokenBuilder {
id,
span: Span::default(),
text,
lemma,
pos,
head,
dep,
xpos: String::new(),
features: std::collections::BTreeMap::new(),
deps: String::new(),
misc: String::new(),
is_punct: false,
}
}
pub fn feat(&self, key: &str) -> Option<&str> {
self.features.get(key).map(String::as_str)
}
}
pub(crate) fn features_from_conllu(column: &str) -> std::collections::BTreeMap<String, String> {
let mut features = std::collections::BTreeMap::new();
for pair in column.split('|') {
if let Some((name, value)) = pair.split_once('=')
&& !name.is_empty()
{
features
.entry(name.to_string())
.or_insert_with(|| value.to_string());
}
}
features
}
pub(crate) fn form_end(text: &str, at: usize, form: &str) -> Option<usize> {
let mut at = at;
let mut wanted = form.chars().peekable();
while let Some(c) = wanted.next() {
if c.is_whitespace() {
while wanted.peek().is_some_and(|c| c.is_whitespace()) {
wanted.next();
}
let tail = text.get(at..)?;
let skipped = tail.len() - tail.trim_start().len();
if skipped == 0 {
return None;
}
at += skipped;
} else {
let found = text.get(at..)?.chars().next()?;
if found != c {
return None;
}
at += found.len_utf8();
}
}
Some(at)
}
pub struct TokenBuilder {
id: usize,
span: Span,
text: String,
lemma: String,
pos: String,
head: usize,
dep: String,
xpos: String,
features: std::collections::BTreeMap<String, String>,
deps: String,
misc: String,
is_punct: bool,
}
impl TokenBuilder {
pub fn span(mut self, span: Span) -> Self {
self.span = span;
self
}
pub fn xpos(mut self, xpos: String) -> Self {
self.xpos = xpos;
self
}
pub fn feats(mut self, feats: String) -> Self {
self.features = features_from_conllu(&feats);
self
}
pub fn deps(mut self, deps: String) -> Self {
self.deps = deps;
self
}
pub fn misc(mut self, misc: String) -> Self {
self.misc = misc;
self
}
pub fn is_punct(mut self, is_punct: bool) -> Self {
self.is_punct = is_punct;
self
}
pub fn build(self) -> Token {
Token {
id: self.id,
span: self.span,
text: self.text,
lemma: self.lemma,
pos: self.pos,
xpos: self.xpos,
features: self.features,
head: self.head,
dep: self.dep,
deps: self.deps,
misc: self.misc,
is_punct: self.is_punct,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Negation {
pub cue_id: usize,
pub cue_lemma: String,
pub head_id: usize,
}
fn detect_negations(tokens: &[Token]) -> Vec<Negation> {
const CUE_LEMMAS: [&str; 5] = ["not", "never", "no", "neither", "nor"];
const CUE_DEPS: [&str; 3] = ["advmod", "det", "cc"];
tokens
.iter()
.filter(|t| CUE_LEMMAS.contains(&t.lemma.as_str()) && CUE_DEPS.contains(&t.dep.as_str()))
.map(|t| Negation {
cue_id: t.id,
cue_lemma: t.lemma.clone(),
head_id: t.head,
})
.collect()
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Modal {
pub aux_id: usize,
pub aux_lemma: String,
pub head_id: usize,
}
fn detect_modals(tokens: &[Token]) -> Vec<Modal> {
const MODAL_LEMMAS: [&str; 10] = [
"can", "could", "may", "might", "must", "ought", "shall", "should", "will", "would",
];
tokens
.iter()
.filter(|t| (t.dep == "aux" || t.pos == "AUX") && MODAL_LEMMAS.contains(&t.lemma.as_str()))
.map(|t| Modal {
aux_id: t.id,
aux_lemma: t.lemma.clone(),
head_id: t.head,
})
.collect()
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct RootMood {
pub root_id: usize,
pub mood: String,
}
fn detect_root_mood(tokens: &[Token]) -> Option<RootMood> {
const FINITE_CARRIER_DEPS: [&str; 3] = ["cop", "aux", "aux:pass"];
let root = tokens.iter().find(|t| t.head == 0)?;
let mood = root.feat("Mood").or_else(|| {
tokens
.iter()
.filter(|t| t.head == root.id && FINITE_CARRIER_DEPS.contains(&t.dep.as_str()))
.find_map(|t| t.feat("Mood"))
})?;
Some(RootMood {
root_id: root.id,
mood: mood.to_string(),
})
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct ClausalComplement {
pub verb_id: usize,
pub verb_lemma: String,
pub ccomp_id: usize,
pub subject_id: Option<usize>,
pub subject_lemma: Option<String>,
}
fn detect_clausal_complements(tokens: &[Token]) -> Vec<ClausalComplement> {
const SUBJECT_DEPS: [&str; 3] = ["nsubj", "nsubj:pass", "nsubjpass"];
tokens
.iter()
.filter(|c| c.dep == "ccomp")
.filter_map(|c| {
let verb = tokens.iter().find(|t| t.id == c.head && t.pos == "VERB")?;
let subject = tokens
.iter()
.find(|t| t.head == verb.id && SUBJECT_DEPS.contains(&t.dep.as_str()));
Some(ClausalComplement {
verb_id: verb.id,
verb_lemma: verb.lemma.clone(),
ccomp_id: c.id,
subject_id: subject.map(|s| s.id),
subject_lemma: subject.map(|s| s.lemma.clone()),
})
})
.collect()
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct RootAdverbial {
pub adv_id: usize,
pub adv_lemma: String,
}
fn detect_root_adverbials(tokens: &[Token]) -> Vec<RootAdverbial> {
let Some(root) = tokens.iter().find(|t| t.head == 0) else {
return Vec::new();
};
tokens
.iter()
.filter(|t| t.dep == "advmod" && t.head == root.id)
.map(|t| RootAdverbial {
adv_id: t.id,
adv_lemma: t.lemma.clone(),
})
.collect()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum HearstPattern {
SuchAs,
SuchNpAs,
Including,
Especially,
AndOther,
OrOther,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct HearstSpan {
pub head_id: usize,
pub head_lemma: String,
pub first_id: usize,
pub last_id: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct HearstPair {
pub pattern: HearstPattern,
pub hypernym: HearstSpan,
pub hyponym: HearstSpan,
}
fn detect_passive(tokens: &[Token]) -> bool {
tokens
.iter()
.any(|t| t.dep == "nsubj:pass" || t.dep == "nsubjpass" || t.dep == "aux:pass")
}
fn measure_tree_depth(tokens: &[Token]) -> Option<usize> {
use std::collections::{HashMap, HashSet};
let head_by_id: HashMap<usize, usize> = tokens.iter().map(|t| (t.id, t.head)).collect();
let mut depth_by_id: HashMap<usize, Option<usize>> = HashMap::new();
fn depth_of(
id: usize,
head_by_id: &HashMap<usize, usize>,
depth_by_id: &mut HashMap<usize, Option<usize>>,
) -> Option<usize> {
if let Some(&d) = depth_by_id.get(&id) {
return d;
}
let mut visited: HashSet<usize> = HashSet::new();
let mut chain: Vec<usize> = Vec::new();
let mut cur = id;
let base_depth = loop {
if !visited.insert(cur) {
for &cid in &chain {
depth_by_id.insert(cid, None);
}
return None;
}
let head = head_by_id.get(&cur).copied().unwrap_or(0);
if head == 0 {
chain.push(cur);
break 0usize;
}
if let Some(&d) = depth_by_id.get(&head) {
let Some(d) = d else {
chain.push(cur);
for &cid in &chain {
depth_by_id.insert(cid, None);
}
return None;
};
chain.push(cur);
break d.saturating_add(1);
}
chain.push(cur);
cur = head;
};
let n = chain.len();
for (i, &cid) in chain.iter().enumerate() {
let d = base_depth.saturating_add(n - 1 - i);
depth_by_id.insert(cid, Some(d));
}
Some(base_depth.saturating_add(n - 1))
}
let mut deepest = 0usize;
for t in tokens {
deepest = deepest.max(depth_of(t.id, &head_by_id, &mut depth_by_id)?);
}
Some(deepest)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Sentence {
pub span: Span,
pub text: String,
#[serde(default)]
pub tokens: Vec<Token>,
pub word_count: usize,
pub tree_depth: Option<usize>,
pub passive: bool,
pub negations: Vec<Negation>,
pub modals: Vec<Modal>,
pub clausal_complements: Vec<ClausalComplement>,
pub root_adverbials: Vec<RootAdverbial>,
pub hearst_pairs: Vec<HearstPair>,
pub root_mood: Option<RootMood>,
}
impl Sentence {
pub fn new(text: String, tokens: Vec<Token>) -> Self {
let span = match (tokens.first(), tokens.last()) {
(Some(first), Some(last)) => Span::new(first.span.byte_start, last.span.byte_end),
_ => Span::default(),
};
let word_count = tokens.iter().filter(|t| !t.is_punct).count();
let tree_depth = measure_tree_depth(&tokens);
let passive = detect_passive(&tokens);
let negations = detect_negations(&tokens);
let modals = detect_modals(&tokens);
let clausal_complements = detect_clausal_complements(&tokens);
let root_adverbials = detect_root_adverbials(&tokens);
let root_mood = detect_root_mood(&tokens);
Self {
span,
text,
tokens,
word_count,
tree_depth,
passive,
negations,
modals,
clausal_complements,
root_adverbials,
hearst_pairs: Vec::new(),
root_mood,
}
}
pub(crate) fn from_tokens(tokens: Vec<Token>) -> Self {
let mut text = String::new();
for (i, token) in tokens.iter().enumerate() {
text.push_str(&token.text);
if i + 1 < tokens.len() && !token.misc.contains("SpaceAfter=No") {
text.push(' ');
}
}
Self::new(text, tokens)
}
pub fn clausal_complements_in(&self, lexicon: &[&str]) -> Vec<&ClausalComplement> {
self.clausal_complements
.iter()
.filter(|r| lexicon.contains(&r.verb_lemma.as_str()))
.collect()
}
pub fn root_adverbials_in(&self, lexicon: &[&str]) -> Vec<&RootAdverbial> {
self.root_adverbials
.iter()
.filter(|a| lexicon.contains(&a.adv_lemma.as_str()))
.collect()
}
pub fn content_tokens(&self) -> Vec<&Token> {
self.tokens.iter().filter(|t| !t.is_punct).collect()
}
pub fn root_token(&self) -> Option<&Token> {
self.tokens.iter().find(|t| t.head == 0)
}
pub fn children_of(&self, id: usize) -> Vec<&Token> {
self.tokens.iter().filter(|t| t.head == id).collect()
}
pub fn head_of(&self, id: usize) -> Option<&Token> {
self.tokens.iter().find(|t| t.id == id).and_then(|t| {
if t.head == 0 {
None
} else {
self.tokens.iter().find(|h| h.id == t.head)
}
})
}
pub fn subtree(&self, id: usize) -> Vec<&Token> {
let mut result = Vec::new();
let mut stack = vec![id];
let mut visited = std::collections::HashSet::new();
while let Some(current) = stack.pop() {
if !visited.insert(current) {
continue;
}
if let Some(token) = self.tokens.iter().find(|t| t.id == current) {
result.push(token);
for child in &self.tokens {
if child.head == current && child.id != current {
stack.push(child.id);
}
}
}
}
result.sort_by_key(|t| t.id);
result
}
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct ParagraphMetrics {
pub readability_grade: Option<f64>,
pub lexical_density: Option<f64>,
pub compression_ratio: Option<f64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum ParagraphKind {
Paragraph,
Quote,
ListItem,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Paragraph {
pub span: Span,
pub text: String,
pub kind: ParagraphKind,
#[serde(default)]
pub sentences: Vec<Sentence>,
pub metrics: ParagraphMetrics,
}
impl Paragraph {
pub fn new(text: String, kind: ParagraphKind, span: Span) -> Self {
Self {
span,
text,
kind,
sentences: Vec::new(),
metrics: ParagraphMetrics::default(),
}
}
pub fn word_count(&self) -> usize {
self.sentences
.iter()
.fold(0usize, |total, s| total.saturating_add(s.word_count))
}
pub fn sentence_count(&self) -> usize {
self.sentences.len()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Section {
pub heading: Option<String>,
pub level: usize,
pub span: Option<Span>,
pub paragraphs: Vec<Paragraph>,
}
impl Section {
pub fn new(
heading: Option<String>,
level: usize,
span: Option<Span>,
paragraphs: Vec<Paragraph>,
) -> Self {
Self {
heading,
level,
span,
paragraphs,
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct DocumentMetrics {
pub total_words: usize,
pub total_sentences: usize,
pub paragraph_count: usize,
pub mean_sentence_length: Option<f64>,
pub sentence_length_std: Option<f64>,
pub vocabulary_ttr: Option<f64>,
pub nominalization_ratio: Option<f64>,
pub passive_ratio: Option<f64>,
}
pub const SCHEMA_VERSION: &str = "2.0";
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Identity {
pub name: String,
pub model: Option<String>,
pub sha256: Option<String>,
pub license: Option<String>,
}
impl Identity {
pub fn new(
name: impl Into<String>,
model: Option<String>,
sha256: Option<String>,
license: Option<String>,
) -> Self {
Self {
name: name.into(),
model,
sha256,
license,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct ProvenanceSource {
#[serde(serialize_with = "serialize_path_lossy")]
pub path: Option<PathBuf>,
pub bytes: usize,
pub sha256: String,
}
impl ProvenanceSource {
pub fn new(path: Option<PathBuf>, bytes: usize, sha256: impl Into<String>) -> Self {
Self {
path,
bytes,
sha256: sha256.into(),
}
}
}
fn serialize_path_lossy<S: serde::Serializer>(
path: &Option<PathBuf>,
serializer: S,
) -> std::result::Result<S::Ok, S::Error> {
match path {
Some(p) => serializer.serialize_some(&p.to_string_lossy()),
None => serializer.serialize_none(),
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Provenance {
pub matra: String,
pub schema: String,
pub parser: Identity,
pub format: Format,
pub source: ProvenanceSource,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub method: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub parameters: Option<Parameters>,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Parameters {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub n: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub threshold: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub embedder: Option<Identity>,
}
impl Parameters {
pub(crate) fn count(n: usize) -> Self {
Self {
n: Some(n),
..Self::default()
}
}
pub(crate) fn clustering(threshold: f64, embedder: Identity) -> Self {
Self {
threshold: Some(threshold),
embedder: Some(embedder),
..Self::default()
}
}
}
impl Provenance {
pub fn new(parser: Identity, format: Format, source: ProvenanceSource) -> Self {
Self {
matra: env!("CARGO_PKG_VERSION").to_string(),
schema: SCHEMA_VERSION.to_string(),
parser,
format,
source,
method: None,
parameters: None,
}
}
pub(crate) fn derived(&self, method: &str, parameters: Parameters) -> Self {
Self {
method: Some(method.to_string()),
parameters: Some(parameters),
..self.clone()
}
}
#[cfg(test)]
pub(crate) fn for_test() -> Self {
Self::new(
Identity::new("test", None, None, None),
Format::PlainText,
ProvenanceSource::new(None, 0, ""),
)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Document {
pub provenance: Provenance,
#[serde(default)]
pub sections: Vec<Section>,
pub metrics: DocumentMetrics,
pub skipped: Vec<Skipped>,
}
impl Document {
pub fn new(provenance: Provenance, sections: Vec<Section>, skipped: Vec<Skipped>) -> Self {
Self {
provenance,
sections,
metrics: DocumentMetrics::default(),
skipped,
}
}
pub fn paragraphs(&self) -> impl Iterator<Item = &Paragraph> {
self.sections.iter().flat_map(|s| s.paragraphs.iter())
}
pub fn paragraphs_mut(&mut self) -> impl Iterator<Item = &mut Paragraph> {
self.sections
.iter_mut()
.flat_map(|s| s.paragraphs.iter_mut())
}
pub fn sentences(&self) -> impl Iterator<Item = &Sentence> {
self.sections
.iter()
.flat_map(|s| s.paragraphs.iter())
.flat_map(|p| p.sentences.iter())
}
pub fn tokens(&self) -> impl Iterator<Item = &Token> {
self.sections
.iter()
.flat_map(|s| s.paragraphs.iter())
.flat_map(|p| p.sentences.iter())
.flat_map(|s| s.tokens.iter())
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Summary {
pub provenance: Provenance,
pub sentences: Vec<SummarySentence>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct SummarySentence {
pub index: usize,
pub span: Span,
pub text: String,
pub score: f64,
}
impl SummarySentence {
pub(crate) fn new(index: usize, sentence: &Sentence, score: f64) -> Self {
Self {
index,
span: sentence.span,
text: sentence.text.clone(),
score,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Keyphrases {
pub provenance: Provenance,
pub phrases: Vec<Keyphrase>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Keyphrase {
pub phrase: String,
pub score: f64,
pub occurrences: Vec<Span>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Clusters {
pub provenance: Provenance,
pub clusters: Vec<Cluster>,
pub edges: Vec<SimilarityEdge>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Cluster {
pub members: Vec<usize>,
}
impl Cluster {
pub(crate) fn new(members: Vec<usize>) -> Self {
Self { members }
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct SimilarityEdge {
pub a: usize,
pub b: usize,
pub score: f64,
}
impl SimilarityEdge {
pub(crate) fn new(a: usize, b: usize, score: f64) -> Self {
Self { a, b, score }
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum Format {
Markdown,
PlainText,
Blocks,
Pdf,
Docx,
#[serde(untagged)]
Other(String),
}
impl Format {
pub fn named(name: impl Into<String>) -> Format {
let name = name.into();
match name.as_str() {
"markdown" => Format::Markdown,
"plain_text" => Format::PlainText,
"blocks" => Format::Blocks,
"pdf" => Format::Pdf,
"docx" => Format::Docx,
_ => Format::Other(name),
}
}
pub(crate) fn wire_name(&self) -> &str {
match self {
Format::Markdown => "markdown",
Format::PlainText => "plain_text",
Format::Blocks => "blocks",
Format::Pdf => "pdf",
Format::Docx => "docx",
Format::Other(name) => name,
}
}
pub fn from_path(path: impl AsRef<Path>) -> Format {
match path.as_ref().extension().and_then(|e| e.to_str()) {
Some("md" | "markdown") => Format::Markdown,
Some("pdf") => Format::Pdf,
Some("docx") => Format::Docx,
_ => Format::PlainText,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Block(pub(crate) BlockContent);
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum BlockContent {
Heading {
level: u8,
text: String,
},
Paragraph {
text: String,
},
}
impl Block {
pub fn heading(level: u8, text: impl Into<String>) -> Block {
Block(BlockContent::Heading {
level,
text: text.into(),
})
}
pub fn paragraph(text: impl Into<String>) -> Block {
Block(BlockContent::Paragraph { text: text.into() })
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Span {
pub byte_start: usize,
pub byte_end: usize,
}
impl Span {
pub fn new(byte_start: usize, byte_end: usize) -> Self {
Self {
byte_start,
byte_end,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum SkippedKind {
Table,
Code,
FrontMatter,
Html,
ThematicBreak,
LinkDefinition,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Skipped {
pub kind: SkippedKind,
pub span: Span,
}
impl Skipped {
pub fn new(kind: SkippedKind, span: Span) -> Self {
Self { kind, span }
}
}
#[derive(Debug)]
#[non_exhaustive]
pub struct DocumentError {
pub path: Option<PathBuf>,
pub error: Error,
}
impl DocumentError {
pub fn new(path: Option<PathBuf>, error: Error) -> Self {
Self { path, error }
}
}
impl std::fmt::Display for DocumentError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match &self.path {
Some(p) => write!(f, "{}: {}", p.display(), self.error),
None => write!(f, "{}", self.error),
}
}
}
impl std::error::Error for DocumentError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.error)
}
}
impl Serialize for DocumentError {
fn serialize<S: serde::Serializer>(
&self,
serializer: S,
) -> std::result::Result<S::Ok, S::Error> {
use serde::ser::SerializeStruct;
struct LossyPath<'a>(&'a Option<PathBuf>);
impl Serialize for LossyPath<'_> {
fn serialize<S: serde::Serializer>(
&self,
serializer: S,
) -> std::result::Result<S::Ok, S::Error> {
serialize_path_lossy(self.0, serializer)
}
}
let mut state = serializer.serialize_struct("DocumentError", 4)?;
state.serialize_field("path", &LossyPath(&self.path))?;
state.serialize_field("kind", self.error.kind())?;
state.serialize_field("message", &self.error.to_string())?;
state.serialize_field("details", &self.error.details())?;
state.end()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn embedding_serializes_as_bare_array() {
let e = Embedding(vec![1.0, -0.5, 0.25]);
let json = serde_json::to_string(&e).expect("serialize");
assert_eq!(json, "[1.0,-0.5,0.25]");
let back: Embedding = serde_json::from_str(&json).expect("deserialize");
assert_eq!(back, e);
}
fn make_token(id: usize, text: &str, pos: &str, dep: &str, head: usize) -> Token {
Token {
id,
span: Default::default(),
text: text.to_string(),
lemma: text.to_lowercase(),
pos: pos.to_string(),
xpos: String::new(),
features: std::collections::BTreeMap::new(),
head,
dep: dep.to_string(),
deps: String::new(),
misc: String::new(),
is_punct: pos == "PUNCT",
}
}
fn make_token_feats(
id: usize,
text: &str,
pos: &str,
dep: &str,
head: usize,
feats: &str,
) -> Token {
let mut tok = make_token(id, text, pos, dep, head);
tok.features = features_from_conllu(feats);
tok
}
fn passive_sentence() -> Sentence {
Sentence::new(
"The system was built by the team".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "system", "NOUN", "nsubj:pass", 4),
make_token(3, "was", "AUX", "aux:pass", 4),
make_token(4, "built", "VERB", "root", 0),
make_token(5, "by", "ADP", "case", 7),
make_token(6, "the", "DET", "det", 7),
make_token(7, "team", "NOUN", "obl", 4),
],
)
}
fn active_sentence() -> Sentence {
Sentence::new(
"The team shipped the product".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "team", "NOUN", "nsubj", 3),
make_token(3, "shipped", "VERB", "root", 0),
make_token(4, "the", "DET", "det", 5),
make_token(5, "product", "NOUN", "obj", 3),
],
)
}
#[test]
fn content_tokens_excludes_punct() {
let sent = Sentence::new(
"Hello, world!".to_string(),
vec![
make_token(1, "Hello", "INTJ", "root", 0),
make_token(2, ",", "PUNCT", "punct", 1),
make_token(3, "world", "NOUN", "vocative", 1),
make_token(4, "!", "PUNCT", "punct", 1),
],
);
assert_eq!(sent.content_tokens().len(), 2);
assert_eq!(sent.word_count, 2);
}
#[test]
fn passive_detects_passive() {
assert!(passive_sentence().passive);
assert!(!active_sentence().passive);
}
fn token_with_feats(feats: &str) -> Token {
Token::builder(
1,
"x".to_string(),
"x".to_string(),
"NOUN".to_string(),
0,
"root".to_string(),
)
.feats(feats.to_string())
.build()
}
const HARVESTED_FEATS: &[&str] = &[
"",
"Case=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs",
"Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin",
"Tense=Past|VerbForm=Part|Voice=Pass",
"Definite=Def|PronType=Art",
"Number=Sing",
"Degree=Pos",
"Mood=Ind|Tense=Past|VerbForm=Fin",
"VerbForm=Fin",
"Case=Nom|Number=Plur|Person=1|PronType=Prs",
"VerbForm=Inf",
"Number=Sing|PronType=Dem",
"Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin",
"Number=Plur",
];
#[test]
fn feat_round_trips_every_harvested_pair() {
for feats in HARVESTED_FEATS {
let tok = token_with_feats(feats);
for pair in feats.split('|').filter(|p| !p.is_empty()) {
let (k, v) = pair.split_once('=').expect("harvested pair has =");
assert_eq!(tok.feat(k), Some(v), "feats {feats:?} key {k:?}");
}
assert_eq!(tok.feat("Absent"), None, "feats {feats:?}");
assert_eq!(tok.feat(""), None, "feats {feats:?}");
}
}
#[test]
fn feat_none_on_empty_and_underscore_placeholder() {
for feats in ["", "_"] {
let tok = token_with_feats(feats);
for key in ["Mood", "VerbForm", "_", ""] {
assert_eq!(tok.feat(key), None, "feats {feats:?} key {key:?}");
}
}
}
#[test]
fn feat_single_feature() {
let tok = token_with_feats("Number=Sing");
assert_eq!(tok.feat("Number"), Some("Sing"));
assert_eq!(tok.feat("number"), None, "keys match exactly");
assert_eq!(tok.feat("Num"), None, "no prefix match");
}
#[test]
fn feat_multi_value_returned_unsplit() {
let tok = token_with_feats("Case=Nom,Acc|Number=Sing");
assert_eq!(tok.feat("Case"), Some("Nom,Acc"));
}
#[test]
fn feat_malformed_input_does_not_panic() {
let tok = token_with_feats("junk|Mood=Ind=extra|Mood=Sub|=orphan|Tense=");
assert_eq!(tok.feat("junk"), None);
assert_eq!(tok.feat("Mood"), Some("Ind=extra"));
assert_eq!(tok.feat("Tense"), Some(""));
assert_eq!(tok.feat("orphan"), None);
}
#[test]
fn negation_detects_not_as_advmod() {
let sent = Sentence::new(
"The plan is not ready.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "plan", "NOUN", "nsubj", 5),
make_token(3, "is", "AUX", "cop", 5),
make_token(4, "not", "PART", "advmod", 5),
make_token(5, "ready", "ADJ", "root", 0),
make_token(6, ".", "PUNCT", "punct", 5),
],
);
assert_eq!(sent.negations.len(), 1);
assert_eq!(sent.negations[0].cue_id, 4);
assert_eq!(sent.negations[0].cue_lemma, "not");
assert_eq!(sent.negations[0].head_id, 5);
}
#[test]
fn negation_detects_never_as_advmod() {
let sent = Sentence::new(
"It was never reviewed.".to_string(),
vec![
make_token(1, "It", "PRON", "nsubj:pass", 4),
make_token(2, "was", "AUX", "aux:pass", 4),
make_token(3, "never", "ADV", "advmod", 4),
make_token(4, "reviewed", "VERB", "root", 0),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.negations.len(), 1);
assert_eq!(sent.negations[0].cue_lemma, "never");
assert_eq!(sent.negations[0].cue_id, 3);
assert_eq!(sent.negations[0].head_id, 4);
}
#[test]
fn negation_detects_no_as_det() {
let sent = Sentence::new(
"No changes were made.".to_string(),
vec![
make_token(1, "No", "DET", "det", 2),
make_token(2, "changes", "NOUN", "nsubj:pass", 4),
make_token(3, "were", "AUX", "aux:pass", 4),
make_token(4, "made", "VERB", "root", 0),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.negations.len(), 1);
assert_eq!(sent.negations[0].cue_lemma, "no");
assert_eq!(sent.negations[0].head_id, 2);
}
#[test]
fn negation_detects_neither_and_nor() {
let sent = Sentence::new(
"Neither option worked, nor did the fallback.".to_string(),
vec![
make_token(1, "Neither", "DET", "det", 2),
make_token(2, "option", "NOUN", "nsubj", 3),
make_token(3, "worked", "VERB", "root", 0),
make_token(4, ",", "PUNCT", "punct", 6),
make_token(5, "nor", "CCONJ", "cc", 6),
make_token(6, "did", "VERB", "conj", 3),
make_token(7, "the", "DET", "det", 8),
make_token(8, "fallback", "NOUN", "obj", 6),
make_token(9, ".", "PUNCT", "punct", 3),
],
);
let lemmas: Vec<&str> = sent
.negations
.iter()
.map(|n| n.cue_lemma.as_str())
.collect();
assert_eq!(lemmas, vec!["neither", "nor"]);
assert_eq!(sent.negations[0].head_id, 2);
assert_eq!(sent.negations[1].head_id, 6);
}
#[test]
fn negation_no_false_positive_on_nothing_as_subject() {
let sent = Sentence::new(
"Nothing happened.".to_string(),
vec![
make_token(1, "Nothing", "PRON", "nsubj", 2),
make_token(2, "happened", "VERB", "root", 0),
make_token(3, ".", "PUNCT", "punct", 2),
],
);
assert!(sent.negations.is_empty());
}
#[test]
fn negation_no_fire_on_pronominal_neither() {
let sent = Sentence::new(
"Neither of them came.".to_string(),
vec![
make_token(1, "Neither", "DET", "nsubj", 4),
make_token(2, "of", "ADP", "case", 3),
make_token(3, "them", "PRON", "nmod", 1),
make_token(4, "came", "VERB", "root", 0),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert!(sent.negations.is_empty());
}
#[test]
fn negation_on_tokenizer_split_cannot_fires_once_on_not() {
let sent = Sentence::new(
"We cannot merge this.".to_string(),
vec![
make_token(1, "We", "PRON", "nsubj", 4),
make_token(2, "can", "AUX", "aux", 4),
make_token(3, "not", "PART", "advmod", 4),
make_token(4, "merge", "VERB", "root", 0),
make_token(5, "this", "PRON", "obj", 4),
make_token(6, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.negations.len(), 1);
assert_eq!(sent.negations[0].cue_id, 3);
assert_eq!(sent.negations[0].cue_lemma, "not");
assert_eq!(sent.negations[0].head_id, 4);
}
#[test]
fn negation_empty_on_unnegated_sentences() {
assert!(active_sentence().negations.is_empty());
assert!(passive_sentence().negations.is_empty());
}
#[test]
fn negations_cross_the_wire() {
let sent = Sentence::new(
"It was never reviewed.".to_string(),
vec![make_token(1, "never", "ADV", "advmod", 0)],
);
let json = serde_json::to_value(&sent).unwrap();
assert_eq!(json["negations"][0]["cue_lemma"], "never");
}
fn modal_clause(lemma: &str) -> Sentence {
Sentence::new(
format!("They {lemma} leave."),
vec![
make_token(1, "They", "PRON", "nsubj", 3),
make_token_feats(2, lemma, "AUX", "aux", 3, "VerbForm=Fin"),
make_token_feats(3, "leave", "VERB", "root", 0, "VerbForm=Inf"),
make_token(4, ".", "PUNCT", "punct", 3),
],
)
}
#[test]
fn modal_every_closed_class_lemma_is_found() {
for lemma in [
"can", "could", "may", "might", "must", "ought", "shall", "should", "will", "would",
] {
let sent = modal_clause(lemma);
assert_eq!(sent.modals.len(), 1, "lemma {lemma:?}");
assert_eq!(sent.modals[0].aux_id, 2, "lemma {lemma:?}");
assert_eq!(sent.modals[0].aux_lemma, lemma);
assert_eq!(sent.modals[0].head_id, 3, "lemma {lemma:?}");
assert_eq!(
sent.root_mood, None,
"the modal carries no Mood, lemma {lemma:?}"
);
}
}
#[test]
fn modal_chain_reports_its_modal_with_the_content_verb_head() {
let sent = Sentence::new(
"It might have been done.".to_string(),
vec![
make_token(1, "It", "PRON", "nsubj:pass", 5),
make_token_feats(2, "might", "AUX", "aux", 5, "VerbForm=Fin"),
make_token_feats(3, "have", "AUX", "aux", 5, "VerbForm=Inf"),
make_token_feats(4, "been", "AUX", "aux:pass", 5, "Tense=Past|VerbForm=Part"),
make_token_feats(
5,
"done",
"VERB",
"root",
0,
"Tense=Past|VerbForm=Part|Voice=Pass",
),
make_token(6, ".", "PUNCT", "punct", 5),
],
);
assert_eq!(sent.modals.len(), 1);
assert_eq!(sent.modals[0].aux_id, 2);
assert_eq!(sent.modals[0].aux_lemma, "might");
assert_eq!(sent.modals[0].head_id, 5);
assert_eq!(sent.root_mood, None, "participle root carries no Mood");
}
#[test]
fn modal_coordinated_clauses_report_both() {
let sent = Sentence::new(
"He can stay, and she may leave.".to_string(),
vec![
make_token(1, "He", "PRON", "nsubj", 3),
make_token_feats(2, "can", "AUX", "aux", 3, "VerbForm=Fin"),
make_token_feats(3, "stay", "VERB", "root", 0, "VerbForm=Inf"),
make_token(4, ",", "PUNCT", "punct", 8),
make_token(5, "and", "CCONJ", "cc", 8),
make_token(6, "she", "PRON", "nsubj", 8),
make_token_feats(7, "may", "AUX", "aux", 8, "VerbForm=Fin"),
make_token_feats(8, "leave", "VERB", "conj", 3, "VerbForm=Inf"),
make_token(9, ".", "PUNCT", "punct", 3),
],
);
let found: Vec<(&str, usize)> = sent
.modals
.iter()
.map(|m| (m.aux_lemma.as_str(), m.head_id))
.collect();
assert_eq!(found, vec![("can", 3), ("may", 8)]);
}
#[test]
fn modal_promoted_to_root_under_vp_ellipsis_is_found() {
let sent = Sentence::new(
"Yes, you should.".to_string(),
vec![
make_token(1, "Yes", "INTJ", "discourse", 4),
make_token(2, ",", "PUNCT", "punct", 4),
make_token(3, "you", "PRON", "nsubj", 4),
make_token_feats(4, "should", "AUX", "root", 0, "VerbForm=Fin"),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.modals.len(), 1);
assert_eq!(sent.modals[0].aux_lemma, "should");
assert_eq!(sent.modals[0].head_id, 0);
assert_eq!(sent.root_mood, None, "a root modal carries no Mood");
}
#[test]
fn modal_coordinated_auxiliaries_report_both() {
let sent = Sentence::new(
"He can and will succeed.".to_string(),
vec![
make_token(1, "He", "PRON", "nsubj", 2),
make_token_feats(2, "can", "AUX", "root", 0, "VerbForm=Fin"),
make_token(3, "and", "CCONJ", "cc", 5),
make_token_feats(4, "will", "AUX", "aux", 5, "VerbForm=Fin"),
make_token_feats(5, "succeed", "VERB", "conj", 2, "VerbForm=Inf"),
make_token(6, ".", "PUNCT", "punct", 2),
],
);
let found: Vec<(&str, usize)> = sent
.modals
.iter()
.map(|m| (m.aux_lemma.as_str(), m.head_id))
.collect();
assert_eq!(found, vec![("can", 0), ("will", 5)]);
assert_eq!(sent.root_mood, None);
}
#[test]
fn modal_second_coordinated_auxiliary_as_conj_is_found() {
let sent = Sentence::new(
"He can, and he will, succeed.".to_string(),
vec![
make_token(1, "He", "PRON", "nsubj", 2),
make_token_feats(2, "can", "AUX", "root", 0, "VerbForm=Fin"),
make_token(3, ",", "PUNCT", "punct", 6),
make_token(4, "and", "CCONJ", "cc", 6),
make_token(5, "he", "PRON", "nsubj", 6),
make_token_feats(6, "will", "AUX", "conj", 2, "VerbForm=Fin"),
make_token(7, ",", "PUNCT", "punct", 8),
make_token_feats(8, "succeed", "VERB", "conj", 2, "Mood=Ind|VerbForm=Fin"),
make_token(9, ".", "PUNCT", "punct", 2),
],
);
let found: Vec<(&str, usize)> = sent
.modals
.iter()
.map(|m| (m.aux_lemma.as_str(), m.head_id))
.collect();
assert_eq!(found, vec![("can", 0), ("will", 2)]);
assert_eq!(sent.root_mood, None, "a root modal carries no Mood");
}
#[test]
fn root_mood_of_a_copular_clause_rides_the_cop() {
let sent = Sentence::new(
"The sky is blue.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "sky", "NOUN", "nsubj", 4),
make_token_feats(
3,
"is",
"AUX",
"cop",
4,
"Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin",
),
make_token_feats(4, "blue", "ADJ", "root", 0, "Degree=Pos"),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert!(sent.modals.is_empty(), "the copula be is outside the class");
assert_eq!(mood_of(&sent), Some((4, "Ind")));
}
#[test]
fn root_mood_of_do_support_rides_the_aux() {
let sent = Sentence::new(
"He did leave.".to_string(),
vec![
make_token(1, "He", "PRON", "nsubj", 3),
make_token_feats(2, "do", "AUX", "aux", 3, "Mood=Ind|Tense=Past|VerbForm=Fin"),
make_token_feats(3, "leave", "VERB", "root", 0, "VerbForm=Inf"),
make_token(4, ".", "PUNCT", "punct", 3),
],
);
assert!(sent.modals.is_empty());
assert_eq!(mood_of(&sent), Some((3, "Ind")));
}
#[test]
fn modal_contraction_fires_on_its_lemma() {
let sent = Sentence::new(
"They won't stop.".to_string(),
vec![
make_token(1, "They", "PRON", "nsubj", 4),
make_token_feats(2, "will", "AUX", "aux", 4, "VerbForm=Fin"),
make_token(3, "not", "PART", "advmod", 4),
make_token_feats(4, "stop", "VERB", "root", 0, "VerbForm=Inf"),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.modals.len(), 1);
assert_eq!(sent.modals[0].aux_lemma, "will");
assert_eq!(sent.negations.len(), 1, "the split not still fires");
}
#[test]
fn modal_no_false_positive_on_lexical_will() {
let sent = Sentence::new(
"The will was signed yesterday.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "will", "NOUN", "nsubj:pass", 4),
make_token_feats(
3,
"was",
"AUX",
"aux:pass",
4,
"Mood=Ind|Tense=Past|VerbForm=Fin",
),
make_token_feats(
4,
"signed",
"VERB",
"root",
0,
"Tense=Past|VerbForm=Part|Voice=Pass",
),
make_token(5, "yesterday", "NOUN", "obl:tmod", 4),
make_token(6, ".", "PUNCT", "punct", 4),
],
);
assert!(sent.modals.is_empty());
assert_eq!(mood_of(&sent), Some((4, "Ind")));
}
#[test]
fn a_modalized_directive_has_no_root_mood() {
let sent = Sentence::new(
"You must complete the form by Friday.".to_string(),
vec![
make_token(1, "You", "PRON", "nsubj", 3),
make_token_feats(2, "must", "AUX", "aux", 3, "VerbForm=Fin"),
make_token_feats(3, "complete", "VERB", "root", 0, "VerbForm=Inf"),
make_token(4, "the", "DET", "det", 5),
make_token(5, "form", "NOUN", "obj", 3),
make_token(6, "by", "ADP", "case", 7),
make_token(7, "Friday", "PROPN", "obl", 3),
make_token(8, ".", "PUNCT", "punct", 3),
],
);
assert_eq!(sent.modals.len(), 1);
assert_eq!(sent.modals[0].aux_lemma, "must");
assert_eq!(sent.root_mood, None);
}
#[test]
fn root_mood_reads_the_indicative_root() {
let sent = Sentence::new(
"The committee approved it.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "committee", "NOUN", "nsubj", 3),
make_token_feats(
3,
"approved",
"VERB",
"root",
0,
"Mood=Ind|Tense=Past|VerbForm=Fin",
),
make_token(4, "it", "PRON", "obj", 3),
make_token(5, ".", "PUNCT", "punct", 3),
],
);
assert!(sent.modals.is_empty());
assert_eq!(mood_of(&sent), Some((3, "Ind")));
}
#[test]
fn root_mood_reads_the_imperative_root() {
let sent = Sentence::new(
"Complete the form by Friday.".to_string(),
vec![
make_token_feats(1, "Complete", "VERB", "root", 0, "Mood=Imp|VerbForm=Fin"),
make_token(2, "the", "DET", "det", 3),
make_token(3, "form", "NOUN", "obj", 1),
make_token(4, "by", "ADP", "case", 5),
make_token(5, "Friday", "PROPN", "obl", 1),
make_token(6, ".", "PUNCT", "punct", 1),
],
);
assert!(sent.modals.is_empty());
assert_eq!(mood_of(&sent), Some((1, "Imp")));
}
#[test]
fn subordinate_modal_is_reported_and_leaves_the_root_mood() {
let sent = Sentence::new(
"I know it might rain.".to_string(),
vec![
make_token(1, "I", "PRON", "nsubj", 2),
make_token_feats(
2,
"know",
"VERB",
"root",
0,
"Mood=Ind|Tense=Pres|VerbForm=Fin",
),
make_token(3, "it", "PRON", "nsubj", 5),
make_token_feats(4, "might", "AUX", "aux", 5, "VerbForm=Fin"),
make_token(5, "rain", "NOUN", "ccomp", 2),
make_token(6, ".", "PUNCT", "punct", 5),
],
);
assert_eq!(sent.modals.len(), 1);
assert_eq!(sent.modals[0].aux_lemma, "might");
assert_eq!(sent.modals[0].head_id, 5);
assert_eq!(mood_of(&sent), Some((2, "Ind")));
}
fn mood_of(sent: &Sentence) -> Option<(usize, &str)> {
sent.root_mood
.as_ref()
.map(|m| (m.root_id, m.mood.as_str()))
}
#[test]
fn modals_and_root_mood_cross_the_wire() {
let sent = modal_clause("might");
let json = serde_json::to_value(&sent).unwrap();
assert_eq!(json["modals"][0]["aux_lemma"], "might");
assert!(json["root_mood"].is_null());
assert!(json.as_object().unwrap().contains_key("root_mood"));
let indicative = Sentence::new(
"It rains.".to_string(),
vec![
make_token(1, "It", "PRON", "expl", 2),
make_token_feats(2, "rains", "VERB", "root", 0, "Mood=Ind|VerbForm=Fin"),
make_token(3, ".", "PUNCT", "punct", 2),
],
);
let json = serde_json::to_value(&indicative).unwrap();
assert_eq!(
json["root_mood"],
serde_json::json!({"root_id": 2, "mood": "Ind"})
);
}
#[test]
fn features_cross_the_wire_as_a_map() {
let tok = token_with_feats("Mood=Ind|Tense=Pres");
let json = serde_json::to_value(&tok).unwrap();
assert_eq!(
json["features"],
serde_json::json!({"Mood": "Ind", "Tense": "Pres"})
);
assert!(json.get("feats").is_none(), "no raw string beside the map");
let empty = serde_json::to_value(token_with_feats("_")).unwrap();
assert_eq!(empty["features"], serde_json::json!({}));
}
#[test]
fn clausal_complement_self_attribution() {
let sent = Sentence::new(
"We show that the method works.".to_string(),
vec![
make_token(1, "We", "PRON", "nsubj", 2),
make_token(2, "show", "VERB", "root", 0),
make_token(3, "that", "SCONJ", "mark", 6),
make_token(4, "the", "DET", "det", 5),
make_token(5, "method", "NOUN", "nsubj", 6),
make_token(6, "works", "VERB", "ccomp", 2),
make_token(7, ".", "PUNCT", "punct", 2),
],
);
assert_eq!(sent.clausal_complements.len(), 1);
let r = &sent.clausal_complements[0];
assert_eq!(r.verb_id, 2);
assert_eq!(r.verb_lemma, "show");
assert_eq!(r.ccomp_id, 6);
assert_eq!(r.subject_id, Some(1));
assert_eq!(r.subject_lemma.as_deref(), Some("we"));
}
#[test]
fn clausal_complement_other_attribution() {
let sent = Sentence::new(
"Smith reported that the effect vanished.".to_string(),
vec![
make_token(1, "Smith", "PROPN", "nsubj", 2),
make_token(2, "reported", "VERB", "root", 0),
make_token(3, "that", "SCONJ", "mark", 6),
make_token(4, "the", "DET", "det", 5),
make_token(5, "effect", "NOUN", "nsubj", 6),
make_token(6, "vanished", "VERB", "ccomp", 2),
make_token(7, ".", "PUNCT", "punct", 2),
],
);
assert_eq!(sent.clausal_complements.len(), 1);
let r = &sent.clausal_complements[0];
assert_eq!((r.verb_id, r.ccomp_id), (2, 6));
assert_eq!(r.verb_lemma, "reported");
assert_eq!(r.subject_id, Some(1));
}
#[test]
fn clausal_complement_impersonal() {
let sent = Sentence::new(
"These results suggest that the mechanism is shared.".to_string(),
vec![
make_token(1, "These", "DET", "det", 2),
make_token(2, "results", "NOUN", "nsubj", 3),
make_token(3, "suggest", "VERB", "root", 0),
make_token(4, "that", "SCONJ", "mark", 8),
make_token(5, "the", "DET", "det", 6),
make_token(6, "mechanism", "NOUN", "nsubj:pass", 8),
make_token(7, "is", "AUX", "aux:pass", 8),
make_token(8, "shared", "VERB", "ccomp", 3),
make_token(9, ".", "PUNCT", "punct", 3),
],
);
assert_eq!(sent.clausal_complements.len(), 1);
let r = &sent.clausal_complements[0];
assert_eq!((r.verb_id, r.ccomp_id), (3, 8));
assert_eq!(r.subject_id, Some(2));
assert_eq!(r.subject_lemma.as_deref(), Some("results"));
}
#[test]
fn clausal_complement_survives_upstream_split_with_no_subject() {
let sent = Sentence::new(
"reported that the effect vanished.".to_string(),
vec![
make_token(1, "reported", "VERB", "root", 0),
make_token(2, "that", "SCONJ", "mark", 5),
make_token(3, "the", "DET", "det", 4),
make_token(4, "effect", "NOUN", "nsubj", 5),
make_token(5, "vanished", "VERB", "ccomp", 1),
make_token(6, ".", "PUNCT", "punct", 1),
],
);
assert_eq!(sent.clausal_complements.len(), 1);
let r = &sent.clausal_complements[0];
assert_eq!((r.verb_id, r.ccomp_id), (1, 5));
assert_eq!(r.subject_id, None);
assert_eq!(r.subject_lemma, None);
}
#[test]
fn clausal_complement_no_fire_without_ccomp() {
let sent = Sentence::new(
"Smith reported a similar finding.".to_string(),
vec![
make_token(1, "Smith", "PROPN", "nsubj", 2),
make_token(2, "reported", "VERB", "root", 0),
make_token(3, "a", "DET", "det", 5),
make_token(4, "similar", "ADJ", "amod", 5),
make_token(5, "finding", "NOUN", "obj", 2),
make_token(6, ".", "PUNCT", "punct", 2),
],
);
assert!(sent.clausal_complements.is_empty());
assert!(active_sentence().clausal_complements.is_empty());
}
#[test]
fn clausal_complement_no_fire_on_non_verb_head() {
let sent = Sentence::new(
"I am sure that it works.".to_string(),
vec![
make_token(1, "I", "PRON", "nsubj", 3),
make_token(2, "am", "AUX", "cop", 3),
make_token(3, "sure", "ADJ", "root", 0),
make_token(4, "that", "SCONJ", "mark", 6),
make_token(5, "it", "PRON", "nsubj", 6),
make_token(6, "works", "VERB", "ccomp", 3),
make_token(7, ".", "PUNCT", "punct", 3),
],
);
assert!(sent.clausal_complements.is_empty());
}
#[test]
fn root_adverbial_detects_sentence_scope_advmod() {
let sent = Sentence::new(
"Reportedly, the deal closed.".to_string(),
vec![
make_token(1, "Reportedly", "ADV", "advmod", 5),
make_token(2, ",", "PUNCT", "punct", 5),
make_token(3, "the", "DET", "det", 4),
make_token(4, "deal", "NOUN", "nsubj", 5),
make_token(5, "closed", "VERB", "root", 0),
make_token(6, ".", "PUNCT", "punct", 5),
],
);
assert_eq!(sent.root_adverbials.len(), 1);
assert_eq!(sent.root_adverbials[0].adv_id, 1);
assert_eq!(sent.root_adverbials[0].adv_lemma, "reportedly");
}
#[test]
fn root_adverbial_reports_manner_adverbs_too() {
let sent = Sentence::new(
"The team quickly shipped.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "team", "NOUN", "nsubj", 4),
make_token(3, "quickly", "ADV", "advmod", 4),
make_token(4, "shipped", "VERB", "root", 0),
make_token(5, ".", "PUNCT", "punct", 4),
],
);
assert_eq!(sent.root_adverbials.len(), 1);
assert_eq!(sent.root_adverbials[0].adv_lemma, "quickly");
assert!(
sent.root_adverbials_in(&["reportedly", "allegedly"])
.is_empty()
);
}
#[test]
fn root_adverbial_ignores_non_root_attachment() {
let sent = Sentence::new(
"She says he ran quickly.".to_string(),
vec![
make_token(1, "She", "PRON", "nsubj", 2),
make_token(2, "says", "VERB", "root", 0),
make_token(3, "he", "PRON", "nsubj", 4),
make_token(4, "ran", "VERB", "ccomp", 2),
make_token(5, "quickly", "ADV", "advmod", 4),
make_token(6, ".", "PUNCT", "punct", 2),
],
);
assert!(sent.root_adverbials.is_empty());
assert_eq!(sent.clausal_complements.len(), 1);
}
#[test]
fn root_adverbial_overlaps_negation_by_design() {
let sent = Sentence::new(
"The plan is not ready.".to_string(),
vec![
make_token(1, "The", "DET", "det", 2),
make_token(2, "plan", "NOUN", "nsubj", 5),
make_token(3, "is", "AUX", "cop", 5),
make_token(4, "not", "PART", "advmod", 5),
make_token(5, "ready", "ADJ", "root", 0),
make_token(6, ".", "PUNCT", "punct", 5),
],
);
assert_eq!(sent.negations.len(), 1);
assert_eq!(sent.root_adverbials.len(), 1);
assert_eq!(sent.root_adverbials[0].adv_id, 4);
}
#[test]
fn clausal_complements_in_filters_by_caller_lexicon() {
let sent = Sentence::new(
"Smith reported that the effect vanished.".to_string(),
vec![
make_token(1, "Smith", "PROPN", "nsubj", 2),
make_token(2, "reported", "VERB", "root", 0),
make_token(3, "that", "SCONJ", "mark", 6),
make_token(4, "the", "DET", "det", 5),
make_token(5, "effect", "NOUN", "nsubj", 6),
make_token(6, "vanished", "VERB", "ccomp", 2),
make_token(7, ".", "PUNCT", "punct", 2),
],
);
assert_eq!(
sent.clausal_complements_in(&["reported", "suggest"]).len(),
1
);
assert!(sent.clausal_complements_in(&["say", "claim"]).is_empty());
}
#[test]
fn clausal_complements_and_root_adverbials_cross_the_wire() {
let sent = Sentence::new(
"Reportedly, it works.".to_string(),
vec![
make_token(1, "Reportedly", "ADV", "advmod", 3),
make_token(2, "it", "PRON", "nsubj", 3),
make_token(3, "works", "VERB", "root", 0),
make_token(4, ".", "PUNCT", "punct", 3),
],
);
let json = serde_json::to_value(&sent).unwrap();
assert_eq!(json["root_adverbials"][0]["adv_lemma"], "reportedly");
assert!(json["clausal_complements"].as_array().unwrap().is_empty());
}
#[test]
fn sentence_counts_cross_the_wire() {
let json = serde_json::to_value(passive_sentence()).unwrap();
assert_eq!(json["word_count"], 7);
assert_eq!(json["passive"], true);
assert_eq!(json["tree_depth"], 2);
}
#[test]
fn tree_depth_computes_max() {
let sent = active_sentence();
assert!(sent.tree_depth.is_some_and(|d| d > 0));
}
#[test]
fn root_token_finds_root() {
let sent = passive_sentence();
let root = sent.root_token().unwrap();
assert_eq!(root.text, "built");
}
#[test]
fn children_of_returns_direct_children() {
let sent = passive_sentence();
let children = sent.children_of(4);
let texts: Vec<&str> = children.iter().map(|t| t.text.as_str()).collect();
assert!(texts.contains(&"system"));
assert!(texts.contains(&"was"));
assert!(texts.contains(&"team"));
}
#[test]
fn head_of_returns_head() {
let sent = passive_sentence();
let head = sent.head_of(2).unwrap();
assert_eq!(head.text, "built");
}
#[test]
fn head_of_root_returns_none() {
let sent = passive_sentence();
assert!(sent.head_of(4).is_none());
}
#[test]
fn subtree_includes_all_descendants() {
let sent = passive_sentence();
let sub = sent.subtree(4);
assert_eq!(sub.len(), sent.tokens.len());
}
#[test]
fn subtree_leaf_returns_single() {
let sent = passive_sentence();
let sub = sent.subtree(1);
assert_eq!(sub.len(), 1);
assert_eq!(sub[0].text, "The");
}
#[test]
fn invalid_id_returns_empty() {
let sent = passive_sentence();
assert!(sent.children_of(99).is_empty());
assert!(sent.head_of(99).is_none());
assert!(sent.subtree(99).is_empty());
}
#[test]
fn tree_depth_with_non_contiguous_ids() {
let sent = Sentence::new(
"non contiguous ids".to_string(),
vec![
make_token(10, "A", "NOUN", "dep", 20),
make_token(20, "B", "NOUN", "dep", 30),
make_token(30, "C", "VERB", "root", 0),
],
);
assert_eq!(sent.tree_depth, Some(2));
assert_eq!(sent.root_token().unwrap().text, "C");
assert_eq!(sent.children_of(30).len(), 1);
assert_eq!(sent.head_of(10).unwrap().text, "B");
}
#[test]
fn subtree_safe_on_cyclic_graph() {
let sent = Sentence::new(
"cyclic".to_string(),
vec![
make_token(1, "A", "NOUN", "dep", 2),
make_token(2, "B", "NOUN", "dep", 1),
],
);
let sub = sent.subtree(1);
assert_eq!(sub.len(), 2);
}
#[test]
fn empty_sentence_is_safe() {
let sent = Sentence::new(String::new(), vec![]);
assert_eq!(sent.word_count, 0);
assert!(!sent.passive);
assert_eq!(sent.tree_depth, Some(0));
assert!(sent.root_token().is_none());
assert!(sent.children_of(1).is_empty());
assert!(sent.head_of(1).is_none());
assert!(sent.subtree(1).is_empty());
}
fn straight_chain(n: usize) -> Sentence {
let tokens: Vec<Token> = (1..=n)
.map(|i| make_token(i, "x", "NOUN", "dep", if i == 1 { 0 } else { i - 1 }))
.collect();
Sentence::new("chain".to_string(), tokens)
}
#[test]
fn tree_depth_25_chain_returns_24() {
let sent = straight_chain(25);
assert_eq!(sent.tree_depth, Some(24));
}
#[test]
fn tree_depth_1000_chain_returns_999_in_bounded_time() {
let tokens: Vec<Token> = (1..=1000)
.map(|i| make_token(i, "x", "NOUN", "dep", if i == 1 { 0 } else { i - 1 }))
.collect();
let start = std::time::Instant::now();
let depth = measure_tree_depth(&tokens);
let elapsed = start.elapsed();
assert_eq!(depth, Some(999));
assert_eq!(straight_chain(1000).tree_depth, Some(999));
assert!(
elapsed < std::time::Duration::from_millis(50),
"tree_depth on 1000-chain took {elapsed:?} (expected < 50ms; suggests non-linear complexity)"
);
}
#[test]
fn document_error_display_with_and_without_path() {
let with_path = DocumentError::new(
Some(PathBuf::from("essay.md")),
Error::ParseFailed {
reason: None,
cause: "boom".to_string(),
},
);
assert_eq!(with_path.to_string(), "essay.md: parse failed: boom");
let without = DocumentError::new(
None,
Error::ParseFailed {
reason: None,
cause: "boom".to_string(),
},
);
assert_eq!(without.to_string(), "parse failed: boom");
}
#[test]
fn document_error_source_is_the_domain_error() {
use std::error::Error as _;
let err = DocumentError::new(
None,
Error::ParseFailed {
reason: None,
cause: "boom".to_string(),
},
);
assert!(err.source().is_some());
}
#[test]
fn unsupported_format_names_the_format_and_the_remedy() {
let pdf = Error::UnsupportedFormat {
format: Some(Format::Pdf),
feature: None,
};
assert_eq!(
pdf.to_string(),
"unsupported format: pdf; no feature of this matra version reads it"
);
let with_feature = Error::UnsupportedFormat {
format: Some(Format::Docx),
feature: Some("docx"),
};
assert_eq!(
with_feature.to_string(),
"unsupported format: docx; enable the `docx` feature to read it"
);
let undecodable = Error::UnsupportedFormat {
format: None,
feature: None,
};
assert!(undecodable.to_string().contains("not UTF-8 text"));
}
#[test]
fn format_wire_name_is_its_serialized_name() {
for format in [
Format::Markdown,
Format::PlainText,
Format::Blocks,
Format::Pdf,
Format::Docx,
Format::Other("html".to_string()),
] {
let wire = serde_json::to_value(&format).unwrap();
assert_eq!(wire, serde_json::json!(format.wire_name()));
assert_eq!(Format::named(format.wire_name()), format);
let back: Format = serde_json::from_value(wire).unwrap();
assert_eq!(back, format, "a name reads back as the variant it names");
}
}
#[test]
fn a_known_name_is_never_other() {
let read: Format = serde_json::from_value(serde_json::json!("markdown")).unwrap();
assert_eq!(read, Format::Markdown);
assert_eq!(Format::named("pdf"), Format::Pdf);
let unknown = Error::UnsupportedFormat {
format: Some(Format::named("html")),
feature: None,
};
assert_eq!(
unknown.to_string(),
"unsupported format: html; no reader is registered for it"
);
assert_eq!(
unknown.details().get("format"),
Some(&DetailValue::Text("html".to_string()))
);
}
#[test]
fn sentence_span_is_derived_from_its_tokens() {
let tok = |id: usize, start: usize, end: usize| {
Token::builder(
id,
"w".to_string(),
"w".to_string(),
"X".to_string(),
0,
"root".to_string(),
)
.span(Span::new(start, end))
.build()
};
let sentence = Sentence::new("a b".to_string(), vec![tok(1, 4, 5), tok(2, 6, 9)]);
assert_eq!(sentence.span, Span::new(4, 9));
assert_eq!(
Sentence::new(String::new(), Vec::new()).span,
Span::default()
);
}
#[test]
fn format_equality() {
assert_eq!(Format::Markdown, Format::Markdown);
assert_ne!(Format::Markdown, Format::PlainText);
}
#[test]
fn format_from_path_reads_the_extension() {
for (path, expected) in [
("notes.md", Format::Markdown),
("notes.markdown", Format::Markdown),
("report.pdf", Format::Pdf),
("report.docx", Format::Docx),
("notes.txt", Format::PlainText),
("notes.rs", Format::PlainText),
("/tmp/deep/notes.md", Format::Markdown),
("archive.md.gz", Format::PlainText),
] {
assert_eq!(Format::from_path(path), expected, "{path}");
}
}
#[test]
fn format_from_path_falls_back_to_plain_text() {
assert_eq!(Format::from_path("README"), Format::PlainText);
assert_eq!(Format::from_path("<stdin>"), Format::PlainText);
assert_eq!(Format::from_path(".gitignore"), Format::PlainText);
assert_eq!(Format::from_path(""), Format::PlainText);
}
#[test]
fn tree_depth_is_none_on_a_cycle() {
let sent = Sentence::new(
"cyclic".to_string(),
vec![
make_token(1, "A", "NOUN", "dep", 2),
make_token(2, "B", "NOUN", "dep", 1),
],
);
assert_eq!(sent.tree_depth, None);
}
#[test]
fn tree_depth_is_none_when_a_chain_leads_into_a_cycle() {
let sent = Sentence::new(
"cyclic tail".to_string(),
vec![
make_token(1, "A", "NOUN", "dep", 2),
make_token(2, "B", "NOUN", "dep", 1),
make_token(3, "C", "NOUN", "dep", 1),
],
);
assert_eq!(sent.tree_depth, None);
}
#[test]
fn provenance_carries_this_builds_versions_and_the_format_vocabulary() {
let p = Provenance::for_test();
assert_eq!(p.schema, SCHEMA_VERSION);
assert_eq!(p.matra, env!("CARGO_PKG_VERSION"));
let json = serde_json::to_value(&p).unwrap();
assert_eq!(json["format"], "plain_text");
assert!(json["source"]["path"].is_null());
assert_eq!(
serde_json::to_value(Format::Markdown).unwrap(),
serde_json::json!("markdown")
);
}
#[cfg(unix)]
#[test]
fn a_non_utf8_source_path_serializes_lossily() {
use std::os::unix::ffi::OsStrExt;
let path = PathBuf::from(std::ffi::OsStr::from_bytes(b"bad\xffname.md"));
let source = ProvenanceSource::new(Some(path), 3, "abc");
let json = serde_json::to_value(&source).expect("serializes");
assert_eq!(json["path"], "bad\u{fffd}name.md");
}
#[test]
fn input_too_large_names_the_main_gate_once() {
let main_gate = Error::InputTooLarge {
limit: 8,
actual: 9,
what: "input",
};
assert_eq!(main_gate.to_string(), "input too large: 9 > limit 8");
let extractor_gate = Error::InputTooLarge {
limit: 2000,
actual: 2001,
what: "tfidf",
};
assert_eq!(
extractor_gate.to_string(),
"tfidf input too large: 2001 > limit 2000"
);
}
#[test]
fn error_kind_names_every_variant_distinctly() {
let cases: [(Error, &str); 7] = [
(
Error::ModelNotFound(PathBuf::from("m.udpipe")),
"model_not_found",
),
(
Error::ModelInvalid("truncated".to_string()),
"model_invalid",
),
(
Error::ParseFailed {
reason: None,
cause: "no parse".to_string(),
},
"parse_failed",
),
(
Error::InputTooLarge {
limit: 1,
actual: 2,
what: "input",
},
"input_too_large",
),
(
Error::UnsupportedFormat {
format: Some(Format::Pdf),
feature: None,
},
"unsupported_format",
),
(
Error::InvalidInput {
reason: "threshold_not_finite",
message: "bad argument".to_string(),
},
"invalid_input",
),
(
Error::Io(std::io::Error::from(std::io::ErrorKind::NotFound)),
"io",
),
];
let mut seen: Vec<&'static str> = Vec::new();
for (error, expected) in &cases {
assert_eq!(error.kind(), *expected, "{error}");
assert!(!seen.contains(expected), "duplicate kind {expected}");
seen.push(expected);
}
}
#[test]
fn the_limits_hold_their_published_values() {
use super::limits::*;
assert_eq!(MAX_INPUT_BYTES, 8_388_608);
assert_eq!(MAX_CONFIG_FILE_BYTES, 65_536);
assert_eq!(MAX_DOWNLOAD_BYTES, 67_108_864);
assert_eq!(MAX_TFIDF_SENTENCES, 2000);
assert_eq!(MAX_TEXTRANK_SENTENCES, 2000);
assert_eq!(MAX_CLUSTER_SENTENCES, MAX_TEXTRANK_SENTENCES);
assert_eq!(MAX_RAKE_TOKENS, 200_000);
assert_eq!(MAX_YAKE_TOKENS, 200_000);
assert_eq!(MAX_COMPRESSION_PARAGRAPH_BYTES, 262_144);
assert_eq!(MIN_READABILITY_WORDS, 11);
assert_eq!(MIN_COMPRESSION_WORDS, 51);
}
fn details_json(error: &Error) -> serde_json::Value {
serde_json::to_value(error.details()).expect("details serialize")
}
#[test]
fn every_dynamic_value_in_a_message_is_a_detail() {
use serde_json::json;
let cases: Vec<(Error, serde_json::Value)> = vec![
(
Error::ModelNotFound(PathBuf::from("models/m.udpipe")),
json!({ "path": "models/m.udpipe" }),
),
(
Error::ModelInvalid("truncated at byte 7".to_string()),
json!({ "cause": "truncated at byte 7" }),
),
(
Error::ParseFailed {
reason: None,
cause: "udpipe panicked: boom".to_string(),
},
json!({ "cause": "udpipe panicked: boom" }),
),
(
Error::ParseFailed {
reason: Some("head_out_of_range"),
cause: "token 3 of sentence 1 has head 9".to_string(),
},
json!({
"reason": "head_out_of_range",
"cause": "token 3 of sentence 1 has head 9",
}),
),
(
Error::InputTooLarge {
limit: 2000,
actual: 2001,
what: "tfidf",
},
json!({ "what": "tfidf", "limit": 2000, "actual": 2001 }),
),
(
Error::UnsupportedFormat {
format: Some(Format::Pdf),
feature: None,
},
json!({ "format": "pdf" }),
),
(
Error::UnsupportedFormat {
format: Some(Format::Docx),
feature: Some("docx"),
},
json!({ "format": "docx", "feature": "docx" }),
),
(
Error::UnsupportedFormat {
format: None,
feature: None,
},
json!({}),
),
(
Error::InvalidInput {
reason: "threshold_not_finite",
message: "threshold must be finite, got NaN".to_string(),
},
json!({ "reason": "threshold_not_finite" }),
),
(
Error::Io(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"denied",
)),
json!({ "io_kind": "permission_denied", "cause": "denied" }),
),
(
Error::Io(IoContext::at_path(
std::io::ErrorKind::NotFound,
Path::new("/models/x"),
"cannot read /models/x: gone",
)),
json!({
"io_kind": "not_found",
"path": "/models/x",
"cause": "cannot read /models/x: gone",
}),
),
(
Error::Io(IoContext::download(
std::io::ErrorKind::TimedOut,
"https://models.example/m",
"download https://models.example/m: timeout",
true,
)),
json!({
"io_kind": "timed_out",
"url": "https://models.example/m",
"retryable": true,
"cause": "download https://models.example/m: timeout",
}),
),
];
for (error, expected) in &cases {
assert_eq!(details_json(error), *expected, "{error}");
}
}
#[test]
fn an_io_context_does_not_change_the_message() {
let plain = Error::Io(std::io::Error::new(
std::io::ErrorKind::NotFound,
"cannot read /m: gone",
));
let with_context = Error::Io(IoContext::at_path(
std::io::ErrorKind::NotFound,
Path::new("/m"),
"cannot read /m: gone",
));
assert_eq!(plain.to_string(), with_context.to_string());
assert_eq!(plain.kind(), with_context.kind());
}
#[test]
fn errors_serialize_as_kind_message_and_details() {
let error = Error::InputTooLarge {
limit: 8,
actual: 9,
what: "input",
};
assert_eq!(
serde_json::to_value(&error).unwrap(),
serde_json::json!({
"kind": "input_too_large",
"message": "input too large: 9 > limit 8",
"details": { "what": "input", "limit": 8, "actual": 9 },
})
);
let on_disk = DocumentError::new(
Some(PathBuf::from("essay.md")),
Error::ParseFailed {
reason: None,
cause: "boom".to_string(),
},
);
assert_eq!(
serde_json::to_value(&on_disk).unwrap(),
serde_json::json!({
"path": "essay.md",
"kind": "parse_failed",
"message": "parse failed: boom",
"details": { "cause": "boom" },
})
);
let in_memory = DocumentError::new(
None,
Error::ParseFailed {
reason: None,
cause: "boom".to_string(),
},
);
assert!(serde_json::to_value(&in_memory).unwrap()["path"].is_null());
}
#[cfg(unix)]
#[test]
fn a_non_utf8_document_error_path_serializes_lossily() {
use std::os::unix::ffi::OsStrExt;
let path = PathBuf::from(std::ffi::OsStr::from_bytes(b"bad\xffname.md"));
let err = DocumentError::new(
Some(path),
Error::ParseFailed {
reason: None,
cause: "boom".to_string(),
},
);
let json = serde_json::to_value(&err).expect("serializes");
assert_eq!(json["path"], "bad\u{fffd}name.md");
}
#[test]
fn io_kinds_are_named_in_snake_case() {
use std::io::ErrorKind;
for (kind, name) in [
(ErrorKind::NotFound, "not_found"),
(ErrorKind::PermissionDenied, "permission_denied"),
(ErrorKind::TimedOut, "timed_out"),
(ErrorKind::NotConnected, "not_connected"),
(ErrorKind::NotADirectory, "not_a_directory"),
(ErrorKind::Other, "other"),
(ErrorKind::WouldBlock, "other"),
] {
assert_eq!(io_kind_name(kind), name, "{kind:?}");
}
}
}