#![doc = include_str!("../README.md")]
pub mod decompose;
pub mod domain;
pub mod extraction;
pub mod hearst;
pub mod metrics;
pub mod nlp;
pub mod source;
mod stopwords;
use std::path::Path;
use decompose::Decomposer;
use domain::{Document, MAX_INPUT_BYTES};
use nlp::NlpProvider;
use source::Source;
fn check_input_size(text: &str) -> domain::Result<()> {
if text.len() > MAX_INPUT_BYTES {
return Err(domain::Error::InputTooLarge {
limit: MAX_INPUT_BYTES,
actual: text.len(),
what: "input",
});
}
Ok(())
}
pub type Ingested = std::result::Result<domain::RawDocument, domain::DocumentError>;
enum Pending {
Ready(domain::RawDocument),
File(std::path::PathBuf),
}
pub struct Ingest {
items: std::vec::IntoIter<Pending>,
}
impl Ingest {
pub fn text(text: impl Into<String>, format: domain::Format) -> Self {
Self {
items: vec![Pending::Ready(domain::RawDocument::new(
text.into(),
None,
format,
))]
.into_iter(),
}
}
pub fn path(input: impl AsRef<Path>) -> domain::Result<Self> {
let input = input.as_ref();
let metadata = std::fs::symlink_metadata(input)?;
let pending = if metadata.file_type().is_dir() {
source::directory::DirectorySource
.candidate_paths(input)?
.into_iter()
.map(Pending::File)
.collect()
} else {
vec![Pending::File(input.to_path_buf())]
};
Ok(Self {
items: pending.into_iter(),
})
}
}
impl Iterator for Ingest {
type Item = Ingested;
fn next(&mut self) -> Option<Ingested> {
let item = self.items.next()?;
Some(match item {
Pending::Ready(doc) => Ok(doc),
Pending::File(path) => match source::file::FileSource.read(&path) {
Ok(docs) => docs.into_iter().next().ok_or_else(|| {
domain::DocumentError::new(
Some(path),
domain::Error::Io(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"source returned no documents",
)),
)
}),
Err(e) => Err(domain::DocumentError::new(Some(path), e)),
},
})
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.items.size_hint()
}
}
pub struct Engine {
nlp: Box<dyn NlpProvider>,
decomposers: decompose::Decomposers,
}
impl Engine {
pub fn new(nlp: Box<dyn NlpProvider>, decomposers: decompose::Decomposers) -> Self {
Self { nlp, decomposers }
}
pub fn analyze<I>(
&self,
input: I,
) -> impl Iterator<Item = std::result::Result<domain::CorpusEntry, domain::DocumentError>>
where
I: IntoIterator<Item = Ingested>,
{
input
.into_iter()
.map(|item| item.and_then(|raw| self.analyze_one(raw)))
}
pub fn analyze_one(
&self,
raw: domain::RawDocument,
) -> std::result::Result<domain::CorpusEntry, domain::DocumentError> {
let path = raw.path.clone();
match self.annotate(&raw) {
Ok(mut doc) => {
self.compose(&mut doc);
Ok(domain::CorpusEntry::new(path, doc))
}
Err(e) => Err(domain::DocumentError::new(path, e)),
}
}
pub fn annotate(&self, raw: &domain::RawDocument) -> domain::Result<Document> {
check_input_size(&raw.text)?;
let decomposer = self
.decomposers
.get(&raw.format)
.ok_or_else(|| domain::Error::UnsupportedFormat(raw.format.clone()))?;
let mut doc = Document::new(decomposer.decompose(&raw.text));
for para in doc.paragraphs_mut() {
if para.in_blockquote {
continue;
}
para.sentences = self.nlp.parse(¶.text)?;
for sentence in &mut para.sentences {
sentence.hearst_pairs = hearst::hypernymy_pairs(&sentence.tokens);
}
}
Ok(doc)
}
pub fn compose(&self, doc: &mut Document) {
let suite = metrics::default_suite();
metrics::run_suite(doc, &suite);
}
}
fn default_decomposer(format: &domain::Format) -> Option<Box<dyn Decomposer>> {
match format {
domain::Format::Markdown => Some(Box::new(decompose::markdown::MarkdownDecomposer)),
domain::Format::PlainText => Some(Box::new(decompose::plain::PlainTextDecomposer)),
domain::Format::Pdf | domain::Format::Docx => None,
}
}
pub fn standard_decomposers() -> decompose::Decomposers {
let mut table = decompose::Decomposers::new();
for format in [
domain::Format::Markdown,
domain::Format::PlainText,
domain::Format::Pdf,
domain::Format::Docx,
] {
if let Some(decomposer) = default_decomposer(&format) {
table = table.with(format, decomposer);
}
}
table
}
#[cfg(feature = "python")]
mod python {
use pyo3::prelude::*;
use pyo3::types::PyAny;
use crate::domain;
use crate::nlp::NlpProvider;
use domain::{Format, RawDocument};
struct MatraError(domain::Error);
impl From<domain::Error> for MatraError {
fn from(e: domain::Error) -> Self {
MatraError(e)
}
}
impl From<MatraError> for PyErr {
fn from(e: MatraError) -> PyErr {
use domain::Error::*;
use pyo3::exceptions::*;
let msg = e.0.to_string();
match e.0 {
ModelNotFound(_) => PyFileNotFoundError::new_err(msg),
InputTooLarge { .. } | UnsupportedFormat(_) => PyValueError::new_err(msg),
Io(_) => PyOSError::new_err(msg),
ModelInvalid(_) | ParseFailed(_) => PyRuntimeError::new_err(msg),
}
}
}
fn to_dict<'py, T: serde::Serialize>(py: Python<'py>, val: &T) -> PyResult<Bound<'py, PyAny>> {
pythonize::pythonize(py, val)
.map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))
}
#[pyclass(unsendable)]
struct Matra {
engine: crate::Engine,
}
impl Matra {
fn from_nlp(nlp: Box<dyn NlpProvider>) -> Self {
Self {
engine: crate::Engine::new(nlp, crate::standard_decomposers()),
}
}
fn document(&self, text: &str, format: Format) -> Result<domain::Document, MatraError> {
let raw = RawDocument::new(text.to_string(), None, format);
self.engine
.analyze_one(raw)
.map(|entry| entry.analysis)
.map_err(|e| MatraError(e.error))
}
fn sentences(&self, text: &str) -> Result<Vec<domain::Sentence>, MatraError> {
let raw = RawDocument::new(text.to_string(), None, Format::PlainText);
let doc = self.engine.annotate(&raw).map_err(MatraError)?;
Ok(doc.sentences().cloned().collect())
}
}
#[pymethods]
impl Matra {
#[staticmethod]
#[cfg(feature = "udpipe")]
fn from_path(model_path: &str) -> PyResult<Self> {
let nlp = crate::nlp::udpipe::Udpipe::from_path(model_path).map_err(MatraError)?;
Ok(Self::from_nlp(Box::new(nlp)))
}
#[staticmethod]
#[cfg(feature = "udpipe")]
fn english(model_dir: &str) -> PyResult<Self> {
let nlp = crate::nlp::udpipe::Udpipe::english(model_dir).map_err(MatraError)?;
Ok(Self::from_nlp(Box::new(nlp)))
}
fn analyze<'py>(&self, py: Python<'py>, text: &str) -> PyResult<Bound<'py, PyAny>> {
let analysis = self.document(text, Format::PlainText)?;
to_dict(py, &analysis)
}
fn analyze_markdown<'py>(
&self,
py: Python<'py>,
text: &str,
) -> PyResult<Bound<'py, PyAny>> {
let analysis = self.document(text, Format::Markdown)?;
to_dict(py, &analysis)
}
fn tfidf_summarize<'py>(
&self,
py: Python<'py>,
text: &str,
n: usize,
) -> PyResult<Bound<'py, PyAny>> {
let sentences = self.sentences(text)?;
let result = crate::extraction::tfidf_summarize(&sentences, n).map_err(MatraError)?;
to_dict(py, &result)
}
fn textrank_summarize<'py>(
&self,
py: Python<'py>,
text: &str,
n: usize,
) -> PyResult<Bound<'py, PyAny>> {
let sentences = self.sentences(text)?;
let result =
crate::extraction::textrank_summarize(&sentences, n).map_err(MatraError)?;
to_dict(py, &result)
}
fn rake_keyphrases<'py>(
&self,
py: Python<'py>,
text: &str,
max_phrases: usize,
) -> PyResult<Bound<'py, PyAny>> {
let sentences = self.sentences(text)?;
let result =
crate::extraction::rake_keyphrases(&sentences, max_phrases).map_err(MatraError)?;
to_dict(py, &result)
}
fn yake_keyphrases<'py>(
&self,
py: Python<'py>,
text: &str,
max_phrases: usize,
) -> PyResult<Bound<'py, PyAny>> {
let sentences = self.sentences(text)?;
let result =
crate::extraction::yake_keyphrases(&sentences, max_phrases).map_err(MatraError)?;
to_dict(py, &result)
}
}
#[pymodule]
#[allow(unreachable_pub)] pub fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<Matra>()?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::nlp::NlpProvider;
use domain::Section;
struct EmptyNlp;
impl NlpProvider for EmptyNlp {
fn parse(&self, _text: &str) -> domain::Result<Vec<domain::Sentence>> {
Ok(Vec::new())
}
}
struct DotSplitNlp;
impl NlpProvider for DotSplitNlp {
fn parse(&self, text: &str) -> domain::Result<Vec<domain::Sentence>> {
Ok(text
.split('.')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(|s| domain::Sentence::new(s.to_string(), Vec::new()))
.collect())
}
}
use std::sync::atomic::{AtomicUsize, Ordering};
struct ObservableNlp {
calls: std::sync::Arc<AtomicUsize>,
max_len_seen: std::sync::Arc<AtomicUsize>,
}
impl ObservableNlp {
fn new() -> (
Self,
std::sync::Arc<AtomicUsize>,
std::sync::Arc<AtomicUsize>,
) {
let calls = std::sync::Arc::new(AtomicUsize::new(0));
let max_len = std::sync::Arc::new(AtomicUsize::new(0));
(
Self {
calls: calls.clone(),
max_len_seen: max_len.clone(),
},
calls,
max_len,
)
}
}
impl NlpProvider for ObservableNlp {
fn parse(&self, text: &str) -> domain::Result<Vec<domain::Sentence>> {
self.calls.fetch_add(1, Ordering::SeqCst);
self.max_len_seen.fetch_max(text.len(), Ordering::SeqCst);
if text.contains("POISON") {
return Err(domain::Error::ParseFailed("poisoned".to_string()));
}
DotSplitNlp.parse(text)
}
}
fn law_engine() -> Engine {
Engine::new(Box::new(DotSplitNlp), standard_decomposers())
}
fn fingerprint(
results: Vec<std::result::Result<domain::CorpusEntry, domain::DocumentError>>,
) -> Vec<String> {
results
.into_iter()
.map(|r| match r {
Ok(entry) => format!("ok:{}", serde_json::to_string(&entry).unwrap()),
Err(e) => format!("err:{e}"),
})
.collect()
}
fn raw_plain(text: &str) -> domain::RawDocument {
domain::RawDocument::new(text.to_string(), None, domain::Format::PlainText)
}
#[test]
fn law_l1_analyze_commutes_with_chain() {
let engine = law_engine();
let a = || Ingest::text("First stream. Has two.", domain::Format::PlainText);
let b = || Ingest::text("Second stream here.", domain::Format::PlainText);
let chained: Vec<_> = engine.analyze(a().chain(b())).collect();
let separate: Vec<_> = engine.analyze(a()).chain(engine.analyze(b())).collect();
assert_eq!(fingerprint(chained), fingerprint(separate));
}
#[test]
fn law_l2_analyze_of_empty_is_empty() {
let engine = law_engine();
assert_eq!(engine.analyze(std::iter::empty()).count(), 0);
}
#[test]
fn law_l3_analyze_commutes_with_singleton_injection() {
let engine = law_engine();
let raw = || raw_plain("One document. Two sentences.");
let streamed: Vec<_> = engine.analyze(std::iter::once(Ok(raw()))).collect();
let direct: Vec<_> = std::iter::once(engine.analyze_one(raw())).collect();
assert_eq!(fingerprint(streamed), fingerprint(direct));
}
#[test]
fn law_l4_analyze_one_is_annotate_then_compose() {
let engine = law_engine();
let raw = raw_plain("Composed of parts. Checked by law.");
let via_analyze_one = engine.analyze_one(raw.clone()).unwrap().analysis;
let via_stages = {
let mut doc = engine.annotate(&raw).unwrap();
engine.compose(&mut doc);
doc
};
assert_eq!(
serde_json::to_string(&via_analyze_one).unwrap(),
serde_json::to_string(&via_stages).unwrap()
);
}
#[test]
fn law_l5_partition_entries_plus_errors_equals_input() {
let (nlp, _, _) = ObservableNlp::new();
let engine = Engine::new(Box::new(nlp), standard_decomposers());
let input: Vec<Ingested> = vec![
Ok(raw_plain("Good document one.")),
Ok(raw_plain("This one is POISON and fails.")),
Err(domain::DocumentError::new(
Some(std::path::PathBuf::from("lost.md")),
domain::Error::ParseFailed("upstream".to_string()),
)),
Ok(raw_plain("Good document two.")),
];
let n = input.len();
let result: domain::CorpusResult = engine.analyze(input).collect();
assert_eq!(result.corpus.entries.len() + result.errors.len(), n);
assert_eq!(result.corpus.entries.len(), 2);
assert_eq!(result.errors.len(), 2);
}
#[test]
fn law_l6_err_input_passes_through_untouched() {
let (nlp, calls, _) = ObservableNlp::new();
let engine = Engine::new(Box::new(nlp), standard_decomposers());
let input: Vec<Ingested> = vec![Err(domain::DocumentError::new(
Some(std::path::PathBuf::from("ghost.md")),
domain::Error::ParseFailed("upstream".to_string()),
))];
let out: Vec<_> = engine.analyze(input).collect();
assert_eq!(out.len(), 1);
let err = out.into_iter().next().unwrap().unwrap_err();
assert_eq!(err.path, Some(std::path::PathBuf::from("ghost.md")));
assert_eq!(err.to_string(), "ghost.md: parse failed: upstream");
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"an Err input item must not reach the parser"
);
}
#[test]
fn law_l6_err_input_never_reaches_the_parser() {
let (nlp, calls, _) = ObservableNlp::new();
let engine = Engine::new(Box::new(nlp), standard_decomposers());
let input: Vec<Ingested> = (0..3)
.map(|i| {
Err(domain::DocumentError::new(
None,
domain::Error::ParseFailed(format!("e{i}")),
))
})
.collect();
let _ = engine.analyze(input).count();
assert_eq!(calls.load(Ordering::SeqCst), 0);
engine.annotate(&raw_plain("Control.")).unwrap();
assert_eq!(calls.load(Ordering::SeqCst), 1, "counter is live");
}
#[test]
fn law_l7_no_oversized_text_reaches_the_parser() {
let (nlp, _, max_len) = ObservableNlp::new();
let engine = Engine::new(Box::new(nlp), standard_decomposers());
let oversized = "a".repeat(MAX_INPUT_BYTES + 1);
let out: Vec<_> = engine
.analyze(Ingest::text(oversized.clone(), domain::Format::PlainText))
.collect();
assert!(matches!(
out[0].as_ref().unwrap_err().error,
domain::Error::InputTooLarge { .. }
));
let raw = raw_plain(&oversized);
assert!(matches!(
engine.annotate(&raw),
Err(domain::Error::InputTooLarge { .. })
));
assert!(
max_len.load(Ordering::SeqCst) <= MAX_INPUT_BYTES,
"the parser never saw text over the cap"
);
}
#[test]
fn annotate_rejects_unregistered_format() {
let engine = law_engine();
let raw = domain::RawDocument::new("text".to_string(), None, domain::Format::Pdf);
assert!(matches!(
engine.annotate(&raw),
Err(domain::Error::UnsupportedFormat(domain::Format::Pdf))
));
}
#[test]
fn annotate_leaves_metrics_none_and_compose_fills_them() {
let engine = law_engine();
let raw = raw_plain("Annotate attaches structure. Compose measures it.");
let mut doc = engine.annotate(&raw).unwrap();
assert!(doc.vocabulary_ttr.is_none(), "annotate does not measure");
assert!(doc.passive_ratio.is_none(), "annotate does not measure");
assert!(doc.total_sentences() > 0, "annotate does attach");
engine.compose(&mut doc);
assert_eq!(
doc.passive_ratio,
Some(0.0),
"compose fills the sentence-derived aggregate"
);
}
#[test]
fn annotate_fills_hearst_pairs_at_the_choke_point() {
struct SuchAsNlp;
impl NlpProvider for SuchAsNlp {
fn parse(&self, _text: &str) -> domain::Result<Vec<domain::Sentence>> {
let tok = |id: usize, text: &str, lemma: &str, pos: &str, dep: &str, head| {
domain::Token::builder(
id,
text.to_string(),
lemma.to_string(),
pos.to_string(),
head,
dep.to_string(),
)
.build()
};
Ok(vec![domain::Sentence::new(
"Animals such as dogs bark".to_string(),
vec![
tok(1, "Animals", "animal", "NOUN", "nsubj", 5),
tok(2, "such", "such", "ADJ", "case", 4),
tok(3, "as", "as", "ADP", "fixed", 2),
tok(4, "dogs", "dog", "NOUN", "nmod", 1),
tok(5, "bark", "bark", "VERB", "root", 0),
],
)])
}
}
let engine = Engine::new(Box::new(SuchAsNlp), standard_decomposers());
let doc = engine.annotate(&raw_plain("any text")).unwrap();
let sentence = doc.sentences().next().expect("one sentence");
assert_eq!(sentence.hearst_pairs.len(), 1, "annotate ran the detector");
let pair = &sentence.hearst_pairs[0];
assert_eq!(pair.pattern, domain::HearstPattern::SuchAs);
assert_eq!(pair.hypernym.head_id, 1);
assert_eq!(pair.hyponym.head_id, 4);
}
#[test]
fn ingest_text_is_a_stream_of_one() {
let items: Vec<_> = Ingest::text("hello", domain::Format::PlainText).collect();
assert_eq!(items.len(), 1);
let doc = items.into_iter().next().unwrap().unwrap();
assert_eq!(doc.text, "hello");
assert_eq!(doc.path, None);
}
#[test]
fn ingest_path_file_is_a_stream_of_one() {
let dir = tempfile::tempdir().unwrap();
let f = dir.path().join("one.md");
std::fs::write(&f, "# One").unwrap();
let items: Vec<_> = Ingest::path(&f).unwrap().collect();
assert_eq!(items.len(), 1);
let doc = items.into_iter().next().unwrap().unwrap();
assert_eq!(doc.path.as_deref(), Some(f.as_path()));
assert!(matches!(doc.format, domain::Format::Markdown));
}
#[test]
fn ingest_path_directory_streams_sorted_entries() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("b.txt"), "B").unwrap();
std::fs::write(dir.path().join("a.md"), "# A").unwrap();
let items: Vec<_> = Ingest::path(dir.path()).unwrap().collect();
assert_eq!(items.len(), 2);
let paths: Vec<_> = items
.into_iter()
.map(|r| r.unwrap().path.unwrap())
.collect();
assert!(paths[0].ends_with("a.md"));
assert!(paths[1].ends_with("b.txt"));
}
#[test]
fn ingest_path_missing_is_a_constructor_error() {
assert!(Ingest::path("/definitely/not/here").is_err());
}
#[cfg(unix)]
#[test]
fn ingest_per_file_failure_is_an_err_item_not_an_abort() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("good.md"), "# Good").unwrap();
let bad = dir.path().join("bad.md");
std::fs::write(&bad, "# Bad").unwrap();
std::fs::set_permissions(&bad, std::fs::Permissions::from_mode(0o000)).unwrap();
let items: Vec<_> = Ingest::path(dir.path()).unwrap().collect();
let _ = std::fs::set_permissions(&bad, std::fs::Permissions::from_mode(0o644));
assert_eq!(items.len(), 2);
assert_eq!(items.iter().filter(|r| r.is_ok()).count(), 1);
let err = items.into_iter().find_map(|r| r.err()).unwrap();
assert!(err.path.unwrap().ends_with("bad.md"));
}
#[test]
fn standard_decomposers_cover_exactly_the_shipping_formats() {
let table = standard_decomposers();
assert!(table.get(&domain::Format::Markdown).is_some());
assert!(table.get(&domain::Format::PlainText).is_some());
assert!(
table.get(&domain::Format::Pdf).is_none(),
"Pdf is reserved, not shipped"
);
assert!(
table.get(&domain::Format::Docx).is_none(),
"Docx is reserved, not shipped"
);
}
#[test]
fn input_at_cap_is_accepted() {
let engine = Engine::new(Box::new(EmptyNlp), standard_decomposers());
let raw = raw_plain(&"a".repeat(MAX_INPUT_BYTES));
assert!(
engine.annotate(&raw).is_ok(),
"input exactly at cap should be accepted"
);
}
#[test]
fn input_one_byte_over_cap_is_rejected() {
let engine = Engine::new(Box::new(EmptyNlp), standard_decomposers());
let raw = raw_plain(&"a".repeat(MAX_INPUT_BYTES + 1));
match engine.annotate(&raw) {
Err(domain::Error::InputTooLarge {
limit,
actual,
what,
}) => {
assert_eq!(limit, MAX_INPUT_BYTES);
assert_eq!(actual, MAX_INPUT_BYTES + 1);
assert_eq!(what, "input");
}
other => panic!("expected InputTooLarge, got {other:?}"),
}
}
#[test]
fn parse_per_paragraph_scopes_sentences_to_originating_paragraph() {
let text = "The system processes input now. Tail one.\n\n\
The system processes input now. Tail two.";
let analysis = law_engine().analyze_one(raw_plain(text)).unwrap().analysis;
let paras: Vec<_> = analysis.paragraphs().collect();
assert_eq!(paras.len(), 2);
assert_eq!(paras[0].sentences.len(), 2);
assert_eq!(paras[1].sentences.len(), 2);
assert!(paras[0].sentences.iter().any(|s| s.text.contains("one")));
assert!(paras[1].sentences.iter().any(|s| s.text.contains("two")));
assert!(!paras[0].sentences.iter().any(|s| s.text.contains("two")));
assert!(!paras[1].sentences.iter().any(|s| s.text.contains("one")));
}
#[test]
fn parse_per_paragraph_no_inner_substring_theft() {
let text = "Outer talks about the special phrase processes input now. End A.\n\n\
The special phrase processes input now. End B.";
let analysis = law_engine().analyze_one(raw_plain(text)).unwrap().analysis;
let paras: Vec<_> = analysis.paragraphs().collect();
assert_eq!(paras.len(), 2);
assert!(paras[1].sentences.iter().any(|s| s.text.contains("End B")));
assert!(!paras[0].sentences.iter().any(|s| s.text.contains("End B")));
}
#[test]
fn empty_paragraph_followed_by_valid_paragraph() {
struct InjectEmptyParagraph;
impl Decomposer for InjectEmptyParagraph {
fn decompose(&self, text: &str) -> Vec<Section> {
let mut sections = decompose::plain::PlainTextDecomposer.decompose(text);
if let Some(section) = sections.first_mut() {
section
.paragraphs
.insert(0, domain::Paragraph::new(String::new(), false));
}
sections
}
}
let table = decompose::Decomposers::new()
.with(domain::Format::PlainText, Box::new(InjectEmptyParagraph));
let engine = Engine::new(Box::new(DotSplitNlp), table);
let analysis = engine
.analyze_one(raw_plain("Real content sentence.\n\nAnother real one."))
.unwrap()
.analysis;
let paras: Vec<_> = analysis.paragraphs().collect();
assert_eq!(paras.len(), 3);
assert_eq!(
paras[0].sentences.len(),
0,
"empty paragraph has zero sentences"
);
assert!(!paras[1].sentences.is_empty());
assert!(!paras[2].sentences.is_empty());
}
}