#![doc = include_str!("../README.md")]
#[cfg(feature = "cli")]
pub mod cli;
pub mod config;
pub mod decompose;
pub mod domain;
pub mod embed;
pub mod extraction;
pub mod hearst;
pub mod metrics;
pub mod nlp;
pub mod source;
mod stopwords;
use std::path::Path;
use decompose::Decomposer;
use domain::{Document, MAX_INPUT_BYTES};
use nlp::NlpProvider;
use source::Source;
fn check_input_size(text: &str) -> domain::Result<()> {
if text.len() > MAX_INPUT_BYTES {
return Err(domain::Error::InputTooLarge {
limit: MAX_INPUT_BYTES,
actual: text.len(),
what: "input",
});
}
Ok(())
}
pub type Ingested = std::result::Result<domain::RawDocument, domain::DocumentError>;
enum Pending {
Ready(domain::RawDocument),
File(std::path::PathBuf),
}
pub struct Ingest {
items: std::vec::IntoIter<Pending>,
}
impl Ingest {
pub fn text(text: impl Into<String>, format: domain::Format) -> Self {
Self {
items: vec![Pending::Ready(domain::RawDocument::new(
text.into(),
None,
format,
))]
.into_iter(),
}
}
pub fn path(input: impl AsRef<Path>) -> domain::Result<Self> {
let input = input.as_ref();
let metadata = std::fs::symlink_metadata(input)?;
let pending = if metadata.file_type().is_dir() {
source::directory::DirectorySource
.candidate_paths(input)?
.into_iter()
.map(Pending::File)
.collect()
} else {
vec![Pending::File(input.to_path_buf())]
};
Ok(Self {
items: pending.into_iter(),
})
}
}
impl Iterator for Ingest {
type Item = Ingested;
fn next(&mut self) -> Option<Ingested> {
let item = self.items.next()?;
Some(match item {
Pending::Ready(doc) => Ok(doc),
Pending::File(path) => match source::file::FileSource.read(&path) {
Ok(docs) => docs.into_iter().next().ok_or_else(|| {
domain::DocumentError::new(
Some(path),
domain::Error::Io(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"source returned no documents",
)),
)
}),
Err(e) => Err(domain::DocumentError::new(Some(path), e)),
},
})
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.items.size_hint()
}
}
pub struct Engine {
nlp: Box<dyn NlpProvider>,
decomposers: decompose::Decomposers,
}
impl Engine {
pub fn new(nlp: Box<dyn NlpProvider>, decomposers: decompose::Decomposers) -> Self {
Self { nlp, decomposers }
}
pub fn analyze<I>(
&self,
input: I,
) -> impl Iterator<Item = std::result::Result<domain::CorpusEntry, domain::DocumentError>>
where
I: IntoIterator<Item = Ingested>,
{
input
.into_iter()
.map(|item| item.and_then(|raw| self.analyze_one(raw)))
}
pub fn analyze_one(
&self,
raw: domain::RawDocument,
) -> std::result::Result<domain::CorpusEntry, domain::DocumentError> {
let path = raw.path.clone();
match self.annotate(&raw) {
Ok(mut doc) => {
self.compose(&mut doc);
Ok(domain::CorpusEntry::new(path, doc))
}
Err(e) => Err(domain::DocumentError::new(path, e)),
}
}
pub fn annotate(&self, raw: &domain::RawDocument) -> domain::Result<Document> {
check_input_size(&raw.text)?;
let decomposer = self
.decomposers
.get(&raw.format)
.ok_or_else(|| domain::Error::UnsupportedFormat(raw.format.clone()))?;
let mut doc = Document::new(decomposer.decompose(&raw.text));
for para in doc.paragraphs_mut() {
if para.in_blockquote {
continue;
}
para.sentences = self.nlp.parse(¶.text)?;
for sentence in &mut para.sentences {
sentence.hearst_pairs = hearst::hypernymy_pairs(&sentence.tokens);
}
}
Ok(doc)
}
pub fn compose(&self, doc: &mut Document) {
let suite = metrics::default_suite();
metrics::run_suite(doc, &suite);
}
}
#[cfg(feature = "udpipe")]
impl Engine {
pub fn from_config(cfg: &config::Config) -> domain::Result<Engine> {
let nlp = nlp::udpipe::Udpipe::from_config(cfg)?;
Ok(Engine::new(Box::new(nlp), standard_decomposers()))
}
pub fn from_config_with_notice(
cfg: &config::Config,
notice: impl FnMut(&domain::ProvisionNotice),
) -> domain::Result<Engine> {
let nlp = nlp::udpipe::Udpipe::from_config_with_notice(cfg, notice)?;
Ok(Engine::new(Box::new(nlp), standard_decomposers()))
}
pub fn with_defaults() -> domain::Result<Engine> {
Engine::from_config(&config::Config::resolve()?)
}
}
fn default_decomposer(format: &domain::Format) -> Option<Box<dyn Decomposer>> {
match format {
domain::Format::Markdown => Some(Box::new(decompose::markdown::MarkdownDecomposer)),
domain::Format::PlainText => Some(Box::new(decompose::plain::PlainTextDecomposer)),
domain::Format::Pdf | domain::Format::Docx => None,
}
}
pub fn standard_decomposers() -> decompose::Decomposers {
let mut table = decompose::Decomposers::new();
for format in [
domain::Format::Markdown,
domain::Format::PlainText,
domain::Format::Pdf,
domain::Format::Docx,
] {
if let Some(decomposer) = default_decomposer(&format) {
table = table.with(format, decomposer);
}
}
table
}
pub fn embed_and_cluster(
doc: &Document,
embedder: &dyn embed::Embedder,
threshold: f32,
) -> domain::Result<domain::SemanticClusters> {
let texts: Vec<&str> = doc.sentences().map(|s| s.text.as_str()).collect();
if texts.len() > extraction::MAX_SEMANTIC_SENTENCES {
return Err(domain::Error::InputTooLarge {
limit: extraction::MAX_SEMANTIC_SENTENCES,
actual: texts.len(),
what: "semantic_clusters",
});
}
let embeddings = embedder.embed(&texts)?;
if embeddings.len() != texts.len() {
return Err(domain::Error::InvalidInput(format!(
"embedder returned {} vectors for {} sentences, violating its contract",
embeddings.len(),
texts.len()
)));
}
extraction::semantic_clusters(&embeddings, threshold, embedder.identity())
}
#[cfg(feature = "python")]
mod python {
use std::path::{Path, PathBuf};
use pyo3::prelude::*;
use pyo3::types::{PyAny, PyDict, PyList, PyString};
use crate::domain;
use crate::nlp::NlpProvider;
use domain::{Format, RawDocument};
struct MatraError(domain::Error);
impl From<domain::Error> for MatraError {
fn from(e: domain::Error) -> Self {
MatraError(e)
}
}
impl From<MatraError> for PyErr {
fn from(e: MatraError) -> PyErr {
use domain::Error::*;
use pyo3::exceptions::*;
let msg = e.0.to_string();
match e.0 {
ModelNotFound(_) => PyFileNotFoundError::new_err(msg),
InputTooLarge { .. } | UnsupportedFormat(_) | InvalidInput(_) => {
PyValueError::new_err(msg)
}
Io(_) => PyOSError::new_err(msg),
ModelInvalid(_) | ParseFailed(_) => PyRuntimeError::new_err(msg),
}
}
}
fn to_dict<'py, T: serde::Serialize>(py: Python<'py>, val: &T) -> PyResult<Bound<'py, PyAny>> {
pythonize::pythonize(py, val)
.map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))
}
#[pyclass(unsendable)]
struct Matra {
engine: crate::Engine,
}
impl Matra {
fn from_nlp(nlp: Box<dyn NlpProvider>) -> Self {
Self {
engine: crate::Engine::new(nlp, crate::standard_decomposers()),
}
}
fn document(&self, text: &str, format: Format) -> Result<domain::Document, MatraError> {
let raw = RawDocument::new(text.to_string(), None, format);
self.engine
.analyze_one(raw)
.map(|entry| entry.analysis)
.map_err(|e| MatraError(e.error))
}
fn annotated(&self, text: &str) -> Result<domain::Document, MatraError> {
let raw = RawDocument::new(text.to_string(), None, Format::PlainText);
self.engine.annotate(&raw).map_err(MatraError)
}
fn sentences(&self, text: &str) -> Result<Vec<domain::Sentence>, MatraError> {
Ok(self.annotated(text)?.sentences().cloned().collect())
}
}
#[pymethods]
impl Matra {
#[staticmethod]
#[cfg(feature = "udpipe")]
fn from_path(model_path: PathBuf) -> PyResult<Self> {
let nlp = crate::nlp::udpipe::Udpipe::from_path(model_path).map_err(MatraError)?;
Ok(Self::from_nlp(Box::new(nlp)))
}
#[staticmethod]
#[cfg(feature = "udpipe")]
#[pyo3(signature = (model_dir=None))]
fn english(model_dir: Option<PathBuf>) -> PyResult<Self> {
let nlp = match model_dir {
Some(dir) => crate::nlp::udpipe::Udpipe::english(dir),
None => {
let cfg = crate::config::Config::resolve().map_err(MatraError)?;
crate::nlp::udpipe::Udpipe::from_config(&cfg)
}
}
.map_err(MatraError)?;
Ok(Self::from_nlp(Box::new(nlp)))
}
fn analyze<'py>(&self, py: Python<'py>, text: &str) -> PyResult<Bound<'py, PyAny>> {
let analysis = self.document(text, Format::PlainText)?;
to_dict(py, &analysis)
}
fn analyze_markdown<'py>(
&self,
py: Python<'py>,
text: &str,
) -> PyResult<Bound<'py, PyAny>> {
let analysis = self.document(text, Format::Markdown)?;
to_dict(py, &analysis)
}
fn tfidf_summarize<'py>(
&self,
py: Python<'py>,
text: &str,
n: usize,
) -> PyResult<Bound<'py, PyAny>> {
let sentences = self.sentences(text)?;
let result = crate::extraction::tfidf_summarize(&sentences, n).map_err(MatraError)?;
to_dict(py, &result)
}
fn textrank_summarize<'py>(
&self,
py: Python<'py>,
text: &str,
n: usize,
) -> PyResult<Bound<'py, PyAny>> {
let sentences = self.sentences(text)?;
let result =
crate::extraction::textrank_summarize(&sentences, n).map_err(MatraError)?;
to_dict(py, &result)
}
fn rake_keyphrases<'py>(
&self,
py: Python<'py>,
text: &str,
max_phrases: usize,
) -> PyResult<Bound<'py, PyAny>> {
let sentences = self.sentences(text)?;
let result =
crate::extraction::rake_keyphrases(&sentences, max_phrases).map_err(MatraError)?;
to_dict(py, &result)
}
fn yake_keyphrases<'py>(
&self,
py: Python<'py>,
text: &str,
max_phrases: usize,
) -> PyResult<Bound<'py, PyAny>> {
let sentences = self.sentences(text)?;
let result =
crate::extraction::yake_keyphrases(&sentences, max_phrases).map_err(MatraError)?;
to_dict(py, &result)
}
fn semantic_clusters<'py>(
&self,
py: Python<'py>,
text: &str,
threshold: f32,
model: &Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
#[cfg(feature = "model2vec")]
if let Ok(loaded) = model.cast::<Model2Vec>() {
let doc = self.annotated(text)?;
let result = crate::embed_and_cluster(&doc, &loaded.get().inner, threshold)
.map_err(MatraError)?;
return to_dict(py, &result);
}
let embedder = PyEmbedder::new(model)?;
let doc = self.annotated(text)?;
let result =
crate::embed_and_cluster(&doc, &embedder, threshold).map_err(MatraError)?;
to_dict(py, &result)
}
fn analyze_path<'py>(&self, py: Python<'py>, path: PathBuf) -> PyResult<Bound<'py, PyAny>> {
let ingest = crate::Ingest::path(&path).map_err(MatraError)?;
let items = PyList::empty(py);
for item in self.engine.analyze(ingest) {
let entry = match item {
Ok(entry) => corpus_entry_dict(py, &entry)?,
Err(err) => document_error_dict(py, &err)?,
};
items.append(entry)?;
}
Ok(items.into_any())
}
}
#[cfg(feature = "model2vec")]
#[pyclass(frozen)]
struct Model2Vec {
inner: crate::embed::model2vec::Model2Vec,
}
#[cfg(feature = "model2vec")]
#[pymethods]
impl Model2Vec {
#[staticmethod]
fn from_dir(dir: PathBuf) -> PyResult<Self> {
let inner = crate::embed::model2vec::Model2Vec::from_dir(dir).map_err(MatraError)?;
Ok(Self { inner })
}
#[staticmethod]
#[pyo3(signature = (dir=None))]
fn potion_base_8m(dir: Option<PathBuf>) -> PyResult<Self> {
let inner = match dir {
Some(dir) => crate::embed::model2vec::Model2Vec::potion_base_8m(dir),
None => {
let cfg = crate::config::Config::resolve().map_err(MatraError)?;
crate::embed::model2vec::Model2Vec::from_config(&cfg)
}
}
.map_err(MatraError)?;
Ok(Self { inner })
}
#[getter]
fn model_hash(&self) -> &str {
self.inner.model_hash()
}
#[getter]
fn dimensions(&self) -> usize {
self.inner.dimensions()
}
fn identity(&self) -> String {
self.inner.model_hash().to_string()
}
fn embed(&self, texts: Vec<String>) -> PyResult<Vec<Vec<f32>>> {
use crate::embed::Embedder;
let refs: Vec<&str> = texts.iter().map(String::as_str).collect();
let out = self.inner.embed(&refs).map_err(MatraError)?;
Ok(out.into_iter().map(|e| e.0).collect())
}
}
#[pyfunction]
#[allow(unreachable_pub)] pub fn semantic_clusters<'py>(
py: Python<'py>,
embeddings: Vec<Vec<f32>>,
threshold: f32,
model_hash: &str,
) -> PyResult<Bound<'py, PyAny>> {
let embeddings: Vec<domain::Embedding> =
embeddings.into_iter().map(domain::Embedding).collect();
let result = crate::extraction::semantic_clusters(&embeddings, threshold, model_hash)
.map_err(MatraError)?;
to_dict(py, &result)
}
#[pyfunction]
#[allow(unreachable_pub)] pub fn cli_main(py: Python<'_>, argv: Vec<Bound<'_, PyAny>>) -> PyResult<i32> {
use std::ffi::OsString;
let fsencode = py.import("os")?.getattr("fsencode")?;
let mut args: Vec<OsString> = Vec::with_capacity(argv.len() + 1);
args.push(OsString::from("matra"));
for arg in argv {
let bytes: Vec<u8> = fsencode.call1((arg,))?.extract()?;
args.push(os_string_of(bytes));
}
let mut out: Vec<u8> = Vec::new();
let mut err = PyStream { py, name: "stderr" };
let code = crate::cli::run(args, &mut out, &mut err);
let _ = write_stream(py, "stdout", &out);
Ok(i32::from(code))
}
struct PyStream<'py> {
py: Python<'py>,
name: &'static str,
}
impl std::io::Write for PyStream<'_> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
write_stream(self.py, self.name, buf)
.map(|()| buf.len())
.map_err(|e| std::io::Error::other(e.to_string()))
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
#[cfg(unix)]
fn os_string_of(bytes: Vec<u8>) -> std::ffi::OsString {
use std::os::unix::ffi::OsStringExt;
std::ffi::OsString::from_vec(bytes)
}
#[cfg(not(unix))]
fn os_string_of(bytes: Vec<u8>) -> std::ffi::OsString {
std::ffi::OsString::from(String::from_utf8_lossy(&bytes).into_owned())
}
fn write_stream(py: Python<'_>, name: &str, bytes: &[u8]) -> PyResult<()> {
if bytes.is_empty() {
return Ok(());
}
let stream = py.import("sys")?.getattr(name)?;
stream.call_method1("write", (String::from_utf8_lossy(bytes).as_ref(),))?;
stream.call_method0("flush")?;
Ok(())
}
struct PyEmbedder {
obj: Py<PyAny>,
identity: String,
}
impl PyEmbedder {
fn new(obj: &Bound<'_, PyAny>) -> PyResult<Self> {
let raw = obj.call_method0("identity").map_err(|e| {
MatraError(domain::Error::InvalidInput(format!(
"calling identity() on the embedder failed: {e}"
)))
})?;
let identity = raw.extract::<String>().map_err(|e| {
MatraError(domain::Error::InvalidInput(format!(
"the embedder's identity() must return a str: {e}"
)))
})?;
Ok(Self {
obj: obj.clone().unbind(),
identity,
})
}
}
impl crate::embed::Embedder for PyEmbedder {
fn embed(&self, texts: &[&str]) -> domain::Result<Vec<domain::Embedding>> {
Python::attach(|py| {
let returned = self
.obj
.bind(py)
.call_method1("embed", (texts.to_vec(),))
.map_err(|e| {
domain::Error::InvalidInput(format!("the embedder's embed() raised {e}"))
})?;
let rows = returned.extract::<Vec<Bound<'_, PyAny>>>().map_err(|e| {
domain::Error::InvalidInput(format!(
"the embedder's embed() must return a sequence of vectors: {e}"
))
})?;
rows.iter()
.enumerate()
.map(|(i, row)| {
let values = row.extract::<Vec<f64>>().map_err(|e| {
domain::Error::InvalidInput(format!(
"the embedder's embed() returned a vector at index {i} that is not a sequence of numbers: {e}"
))
})?;
Ok(domain::Embedding(
values.into_iter().map(|v| v as f32).collect(),
))
})
.collect()
})
}
fn identity(&self) -> &str {
&self.identity
}
}
#[cfg(unix)]
fn fs_path_str<'py>(py: Python<'py>, path: &Path) -> PyResult<Bound<'py, PyString>> {
use std::os::unix::ffi::OsStrExt;
let bytes = pyo3::types::PyBytes::new(py, path.as_os_str().as_bytes());
let decoded = py.import("os")?.getattr("fsdecode")?.call1((bytes,))?;
Ok(decoded.cast_into::<PyString>()?)
}
#[cfg(not(unix))]
fn fs_path_str<'py>(py: Python<'py>, path: &Path) -> PyResult<Bound<'py, PyString>> {
Ok(PyString::new(py, &path.to_string_lossy()))
}
fn fs_path_item<'py>(
py: Python<'py>,
path: Option<&Path>,
) -> PyResult<Option<Bound<'py, PyString>>> {
path.map(|p| fs_path_str(py, p)).transpose()
}
fn corpus_entry_dict<'py>(
py: Python<'py>,
entry: &domain::CorpusEntry,
) -> PyResult<Bound<'py, PyAny>> {
let item = PyDict::new(py);
item.set_item("path", fs_path_item(py, entry.path.as_deref())?)?;
item.set_item("analysis", to_dict(py, &entry.analysis)?)?;
Ok(item.into_any())
}
fn document_error_dict<'py>(
py: Python<'py>,
err: &domain::DocumentError,
) -> PyResult<Bound<'py, PyAny>> {
let error = PyDict::new(py);
error.set_item("kind", err.error.kind())?;
error.set_item("message", err.error.to_string())?;
let item = PyDict::new(py);
item.set_item("path", fs_path_item(py, err.path.as_deref())?)?;
item.set_item("error", error)?;
Ok(item.into_any())
}
#[pymodule]
#[allow(unreachable_pub)] pub fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<Matra>()?;
#[cfg(feature = "model2vec")]
m.add_class::<Model2Vec>()?;
m.add_function(pyo3::wrap_pyfunction!(semantic_clusters, m)?)?;
m.add_function(pyo3::wrap_pyfunction!(cli_main, m)?)?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::nlp::NlpProvider;
use domain::Section;
struct EmptyNlp;
impl NlpProvider for EmptyNlp {
fn parse(&self, _text: &str) -> domain::Result<Vec<domain::Sentence>> {
Ok(Vec::new())
}
}
struct DotSplitNlp;
impl NlpProvider for DotSplitNlp {
fn parse(&self, text: &str) -> domain::Result<Vec<domain::Sentence>> {
Ok(text
.split('.')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(|s| domain::Sentence::new(s.to_string(), Vec::new()))
.collect())
}
}
use std::sync::atomic::{AtomicUsize, Ordering};
struct ObservableNlp {
calls: std::sync::Arc<AtomicUsize>,
max_len_seen: std::sync::Arc<AtomicUsize>,
}
impl ObservableNlp {
fn new() -> (
Self,
std::sync::Arc<AtomicUsize>,
std::sync::Arc<AtomicUsize>,
) {
let calls = std::sync::Arc::new(AtomicUsize::new(0));
let max_len = std::sync::Arc::new(AtomicUsize::new(0));
(
Self {
calls: calls.clone(),
max_len_seen: max_len.clone(),
},
calls,
max_len,
)
}
}
impl NlpProvider for ObservableNlp {
fn parse(&self, text: &str) -> domain::Result<Vec<domain::Sentence>> {
self.calls.fetch_add(1, Ordering::SeqCst);
self.max_len_seen.fetch_max(text.len(), Ordering::SeqCst);
if text.contains("POISON") {
return Err(domain::Error::ParseFailed("poisoned".to_string()));
}
DotSplitNlp.parse(text)
}
}
fn law_engine() -> Engine {
Engine::new(Box::new(DotSplitNlp), standard_decomposers())
}
fn fingerprint(
results: Vec<std::result::Result<domain::CorpusEntry, domain::DocumentError>>,
) -> Vec<String> {
results
.into_iter()
.map(|r| match r {
Ok(entry) => format!("ok:{}", serde_json::to_string(&entry).unwrap()),
Err(e) => format!("err:{e}"),
})
.collect()
}
fn raw_plain(text: &str) -> domain::RawDocument {
domain::RawDocument::new(text.to_string(), None, domain::Format::PlainText)
}
#[test]
fn law_l1_analyze_commutes_with_chain() {
let engine = law_engine();
let a = || Ingest::text("First stream. Has two.", domain::Format::PlainText);
let b = || Ingest::text("Second stream here.", domain::Format::PlainText);
let chained: Vec<_> = engine.analyze(a().chain(b())).collect();
let separate: Vec<_> = engine.analyze(a()).chain(engine.analyze(b())).collect();
assert_eq!(fingerprint(chained), fingerprint(separate));
}
#[test]
fn law_l2_analyze_of_empty_is_empty() {
let engine = law_engine();
assert_eq!(engine.analyze(std::iter::empty()).count(), 0);
}
#[test]
fn law_l3_analyze_commutes_with_singleton_injection() {
let engine = law_engine();
let raw = || raw_plain("One document. Two sentences.");
let streamed: Vec<_> = engine.analyze(std::iter::once(Ok(raw()))).collect();
let direct: Vec<_> = std::iter::once(engine.analyze_one(raw())).collect();
assert_eq!(fingerprint(streamed), fingerprint(direct));
}
#[test]
fn law_l4_analyze_one_is_annotate_then_compose() {
let engine = law_engine();
let raw = raw_plain("Composed of parts. Checked by law.");
let via_analyze_one = engine.analyze_one(raw.clone()).unwrap().analysis;
let via_stages = {
let mut doc = engine.annotate(&raw).unwrap();
engine.compose(&mut doc);
doc
};
assert_eq!(
serde_json::to_string(&via_analyze_one).unwrap(),
serde_json::to_string(&via_stages).unwrap()
);
}
#[test]
fn law_l5_partition_entries_plus_errors_equals_input() {
let (nlp, _, _) = ObservableNlp::new();
let engine = Engine::new(Box::new(nlp), standard_decomposers());
let input: Vec<Ingested> = vec![
Ok(raw_plain("Good document one.")),
Ok(raw_plain("This one is POISON and fails.")),
Err(domain::DocumentError::new(
Some(std::path::PathBuf::from("lost.md")),
domain::Error::ParseFailed("upstream".to_string()),
)),
Ok(raw_plain("Good document two.")),
];
let n = input.len();
let result: domain::CorpusResult = engine.analyze(input).collect();
assert_eq!(result.corpus.entries.len() + result.errors.len(), n);
assert_eq!(result.corpus.entries.len(), 2);
assert_eq!(result.errors.len(), 2);
}
#[test]
fn law_l6_err_input_passes_through_untouched() {
let (nlp, calls, _) = ObservableNlp::new();
let engine = Engine::new(Box::new(nlp), standard_decomposers());
let input: Vec<Ingested> = vec![Err(domain::DocumentError::new(
Some(std::path::PathBuf::from("ghost.md")),
domain::Error::ParseFailed("upstream".to_string()),
))];
let out: Vec<_> = engine.analyze(input).collect();
assert_eq!(out.len(), 1);
let err = out.into_iter().next().unwrap().unwrap_err();
assert_eq!(err.path, Some(std::path::PathBuf::from("ghost.md")));
assert_eq!(err.to_string(), "ghost.md: parse failed: upstream");
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"an Err input item must not reach the parser"
);
}
#[test]
fn law_l6_err_input_never_reaches_the_parser() {
let (nlp, calls, _) = ObservableNlp::new();
let engine = Engine::new(Box::new(nlp), standard_decomposers());
let input: Vec<Ingested> = (0..3)
.map(|i| {
Err(domain::DocumentError::new(
None,
domain::Error::ParseFailed(format!("e{i}")),
))
})
.collect();
let _ = engine.analyze(input).count();
assert_eq!(calls.load(Ordering::SeqCst), 0);
engine.annotate(&raw_plain("Control.")).unwrap();
assert_eq!(calls.load(Ordering::SeqCst), 1, "counter is live");
}
#[test]
fn law_l7_no_oversized_text_reaches_the_parser() {
let (nlp, _, max_len) = ObservableNlp::new();
let engine = Engine::new(Box::new(nlp), standard_decomposers());
let oversized = "a".repeat(MAX_INPUT_BYTES + 1);
let out: Vec<_> = engine
.analyze(Ingest::text(oversized.clone(), domain::Format::PlainText))
.collect();
assert!(matches!(
out[0].as_ref().unwrap_err().error,
domain::Error::InputTooLarge { .. }
));
let raw = raw_plain(&oversized);
assert!(matches!(
engine.annotate(&raw),
Err(domain::Error::InputTooLarge { .. })
));
assert!(
max_len.load(Ordering::SeqCst) <= MAX_INPUT_BYTES,
"the parser never saw text over the cap"
);
}
#[test]
fn annotate_rejects_unregistered_format() {
let engine = law_engine();
let raw = domain::RawDocument::new("text".to_string(), None, domain::Format::Pdf);
assert!(matches!(
engine.annotate(&raw),
Err(domain::Error::UnsupportedFormat(domain::Format::Pdf))
));
}
#[test]
fn annotate_leaves_metrics_none_and_compose_fills_them() {
let engine = law_engine();
let raw = raw_plain("Annotate attaches structure. Compose measures it.");
let mut doc = engine.annotate(&raw).unwrap();
assert!(doc.vocabulary_ttr.is_none(), "annotate does not measure");
assert!(doc.passive_ratio.is_none(), "annotate does not measure");
assert!(doc.total_sentences() > 0, "annotate does attach");
engine.compose(&mut doc);
assert_eq!(
doc.passive_ratio,
Some(0.0),
"compose fills the sentence-derived aggregate"
);
}
#[test]
fn annotate_fills_hearst_pairs_at_the_choke_point() {
struct SuchAsNlp;
impl NlpProvider for SuchAsNlp {
fn parse(&self, _text: &str) -> domain::Result<Vec<domain::Sentence>> {
let tok = |id: usize, text: &str, lemma: &str, pos: &str, dep: &str, head| {
domain::Token::builder(
id,
text.to_string(),
lemma.to_string(),
pos.to_string(),
head,
dep.to_string(),
)
.build()
};
Ok(vec![domain::Sentence::new(
"Animals such as dogs bark".to_string(),
vec![
tok(1, "Animals", "animal", "NOUN", "nsubj", 5),
tok(2, "such", "such", "ADJ", "case", 4),
tok(3, "as", "as", "ADP", "fixed", 2),
tok(4, "dogs", "dog", "NOUN", "nmod", 1),
tok(5, "bark", "bark", "VERB", "root", 0),
],
)])
}
}
let engine = Engine::new(Box::new(SuchAsNlp), standard_decomposers());
let doc = engine.annotate(&raw_plain("any text")).unwrap();
let sentence = doc.sentences().next().expect("one sentence");
assert_eq!(sentence.hearst_pairs.len(), 1, "annotate ran the detector");
let pair = &sentence.hearst_pairs[0];
assert_eq!(pair.pattern, domain::HearstPattern::SuchAs);
assert_eq!(pair.hypernym.head_id, 1);
assert_eq!(pair.hyponym.head_id, 4);
}
#[test]
fn ingest_text_is_a_stream_of_one() {
let items: Vec<_> = Ingest::text("hello", domain::Format::PlainText).collect();
assert_eq!(items.len(), 1);
let doc = items.into_iter().next().unwrap().unwrap();
assert_eq!(doc.text, "hello");
assert_eq!(doc.path, None);
}
#[test]
fn ingest_path_file_is_a_stream_of_one() {
let dir = tempfile::tempdir().unwrap();
let f = dir.path().join("one.md");
std::fs::write(&f, "# One").unwrap();
let items: Vec<_> = Ingest::path(&f).unwrap().collect();
assert_eq!(items.len(), 1);
let doc = items.into_iter().next().unwrap().unwrap();
assert_eq!(doc.path.as_deref(), Some(f.as_path()));
assert!(matches!(doc.format, domain::Format::Markdown));
}
#[test]
fn ingest_path_directory_streams_sorted_entries() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("b.txt"), "B").unwrap();
std::fs::write(dir.path().join("a.md"), "# A").unwrap();
let items: Vec<_> = Ingest::path(dir.path()).unwrap().collect();
assert_eq!(items.len(), 2);
let paths: Vec<_> = items
.into_iter()
.map(|r| r.unwrap().path.unwrap())
.collect();
assert!(paths[0].ends_with("a.md"));
assert!(paths[1].ends_with("b.txt"));
}
#[test]
fn ingest_path_missing_is_a_constructor_error() {
assert!(Ingest::path("/definitely/not/here").is_err());
}
#[cfg(unix)]
#[test]
fn ingest_per_file_failure_is_an_err_item_not_an_abort() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("good.md"), "# Good").unwrap();
let bad = dir.path().join("bad.md");
std::fs::write(&bad, "# Bad").unwrap();
std::fs::set_permissions(&bad, std::fs::Permissions::from_mode(0o000)).unwrap();
let items: Vec<_> = Ingest::path(dir.path()).unwrap().collect();
let _ = std::fs::set_permissions(&bad, std::fs::Permissions::from_mode(0o644));
assert_eq!(items.len(), 2);
assert_eq!(items.iter().filter(|r| r.is_ok()).count(), 1);
let err = items.into_iter().find_map(|r| r.err()).unwrap();
assert!(err.path.unwrap().ends_with("bad.md"));
}
#[test]
fn standard_decomposers_cover_exactly_the_shipping_formats() {
let table = standard_decomposers();
assert!(table.get(&domain::Format::Markdown).is_some());
assert!(table.get(&domain::Format::PlainText).is_some());
assert!(
table.get(&domain::Format::Pdf).is_none(),
"Pdf is reserved, not shipped"
);
assert!(
table.get(&domain::Format::Docx).is_none(),
"Docx is reserved, not shipped"
);
}
#[test]
fn input_at_cap_is_accepted() {
let engine = Engine::new(Box::new(EmptyNlp), standard_decomposers());
let raw = raw_plain(&"a".repeat(MAX_INPUT_BYTES));
assert!(
engine.annotate(&raw).is_ok(),
"input exactly at cap should be accepted"
);
}
#[test]
fn input_one_byte_over_cap_is_rejected() {
let engine = Engine::new(Box::new(EmptyNlp), standard_decomposers());
let raw = raw_plain(&"a".repeat(MAX_INPUT_BYTES + 1));
match engine.annotate(&raw) {
Err(domain::Error::InputTooLarge {
limit,
actual,
what,
}) => {
assert_eq!(limit, MAX_INPUT_BYTES);
assert_eq!(actual, MAX_INPUT_BYTES + 1);
assert_eq!(what, "input");
}
other => panic!("expected InputTooLarge, got {other:?}"),
}
}
#[test]
fn parse_per_paragraph_scopes_sentences_to_originating_paragraph() {
let text = "The system processes input now. Tail one.\n\n\
The system processes input now. Tail two.";
let analysis = law_engine().analyze_one(raw_plain(text)).unwrap().analysis;
let paras: Vec<_> = analysis.paragraphs().collect();
assert_eq!(paras.len(), 2);
assert_eq!(paras[0].sentences.len(), 2);
assert_eq!(paras[1].sentences.len(), 2);
assert!(paras[0].sentences.iter().any(|s| s.text.contains("one")));
assert!(paras[1].sentences.iter().any(|s| s.text.contains("two")));
assert!(!paras[0].sentences.iter().any(|s| s.text.contains("two")));
assert!(!paras[1].sentences.iter().any(|s| s.text.contains("one")));
}
#[test]
fn parse_per_paragraph_no_inner_substring_theft() {
let text = "Outer talks about the special phrase processes input now. End A.\n\n\
The special phrase processes input now. End B.";
let analysis = law_engine().analyze_one(raw_plain(text)).unwrap().analysis;
let paras: Vec<_> = analysis.paragraphs().collect();
assert_eq!(paras.len(), 2);
assert!(paras[1].sentences.iter().any(|s| s.text.contains("End B")));
assert!(!paras[0].sentences.iter().any(|s| s.text.contains("End B")));
}
#[test]
fn empty_paragraph_followed_by_valid_paragraph() {
struct InjectEmptyParagraph;
impl Decomposer for InjectEmptyParagraph {
fn decompose(&self, text: &str) -> Vec<Section> {
let mut sections = decompose::plain::PlainTextDecomposer.decompose(text);
if let Some(section) = sections.first_mut() {
section
.paragraphs
.insert(0, domain::Paragraph::new(String::new(), false));
}
sections
}
}
let table = decompose::Decomposers::new()
.with(domain::Format::PlainText, Box::new(InjectEmptyParagraph));
let engine = Engine::new(Box::new(DotSplitNlp), table);
let analysis = engine
.analyze_one(raw_plain("Real content sentence.\n\nAnother real one."))
.unwrap()
.analysis;
let paras: Vec<_> = analysis.paragraphs().collect();
assert_eq!(paras.len(), 3);
assert_eq!(
paras[0].sentences.len(),
0,
"empty paragraph has zero sentences"
);
assert!(!paras[1].sentences.is_empty());
assert!(!paras[2].sentences.is_empty());
}
struct FixedEmbedder {
identity: &'static str,
force_count: Option<usize>,
}
impl embed::Embedder for FixedEmbedder {
fn embed(&self, texts: &[&str]) -> domain::Result<Vec<domain::Embedding>> {
let n = self.force_count.unwrap_or(texts.len());
Ok((0..n).map(|_| domain::Embedding(vec![1.0, 0.0])).collect())
}
fn identity(&self) -> &str {
self.identity
}
}
fn two_sentence_doc() -> Document {
let engine = Engine::new(Box::new(DotSplitNlp), standard_decomposers());
let raw = domain::RawDocument::new(
"One sentence. Another sentence.".to_string(),
None,
domain::Format::PlainText,
);
engine.annotate(&raw).expect("annotate")
}
#[test]
fn embed_and_cluster_carries_the_embedder_identity() {
let doc = two_sentence_doc();
let embedder = FixedEmbedder {
identity: "the-real-model",
force_count: None,
};
let out = embed_and_cluster(&doc, &embedder, 0.9).expect("cluster");
assert_eq!(out.model_hash, "the-real-model");
assert_eq!(out.clusters.len(), 1);
}
#[test]
fn embed_and_cluster_rejects_a_length_contract_violation() {
let doc = two_sentence_doc();
let embedder = FixedEmbedder {
identity: "liar",
force_count: Some(1),
};
match embed_and_cluster(&doc, &embedder, 0.9) {
Err(domain::Error::InvalidInput(msg)) => {
assert!(msg.contains("violating its contract"));
}
other => panic!("expected InvalidInput, got {:?}", other.map(|_| ())),
}
}
#[test]
fn embed_and_cluster_caps_before_embedding() {
struct MustNotEmbed;
impl embed::Embedder for MustNotEmbed {
fn embed(&self, _texts: &[&str]) -> domain::Result<Vec<domain::Embedding>> {
panic!("embed ran on an over-cap document");
}
fn identity(&self) -> &str {
"unreached"
}
}
let many = "Word. ".repeat(extraction::MAX_SEMANTIC_SENTENCES + 1);
let engine = Engine::new(Box::new(DotSplitNlp), standard_decomposers());
let raw = domain::RawDocument::new(many, None, domain::Format::PlainText);
let doc = engine.annotate(&raw).expect("annotate");
match embed_and_cluster(&doc, &MustNotEmbed, 0.9) {
Err(domain::Error::InputTooLarge { what, .. }) => {
assert_eq!(what, "semantic_clusters");
}
other => panic!("expected InputTooLarge, got {:?}", other.map(|_| ())),
}
}
}