#![doc = include_str!("../README.md")]
#[cfg(feature = "cli")]
pub mod cli;
pub mod config;
mod conform;
pub mod decompose;
mod detect;
mod digest;
pub mod domain;
pub mod embed;
mod extraction;
mod hearst;
mod metrics;
pub mod nlp;
pub mod source;
mod stopwords;
mod structure;
#[cfg(test)]
#[path = "../tests/document_schema/mod.rs"]
mod document_schema;
use std::path::{Path, PathBuf};
use decompose::Decomposer;
pub use domain::Block;
use domain::limits::MAX_INPUT_BYTES;
use domain::{Document, Format, ParagraphKind};
pub use extraction::{
KeyphraseMethod, SummaryMethod, keyphrases, keyphrases_with, summarize, summarize_with,
};
use nlp::NlpProvider;
use source::Source;
fn check_input_size(len: usize) -> domain::Result<()> {
if len > MAX_INPUT_BYTES {
return Err(domain::Error::InputTooLarge {
limit: MAX_INPUT_BYTES,
actual: len,
what: "input",
});
}
Ok(())
}
#[derive(Debug, Clone)]
pub struct Input(InputKind);
#[derive(Debug, Clone)]
enum InputKind {
Text {
text: String,
format: Format,
},
Bytes {
data: Vec<u8>,
format: Option<Format>,
},
File {
path: PathBuf,
format: Option<Format>,
},
Blocks(Vec<Block>),
}
impl Input {
pub fn text(text: impl Into<String>, format: Format) -> Input {
Input(InputKind::Text {
text: text.into(),
format,
})
}
pub fn bytes(data: impl Into<Vec<u8>>, format: Option<Format>) -> Input {
Input(InputKind::Bytes {
data: data.into(),
format,
})
}
pub fn file(path: impl AsRef<Path>) -> Input {
Input(InputKind::File {
path: path.as_ref().to_path_buf(),
format: None,
})
}
pub fn file_as(path: impl AsRef<Path>, format: Format) -> Input {
Input(InputKind::File {
path: path.as_ref().to_path_buf(),
format: Some(format),
})
}
pub fn blocks(blocks: impl IntoIterator<Item = Block>) -> Input {
Input(InputKind::Blocks(blocks.into_iter().collect()))
}
fn path(&self) -> Option<&Path> {
match &self.0 {
InputKind::File { path, .. } => Some(path),
InputKind::Text { .. } | InputKind::Bytes { .. } | InputKind::Blocks(_) => None,
}
}
}
#[derive(Debug)]
pub struct Inputs {
paths: std::vec::IntoIter<PathBuf>,
}
impl Inputs {
pub fn dir(path: impl AsRef<Path>) -> domain::Result<Inputs> {
let path = path.as_ref();
let file_type = std::fs::symlink_metadata(path)
.map_err(|e| domain::Error::Io(domain::IoContext::cannot("read", path, &e)))?
.file_type();
if file_type.is_symlink() {
return Err(domain::Error::Io(domain::IoContext::at_path(
std::io::ErrorKind::Unsupported,
path,
format!("refusing to read symlink: {}", path.display()),
)));
}
if !file_type.is_dir() {
return Err(domain::Error::InvalidInput {
reason: "not_a_directory",
message: format!(
"{} is not a directory; read a file with Input::file",
path.display()
),
});
}
let paths = source::directory::DirectorySource.candidate_paths(path)?;
Ok(Inputs {
paths: paths.into_iter(),
})
}
}
impl Iterator for Inputs {
type Item = Input;
fn next(&mut self) -> Option<Input> {
self.paths.next().map(Input::file)
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.paths.size_hint()
}
}
impl ExactSizeIterator for Inputs {}
pub struct Engine {
parser: Box<dyn NlpProvider>,
readers: decompose::Decomposers,
source: Box<dyn Source>,
}
impl Engine {
pub fn with_parser(parser: impl NlpProvider + 'static) -> Engine {
Engine {
parser: Box::new(parser),
readers: standard_readers(),
source: Box::new(source::file::FileSource),
}
}
#[must_use]
pub fn with_reader(mut self, format: Format, reader: impl Decomposer + 'static) -> Engine {
self.readers = self.readers.with(format, Box::new(reader));
self
}
#[must_use]
pub fn with_source(mut self, source: impl Source + 'static) -> Engine {
self.source = Box::new(source);
self
}
pub fn analyze(&self, input: Input) -> domain::Result<Document> {
let mut doc = self.annotate(input)?;
self.compose(&mut doc);
Ok(doc)
}
pub fn analyze_all<I>(
&self,
inputs: I,
) -> impl Iterator<Item = std::result::Result<Document, domain::DocumentError>>
where
I: IntoIterator<Item = Input>,
{
inputs.into_iter().map(|input| {
let path = input.path().map(Path::to_path_buf);
self.analyze(input)
.map_err(|e| domain::DocumentError::new(path, e))
})
}
pub(crate) fn annotate(&self, input: Input) -> domain::Result<Document> {
let path = input.path().map(Path::to_path_buf);
let (format, input_text, mut read) = match input.0 {
InputKind::Text { text, format } => {
check_input_size(text.len())?;
let read = self.reader(&format)?.decompose(text.as_bytes())?;
(format, text, read)
}
InputKind::Bytes { data, format } => self.read_bytes(data, format.as_ref(), None)?,
InputKind::File { path, format } => {
let bytes = self.source.read(&path)?;
self.read_bytes(bytes, format.as_ref(), Some(&path))?
}
InputKind::Blocks(blocks) => (
Format::Blocks,
String::new(),
decompose::blocks::assemble(&blocks)?,
),
};
let text = read.text.take().unwrap_or(input_text);
check_input_size(text.len())?;
conform::read(&text, &read)?;
let provenance = domain::Provenance::new(
self.parser.identity(),
format,
domain::ProvenanceSource::new(path, text.len(), digest::sha256_hex(text.as_bytes())),
);
let (sections, maps) = structure::sections(read.blocks);
let mut doc = Document::new(provenance, sections, read.skipped);
for (index, (para, map)) in doc.paragraphs_mut().zip(&maps).enumerate() {
if para.kind == ParagraphKind::Quote {
continue;
}
let parsed = self.parser.parse(¶.text)?;
conform::parse(index + 1, ¶.text, &parsed)?;
para.sentences = parsed
.into_iter()
.map(|mut tokens| {
for token in &mut tokens {
token.span = map.map(token.span);
}
let mut sentence = domain::Sentence::from_tokens(tokens);
sentence.hearst_pairs = hearst::hypernymy_pairs(&sentence.tokens);
sentence
})
.collect();
}
Ok(doc)
}
fn read_bytes(
&self,
bytes: Vec<u8>,
declared: Option<&Format>,
name: Option<&Path>,
) -> domain::Result<(Format, String, decompose::Read)> {
check_input_size(bytes.len())?;
let format = detect::format_of(&bytes, declared, name);
let read = self.reader(&format)?.decompose(&bytes)?;
let text = if read.text.is_some() {
String::new()
} else {
detect::decode(bytes)?
};
Ok((format, text, read))
}
fn reader(&self, format: &Format) -> domain::Result<&dyn Decomposer> {
self.readers
.get(format)
.ok_or_else(|| domain::Error::UnsupportedFormat {
format: Some(format.clone()),
feature: reader_feature(format),
})
}
pub(crate) fn compose(&self, doc: &mut Document) {
let suite = metrics::default_suite();
metrics::run_suite(doc, &suite);
}
}
#[cfg(feature = "udpipe")]
impl Engine {
pub fn new() -> domain::Result<Engine> {
let cfg = config::Config::from_sources(|key| std::env::var(key).ok(), None)?;
Engine::from_config(&cfg)
}
pub fn from_config(cfg: &config::Config) -> domain::Result<Engine> {
Ok(Engine::with_parser(nlp::udpipe::Udpipe::from_config(cfg)?))
}
pub fn from_config_with_notice(
cfg: &config::Config,
notice: impl FnMut(&domain::ProvisionNotice),
) -> domain::Result<Engine> {
Ok(Engine::with_parser(
nlp::udpipe::Udpipe::from_config_with_notice(cfg, notice)?,
))
}
}
fn default_reader(format: &Format) -> Option<Box<dyn Decomposer>> {
match format {
Format::Markdown => Some(Box::new(decompose::markdown::MarkdownDecomposer)),
Format::PlainText => Some(Box::new(decompose::plain::PlainTextDecomposer)),
Format::Blocks => Some(Box::new(decompose::blocks::BlocksDecomposer)),
Format::Pdf | Format::Docx | Format::Other(_) => None,
}
}
fn reader_feature(format: &Format) -> Option<&'static str> {
match format {
Format::Markdown
| Format::PlainText
| Format::Blocks
| Format::Pdf
| Format::Docx
| Format::Other(_) => None,
}
}
fn standard_readers() -> decompose::Decomposers {
let mut table = decompose::Decomposers::new();
for format in [
Format::Markdown,
Format::PlainText,
Format::Blocks,
Format::Pdf,
Format::Docx,
] {
if let Some(reader) = default_reader(&format) {
table = table.with(format, reader);
}
}
table
}
pub fn clusters(
doc: &Document,
embedder: &dyn embed::Embedder,
threshold: f64,
) -> domain::Result<domain::Clusters> {
let texts: Vec<&str> = doc.sentences().map(|s| s.text.as_str()).collect();
if texts.len() > domain::limits::MAX_CLUSTER_SENTENCES {
return Err(domain::Error::InputTooLarge {
limit: domain::limits::MAX_CLUSTER_SENTENCES,
actual: texts.len(),
what: "clusters",
});
}
if !threshold.is_finite() {
return Err(domain::Error::InvalidInput {
reason: "threshold_not_finite",
message: format!("threshold must be finite, got {threshold}"),
});
}
let embeddings = embedder.embed(&texts)?;
if embeddings.len() != texts.len() {
return Err(domain::Error::InvalidInput {
reason: "embedder_vector_count",
message: format!(
"embedder returned {} for {}, violating its contract",
counted(embeddings.len(), "vector"),
counted(texts.len(), "sentence"),
),
});
}
let (clusters, edges) = extraction::cluster(&embeddings, threshold)?;
Ok(domain::Clusters {
provenance: doc.provenance.derived(
CLUSTERS_METHOD,
domain::Parameters::clustering(threshold, embedder.identity()),
),
clusters,
edges,
})
}
fn counted(n: usize, noun: &str) -> String {
if n == 1 {
format!("1 {noun}")
} else {
format!("{n} {noun}s")
}
}
const CLUSTERS_METHOD: &str = "cosine";
#[cfg(feature = "python")]
mod python;
#[cfg(test)]
mod tests {
use super::*;
use crate::nlp::NlpProvider;
use domain::{Section, Span};
#[test]
fn an_engine_is_send() {
fn assert_send<T: Send>() {}
assert_send::<Engine>();
}
fn flat_sentence(text: &str, from: usize, to: usize) -> Vec<domain::Token> {
let mut tokens = Vec::new();
let mut at = from;
for word in text[from..to].split_whitespace() {
let start = at + text[at..to].find(word).unwrap_or(0);
at = start + word.len();
let id = tokens.len() + 1;
tokens.push(
domain::Token::builder(
id,
word.to_string(),
word.to_lowercase(),
"X".to_string(),
usize::from(id > 1),
"dep".to_string(),
)
.span(Span::new(start, at))
.build(),
);
}
tokens
}
fn placed(text: &str, mut tokens: Vec<domain::Token>) -> Vec<domain::Token> {
let mut at = 0;
for token in &mut tokens {
let start = at
+ text[at..]
.find(&token.text)
.expect("the form is in the text");
at = start + token.text.len();
token.span = Span::new(start, at);
}
tokens
}
struct EmptyNlp;
impl NlpProvider for EmptyNlp {
fn parse(&self, _text: &str) -> domain::Result<Vec<Vec<domain::Token>>> {
Ok(Vec::new())
}
fn identity(&self) -> domain::Identity {
domain::Identity::new("empty", None, None, None)
}
}
struct DotSplitNlp;
impl NlpProvider for DotSplitNlp {
fn parse(&self, text: &str) -> domain::Result<Vec<Vec<domain::Token>>> {
let mut sentences = Vec::new();
let mut from = 0;
for piece in text.split('.') {
let sentence = flat_sentence(text, from, from + piece.len());
if !sentence.is_empty() {
sentences.push(sentence);
}
from += piece.len() + 1;
}
Ok(sentences)
}
fn identity(&self) -> domain::Identity {
domain::Identity::new("dot-split", None, None, None)
}
}
struct WordNlp;
impl NlpProvider for WordNlp {
fn parse(&self, text: &str) -> domain::Result<Vec<Vec<domain::Token>>> {
let sentence = flat_sentence(text, 0, text.len());
Ok(if sentence.is_empty() {
Vec::new()
} else {
vec![sentence]
})
}
fn identity(&self) -> domain::Identity {
domain::Identity::new("words", None, None, None)
}
}
use std::sync::atomic::{AtomicUsize, Ordering};
struct ObservableNlp {
calls: std::sync::Arc<AtomicUsize>,
max_len_seen: std::sync::Arc<AtomicUsize>,
}
impl ObservableNlp {
fn new() -> (
Self,
std::sync::Arc<AtomicUsize>,
std::sync::Arc<AtomicUsize>,
) {
let calls = std::sync::Arc::new(AtomicUsize::new(0));
let max_len = std::sync::Arc::new(AtomicUsize::new(0));
(
Self {
calls: calls.clone(),
max_len_seen: max_len.clone(),
},
calls,
max_len,
)
}
}
impl NlpProvider for ObservableNlp {
fn parse(&self, text: &str) -> domain::Result<Vec<Vec<domain::Token>>> {
self.calls.fetch_add(1, Ordering::SeqCst);
self.max_len_seen.fetch_max(text.len(), Ordering::SeqCst);
if text.contains("POISON") {
return Err(domain::Error::ParseFailed {
reason: None,
cause: "poisoned".to_string(),
});
}
DotSplitNlp.parse(text)
}
fn identity(&self) -> domain::Identity {
domain::Identity::new("observable", None, None, None)
}
}
fn law_engine() -> Engine {
Engine::with_parser(DotSplitNlp)
}
fn fingerprint(
results: Vec<std::result::Result<Document, domain::DocumentError>>,
) -> Vec<String> {
results
.into_iter()
.map(|r| match r {
Ok(doc) => format!("ok:{}", serde_json::to_string(&doc).unwrap()),
Err(e) => format!("err:{e}"),
})
.collect()
}
fn constructors(text: &str, dir: &Path) -> Vec<(&'static str, Input)> {
let file = dir.join(format!("{}.txt", text.len()));
std::fs::write(&file, text).unwrap();
vec![
("text", Input::text(text, Format::PlainText)),
("bytes", Input::bytes(text.as_bytes(), None)),
("file", Input::file(&file)),
("file_as", Input::file_as(&file, Format::PlainText)),
(
"blocks",
Input::blocks(text.split("\n\n").map(Block::paragraph)),
),
]
}
#[test]
fn law_l1_analyze_all_commutes_with_chain() {
let engine = law_engine();
let dir = tempfile::tempdir().unwrap();
let firsts = constructors("First stream. Has two.", dir.path());
let seconds = constructors("Second stream here.", dir.path());
for ((name, a), (_, b)) in firsts.into_iter().zip(seconds) {
let chained: Vec<_> = engine
.analyze_all([a.clone()].into_iter().chain([b.clone()]))
.collect();
let separate: Vec<_> = engine
.analyze_all([a])
.chain(engine.analyze_all([b]))
.collect();
assert_eq!(fingerprint(chained), fingerprint(separate), "{name}");
}
}
#[test]
fn law_l2_analyze_all_of_empty_is_empty() {
let engine = law_engine();
assert_eq!(engine.analyze_all(std::iter::empty()).count(), 0);
}
#[test]
fn law_l3_analyze_all_of_one_is_analyze() {
let engine = law_engine();
let dir = tempfile::tempdir().unwrap();
for (name, input) in constructors("One document. Two sentences.", dir.path()) {
let path = input.path().map(Path::to_path_buf);
let streamed: Vec<_> = engine.analyze_all([input.clone()]).collect();
let direct = vec![
engine
.analyze(input)
.map_err(|e| domain::DocumentError::new(path, e)),
];
assert_eq!(fingerprint(streamed), fingerprint(direct), "{name}");
}
}
#[test]
fn law_l4_analyze_is_annotate_then_compose() {
let engine = law_engine();
let dir = tempfile::tempdir().unwrap();
for (name, input) in constructors("Composed of parts. Checked by law.", dir.path()) {
let via_analyze = engine.analyze(input.clone()).unwrap();
let via_stages = {
let mut doc = engine.annotate(input).unwrap();
engine.compose(&mut doc);
doc
};
assert_eq!(
serde_json::to_string(&via_analyze).unwrap(),
serde_json::to_string(&via_stages).unwrap(),
"{name}"
);
}
}
#[test]
fn law_l5_partition_documents_plus_errors_equals_inputs() {
let (nlp, _, _) = ObservableNlp::new();
let engine = Engine::with_parser(nlp);
let dir = tempfile::tempdir().unwrap();
let mut inputs: Vec<Input> = constructors("Good document one.", dir.path())
.into_iter()
.map(|(_, input)| input)
.collect();
let good = inputs.len();
inputs.extend(
constructors("This one is POISON and fails.", dir.path())
.into_iter()
.map(|(_, input)| input),
);
inputs.push(Input::file(dir.path().join("lost.md")));
let n = inputs.len();
let (documents, errors): (Vec<_>, Vec<_>) =
engine.analyze_all(inputs).partition(Result::is_ok);
assert_eq!(documents.len() + errors.len(), n);
assert_eq!(documents.len(), good);
assert_eq!(errors.len(), good + 1);
}
#[test]
fn law_l6_a_failed_read_never_reaches_the_parser() {
let (nlp, calls, _) = ObservableNlp::new();
let engine = Engine::with_parser(nlp);
let dir = tempfile::tempdir().unwrap();
let ghost = dir.path().join("ghost.md");
let inputs = vec![
Input::file(&ghost),
Input::file(dir.path()),
Input::bytes(vec![0xff, 0xfe, b'x'], None),
Input::bytes(b"%PDF-1.7".to_vec(), Some(Format::PlainText)),
Input::text("[{\"kind\": \"table\"}]", Format::Blocks),
];
let out: Vec<_> = engine.analyze_all(inputs).collect();
assert_eq!(out.len(), 5);
assert!(out.iter().all(Result::is_err));
let first = out[0].as_ref().unwrap_err();
assert_eq!(first.path.as_deref(), Some(ghost.as_path()));
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"a failed read must not reach the parser"
);
engine
.annotate(Input::text("Control.", Format::PlainText))
.unwrap();
assert_eq!(calls.load(Ordering::SeqCst), 1, "counter is live");
}
#[test]
fn law_l7_no_oversized_input_reaches_the_parser() {
let (nlp, _, max_len) = ObservableNlp::new();
let engine = Engine::with_parser(nlp);
let dir = tempfile::tempdir().unwrap();
let oversized = "a".repeat(MAX_INPUT_BYTES + 1);
for (name, input) in constructors(&oversized, dir.path()) {
let out: Vec<_> = engine.analyze_all([input.clone()]).collect();
assert!(
matches!(
out[0].as_ref().unwrap_err().error,
domain::Error::InputTooLarge { .. }
),
"{name} through the stream"
);
assert!(
matches!(
engine.annotate(input),
Err(domain::Error::InputTooLarge { .. })
),
"{name} through annotate"
);
}
let half = "b".repeat(MAX_INPUT_BYTES / 2 + 1);
let joined = Input::blocks([Block::paragraph(half.clone()), Block::paragraph(half)]);
assert!(matches!(
engine.annotate(joined),
Err(domain::Error::InputTooLarge { .. })
));
assert!(
max_len.load(Ordering::SeqCst) <= MAX_INPUT_BYTES,
"the parser never saw text over the cap"
);
}
fn accounting(text: &str, doc: &Document) -> std::result::Result<(), String> {
let mut owners = vec![0u8; text.len()];
let spans = doc
.sections
.iter()
.filter_map(|s| s.span)
.chain(doc.paragraphs().map(|p| p.span))
.chain(doc.skipped.iter().map(|s| s.span));
for span in spans {
if span.byte_start > span.byte_end || span.byte_end > text.len() {
return Err(format!("span {span:?} is outside the text"));
}
for owner in &mut owners[span.byte_start..span.byte_end] {
*owner = owner.saturating_add(1);
}
}
for (at, c) in text.char_indices() {
if !c.is_whitespace() && owners[at] != 1 {
let mut from = at.saturating_sub(20);
while !text.is_char_boundary(from) {
from -= 1;
}
let mut to = (at + 20).min(text.len());
while !text.is_char_boundary(to) {
to += 1;
}
return Err(format!(
"byte {at} ({c:?}) lies in {} spans, near {:?}",
owners[at],
&text[from..to]
));
}
}
Ok(())
}
fn accounting_engine() -> Engine {
Engine::with_parser(EmptyNlp)
}
#[test]
fn law_l9_holds_over_every_conformance_fixture() {
let dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("spec/tests");
let engine = accounting_engine();
let mut checked = 0;
for entry in std::fs::read_dir(&dir).unwrap() {
let path = entry.unwrap().path();
if path.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
let fixture: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
let input = fixture["input"].as_str().unwrap();
let format: Format = serde_json::from_value(fixture["format"].clone()).unwrap();
let doc = engine.annotate(Input::text(input, format)).unwrap();
accounting(input, &doc).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
checked += 1;
}
assert!(checked > 0, "no fixtures in {}", dir.display());
}
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
self.0 ^= self.0 << 13;
self.0 ^= self.0 >> 7;
self.0 ^= self.0 << 17;
self.0
}
fn pick<'a>(&mut self, items: &[&'a str]) -> &'a str {
items[usize::try_from(self.next() % items.len() as u64).unwrap()]
}
}
const MARKDOWN_PIECES: &[&str] = &[
"# Heading",
"## References",
"### Deep heading ###",
"#hashtag opens this line",
"Plain prose with **bold**, *em* and `code`.",
"Text with [a link](https://x.test) and  & \\* and <br> html.",
"> quoted line\n> more quote",
"> one\n>\n> two",
">",
"> # quoted heading",
"> - quoted item",
"- item one\n- item two\n continued",
"1. first\n\n2. second",
"1) paren item",
"+ plus item",
" * star item",
"- [ ] open task\n- [x] done task",
"-",
"1.",
"- > quote in a list",
"- ```\n code in a list\n ```",
"| a | b |\n|---|---|\n| 1 | 2 |",
"```rust\nfn main() {}\n```",
"~~~\ntilde fence\n~~~",
" indented code",
"<div>\nhtml block\n</div>",
"<!-- a comment -->",
"---",
"***",
"___",
"[x]: https://example.com \"title\"",
"> [y]: /quoted-def",
"Setext heading\n===",
"Another setext\n---",
"line one\r\nline two",
"café, naïve, 日本語 and an emoji 🙂",
"~~struck~~ through",
"trailing hard break \nnext line",
"a line ending in a backslash\\\nnext",
"[^1] footnote-looking text",
"\t\ttabbed text",
"*",
"> > nested quote",
];
const SEPARATORS: &[&str] = &["\n\n", "\n", "\r\n\r\n", "\n\n\n", "\n \n"];
fn generated_markdown(rng: &mut Rng) -> String {
let mut doc = String::new();
if rng.next().is_multiple_of(5) {
doc.push_str(rng.pick(&["---\ntitle: x\n---\n\n", "+++\ntitle = 1\n+++\n\n"]));
}
let pieces = 1 + rng.next() % 8;
for i in 0..pieces {
if i > 0 {
doc.push_str(rng.pick(SEPARATORS));
}
doc.push_str(rng.pick(MARKDOWN_PIECES));
}
if rng.next().is_multiple_of(2) {
doc.push('\n');
}
doc
}
const SOUP: &[&str] = &[
"#", "# ", "## ", ">", "> ", "-", "- ", "*", "* ", "+", "1.", "1)", "|", " | ", "`", "```",
"~~~", "[", "]", "(", ")", "![", "<", "</", "&", "&", "\\", "_", "__", "=", "===",
"---", "+++", "***", ":", "\"", "[x]: ", "word", "text", "é", " ", " ", " ", "\t",
"\n", "\n", "\n\n", "\r\n",
];
fn generated_soup(rng: &mut Rng) -> String {
let pieces = 1 + rng.next() % 40;
(0..pieces).map(|_| rng.pick(SOUP)).collect()
}
#[test]
fn law_l9_holds_over_generated_documents() {
let engine = accounting_engine();
for seed in 1..=6000_u64 {
let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15));
let markdown = if seed % 2 == 0 {
generated_markdown(&mut rng)
} else {
generated_soup(&mut rng)
};
let doc = engine
.annotate(Input::text(markdown.clone(), Format::Markdown))
.unwrap();
accounting(&markdown, &doc)
.unwrap_or_else(|e| panic!("seed {seed}, markdown {markdown:?}: {e}"));
let doc = engine
.annotate(Input::text(markdown.clone(), Format::PlainText))
.unwrap();
accounting(&markdown, &doc)
.unwrap_or_else(|e| panic!("seed {seed}, plain text {markdown:?}: {e}"));
}
}
#[test]
fn token_and_sentence_spans_point_into_the_analysed_text() {
let engine = Engine::with_parser(WordNlp);
let md = "# Title\n\nSome **bold** and `code` here.\n\n- item text";
let doc = engine.analyze(Input::text(md, Format::Markdown)).unwrap();
let tokens: Vec<&str> = doc
.tokens()
.map(|t| &md[t.span.byte_start..t.span.byte_end])
.collect();
assert_eq!(
tokens,
["Some", "bold", "and", "code", "here.", "item", "text"]
);
let sentence = doc.sentences().next().unwrap();
assert_eq!(
&md[sentence.span.byte_start..sentence.span.byte_end],
"Some **bold** and `code` here."
);
}
#[test]
fn spans_count_bytes_not_characters() {
let engine = Engine::with_parser(WordNlp);
let text = "Café naïve.\n\nÜber.";
let doc = engine
.analyze(Input::text(text, Format::PlainText))
.unwrap();
for token in doc.tokens() {
assert_eq!(
&text[token.span.byte_start..token.span.byte_end],
token.text
);
}
let second = doc.paragraphs().nth(1).unwrap();
assert_eq!(second.span, Span::new(15, 21));
}
struct SentenceNlp;
impl NlpProvider for SentenceNlp {
fn parse(&self, text: &str) -> domain::Result<Vec<Vec<domain::Token>>> {
let mut sentences = Vec::new();
let mut from = 0;
let mut at = 0;
for word in text.split_whitespace() {
at += text[at..].find(word).unwrap_or(0) + word.len();
if word.ends_with('.') {
sentences.push(flat_sentence(text, from, at));
from = at;
}
}
Ok(sentences)
}
fn identity(&self) -> domain::Identity {
domain::Identity::new("sentences", None, None, None)
}
}
#[test]
fn non_ascii_sentence_spans_slice_as_bytes() {
let engine = Engine::with_parser(SentenceNlp);
let text = "Café ☕ is open. Naïve plan.";
let doc = engine
.analyze(Input::text(text, Format::PlainText))
.unwrap();
let sentences: Vec<_> = doc.sentences().collect();
assert_eq!(sentences.len(), 2);
for sentence in &sentences {
let span = sentence.span;
assert_eq!(&text[span.byte_start..span.byte_end], sentence.text);
}
let second = sentences[1].span;
assert_eq!(second, Span::new(19, 31));
let by_chars: String = text
.chars()
.skip(second.byte_start)
.take(second.byte_end - second.byte_start)
.collect();
assert_ne!(by_chars, sentences[1].text);
}
#[test]
fn block_input_spans_point_into_the_joined_text() {
let engine = Engine::with_parser(WordNlp);
let doc = engine
.analyze(Input::blocks([
Block::heading(1, "Field notes"),
Block::paragraph("The river rose overnight."),
]))
.unwrap();
let joined = "Field notes\n\nThe river rose overnight.";
assert_eq!(doc.provenance.format, Format::Blocks);
assert_eq!(doc.provenance.source.bytes, joined.len());
assert_eq!(
doc.provenance.source.sha256,
digest::sha256_hex(joined.as_bytes())
);
assert_eq!(doc.sections[0].heading.as_deref(), Some("Field notes"));
assert_eq!(doc.sections[0].level, 1);
assert_eq!(doc.sections[0].span, Some(Span::new(0, 11)));
let token = doc.tokens().nth(1).unwrap();
assert_eq!(&joined[token.span.byte_start..token.span.byte_end], "river");
}
#[test]
fn a_quote_is_not_parsed() {
let (nlp, calls, _) = ObservableNlp::new();
let engine = Engine::with_parser(nlp);
let doc = engine
.analyze(Input::text("> Quoted words.", Format::Markdown))
.unwrap();
let para = doc.paragraphs().next().unwrap();
assert_eq!(para.kind, ParagraphKind::Quote);
assert!(para.sentences.is_empty());
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
#[test]
fn what_the_reader_skipped_reaches_the_document() {
let doc = accounting_engine()
.analyze(Input::text(
"Text.\n\n```\ncode\n```\n\n| a |\n|---|\n| 1 |",
Format::Markdown,
))
.unwrap();
let kinds: Vec<domain::SkippedKind> = doc.skipped.iter().map(|s| s.kind).collect();
assert_eq!(
kinds,
[domain::SkippedKind::Code, domain::SkippedKind::Table]
);
}
#[test]
fn annotate_records_the_provenance() {
let engine = law_engine();
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("notes.md");
std::fs::write(&file, "abc").unwrap();
let doc = engine.annotate(Input::file(&file)).unwrap();
let p = &doc.provenance;
assert_eq!(p.parser.name, "dot-split", "the parser names itself");
assert_eq!(p.format, Format::Markdown);
assert_eq!(p.schema, domain::SCHEMA_VERSION);
assert_eq!(p.matra, env!("CARGO_PKG_VERSION"));
assert_eq!(p.source.path.as_deref(), Some(file.as_path()));
assert_eq!(p.source.bytes, 3);
assert_eq!(
p.source.sha256,
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
);
}
#[test]
fn a_recognised_format_with_no_reader_is_unsupported() {
let engine = law_engine();
for (input, format) in [
(Input::text("text", Format::Pdf), Format::Pdf),
(Input::bytes(b"%PDF-1.4 ...".to_vec(), None), Format::Pdf),
(
Input::bytes(b"PK\x03\x04..[Content_Types].xml..".to_vec(), None),
Format::Docx,
),
] {
match engine.annotate(input) {
Err(domain::Error::UnsupportedFormat {
format: Some(got),
feature: None,
}) => assert_eq!(got, format),
other => panic!("expected UnsupportedFormat, got {:?}", other.map(|_| ())),
}
}
}
#[test]
fn bytes_that_are_not_text_are_unsupported() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("binary.txt");
std::fs::write(&file, [0xff, 0xfe, 0x00, 0x01]).unwrap();
for input in [Input::file(&file), Input::bytes(vec![0xc3, 0x28], None)] {
assert!(matches!(
law_engine().annotate(input),
Err(domain::Error::UnsupportedFormat { format: None, .. })
));
}
}
#[test]
fn an_unknown_extension_reads_as_plain_text() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("notes.log");
std::fs::write(&file, "# not a heading here").unwrap();
let doc = law_engine().analyze(Input::file(&file)).unwrap();
assert_eq!(doc.provenance.format, Format::PlainText);
assert!(doc.sections[0].heading.is_none());
}
#[test]
fn a_file_reads_in_the_format_the_caller_names() {
let dir = tempfile::tempdir().unwrap();
let blocks = dir.path().join("notes.json");
std::fs::write(
&blocks,
r#"[{"kind": "heading", "level": 1, "text": "Field notes"},
{"kind": "paragraph", "text": "The river rose."}]"#,
)
.unwrap();
let doc = law_engine()
.analyze(Input::file_as(&blocks, Format::Blocks))
.unwrap();
assert_eq!(doc.provenance.format, Format::Blocks);
assert_eq!(doc.sections[0].heading.as_deref(), Some("Field notes"));
assert_eq!(
doc.provenance.source.path.as_deref(),
Some(blocks.as_path())
);
let markdown = dir.path().join("notes.md");
std::fs::write(&markdown, "# not a heading here").unwrap();
let doc = law_engine()
.analyze(Input::file_as(&markdown, Format::PlainText))
.unwrap();
assert_eq!(doc.provenance.format, Format::PlainText);
assert!(doc.sections[0].heading.is_none());
}
#[test]
fn a_signature_outranks_the_format_a_file_is_read_as() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("scan.txt");
std::fs::write(&file, b"%PDF-1.7\n").unwrap();
match law_engine().analyze(Input::file_as(&file, Format::Markdown)) {
Err(domain::Error::UnsupportedFormat {
format: Some(Format::Pdf),
..
}) => {}
other => panic!(
"expected UnsupportedFormat(pdf), got {:?}",
other.map(|_| ())
),
}
}
#[test]
fn a_markdown_file_reads_as_markdown() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("notes.md");
std::fs::write(&file, "# Title\n\nBody.").unwrap();
let doc = law_engine().analyze(Input::file(&file)).unwrap();
assert_eq!(doc.provenance.format, Format::Markdown);
assert_eq!(doc.sections[0].heading.as_deref(), Some("Title"));
}
#[test]
fn input_file_on_a_directory_names_inputs_dir() {
let dir = tempfile::tempdir().unwrap();
match law_engine().analyze(Input::file(dir.path())) {
Err(domain::Error::InvalidInput { message, .. }) => {
assert!(message.contains("Inputs::dir"), "{message}");
}
other => panic!("expected InvalidInput, got {:?}", other.map(|_| ())),
}
}
#[test]
fn inputs_dir_lists_sorted_files_and_reads_them_lazily() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("b.txt"), "B").unwrap();
std::fs::write(dir.path().join("a.md"), "# A").unwrap();
let inputs = Inputs::dir(dir.path()).unwrap();
assert_eq!(inputs.len(), 2);
let docs: Vec<Document> = law_engine()
.analyze_all(inputs)
.map(Result::unwrap)
.collect();
let paths: Vec<_> = docs
.iter()
.map(|d| d.provenance.source.path.clone().unwrap())
.collect();
assert!(paths[0].ends_with("a.md"));
assert!(paths[1].ends_with("b.txt"));
assert_eq!(docs[0].provenance.format, Format::Markdown);
}
#[test]
fn inputs_dir_on_a_missing_path_or_a_file_is_a_constructor_error() {
let missing = Path::new("/definitely/not/here");
let err = Inputs::dir(missing).unwrap_err();
assert_eq!(err.kind(), "io");
let details = err.details();
assert_eq!(
details.get("io_kind"),
Some(&domain::DetailValue::Text("not_found".to_string()))
);
assert_eq!(
details.get("path"),
Some(&domain::DetailValue::Text(missing.display().to_string()))
);
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("one.txt");
std::fs::write(&file, "x").unwrap();
match Inputs::dir(&file) {
Err(domain::Error::InvalidInput { message, .. }) => {
assert!(message.contains("Input::file"), "{message}");
}
other => panic!("expected InvalidInput, got {:?}", other.map(|_| ())),
}
}
#[cfg(unix)]
#[test]
fn inputs_dir_skips_a_symlink() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("real.md"), "# Real").unwrap();
let outside = tempfile::tempdir().unwrap();
let target = outside.path().join("outside.md");
std::fs::write(&target, "# Outside").unwrap();
std::os::unix::fs::symlink(&target, dir.path().join("link.md")).unwrap();
let items: Vec<_> = law_engine()
.analyze_all(Inputs::dir(dir.path()).unwrap())
.collect();
assert_eq!(items.len(), 1);
let doc = items.into_iter().next().unwrap().unwrap();
assert!(doc.provenance.source.path.unwrap().ends_with("real.md"));
}
#[cfg(unix)]
#[test]
fn a_per_file_failure_is_an_err_item_not_an_abort() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("good.md"), "# Good").unwrap();
let bad = dir.path().join("bad.md");
std::fs::write(&bad, "# Bad").unwrap();
std::fs::set_permissions(&bad, std::fs::Permissions::from_mode(0o000)).unwrap();
let items: Vec<_> = law_engine()
.analyze_all(Inputs::dir(dir.path()).unwrap())
.collect();
let _ = std::fs::set_permissions(&bad, std::fs::Permissions::from_mode(0o644));
assert_eq!(items.len(), 2);
assert_eq!(items.iter().filter(|r| r.is_ok()).count(), 1);
let err = items.into_iter().find_map(|r| r.err()).unwrap();
assert!(err.path.unwrap().ends_with("bad.md"));
}
#[test]
fn standard_readers_cover_exactly_the_shipping_formats() {
let table = standard_readers();
assert!(table.get(&Format::Markdown).is_some());
assert!(table.get(&Format::PlainText).is_some());
assert!(table.get(&Format::Blocks).is_some());
assert!(
table.get(&Format::Pdf).is_none(),
"Pdf is recognised, not read"
);
assert!(
table.get(&Format::Docx).is_none(),
"Docx is recognised, not read"
);
}
#[test]
fn input_at_cap_is_accepted() {
let engine = Engine::with_parser(EmptyNlp);
assert!(
engine
.annotate(Input::text("a".repeat(MAX_INPUT_BYTES), Format::PlainText))
.is_ok(),
"input exactly at cap should be accepted"
);
}
#[test]
fn input_one_byte_over_cap_is_rejected() {
let engine = Engine::with_parser(EmptyNlp);
match engine.annotate(Input::text(
"a".repeat(MAX_INPUT_BYTES + 1),
Format::PlainText,
)) {
Err(domain::Error::InputTooLarge {
limit,
actual,
what,
}) => {
assert_eq!(limit, MAX_INPUT_BYTES);
assert_eq!(actual, MAX_INPUT_BYTES + 1);
assert_eq!(what, "input");
}
other => panic!("expected InputTooLarge, got {:?}", other.map(|_| ())),
}
}
#[test]
fn annotate_leaves_metrics_none_and_compose_fills_them() {
let engine = law_engine();
let mut doc = engine
.annotate(Input::text(
"Annotate attaches structure. Compose measures it.",
Format::PlainText,
))
.unwrap();
assert_eq!(
doc.metrics,
domain::DocumentMetrics::default(),
"annotate does not measure"
);
assert_eq!(doc.sentences().count(), 2, "annotate does attach");
engine.compose(&mut doc);
assert_eq!(doc.metrics.total_sentences, 2, "compose counts");
assert_eq!(doc.metrics.paragraph_count, 1, "compose counts");
assert_eq!(
doc.metrics.passive_ratio,
Some(0.0),
"compose fills the sentence-derived aggregate"
);
}
#[test]
fn annotate_fills_hearst_pairs_at_the_choke_point() {
struct SuchAsNlp;
impl NlpProvider for SuchAsNlp {
fn parse(&self, text: &str) -> domain::Result<Vec<Vec<domain::Token>>> {
let tok = |id: usize, text: &str, lemma: &str, pos: &str, dep: &str, head| {
domain::Token::builder(
id,
text.to_string(),
lemma.to_string(),
pos.to_string(),
head,
dep.to_string(),
)
.build()
};
Ok(vec![placed(
text,
vec![
tok(1, "Animals", "animal", "NOUN", "nsubj", 5),
tok(2, "such", "such", "ADJ", "case", 4),
tok(3, "as", "as", "ADP", "fixed", 2),
tok(4, "dogs", "dog", "NOUN", "nmod", 1),
tok(5, "bark", "bark", "VERB", "root", 0),
],
)])
}
fn identity(&self) -> domain::Identity {
domain::Identity::new("such-as", None, None, None)
}
}
let engine = Engine::with_parser(SuchAsNlp);
let doc = engine
.annotate(Input::text("Animals such as dogs bark", Format::PlainText))
.unwrap();
let sentence = doc.sentences().next().expect("one sentence");
assert_eq!(sentence.hearst_pairs.len(), 1, "annotate ran the detector");
let pair = &sentence.hearst_pairs[0];
assert_eq!(pair.pattern, domain::HearstPattern::SuchAs);
assert_eq!(pair.hypernym.head_id, 1);
assert_eq!(pair.hyponym.head_id, 4);
}
#[test]
fn parse_per_paragraph_scopes_sentences_to_originating_paragraph() {
let text = "The system processes input now. Tail one.\n\n\
The system processes input now. Tail two.";
let analysis = law_engine()
.analyze(Input::text(text, Format::PlainText))
.unwrap();
let paras: Vec<_> = analysis.paragraphs().collect();
assert_eq!(paras.len(), 2);
assert_eq!(paras[0].sentences.len(), 2);
assert_eq!(paras[1].sentences.len(), 2);
assert!(paras[0].sentences.iter().any(|s| s.text.contains("one")));
assert!(paras[1].sentences.iter().any(|s| s.text.contains("two")));
assert!(!paras[0].sentences.iter().any(|s| s.text.contains("two")));
assert!(!paras[1].sentences.iter().any(|s| s.text.contains("one")));
}
#[test]
fn parse_per_paragraph_no_inner_substring_theft() {
let text = "Outer talks about the special phrase processes input now. End A.\n\n\
The special phrase processes input now. End B.";
let analysis = law_engine()
.analyze(Input::text(text, Format::PlainText))
.unwrap();
let paras: Vec<_> = analysis.paragraphs().collect();
assert_eq!(paras.len(), 2);
assert!(paras[1].sentences.iter().any(|s| s.text.contains("End B")));
assert!(!paras[0].sentences.iter().any(|s| s.text.contains("End B")));
}
#[test]
fn empty_paragraph_followed_by_valid_paragraph() {
struct InjectEmptyParagraph;
impl Decomposer for InjectEmptyParagraph {
fn decompose(&self, input: &[u8]) -> domain::Result<decompose::Read> {
let mut read = decompose::plain::PlainTextDecomposer.decompose(input)?;
read.blocks.insert(
0,
decompose::ReadBlock::paragraph(ParagraphKind::Paragraph, "", Span::default()),
);
Ok(read)
}
}
let engine =
Engine::with_parser(DotSplitNlp).with_reader(Format::PlainText, InjectEmptyParagraph);
let analysis = engine
.analyze(Input::text(
"Real content sentence.\n\nAnother real one.",
Format::PlainText,
))
.unwrap();
let paras: Vec<_> = analysis.paragraphs().collect();
assert_eq!(paras.len(), 3);
assert_eq!(
paras[0].sentences.len(),
0,
"empty paragraph has zero sentences"
);
assert!(!paras[1].sentences.is_empty());
assert!(!paras[2].sentences.is_empty());
let _: &[Section] = &analysis.sections;
}
struct FixedEmbedder {
identity: &'static str,
force_count: Option<usize>,
}
impl embed::Embedder for FixedEmbedder {
fn embed(&self, texts: &[&str]) -> domain::Result<Vec<domain::Embedding>> {
let n = self.force_count.unwrap_or(texts.len());
Ok((0..n).map(|_| domain::Embedding(vec![1.0, 0.0])).collect())
}
fn identity(&self) -> domain::Identity {
domain::Identity::new(self.identity, None, None, None)
}
}
struct MustNotEmbed;
impl embed::Embedder for MustNotEmbed {
fn embed(&self, _texts: &[&str]) -> domain::Result<Vec<domain::Embedding>> {
panic!("embed ran when a gate should have refused first");
}
fn identity(&self) -> domain::Identity {
domain::Identity::new("unreached", None, None, None)
}
}
fn two_sentence_doc() -> Document {
let engine = Engine::with_parser(DotSplitNlp);
engine
.annotate(Input::text(
"One sentence. Another sentence.",
Format::PlainText,
))
.expect("annotate")
}
#[test]
fn clusters_records_the_embedder_and_the_threshold() {
let doc = two_sentence_doc();
let embedder = FixedEmbedder {
identity: "the-real-model",
force_count: None,
};
let out = clusters(&doc, &embedder, 0.9).expect("cluster");
let p = &out.provenance;
assert_eq!(p.method.as_deref(), Some(CLUSTERS_METHOD));
let parameters = p.parameters.as_ref().expect("parameters");
assert_eq!(
parameters.embedder,
Some(domain::Identity::new("the-real-model", None, None, None))
);
assert_eq!(parameters.threshold, Some(0.9));
assert_eq!(parameters.n, None);
assert_eq!(p.source, doc.provenance.source, "the document's input");
assert_eq!(p.parser, doc.provenance.parser, "the document's parser");
assert_eq!(out.clusters.len(), 1);
assert_eq!(out.clusters[0].members, [0, 1]);
assert_eq!(out.edges.len(), 1);
}
#[test]
fn clusters_rejects_a_length_contract_violation() {
let doc = two_sentence_doc();
let embedder = FixedEmbedder {
identity: "liar",
force_count: Some(1),
};
match clusters(&doc, &embedder, 0.9) {
Err(domain::Error::InvalidInput { message: msg, .. }) => {
assert_eq!(
msg,
"embedder returned 1 vector for 2 sentences, violating its contract"
);
}
other => panic!("expected InvalidInput, got {:?}", other.map(|_| ())),
}
}
#[test]
fn clusters_caps_before_embedding() {
let many = "Word. ".repeat(domain::limits::MAX_CLUSTER_SENTENCES + 1);
let engine = Engine::with_parser(DotSplitNlp);
let doc = engine
.annotate(Input::text(many, Format::PlainText))
.expect("annotate");
match clusters(&doc, &MustNotEmbed, 0.9) {
Err(domain::Error::InputTooLarge { what, .. }) => {
assert_eq!(what, "clusters");
}
other => panic!("expected InputTooLarge, got {:?}", other.map(|_| ())),
}
}
#[test]
fn clusters_refuses_a_non_finite_threshold_before_embedding() {
let doc = two_sentence_doc();
for threshold in [f64::NAN, f64::INFINITY] {
assert!(matches!(
clusters(&doc, &MustNotEmbed, threshold),
Err(domain::Error::InvalidInput { .. })
));
}
}
#[test]
fn outcomes_record_the_method_and_n() {
let doc = Engine::with_parser(WordNlp)
.analyze(Input::text(
"Rivers carry silt.\n\nSilt builds deltas.",
Format::PlainText,
))
.unwrap();
let summary = summarize(&doc, 1).unwrap();
assert_eq!(summary.provenance.method.as_deref(), Some("tfidf"));
let summary_with = summarize_with(&doc, 1, SummaryMethod::TextRank).unwrap();
assert_eq!(summary_with.provenance.method.as_deref(), Some("textrank"));
let phrases = keyphrases(&doc, 4).unwrap();
assert_eq!(phrases.provenance.method.as_deref(), Some("rake"));
let phrases_with = keyphrases_with(&doc, 4, KeyphraseMethod::Yake).unwrap();
assert_eq!(phrases_with.provenance.method.as_deref(), Some("yake"));
for (provenance, n) in [
(&summary.provenance, 1),
(&summary_with.provenance, 1),
(&phrases.provenance, 4),
(&phrases_with.provenance, 4),
] {
let parameters = provenance.parameters.as_ref().unwrap();
assert_eq!(parameters.n, Some(n));
assert_eq!(parameters.threshold, None);
assert_eq!(parameters.embedder, None);
assert_eq!(provenance.source, doc.provenance.source);
assert_eq!(provenance.schema, doc.provenance.schema);
}
assert!(doc.provenance.method.is_none(), "a document has no method");
let json = serde_json::to_value(&summary).unwrap();
assert_eq!(
json["provenance"]["parameters"],
serde_json::json!({ "n": 1 }),
"only the parameters the method takes"
);
let json = serde_json::to_value(&doc).unwrap();
assert!(json["provenance"].get("method").is_none());
assert!(json["provenance"].get("parameters").is_none());
}
#[test]
fn a_summary_sentence_points_into_the_document() {
let text = "Rivers carry silt downstream.\n\nThe silt settles where rivers slow.";
let doc = Engine::with_parser(WordNlp)
.analyze(Input::text(text, Format::PlainText))
.unwrap();
let summary = summarize(&doc, 2).unwrap();
assert_eq!(summary.sentences.len(), 2);
for picked in &summary.sentences {
let sentence = doc.sentences().nth(picked.index).unwrap();
assert_eq!(picked.span, sentence.span);
assert_eq!(picked.text, sentence.text);
assert_eq!(
&text[picked.span.byte_start..picked.span.byte_end],
picked.text
);
}
}
#[test]
fn the_default_methods_are_the_shipped_configuration_defaults() {
let cfg = config::Config::from_sources(
|key| (key == "HOME").then(|| "/nonexistent/matra-test-home".to_string()),
None,
)
.unwrap();
assert_eq!(
extraction::DEFAULT_SUMMARY_METHOD.name(),
cfg.summary_method()
);
assert_eq!(
extraction::DEFAULT_KEYPHRASE_METHOD.name(),
cfg.keyphrases_method()
);
}
use crate::document_schema;
struct RecordParser;
const RECORD_SENTENCE: &str = "Smith never said that animals such as dogs may bark.";
impl NlpProvider for RecordParser {
fn parse(&self, text: &str) -> domain::Result<Vec<Vec<domain::Token>>> {
if text != RECORD_SENTENCE {
return WordNlp.parse(text);
}
let tok = |id: usize, text: &str, pos: &str, dep: &str, head: usize, feats: &str| {
domain::Token::builder(
id,
text.to_string(),
text.to_lowercase(),
pos.to_string(),
head,
dep.to_string(),
)
.feats(feats.to_string())
.is_punct(pos == "PUNCT")
.build()
};
Ok(vec![placed(
text,
vec![
tok(1, "Smith", "PROPN", "nsubj", 3, "Number=Sing"),
tok(2, "never", "ADV", "advmod", 3, "_"),
tok(
3,
"said",
"VERB",
"root",
0,
"Mood=Ind|Tense=Past|VerbForm=Fin",
),
tok(4, "that", "SCONJ", "mark", 10, "_"),
tok(5, "animals", "NOUN", "nsubj", 10, "Number=Plur"),
tok(6, "such", "ADJ", "case", 8, "Degree=Pos"),
tok(7, "as", "ADP", "fixed", 6, "_"),
tok(8, "dogs", "NOUN", "nmod", 5, "Number=Plur"),
tok(9, "may", "AUX", "aux", 10, "VerbForm=Fin"),
tok(10, "bark", "VERB", "ccomp", 3, "VerbForm=Inf"),
tok(11, ".", "PUNCT", "punct", 3, "_"),
],
)])
}
fn identity(&self) -> domain::Identity {
domain::Identity::new(
"record-parser",
Some("fixture".to_string()),
Some("0".repeat(64)),
Some("MIT".to_string()),
)
}
}
fn record_document(text: &str, format: Format) -> Document {
Engine::with_parser(RecordParser)
.analyze(Input::text(text, format))
.expect("analyze")
}
fn record_json(text: &str, format: Format) -> serde_json::Value {
serde_json::to_value(record_document(text, format)).expect("a document serializes")
}
#[test]
fn a_plain_text_document_matches_the_schema() {
let doc = record_json(RECORD_SENTENCE, Format::PlainText);
let sentence = &doc["sections"][0]["paragraphs"][0]["sentences"][0];
for field in [
"negations",
"modals",
"clausal_complements",
"root_adverbials",
"hearst_pairs",
] {
assert!(
!sentence[field].as_array().expect(field).is_empty(),
"the stand-in parse populates {field}"
);
}
assert!(sentence["root_mood"].is_object());
document_schema::assert_valid("document", &doc, "plain text");
}
#[test]
fn a_markdown_document_with_a_blockquote_matches_the_schema() {
let doc = record_json(
"# Title\n\nA paragraph.\n\n> A quotation, never parsed.\n\n## Second\n\nMore prose.\n",
Format::Markdown,
);
assert_eq!(doc["provenance"]["format"], "markdown");
document_schema::assert_valid("document", &doc, "markdown");
}
#[test]
fn a_markdown_document_with_skipped_constructs_matches_the_schema() {
let doc = record_json(
"---\ntitle: x\n---\n\n# Title\n\n- an item\n- another\n\n| a | b |\n|---|---|\n| 1 | 2 |\n\n```\ncode\n```\n",
Format::Markdown,
);
let kinds: Vec<&str> = doc["skipped"]
.as_array()
.expect("skipped")
.iter()
.map(|s| s["kind"].as_str().expect("kind"))
.collect();
assert_eq!(kinds, ["front_matter", "table", "code"]);
assert_eq!(
doc["sections"][0]["paragraphs"][0]["kind"], "list_item",
"a list item is a paragraph of its own kind"
);
document_schema::assert_valid("document", &doc, "markdown with skipped constructs");
}
#[test]
fn block_input_matches_its_schema_and_reads_into_a_document() {
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("spec/schema/v2/blocks.schema.json");
let schema: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(path).expect("read")).expect("parse");
let validator = jsonschema::draft202012::new(&schema).expect("a valid JSON Schema");
let good = serde_json::json!([
{"kind": "heading", "level": 1, "text": "Field notes"},
{"kind": "paragraph", "text": "The river rose overnight."}
]);
assert!(validator.is_valid(&good));
let doc = record_json(&good.to_string(), Format::Blocks);
assert_eq!(doc["provenance"]["format"], "blocks");
assert_eq!(doc["sections"][0]["heading"], "Field notes");
document_schema::assert_valid("document", &doc, "block input");
let engine = Engine::with_parser(RecordParser);
for bad in [
serde_json::json!([{"kind": "table", "text": "x"}]),
serde_json::json!([{"kind": "heading", "level": 0, "text": "x"}]),
serde_json::json!([{"kind": "paragraph", "text": "x", "style": "bold"}]),
] {
assert!(!validator.is_valid(&bad), "the schema refuses {bad}");
let err = engine
.analyze(Input::text(bad.to_string(), Format::Blocks))
.expect_err("the reader refuses it too");
assert_eq!(err.kind(), "invalid_input", "{bad}: {err}");
}
}
#[test]
fn an_empty_document_matches_the_schema() {
let doc = record_json("", Format::PlainText);
assert!(doc["metrics"]["mean_sentence_length"].is_null());
document_schema::assert_valid("document", &doc, "empty");
}
#[test]
#[should_panic(expected = "does not match schema 2")]
fn an_undocumented_field_fails_the_check() {
let mut doc = record_json("Prose.", Format::PlainText);
doc["metrics"]["undocumented"] = serde_json::json!(1);
document_schema::assert_valid("document", &doc, "undocumented field");
}
#[test]
fn every_outcome_of_a_parsed_document_matches_its_schema() {
let text = format!("{RECORD_SENTENCE}\n\n{RECORD_SENTENCE}");
let doc = record_document(&text, Format::PlainText);
for method in [SummaryMethod::TfIdf, SummaryMethod::TextRank] {
let value = serde_json::to_value(summarize_with(&doc, 1, method).unwrap()).unwrap();
document_schema::assert_valid("summary", &value, method.name());
}
for method in [KeyphraseMethod::Rake, KeyphraseMethod::Yake] {
let value = serde_json::to_value(keyphrases_with(&doc, 5, method).unwrap()).unwrap();
assert!(
!value["phrases"].as_array().unwrap().is_empty(),
"{}: the check runs over populated phrases",
method.name()
);
document_schema::assert_valid("keyphrases", &value, method.name());
}
let embedder = FixedEmbedder {
identity: "fixed",
force_count: None,
};
let value = serde_json::to_value(clusters(&doc, &embedder, 0.5).unwrap()).unwrap();
assert!(!value["edges"].as_array().unwrap().is_empty());
document_schema::assert_valid("clusters", &value, "clusters");
}
#[test]
fn every_io_detail_matches_the_error_schema() {
let cases = [
domain::Error::Io(domain::IoContext::at_path(
std::io::ErrorKind::Unsupported,
Path::new("link.md"),
"refusing to read symlink: link.md",
)),
domain::Error::Io(domain::IoContext::download(
std::io::ErrorKind::TimedOut,
"https://models.example/m",
"download https://models.example/m: timeout",
true,
)),
match Inputs::dir(Path::new("/nonexistent/matra/dir")) {
Err(err) => err,
Ok(_) => panic!("a missing directory lists nothing"),
},
];
for error in &cases {
let value = serde_json::to_value(error).unwrap();
assert_eq!(value["kind"], "io");
document_schema::assert_valid("error", &value, &error.to_string());
}
}
#[test]
#[should_panic(expected = "does not match schema 2")]
fn an_undocumented_detail_fails_the_check() {
let mut value = serde_json::to_value(domain::Error::InputTooLarge {
limit: 1,
actual: 2,
what: "input",
})
.unwrap();
value["details"]["undocumented"] = serde_json::json!(1);
document_schema::assert_valid("error", &value, "undocumented detail");
}
#[test]
fn a_directory_walk_yields_one_item_per_file() {
#[derive(serde::Deserialize)]
struct Fixture {
directory: Directory,
item_shapes: ItemShapes,
}
#[derive(serde::Deserialize)]
struct ItemShapes {
error: Vec<String>,
}
#[derive(serde::Deserialize)]
struct Directory {
files: Vec<FixtureFile>,
expect: Vec<Expect>,
}
#[derive(serde::Deserialize)]
struct FixtureFile {
name: String,
text: Option<String>,
bytes: Option<Vec<u8>>,
}
#[derive(serde::Deserialize)]
struct Expect {
name: String,
shape: String,
kind: Option<String>,
details: Option<serde_json::Value>,
}
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("spec/tests/corpus/items.json");
let fixture: Fixture =
serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();
let dir = tempfile::tempdir().unwrap();
for file in &fixture.directory.files {
let path = dir.path().join(&file.name);
match (&file.text, &file.bytes) {
(Some(text), None) => std::fs::write(&path, text).unwrap(),
(None, Some(bytes)) => std::fs::write(&path, bytes).unwrap(),
_ => panic!(
"fixture file {} needs exactly one of text, bytes",
file.name
),
}
}
let items: Vec<_> = Engine::with_parser(EmptyNlp)
.analyze_all(Inputs::dir(dir.path()).unwrap())
.collect();
assert_eq!(items.len(), fixture.directory.expect.len());
for (item, expect) in items.iter().zip(&fixture.directory.expect) {
match (item, expect.shape.as_str()) {
(Ok(document), "document") => {
let path = document.provenance.source.path.as_ref().unwrap();
assert_eq!(path.file_name().unwrap(), expect.name.as_str());
document_schema::assert_valid(
"document",
&serde_json::to_value(document).unwrap(),
&expect.name,
);
}
(Err(err), "error") => {
let path = err.path.as_ref().unwrap();
assert_eq!(path.file_name().unwrap(), expect.name.as_str());
assert_eq!(err.error.kind(), expect.kind.as_deref().unwrap());
assert!(!err.error.to_string().is_empty());
let value = serde_json::to_value(err).unwrap();
let mut keys: Vec<&String> = value.as_object().unwrap().keys().collect();
keys.sort();
let mut shape = fixture.item_shapes.error.iter().collect::<Vec<_>>();
shape.sort();
assert_eq!(keys, shape, "{}", expect.name);
assert_eq!(Some(&value["details"]), expect.details.as_ref());
document_schema::assert_valid("error", &value, &expect.name);
}
(got, shape) => panic!(
"{} expected shape {shape}, got {}",
expect.name,
if got.is_ok() { "document" } else { "error" }
),
}
}
}
}