use matra::domain::{Format, Identity, Result, Span, Token};
use matra::nlp::NlpProvider;
use matra::{Engine, Input, summarize};
struct Whitespace;
impl NlpProvider for Whitespace {
fn parse(&self, text: &str) -> Result<Vec<Vec<Token>>> {
let mut sentences = Vec::new();
let mut tokens: Vec<Token> = Vec::new();
let mut cursor = 0;
for word in text.split_whitespace() {
let start = cursor + text[cursor..].find(word).unwrap_or(0);
cursor = start + word.len();
let id = tokens.len() + 1;
let head = if id == 1 { 0 } else { 1 };
let token = Token::builder(
id,
word.to_string(),
word.to_lowercase(),
"X".to_string(),
head,
if head == 0 { "root" } else { "dep" }.to_string(),
)
.span(Span::new(start, cursor))
.build();
tokens.push(token);
if word.ends_with(['.', '!', '?']) {
sentences.push(std::mem::take(&mut tokens));
}
}
if !tokens.is_empty() {
sentences.push(tokens);
}
Ok(sentences)
}
fn identity(&self) -> Identity {
Identity::new("whitespace", Some("toy".to_string()), None, None)
}
}
fn main() -> Result<()> {
let text = "# Field notes\n\n\
The river rose overnight. By morning the low road was under water.\n\n\
The bridge held. Nobody crossed it until noon.\n";
let engine = Engine::with_parser(Whitespace);
let doc = engine.analyze(Input::text(text, Format::Markdown))?;
println!(
"parser: {} ({})",
doc.provenance.parser.name,
doc.provenance.parser.model.as_deref().unwrap_or("-")
);
println!(
"{} sentences, {} words",
doc.metrics.total_sentences, doc.metrics.total_words
);
for sentence in doc.sentences() {
let source = &text[sentence.span.byte_start..sentence.span.byte_end];
println!(
" {:>3}..{:<3} depth {:?} {source}",
sentence.span.byte_start, sentence.span.byte_end, sentence.tree_depth
);
}
let summary = summarize(&doc, 1)?;
for picked in &summary.sentences {
println!("summary: {}", picked.text);
}
Ok(())
}