nl3/
tagger.rs

1//! Part-of-speech tagging. Ports the tagging half of `lib/text/classify.js`.
2//!
3//! The original used the npm `pos` package (a Brill tagger). Analysis of the
4//! pipeline shows POS tags only ever change behavior in two ways: `reduce_parts`
5//! skips prepositions (`IN`) and WH-words (any tag starting with `W`), and
6//! numbers stay as values (`CD`). Entity *types* come from the grammar lexicon
7//! and predicates from vocabulary stemming — not from tags. So a small
8//! closed-class lexicon tagger reproduces every documented example.
9//!
10//! [`Tagger`] keeps this pluggable: a caller wanting general English POS
11//! tagging can implement the trait over, e.g., `rust-bert`'s `POSModel`.
12
13/// Tags text into `(token, Penn-Treebank-style tag)` pairs.
14pub trait Tagger {
15    /// Tag `text`, returning one `(token, tag)` pair per whitespace-separated
16    /// token in order.
17    fn tag(&self, text: &str) -> Vec<(String, String)>;
18}
19
20/// The default tagger: a closed-class lexicon. Recognizes prepositions (`IN`),
21/// WH-words (`WDT`), and cardinal numbers (`CD`); everything else is `NN` and
22/// falls through to the grammar/vocabulary logic.
23#[derive(Debug, Clone, Copy, Default)]
24pub struct LexiconTagger;
25
26impl Tagger for LexiconTagger {
27    fn tag(&self, text: &str) -> Vec<(String, String)> {
28        text.split_whitespace()
29            .map(|token| (token.to_string(), tag_word(token).to_string()))
30            .collect()
31    }
32}
33
34fn tag_word(word: &str) -> &'static str {
35    if !word.is_empty() && word.bytes().all(|b| b.is_ascii_digit()) {
36        return "CD";
37    }
38    match word.to_lowercase().as_str() {
39        "by" | "from" | "on" | "for" | "to" | "with" | "of" | "in" | "at" | "into" | "onto"
40        | "about" | "as" | "per" | "via" => "IN",
41        "who" | "which" | "that" | "what" | "whom" | "whose" | "where" | "when" | "why" | "how" => {
42            "WDT"
43        }
44        _ => "NN",
45    }
46}
nl3/tagger.rs

nl3/
tagger.rs
