pos-tagger 0.2.0

//! Embedded English lexicon mapping words to candidate part-of-speech tags.
//!
//! The data ships as a compact JSON object literal compiled into the binary.
//! Each key is a word or token. Each value is a non-empty ordered list of tag
//! strings, most likely first. The tagger reads only the first tag, but the
//! full list is preserved so callers can inspect it.

use std::collections::HashMap;

/// Embedded lexicon source. A JSON object of the form `{"word":["TAG",...]}`.
const LEXICON_JSON: &str = include_str!("lexicon.json");

/// Parse the embedded lexicon into an owned map.
///
/// This parser targets one fixed asset, not arbitrary JSON. The asset is a
/// compact object with string keys and arrays of string values. Its only escape
/// sequences are `\"` and `\\`, so those are the only ones decoded. Any other
/// escape, or any structural surprise, means the asset changed shape, and the
/// parser panics so the build fails loudly. The parser assumes valid input and
/// does not bounds-check, so malformed input panics rather than returning an
/// error.
pub(crate) fn load() -> HashMap<String, Vec<String>> {
    let mut parser = Parser::new(LEXICON_JSON);
    parser.parse_object()
}

struct Parser<'a> {
    bytes: &'a [u8],
    pos: usize,
}

impl<'a> Parser<'a> {
    fn new(input: &'a str) -> Self {
        Parser {
            bytes: input.as_bytes(),
            pos: 0,
        }
    }

    fn skip_ws(&mut self) {
        while self.pos < self.bytes.len() {
            match self.bytes[self.pos] {
                b' ' | b'\t' | b'\n' | b'\r' => self.pos += 1,
                _ => break,
            }
        }
    }

    fn expect(&mut self, ch: u8) {
        self.skip_ws();
        assert_eq!(self.bytes[self.pos], ch);
        self.pos += 1;
    }

    fn peek(&mut self) -> u8 {
        self.skip_ws();
        self.bytes[self.pos]
    }

    fn parse_object(&mut self) -> HashMap<String, Vec<String>> {
        let mut map = HashMap::new();
        self.expect(b'{');
        if self.peek() == b'}' {
            self.pos += 1;
            return map;
        }
        loop {
            let key = self.parse_string();
            self.expect(b':');
            let value = self.parse_array();
            map.insert(key, value);
            match self.peek() {
                b',' => {
                    self.pos += 1;
                }
                b'}' => {
                    self.pos += 1;
                    break;
                }
                other => panic!("lexicon JSON expected ',' or '}}', got {}", other as char),
            }
        }
        map
    }

    fn parse_array(&mut self) -> Vec<String> {
        let mut items = Vec::new();
        self.expect(b'[');
        if self.peek() == b']' {
            self.pos += 1;
            return items;
        }
        loop {
            items.push(self.parse_string());
            match self.peek() {
                b',' => {
                    self.pos += 1;
                }
                b']' => {
                    self.pos += 1;
                    break;
                }
                other => panic!("lexicon JSON expected ',' or ']', got {}", other as char),
            }
        }
        items
    }

    fn parse_string(&mut self) -> String {
        self.expect(b'"');
        let mut out = String::new();
        loop {
            let b = self.bytes[self.pos];
            match b {
                b'"' => {
                    self.pos += 1;
                    break;
                }
                b'\\' => {
                    self.pos += 1;
                    let esc = self.bytes[self.pos];
                    self.pos += 1;
                    // The asset uses only these two escapes. Any other escape
                    // means the data changed shape, so fail loudly rather than
                    // decode forms that never appear.
                    match esc {
                        b'"' => out.push('"'),
                        b'\\' => out.push('\\'),
                        other => panic!("lexicon JSON has unexpected escape \\{}", other as char),
                    }
                }
                _ => {
                    // Copy a run of UTF-8 bytes that are not quote or backslash.
                    let start = self.pos;
                    while self.pos < self.bytes.len()
                        && self.bytes[self.pos] != b'"'
                        && self.bytes[self.pos] != b'\\'
                    {
                        self.pos += 1;
                    }
                    out.push_str(
                        std::str::from_utf8(&self.bytes[start..self.pos])
                            .expect("lexicon JSON is valid UTF-8"),
                    );
                }
            }
        }
        out
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn loads_all_entries() {
        let map = load();
        assert_eq!(map.len(), 92662);
    }

    #[test]
    fn spot_check_known_keys() {
        let map = load();
        assert_eq!(map.get("Mr").unwrap(), &["NNP"]);
        assert_eq!(map.get("would").unwrap(), &["MD"]);
        assert_eq!(map.get("@").unwrap(), &["IN", "SYM"]);
        assert_eq!(map.get("").unwrap(), &Vec::<String>::new());
        assert!(!map.contains_key("Obama"));
    }

    #[test]
    fn parses_escaped_keys() {
        let map = load();
        // The quote and backslash keys exist with self-referential tags.
        assert!(map.contains_key("\""));
        assert!(map.contains_key("'"));
    }
}