tokmat 0.3.3

Standalone high-performance Canadian address parsing engine core
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

use std::path::Path;

use tokmat::extractor::{Extractor, MatchMode};
use tokmat::tokenizer::{load_token_class_list, load_token_definitions, split_input_tokens};

fn load_extractor() -> Extractor {
    let base_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/model_1");

    let token_definitions =
        load_token_definitions(base_path.join("TOKENDEFINITION/TOKENDEFINITONS.param2"))
            .expect("token definitions");
    let token_class_list =
        load_token_class_list(base_path.join("TOKENCLASS")).expect("token class list");
    Extractor::new(token_definitions, token_class_list)
}

#[test]
fn test_tokenizer_keeps_extended_word_boundaries() {
    assert_eq!(
        split_input_tokens("APT-210 O'CONNOR"),
        vec!["APT-210", " ", "O'CONNOR"]
    );
}

#[test]
fn test_extractor_requires_percent_for_extended_alpha() {
    let extractor = load_extractor();
    let tokens = vec!["Jean-Paul".to_string()];
    let classes = vec!["ALPHA_EXTENDED".to_string()];

    let without_percent = extractor
        .parse_tokens("1", &tokens, &classes, "<<NAME@>>", MatchMode::Whole)
        .expect("pattern should parse");
    assert!(without_percent.fields.is_empty());

    let with_percent = extractor
        .parse_tokens("1", &tokens, &classes, "<<NAME@%>>", MatchMode::Whole)
        .expect("pattern should parse");
    assert_eq!(
        with_percent.fields.get("NAME").map(String::as_str),
        Some("Jean-Paul")
    );
}

#[test]
fn test_extractor_preserves_leading_space_in_complement() {
    let extractor = load_extractor();
    let output = extractor
        .parse_tokens(
            "1",
            &[" ".to_string(), "NS".to_string()],
            &[" ".to_string(), "PROV".to_string()],
            "<<PROV::PROV>>",
            MatchMode::Whole,
        )
        .expect("pattern should parse");

    assert_eq!(output.fields.get("PROV").map(String::as_str), Some("NS"));
    assert_eq!(output.complement, " ");
}

#[test]
fn test_extractor_handles_literal_quotes() {
    let extractor = load_extractor();
    let output = extractor
        .parse_tokens(
            "1",
            &[
                "NOAH".to_string(),
                " ".to_string(),
                "\"".to_string(),
                "AWESOME".to_string(),
                "\"".to_string(),
                " ".to_string(),
                "STEVENS".to_string(),
            ],
            &[
                "ALPHA".to_string(),
                " ".to_string(),
                "\"".to_string(),
                "ALPHA".to_string(),
                "\"".to_string(),
                " ".to_string(),
                "ALPHA".to_string(),
            ],
            "<<FIRST>> \"<<TITLE>>\" <<LAST>>",
            MatchMode::Whole,
        )
        .expect("pattern should parse");

    assert_eq!(output.fields.get("FIRST").map(String::as_str), Some("NOAH"));
    assert_eq!(
        output.fields.get("TITLE").map(String::as_str),
        Some("AWESOME")
    );
    assert_eq!(
        output.fields.get("LAST").map(String::as_str),
        Some("STEVENS")
    );
}