ainu-utils 0.5.1

A collection of utilities for the Ainu language
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

use super::tokenizer::tokenize;

#[test]
fn test_tokenize() {
    let text = "irankarapte! eyami yak a=ye aeywankep ku=kar wa k=an.";
    let tokens = tokenize(text, false);

    assert_eq!(
        tokens,
        vec![
            "irankarapte",
            "!",
            "eyami",
            "yak",
            "a=",
            "ye",
            "aeywankep",
            "ku=",
            "kar",
            "wa",
            "k=",
            "an",
            "."
        ]
    );
}

#[test]
fn test_tokenize_suffix() {
    let text = "soyenpa=an wa sinot=an ro!";
    let tokens = tokenize(text, false);

    assert_eq!(
        tokens,
        vec!["soyenpa", "=an", "wa", "sinot", "=an", "ro", "!"]
    );
}

#[test]
fn test_sentence_does_not_end_with_period() {
    let text = "a=nukar hike i=yaykohaytare i=yaypokaste wa iki pe";
    let tokens = tokenize(text, false);

    assert_eq!(
        tokens,
        vec![
            "a=",
            "nukar",
            "hike",
            "i=",
            "yaykohaytare",
            "i=",
            "yaypokaste",
            "wa",
            "iki",
            "pe"
        ]
    );
}

#[test]
fn test_sentence_ending_with_a_fixed_word() {
    let text = "neno a=ye itak pirka a=ye itak i=koynu wa ... i=konu wa i=kore";
    let tokens = tokenize(text, false);

    assert_eq!(
        tokens,
        vec![
            "neno", "a=", "ye", "itak", "pirka", "a=", "ye", "itak", "i=", "koynu", "wa", ".", ".",
            ".", "i=", "konu", "wa", "i=", "kore"
        ]
    );
}

#[test]
fn test_parse_numbers() {
    let text = "1000 yen ku=kor";
    let tokens = tokenize(text, false);

    assert_eq!(tokens, vec!["1000", "yen", "ku=", "kor"]);
}

#[test]
fn test_handles_hyphen_within_word() {
    let text = "cep-koyki wa e";
    let tokens = tokenize(text, false);
    assert_eq!(tokens, vec!["cep-koyki", "wa", "e"]);
}

#[test]
fn test_handles_double_prefixes() {
    let text = "niwen seta ne kusu a=e=kupa na.";
    let tokens = tokenize(text, false);
    assert_eq!(
        tokens,
        vec!["niwen", "seta", "ne", "kusu", "a=", "e=", "kupa", "na", "."]
    );
}

#[test]
fn test_handles_glottal_stop() {
    let text = "ku=kor irwak'utari";
    let tokens = tokenize(text, false);
    assert_eq!(tokens, vec!["ku=", "kor", "irwak'utari"]);

    let text = "'ku=kor rusuy!' sekor hawean";
    let tokens = tokenize(text, false);
    assert_eq!(
        tokens,
        vec!["'", "ku=", "kor", "rusuy", "!", "'", "sekor", "hawean"]
    );
}

#[test]
fn test_keep_whitespace() {
    let text = "irankarapte. tanto sirpirka ne.";
    let tokens = tokenize(text, true);
    assert_eq!(
        tokens,
        vec![
            "irankarapte",
            ".",
            " ",
            "tanto",
            " ",
            "sirpirka",
            " ",
            "ne",
            "."
        ]
    );
}