rsnltk 0.1.3

Rust-based Natural Language Toolkit
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

/////////////////////////////////////////////////////////
// TokenIter
//

use crate::native::chardata;

static IN_TOKEN: u8 = 1;
static BTWN_TOKS: u8 = 0;

/*  TODO: Probably there should not be a String member here.
    We should either borrow a string slice from the original text,
    or else leave it out, and provide some other facility for
    converting Tokens to Strings, given the underlying string.
    In a full fledged parser, the parsed-document representation
    would handle that.
 */
#[derive(Debug)]
pub struct Token {
    pub text: String,
    pub byte_offsets: (usize, usize),
    pub char_offsets: (usize, usize),
    pub token_offset: usize
}

impl Token {
    fn new() -> Token {
        Token {
            text: "".to_string(),
            byte_offsets: (0, 0),
            char_offsets: (0, 0),
            token_offset: 0,
        }
    }
}

pub struct TokenIter<'a> {
    chdat_stream: &'a mut chardata::CharDataIter<'a>,
    curr_tok_offset: usize,
    state: u8,
}

impl<'a> TokenIter<'a> {
    pub fn new(chdats: &'a mut chardata::CharDataIter<'a>) -> Self {
        TokenIter {
            chdat_stream: chdats,
            curr_tok_offset: 0,
            state: BTWN_TOKS,
        }
    }

    fn is_boundary_char(ch: char) -> bool {
        if ch == chardata::END_OF_STRING {
            true
        } else if ch.is_whitespace() {
            true
        } else {
            false
        }
    }
}

/*  Always start out BTWN_TOKS, and therefore always end in BTWN_TOKS.
    Start by skipping characters until state changes to IN_TOKEN.
    Then (1) set the token start offsets; (2) march the char data iter forward
    until state changes to BTWN_TOKS, then fix the end offsets of the token
    under construction. Update the current token offset.
    Leave the resulting Token as the return value of next().
    If the underlying chardata::CharDataIter yields END_OF_SENTENCE:
        IN_TOKEN --> ship the current token
        BTWN_TOKS --> return None
    In the first case, the next call to next() will immediately trigger
    the second case.
*/

impl<'a> Iterator for TokenIter<'a> {
    type Item = Token;

    fn next(&mut self) -> Option<Self::Item> {
        assert_eq!(self.state, BTWN_TOKS);
        let mut curr_tok = Token::new();
        loop {
            match self.chdat_stream.next() {

                Some( chardata::CharData {ch, byte_offset, char_offset} ) => {

                    if TokenIter::is_boundary_char(ch) {
                        if self.state == IN_TOKEN {
                            // ship token
                            curr_tok.byte_offsets.1 = byte_offset;
                            curr_tok.char_offsets.1 = char_offset;
                            self.state = BTWN_TOKS;
                            self.curr_tok_offset += 1;
                            return Some(curr_tok);
                        }
                        // else do nothing -- skip boundary chars
                    } else {
                        if self.state == BTWN_TOKS {
                            // start token
                            curr_tok.token_offset = self.curr_tok_offset;
                            curr_tok.byte_offsets.0 = byte_offset;
                            curr_tok.char_offsets.0 = char_offset;
                            self.state = IN_TOKEN;
                        }
                        // Accumulate characters
                        curr_tok.text.push(ch);
                        curr_tok.byte_offsets.1 = byte_offset;
                        curr_tok.char_offsets.1 = char_offset;
                    }
                },

                None => {
                    // May need to ship a token here!
                    if self.state == IN_TOKEN {
                        self.state = BTWN_TOKS;
                        return Some(curr_tok);
                    }
                    return None;
                }
            }
        }
    }
}
//
// TokenIter
/////////////////////////////////////////////////////////