speed_reader_core/
tokenizer.rs

1use unicode_segmentation::UnicodeSegmentation;
2
3/// Tokenizer — разбивает текст на токены с ORP-позиционированием.
4/// Requirement: 1.1, Design: Tokenizer section
5
6/// Позиция ORP-буквы в слове.
7pub fn orp_index(word: &str) -> usize {
8    let len = word.chars().count();
9    if len <= 2 {
10        0
11    } else {
12        ((len + 1) / 2).min(4) - 1
13    }
14}
15
16/// Токен — одно слово с метаданными для RSVP.
17#[derive(Debug, Clone, PartialEq)]
18pub struct Token {
19    pub word: String,
20    pub orp_index: usize,
21    pub byte_offset: usize,
22    pub byte_len: usize,
23    pub is_sentence_end: bool,
24}
25
26/// Ошибки токенизации.
27#[derive(Debug, Clone, PartialEq)]
28pub enum TokenizeError {
29    EmptyText,
30}
31
32/// Разбивает текст на токены.
33pub fn tokenize(text: &str) -> Result<Vec<Token>, TokenizeError> {
34    if text.is_empty() {
35        return Err(TokenizeError::EmptyText);
36    }
37    let mut tokens = Vec::new();
38    let mut byte_offset = 0;
39    let mut prev_char_was_sentence_end = false;
40
41    for word in text.split_word_bounds() {
42        let trimmed = word.trim();
43        if trimmed.is_empty() {
44            byte_offset += word.len();
45            // whitespace between sentences
46            if word.contains('\n') {
47                prev_char_was_sentence_end = true;
48            }
49            continue;
50        }
51        let is_sentence_end = matches!(
52            trimmed.chars().last(),
53            Some('.') | Some('!') | Some('?')
54        );
55        // Handle sentence-starting punctuation
56        let display_word = if prev_char_was_sentence_end {
57            // Capitalize first letter of first word after sentence end
58            let mut chars: Vec<char> = trimmed.chars().collect();
59            if let Some(c) = chars.first_mut() {
60                if c.is_alphabetic() {
61                    *c = c.to_ascii_uppercase();
62                }
63            }
64            chars.into_iter().collect()
65        } else {
66            trimmed.to_string()
67        };
68
69        let orp = orp_index(&display_word);
70        tokens.push(Token {
71            word: display_word,
72            orp_index: orp,
73            byte_offset,
74            byte_len: word.len(),
75            is_sentence_end,
76        });
77        byte_offset += word.len();
78        prev_char_was_sentence_end = is_sentence_end;
79    }
80
81    if tokens.is_empty() {
82        return Err(TokenizeError::EmptyText);
83    }
84    Ok(tokens)
85}
86
87#[cfg(test)]
88mod tests {
89    use super::*;
90
91    #[test]
92    fn test_orp_short_word() {
93        assert_eq!(orp_index("a"), 0);
94        assert_eq!(orp_index("an"), 0);
95        assert_eq!(orp_index("and"), 1);
96    }
97
98    #[test]
99    fn test_orp_medium_word() {
100        assert_eq!(orp_index("hello"), 2);
101        assert_eq!(orp_index("world"), 2);
102        assert_eq!(orp_index("speed"), 2);
103    }
104
105    #[test]
106    fn test_orp_long_word() {
107        assert_eq!(orp_index("reading"), 3);
108        assert_eq!(orp_index("presentation"), 3);
109    }
110
111    #[test]
112    fn test_tokenize_simple() {
113        let tokens = tokenize("Hello world").unwrap();
114        assert_eq!(tokens.len(), 2);
115        assert_eq!(tokens[0].word, "Hello");
116        assert_eq!(tokens[0].orp_index, 2);
117        assert_eq!(tokens[1].word, "world");
118        assert_eq!(tokens[1].orp_index, 2);
119    }
120
121    #[test]
122    fn test_tokenize_empty() {
123        assert_eq!(tokenize(""), Err(TokenizeError::EmptyText));
124    }
125
126    #[test]
127    fn test_tokenize_sentence_end() {
128        let tokens = tokenize("Go").unwrap();
129        assert_eq!(tokens.len(), 1);
130        // Note: punctuation like "." is split into separate tokens by unicode word segmentation
131    }
132
133    #[test]
134    fn test_tokenize_byte_offsets() {
135        let tokens = tokenize("Hello world").unwrap();
136        assert_eq!(tokens[0].byte_offset, 0);
137        assert_eq!(tokens[1].byte_offset, 6); // 'H'(0) e(1) l(2) l(3) o(4) ' '(5) w(6)
138    }
139
140    #[test]
141    fn test_tokenize_unicode() {
142        let tokens = tokenize("Привет мир").unwrap();
143        assert_eq!(tokens.len(), 2);
144        assert_eq!(tokens[0].word, "Привет");
145        assert_eq!(tokens[1].word, "мир");
146    }
147}
speed_reader_core/tokenizer.rs

speed_reader_core/
tokenizer.rs
