1use crate::token::{self, Position, Token, TokenWithContext};
2use auto_correct_n_suggest;
3use std::collections::HashMap;
4use std::fmt;
5use std::iter::Peekable;
6use std::str;
7
8#[derive(Debug, PartialEq, Eq)]
9pub enum ScannerError {
10 UnexpectedCharacter(char),
11 NumberParsingError(String),
12 DidYouMean(String),
13 UnknownKeyWord(String),
14}
15
16impl fmt::Display for ScannerError {
17 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
18 match self {
19 ScannerError::DidYouMean(suggestion) => write!(f, "Did you mean {} ?", suggestion),
20 ScannerError::NumberParsingError(number) => {
21 write!(f, "Unrecognized number {}", number)
22 }
23 ScannerError::UnexpectedCharacter(c) => write!(f, "Unexpected character {}", c),
24 ScannerError::UnknownKeyWord(keyword) => write!(f, "Unknown keyword {}", keyword),
25 }
26 }
27}
28
29struct Scanner<'a> {
30 keywords: HashMap<String, token::Token>,
31 dictionary: auto_correct_n_suggest::Dictionary,
32 current_lexeme: String,
33 current_position: Position,
34 source: Peekable<str::Chars<'a>>,
35}
36
37impl<'a> Scanner<'a> {
38 fn initialize(source: &'a str) -> Scanner {
39 let mut keywords = HashMap::new();
40 let mut dictionary = auto_correct_n_suggest::Dictionary::new();
41 Scanner::add_keywords_to_hashmap(&mut keywords);
42 Scanner::insert_keywords_to_dictionary(&keywords, &mut dictionary);
43 Scanner {
44 dictionary,
45 keywords,
46 current_lexeme: "".into(),
47 current_position: Position::initial(),
48 source: source.chars().into_iter().peekable(),
49 }
50 }
51
52 fn add_keywords_to_hashmap(keywords: &mut HashMap<String, Token>) {
53 keywords.insert("plus".to_string(), token::Token::Addition);
54 keywords.insert("minus".to_string(), token::Token::Subtraction);
55 keywords.insert("multiplication".to_string(), token::Token::Multiply);
56 keywords.insert("division".to_string(), token::Token::Division);
57 }
58
59 fn insert_keywords_to_dictionary(
60 keywords: &HashMap<String, Token>,
61 dictionary: &mut auto_correct_n_suggest::Dictionary,
62 ) {
63 for keyword in keywords.keys() {
64 dictionary.insert(keyword.to_string())
65 }
66 }
67
68 fn advance(&mut self) -> Option<char> {
69 let next = self.source.next();
70 if let Some(c) = next {
71 self.current_lexeme.push(c);
72 self.current_position.increment_column();
73 }
74 next
75 }
76
77 fn add_context(&mut self, token: Token, initial_position: Position) -> TokenWithContext {
78 TokenWithContext {
79 token,
80 lexeme: self.current_lexeme.clone(),
81 position: initial_position,
82 }
83 }
84
85 fn scan_next(&mut self) -> Option<Result<TokenWithContext, ScannerError>> {
86 let initial_position = self.current_position;
87 self.current_lexeme.clear();
88 let next_char = match self.advance() {
89 Some(c) => c,
90 None => return None,
91 };
92
93 let result = match next_char {
94 '*' => Ok(Token::Multiply),
95 '-' => Ok(Token::Subtraction),
96 '+' => Ok(Token::Addition),
97 '/' => Ok(Token::Division),
98 '(' => Ok(Token::OpeningBracket),
99 ')' => Ok(Token::ClosingBracket),
100 c if token::is_whitespace(c) => Ok(Token::WhiteSpace),
101 c if token::is_digit(c) => self.digit(),
102 c if token::is_alpha(c) => self.keyword(),
103 c => Err(ScannerError::UnexpectedCharacter(c)),
104 };
105
106 Some(result.map(|token| self.add_context(token, initial_position)))
107 }
108 fn peek_check(&mut self, check: &dyn Fn(char) -> bool) -> bool {
109 match self.source.peek() {
110 Some(&c) => check(c),
111 None => false,
112 }
113 }
114
115 fn advance_while(&mut self, condition: &dyn Fn(char) -> bool) {
116 while self.peek_check(condition) {
117 self.advance();
118 }
119 }
120
121 fn digit(&mut self) -> Result<token::Token, ScannerError> {
122 self.advance_while(&|c| token::is_digit(c));
123 let literal_length = self.current_lexeme.len();
124 let num: String = self.current_lexeme.chars().take(literal_length).collect();
125 let num = num
126 .parse::<f64>()
127 .map_err(|_| ScannerError::NumberParsingError(num))?;
128 Ok(Token::DigitLiteral(num))
129 }
130
131 fn keyword(&mut self) -> Result<token::Token, ScannerError> {
132 self.advance_while(&|c| token::is_alpha(c));
133 let literal_length = self.current_lexeme.len();
134 let mut keyword: String = self.current_lexeme.chars().take(literal_length).collect();
135 keyword.make_ascii_lowercase();
136 match self.keywords.get(&keyword) {
137 Some(token) => Ok(*token),
138 None => Err(self.attempt_to_suggest_word(&keyword)),
139 }
140 }
141 fn attempt_to_suggest_word(&mut self, keyword: &str) -> ScannerError {
142 let auto_suggested_word = self
143 .dictionary
144 .auto_correct(keyword.to_string())
145 .and_then(|suggestions| suggestions.first().cloned());
146 match auto_suggested_word {
147 Some(word) => ScannerError::DidYouMean(word),
148 None => ScannerError::UnknownKeyWord(keyword.to_string()),
149 }
150 }
151}
152
153struct TokensIterator<'a> {
154 scanner: Scanner<'a>,
155}
156
157impl<'a> Iterator for TokensIterator<'a> {
158 type Item = Result<TokenWithContext, ScannerError>;
159 fn next(&mut self) -> Option<Self::Item> {
160 self.scanner.scan_next()
161 }
162}
163
164pub fn scan_into_iterator<'a>(
165 source: &'a str,
166) -> impl Iterator<Item = Result<TokenWithContext, ScannerError>> + 'a {
167 TokensIterator {
168 scanner: Scanner::initialize(source),
169 }
170}
171
172pub fn scan(source: &str) -> Result<Vec<TokenWithContext>, Vec<String>> {
173 let mut tokens = Vec::new();
174 let mut errors = Vec::new();
175
176 for result in scan_into_iterator(source) {
177 match result {
178 Ok(token_with_context) => match token_with_context.token {
179 Token::WhiteSpace => {}
180 _ => tokens.push(token_with_context),
181 },
182 Err(error) => errors.push(format!("{}", error)),
183 }
184 }
185 if errors.is_empty() {
186 Ok(tokens)
187 } else {
188 Err(errors)
189 }
190}
191
192#[cfg(test)]
193mod tests {
194 use super::*;
195 #[test]
196 fn test_can_scan_addition_expression() {
197 let source = r#"1+1"#;
198 let scanned_tokens = scan(source).expect("1 + 1 was scanned with an error");
199 assert_eq!(scanned_tokens.len(), 3);
200 }
201
202 #[test]
203 fn test_can_scan_with_keywords() {
204 let source = r#"1 plus 1"#;
205 let scanned_tokens = scan(source).expect("1 plus 1 was scanned with an error");
206 assert_eq!(scanned_tokens.len(), 3);
207 }
208
209 #[test]
210 fn test_scanner_can_recognize_auto_capitalized_keywords() {
211 let source = r#"1 PLUS 1"#;
212 let scanned_tokens = scan(source).expect("1 PLUS 1 was scanned with an error");
213 assert_eq!(scanned_tokens.len(), 3);
214 }
215
216 #[test]
217 fn test_scanner_can_auto_suggest_keyword() {
218 let source = r#"1 plux 1"#;
219 let scanned_tokens = scan(source);
220 assert!(scanned_tokens.is_err());
221 let err = scanned_tokens.unwrap_err();
222 assert_eq!(vec!["Did you mean plus ?".to_string()], err)
223 }
224}