code_tokenizer/
lib.rs

1//! # Code Tokenizer
2//!
3//! A string/code tokenizer. Transforms a string with a vector of operators into a vector of keywords.
4
5/// Creates a vector of tokens from a String and a vector of operators.
6///
7/// # Examples
8/// This code :
9/// ```
10/// let operators = vec![String::from("+"), String::from("+="), String::from("===")];
11/// let tokens = get_tokens(String::from("0 +1+= 2\n6=== 7 \n"), operators.clone());
12/// for s in tokens.iter() {
13///     println!("{}", s);
14/// }
15/// ```
16/// Will print :
17/// ```
18/// 0
19/// +
20/// +
21/// +=
22/// 2
23/// 6
24/// ===
25/// 7
26/// ```
27///
28/// This code :
29/// ```
30/// let operators: Vec<String> = Vec::new();
31/// let tokens = get_tokens(String::from("a b \"c d e\" f g"), operators.clone());
32/// for s in tokens.iter() {
33///     println!("{}", s);
34/// }
35/// ```
36/// Will print :
37/// ```
38/// a
39/// b
40/// "
41/// c d e
42/// "
43/// f
44/// g
45/// ```
46pub fn get_tokens(input: String, operators: Vec<String>) -> Vec<String> {
47    let mut result: Vec<String> = Vec::new();
48    let mut new_string = false;
49    let mut quote = false;
50    let mut buffer: String = String::from("");
51    let mut to_skip = 0;
52
53    for i in 0..input.len() {
54        if to_skip > 0 {
55            to_skip -= 1;
56            continue;
57        }
58        let c = input.chars().nth(i).unwrap();
59        if c == '"' || c == '\'' {
60            quote = !quote;
61            new_string = true;
62            if buffer != String::from("") {
63                result.push(buffer.clone());
64            }
65            buffer = String::from("");
66            if quote {
67                result.push(String::from(c.to_string()));
68            }
69        }
70
71        if !quote {
72            if c == '\n' || c == ' ' || c == '\t' {
73                if !new_string {
74                    new_string = true;
75                    result.push(buffer.clone());
76                    buffer = String::from("");
77                }
78            } else {
79                let mut is_operator: bool;
80                let mut match_operator = false;
81                let mut op: String = String::from("");
82
83                for s in operators.iter() {
84                    is_operator = true;
85                    for j in 0..s.len() {
86                        if s.chars().nth(j) != input.chars().nth(i + j) {
87                            is_operator = false;
88                        }
89                    }
90                    if is_operator {
91                        if s.len() > op.len() {
92                            match_operator = true;
93                            op = s.clone();
94                        }
95                    }
96                }
97                if !match_operator {
98                    new_string = false;
99                    buffer.push(c.clone());
100                }
101                if match_operator {
102                    if !new_string {
103                        result.push(buffer.clone());
104                    }
105                    buffer = String::from("");
106                    new_string = true;
107                    result.push(op.clone());
108                    to_skip += op.len() - 1;
109                }
110            }
111        } else {
112            if !(c == '"' || c == '\'') {
113                buffer.push(c.clone());
114            }
115        }
116    }
117    if !new_string {
118        result.push(buffer.clone());
119    }
120
121    return result;
122}
code_tokenizer/lib.rs

code_tokenizer/
lib.rs
