pub fn tokenize(input: &str) -> Vec<String>Expand description
Split a string into tokens. Special characters will be included in tokens. However, whitespace will not.
Examples found in repository?
examples/example.rs (line 7)
6fn main() {
7 let mut tokens: Vec<String> = tokenize("true 42 123.0 \"string\" cool_identifier");
8 tokens.reverse();
9
10 let lit_bool: Literal = Literal::parse(&mut tokens).unwrap();
11 let lit_int: Literal = Literal::parse(&mut tokens).unwrap();
12 let lit_float: Literal = Literal::parse(&mut tokens).unwrap();
13 let lit_str: Literal = Literal::parse(&mut tokens).unwrap();
14 let ident: Identifier = Identifier::parse(&mut tokens).unwrap();
15
16 println!("{lit_bool:?} {lit_int:?} {lit_float:?} {lit_str:?} {ident:?}");
17}