Function specmc_base::tokenize::tokenize

source ยท 
pub fn tokenize(input: &str) -> Vec<String>
Expand description

Split a string into tokens. Special characters will be included in tokens. However, whitespace will not.

Examples found in repository?
examples/example.rs (line 9)
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

fn main() {
    use specmc_base::{
        parse::{Identifier, Literal},
        tokenize::tokenize,
    };

    let mut tokens = tokenize("true 42 123.0 \"string\" cool_identifier");
    tokens.reverse();
    let lit_bool = Literal::parse(&mut tokens).unwrap();
    let lit_int = Literal::parse(&mut tokens).unwrap();
    let lit_float = Literal::parse(&mut tokens).unwrap();
    let lit_str = Literal::parse(&mut tokens).unwrap();
    let ident = Identifier::parse(&mut tokens).unwrap();
    println!("{lit_bool:?} {lit_int:?} {lit_float:?} {lit_str:?} {ident:?}");
}