Function code_tokenizer::get_tokens

pub fn get_tokens(input: String, operators: Vec<String>) -> Vec<String>

Creates a vector of tokens from a String and a vector of operators.

Examples

This code :

let operators = vec![String::from("+"), String::from("+="), String::from("===")];
let tokens = get_tokens(String::from("0 +1+= 2\n6=== 7 \n"), operators.clone());
for s in tokens.iter() {
    println!("{}", s);
}

Will print :

0
+
+
+=
2
6
===
7

This code :

let operators: Vec<String> = Vec::new();
let tokens = get_tokens(String::from("a b \"c d e\" f g"), operators.clone());
for s in tokens.iter() {
    println!("{}", s);
}

Will print :

a
b
"
c d e
"
f
g