1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

//! # Code Tokenizer
//!
//! A string/code tokenizer. Transforms a string with a vector of operators into a vector of keywords.

/// Creates a vector of tokens from a String and a vector of operators.
///
/// # Examples
/// This code :
/// ```
/// let operators = vec![String::from("+"), String::from("+="), String::from("===")];
/// let tokens = get_tokens(String::from("0 +1+= 2\n6=== 7 \n"), operators.clone());
/// for s in tokens.iter() {
///     println!("{}", s);
/// }
/// ```
/// Will print :
/// ```
/// 0
/// +
/// +
/// +=
/// 2
/// 6
/// ===
/// 7
/// ```
///
/// This code :
/// ```
/// let operators: Vec<String> = Vec::new();
/// let tokens = get_tokens(String::from("a b \"c d e\" f g"), operators.clone());
/// for s in tokens.iter() {
///     println!("{}", s);
/// }
/// ```
/// Will print :
/// ```
/// a
/// b
/// "
/// c d e
/// "
/// f
/// g
/// ```
pub fn get_tokens(input: String, operators: Vec<String>) -> Vec<String> {
    let mut result: Vec<String> = Vec::new();
    let mut new_string = false;
    let mut quote = false;
    let mut buffer: String = String::from("");
    let mut to_skip = 0;

    for i in 0..input.len() {
        if to_skip > 0 {
            to_skip -= 1;
            continue;
        }
        let c = input.chars().nth(i).unwrap();
        if c == '"' || c == '\'' {
            quote = !quote;
            new_string = true;
            if buffer != String::from("") {
                result.push(buffer.clone());
            }
            buffer = String::from("");
            if quote {
                result.push(String::from(c.to_string()));
            }
        }

        if !quote {
            if c == '\n' || c == ' ' || c == '\t' {
                if !new_string {
                    new_string = true;
                    result.push(buffer.clone());
                    buffer = String::from("");
                }
            } else {
                let mut is_operator: bool;
                let mut match_operator = false;
                let mut op: String = String::from("");

                for s in operators.iter() {
                    is_operator = true;
                    for j in 0..s.len() {
                        if s.chars().nth(j) != input.chars().nth(i + j) {
                            is_operator = false;
                        }
                    }
                    if is_operator {
                        if s.len() > op.len() {
                            match_operator = true;
                            op = s.clone();
                        }
                    }
                }
                if !match_operator {
                    new_string = false;
                    buffer.push(c.clone());
                }
                if match_operator {
                    if !new_string {
                        result.push(buffer.clone());
                    }
                    buffer = String::from("");
                    new_string = true;
                    result.push(op.clone());
                    to_skip += op.len() - 1;
                }
            }
        } else {
            if !(c == '"' || c == '\'') {
                buffer.push(c.clone());
            }
        }
    }
    if !new_string {
        result.push(buffer.clone());
    }

    return result;
}