code-tokenizer 0.1.0

//! # Code Tokenizer
//!
//! A string/code tokenizer. Transforms a string with a vector of operators into a vector of keywords.

/// Creates a vector of tokens from a String and a vector of operators.
///
/// # Examples
/// This code :
/// ```
/// let operators = vec![String::from("+"), String::from("+="), String::from("===")];
/// let tokens = get_tokens(String::from("0 +1+= 2\n6=== 7 \n"), operators.clone());
/// for s in tokens.iter() {
///     println!("{}", s);
/// }
/// ```
/// Will print :
/// ```
/// 0
/// +
/// +
/// +=
/// 2
/// 6
/// ===
/// 7
/// ```
///
/// This code :
/// ```
/// let operators: Vec<String> = Vec::new();
/// let tokens = get_tokens(String::from("a b \"c d e\" f g"), operators.clone());
/// for s in tokens.iter() {
///     println!("{}", s);
/// }
/// ```
/// Will print :
/// ```
/// a
/// b
/// "
/// c d e
/// "
/// f
/// g
/// ```
pub fn get_tokens(input: String, operators: Vec<String>) -> Vec<String> {
    let mut result: Vec<String> = Vec::new();
    let mut new_string = false;
    let mut quote = false;
    let mut buffer: String = String::from("");
    let mut to_skip = 0;

    for i in 0..input.len() {
        if to_skip > 0 {
            to_skip -= 1;
            continue;
        }
        let c = input.chars().nth(i).unwrap();
        if c == '"' || c == '\'' {
            quote = !quote;
            new_string = true;
            if buffer != String::from("") {
                result.push(buffer.clone());
            }
            buffer = String::from("");
            if quote {
                result.push(String::from(c.to_string()));
            }
        }

        if !quote {
            if c == '\n' || c == ' ' || c == '\t' {
                if !new_string {
                    new_string = true;
                    result.push(buffer.clone());
                    buffer = String::from("");
                }
            } else {
                let mut is_operator: bool;
                let mut match_operator = false;
                let mut op: String = String::from("");

                for s in operators.iter() {
                    is_operator = true;
                    for j in 0..s.len() {
                        if s.chars().nth(j) != input.chars().nth(i + j) {
                            is_operator = false;
                        }
                    }
                    if is_operator {
                        if s.len() > op.len() {
                            match_operator = true;
                            op = s.clone();
                        }
                    }
                }
                if !match_operator {
                    new_string = false;
                    buffer.push(c.clone());
                }
                if match_operator {
                    if !new_string {
                        result.push(buffer.clone());
                    }
                    buffer = String::from("");
                    new_string = true;
                    result.push(op.clone());
                    to_skip += op.len() - 1;
                }
            }
        } else {
            if !(c == '"' || c == '\'') {
                buffer.push(c.clone());
            }
        }
    }
    if !new_string {
        result.push(buffer.clone());
    }

    return result;
}