sqlite_fsr/command/sql/parser/
sql_token.rs

1#[derive(Debug)]
2pub enum SQLToken {
3    Keyword(String),
4    Identifier(String),
5    Symbol(Symbol)
6}
7
8#[derive(Debug)]
9pub enum Symbol {
10    LeftParenthesis,
11    RightParenthesis,
12    Comma,
13    Semicolon
14}
15
16impl Symbol {
17    pub fn as_str(&self) -> &'static str {
18        match self {
19            Symbol::LeftParenthesis => "(",
20            Symbol::RightParenthesis => ")",
21            Symbol::Comma => ",",
22            Symbol::Semicolon => ";"
23        }
24    }
25
26    pub const ALL: [Symbol; 4] = [
27        Symbol::LeftParenthesis,
28        Symbol::RightParenthesis,
29        Symbol::Comma,
30        Symbol::Semicolon,
31    ];
32}
33
34
35
36pub trait Tokenize {
37    fn tokenize(&self) -> Vec<SQLToken>;
38} 
39impl Tokenize for str {
40    fn tokenize(&self) -> Vec<SQLToken> {
41        let mut tokens: Vec<SQLToken> = Vec::new();
42        let binding = self.trim().replace("\n", " ");
43        let components = binding.split_whitespace();
44        for component in components.into_iter() {
45            let mut component_tokens = match component {
46                                        s if (s == "SELECT") | (s == "CREATE") | (s == "FROM") => vec![SQLToken::Keyword((*s).to_string())],
47                                        s if s.len() == 1 => vec![s.chars().next().unwrap().to_sql_token()], //this is ugly as hell but rust std doesnt support character indexing🤷🏾‍♂️ 
48                                        _ => tokenize_component(component)
49                                        };
50            tokens.append(&mut component_tokens);
51        }
52        return tokens;
53    }
54}
55
56pub trait ToSQLToken {
57    fn to_sql_token(&self) -> SQLToken;
58} 
59impl ToSQLToken for char {
60    fn to_sql_token(&self) -> SQLToken {
61        let token = match self {
62                        '(' => SQLToken::Symbol(Symbol::LeftParenthesis),
63                        ')' => SQLToken::Symbol(Symbol::RightParenthesis),
64                        ',' => SQLToken::Symbol(Symbol::Comma),
65                        ';' => SQLToken::Symbol(Symbol::Semicolon),
66                        _ => SQLToken::Identifier(self.to_string())
67                    };
68        return token;
69    }
70}
71
72pub fn tokenize_component(value: &str) -> Vec<SQLToken> {
73    let mut tokens: Vec<SQLToken> = Vec::new();
74    let mut current = String::new();
75
76    for character in value.chars() {
77        // Check if character is any known symbol
78        if Symbol::ALL.iter().any(|s| s.as_str() == character.to_string()) {
79            if !current.is_empty() {
80                tokens.push(SQLToken::Identifier(current.clone()));
81                current.clear();
82            }
83            tokens.push(character.to_sql_token());
84        } else {
85            current.push(character);
86        }
87    }
88
89    if !current.is_empty() {
90        tokens.push(SQLToken::Identifier(current));
91    }
92
93    
94
95    return tokens
96}
sqlite_fsr/command/sql/parser/sql_token.rs

sqlite_fsr/command/sql/parser/
sql_token.rs
