ddb_core/lexer/
tokenizer.rs

1use super::types::{Token, TokenType};
2use crate::error::{DdbError, Result};
3use nom::{
4    branch::alt,
5    bytes::complete::{is_not, tag, take_while},
6    character::complete::{alpha1, alphanumeric1, char, digit1, multispace1},
7    combinator::{map, opt, recognize, value},
8    multi::many0,
9    sequence::{delimited, pair, preceded},
10    IResult,
11};
12
13/// Tokenizer for SQL queries
14pub struct Tokenizer {
15    position: usize,
16    line: usize,
17    column: usize,
18}
19
20impl Tokenizer {
21    pub fn new() -> Self {
22        Self {
23            position: 0,
24            line: 1,
25            column: 1,
26        }
27    }
28
29    /// Tokenize a SQL query string
30    pub fn tokenize(&mut self, input: &str) -> Result<Vec<Token>> {
31        let mut tokens = Vec::new();
32        let mut remaining = input;
33
34        while !remaining.is_empty() {
35            // Try to parse a token
36            match self.parse_token(remaining) {
37                Ok((rest, token_type)) => {
38                    let consumed = remaining.len() - rest.len();
39
40                    // Skip whitespace tokens unless we need them
41                    if !matches!(token_type, TokenType::Whitespace) {
42                        let token = Token::new(
43                            token_type,
44                            self.position,
45                            self.line,
46                            self.column,
47                        );
48                        tokens.push(token);
49                    }
50
51                    // Update position tracking
52                    for ch in remaining[..consumed].chars() {
53                        self.position += 1;
54                        if ch == '\n' {
55                            self.line += 1;
56                            self.column = 1;
57                        } else {
58                            self.column += 1;
59                        }
60                    }
61
62                    remaining = rest;
63                }
64                Err(_) => {
65                    return Err(DdbError::ParseError(format!(
66                        "Unexpected character at line {}, column {}: '{}'",
67                        self.line,
68                        self.column,
69                        remaining.chars().next().unwrap_or(' ')
70                    )));
71                }
72            }
73        }
74
75        // Add EOF token
76        tokens.push(Token::new(TokenType::Eof, self.position, self.line, self.column));
77
78        Ok(tokens)
79    }
80
81    fn parse_token<'a>(&self, input: &'a str) -> IResult<&'a str, TokenType> {
82        alt((
83            parse_whitespace,
84            parse_comment,
85            parse_multi_word_keyword,
86            parse_keyword,
87            parse_operator,
88            parse_delimiter,
89            parse_string,
90            parse_number,
91            parse_identifier,
92        ))(input)
93    }
94}
95
96// Free functions for nom parsers
97fn parse_whitespace(input: &str) -> IResult<&str, TokenType> {
98    map(multispace1, |_| TokenType::Whitespace)(input)
99}
100
101fn parse_comment(input: &str) -> IResult<&str, TokenType> {
102    alt((
103        // Line comment: -- comment
104        map(
105            preceded(tag("--"), is_not("\n")),
106            |s: &str| TokenType::Comment(s.to_string())
107        ),
108        // Block comment: /* comment */
109        map(
110            delimited(tag("/*"), is_not("*/"), tag("*/")),
111            |s: &str| TokenType::Comment(s.to_string())
112        ),
113    ))(input)
114}
115
116fn parse_keyword(input: &str) -> IResult<&str, TokenType> {
117    // Try to match an identifier first
118    let (remaining, word) = recognize(pair(
119        alt((alpha1, tag("_"))),
120        many0(alt((alphanumeric1, tag("_"))))
121    ))(input)?;
122
123    // Check for word boundary
124    if let Some(ch) = remaining.chars().next() {
125        if ch.is_alphanumeric() || ch == '_' {
126            return Err(nom::Err::Error(nom::error::Error::new(
127                input,
128                nom::error::ErrorKind::Tag,
129            )));
130        }
131    }
132
133    // Convert to lowercase and match against keywords
134    let token_type = match word.to_lowercase().as_str() {
135        "select" => TokenType::Select,
136        "insert" => TokenType::Insert,
137        "update" => TokenType::Update,
138        "delete" => TokenType::Delete,
139        "create" => TokenType::Create,
140        "drop" => TokenType::Drop,
141        "use" => TokenType::Use,
142        "show" => TokenType::Show,
143        "set" => TokenType::Set,
144        "from" => TokenType::From,
145        "where" => TokenType::Where,
146        "limit" => TokenType::Limit,
147        "distinct" => TokenType::Distinct,
148        "as" => TokenType::As,
149        "into" => TokenType::Into,
150        "values" => TokenType::Values,
151        "table" => TokenType::Table,
152        "tables" => TokenType::Tables,
153        "columns" => TokenType::Columns,
154        "database" => TokenType::Database,
155        "begin" => TokenType::Begin,
156        "commit" => TokenType::Commit,
157        "rollback" => TokenType::Rollback,
158        "describe" => TokenType::Describe,
159        "and" => TokenType::And,
160        "or" => TokenType::Or,
161        "not" => TokenType::Not,
162        "like" => TokenType::Like,
163        "is" => TokenType::Is,
164        "null" => TokenType::Null,
165        "true" => TokenType::True,
166        "false" => TokenType::False,
167        "in" => TokenType::In,
168        "between" => TokenType::Between,
169        "asc" => TokenType::Asc,
170        "desc" => TokenType::Desc,
171        "order" => TokenType::Order,
172        "by" => TokenType::By,
173        "file" => TokenType::File,
174        "delimiter" => TokenType::Delimiter,
175        "variables" => TokenType::Variables,
176        "offset" => TokenType::Offset,
177        _ => {
178            // Not a keyword, fail to match
179            return Err(nom::Err::Error(nom::error::Error::new(
180                input,
181                nom::error::ErrorKind::Tag,
182            )));
183        }
184    };
185
186    Ok((remaining, token_type))
187}
188
189fn parse_multi_word_keyword(input: &str) -> IResult<&str, TokenType> {
190    alt((
191        map(
192            alt((tag("ORDER BY"), tag("order by"), tag("Order By"))),
193            |_| TokenType::OrderBy
194        ),
195        map(
196            alt((tag("GROUP BY"), tag("group by"), tag("Group By"))),
197            |_| TokenType::GroupBy
198        ),
199    ))(input)
200}
201
202fn parse_operator(input: &str) -> IResult<&str, TokenType> {
203    alt((
204        value(TokenType::LessEqual, tag("<=")),
205        value(TokenType::GreaterEqual, tag(">=")),
206        value(TokenType::NotEqual, alt((tag("!="), tag("<>")))),
207        value(TokenType::Equal, tag("=")),
208        value(TokenType::LessThan, tag("<")),
209        value(TokenType::GreaterThan, tag(">")),
210    ))(input)
211}
212
213fn parse_delimiter(input: &str) -> IResult<&str, TokenType> {
214    alt((
215        value(TokenType::Comma, char(',')),
216        value(TokenType::Semicolon, char(';')),
217        value(TokenType::LeftParen, char('(')),
218        value(TokenType::RightParen, char(')')),
219        value(TokenType::LeftBracket, char('[')),
220        value(TokenType::RightBracket, char(']')),
221        value(TokenType::Dot, char('.')),
222        value(TokenType::Star, char('*')),
223    ))(input)
224}
225
226fn parse_string(input: &str) -> IResult<&str, TokenType> {
227    alt((
228        // Single-quoted string
229        map(
230            delimited(char('\''), take_while(|c| c != '\''), char('\'')),
231            |s: &str| TokenType::String(s.to_string())
232        ),
233        // Double-quoted string
234        map(
235            delimited(char('"'), take_while(|c| c != '"'), char('"')),
236            |s: &str| TokenType::String(s.to_string())
237        ),
238    ))(input)
239}
240
241fn parse_number(input: &str) -> IResult<&str, TokenType> {
242    map(
243        recognize(pair(
244            digit1,
245            opt(pair(char('.'), digit1))
246        )),
247        |s: &str| TokenType::Number(s.to_string())
248    )(input)
249}
250
251fn parse_identifier(input: &str) -> IResult<&str, TokenType> {
252    map(
253        recognize(pair(
254            alt((
255                tag("@@"),  // System variables (@@VERSION, @@ROWCOUNT, etc.)
256                alpha1,
257                tag("_")
258            )),
259            many0(alt((alphanumeric1, tag("_"))))
260        )),
261        |s: &str| TokenType::Identifier(s.to_string())
262    )(input)
263}
264
265impl Default for Tokenizer {
266    fn default() -> Self {
267        Self::new()
268    }
269}
270
271#[cfg(test)]
272mod tests {
273    use super::*;
274
275    #[test]
276    fn test_tokenize_select() {
277        let mut tokenizer = Tokenizer::new();
278        let tokens = tokenizer.tokenize("SELECT * FROM users").unwrap();
279
280        assert_eq!(tokens[0].token_type, TokenType::Select);
281        assert_eq!(tokens[1].token_type, TokenType::Star);
282        assert_eq!(tokens[2].token_type, TokenType::From);
283        assert!(matches!(tokens[3].token_type, TokenType::Identifier(_)));
284    }
285
286    #[test]
287    fn test_tokenize_where_clause() {
288        let mut tokenizer = Tokenizer::new();
289        let tokens = tokenizer.tokenize("WHERE id = 123 AND name LIKE 'test%'").unwrap();
290
291        assert_eq!(tokens[0].token_type, TokenType::Where);
292        assert!(matches!(tokens[1].token_type, TokenType::Identifier(_)));
293        assert_eq!(tokens[2].token_type, TokenType::Equal);
294        assert!(matches!(tokens[3].token_type, TokenType::Number(_)));
295    }
296
297    #[test]
298    fn test_tokenize_strings() {
299        let mut tokenizer = Tokenizer::new();
300        let tokens = tokenizer.tokenize("'single' \"double\"").unwrap();
301
302        assert!(matches!(tokens[0].token_type, TokenType::String(ref s) if s == "single"));
303        assert!(matches!(tokens[1].token_type, TokenType::String(ref s) if s == "double"));
304    }
305}
ddb_core/lexer/tokenizer.rs

ddb_core/lexer/
tokenizer.rs
