sqlexpr-congo-rust 1.0.0

//! Lexer implementation. Generated by CongoCC Parser Generator. Do not edit.

use crate::error::{ParseError, ParseResult};
use crate::tokens::{Token, TokenType, LexicalState, TokenSource};

/// The lexer/tokenizer for SqlExprParser
pub struct Lexer {
    /// The input string being tokenized
    input: String,
    /// Current position in the input
    position: usize,
    /// Current lexical state
    state: LexicalState,
    /// All tokens generated so far
    tokens: Vec<Token>,
    /// Current line number (1-indexed)
    current_line: usize,
    /// Current column number (1-indexed)
    current_column: usize,
    /// Line start offsets for quick line/column lookup
    line_starts: Vec<usize>,
}

impl Lexer {
    /// Create a new lexer for the given input
    pub fn new(input: String) -> Self {
        let mut line_starts = vec![0];
        for (i, ch) in input.char_indices() {
            if ch == '\n' {
                line_starts.push(i + 1);
            }
        }

        Lexer {
            input,
            position: 0,
            state: LexicalState::DEFAULT,
            tokens: Vec::new(),
            current_line: 1,
            current_column: 1,
            line_starts,
        }
    }

    /// Get the next token from the input
    pub fn next_token(&mut self) -> ParseResult<Token> {
        // Skip whitespace and comments based on lexical state
        self.skip_ignored()?;

        if self.position >= self.input.len() {
            // Return EOF token
            return Ok(Token::new(
                TokenType::EOF,
                String::new(),
                self.position,
                self.position,
            ));
        }

        let start_pos = self.position;
        let start_line = self.current_line;
        let start_column = self.current_column;

        // Try to match each token type in order
        if let Some(token) = self.try_match_token(start_pos)? {
            return Ok(token);
        }

        // If no token matched, it's an error
        Err(ParseError::at_location(
            format!("Unexpected character: '{}'", self.current_char()),
            start_line,
            start_column,
        ))
    }

    /// Try to match a token at the current position
    fn try_match_token(&mut self, start_pos: usize) -> ParseResult<Option<Token>> {
        let ch = self.current_char();

        if self.matches_string("!=") {
            return Ok(Some(self.consume_literal(TokenType::NE, "!=", start_pos)));
        }
        if self.matches_string("<>") {
            return Ok(Some(self.consume_literal(TokenType::NE, "<>", start_pos)));
        }
        if self.matches_string(">=") {
            return Ok(Some(self.consume_literal(TokenType::GE, ">=", start_pos)));
        }
        if self.matches_string("<=") {
            return Ok(Some(self.consume_literal(TokenType::LE, "<=", start_pos)));
        }
        match ch {
            ' ' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::SPACE,
                    " ".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '\t' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::TAB,
                    "\t".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '\n' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::NEWLINE,
                    "\n".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '\r' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::CR,
                    "\r".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '\x0c' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::FORM_FEED,
                    "\x0c".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '=' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::EQ,
                    "=".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '>' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::GT,
                    ">".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '<' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::LT,
                    "<".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '(' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::LPAREN,
                    "(".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            ',' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::COMMA,
                    ",".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            ')' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::RPAREN,
                    ")".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '+' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::PLUS,
                    "+".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '-' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::MINUS,
                    "-".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '*' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::STAR,
                    "*".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '/' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::SLASH,
                    "/".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            '%' => {
                self.advance();
                return Ok(Some(Token::new(
                    TokenType::PERCENT,
                    "%".to_string(),
                    start_pos,
                    self.position,
                )));
            }
            _ => {}
        }

        // String literal (single-quoted)
        if ch == '\'' {
            return self.match_string_literal(start_pos);
        }

        // Numeric literals (including leading-dot floats like .5)
        if ch.is_ascii_digit()
            || (ch == '.' && self.peek(1).is_some_and(|c| c.is_ascii_digit()))
        {
            return self.match_number(start_pos);
        }

        // Identifiers and keywords
        if ch.is_ascii_alphabetic() || ch == '_' || ch == '$' {
            return self.match_identifier_or_keyword(start_pos);
        }

        // No token matched
        Ok(None)
    }

    /// Consume a literal string token
    fn consume_literal(&mut self, token_type: TokenType, literal: &str, start_pos: usize) -> Token {
        for _ in 0..literal.len() {
            self.advance();
        }
        Token::new(token_type, literal.to_string(), start_pos, self.position)
    }

    /// Match a single-quoted string literal
    fn match_string_literal(&mut self, start_pos: usize) -> ParseResult<Option<Token>> {
        // Consume opening quote
        self.advance();
        while self.position < self.input.len() {
            let ch = self.current_char();
            if ch == '\'' {
                // Check for escaped quote ('')
                if self.peek(1) == Some('\'') {
                    self.advance(); // consume first '
                    self.advance(); // consume second '
                    continue;
                }
                self.advance(); // consume closing quote
                let image = self.input[start_pos..self.position].to_string();
                return Ok(Some(Token::new(
                    TokenType::STRING_LITERAL,
                    image,
                    start_pos,
                    self.position,
                )));
            }
            self.advance();
        }
        // Unterminated string literal
        Err(ParseError::at_position(
            "Unterminated string literal".to_string(),
            start_pos,
        ))
    }

    /// Match a numeric literal (integer, hex, octal, or decimal/float)
    fn match_number(&mut self, start_pos: usize) -> ParseResult<Option<Token>> {
        // Check for hex (0x/0X) or octal (leading 0 + digits) prefix
        if self.current_char() == '0' {
            if self.peek(1).is_some_and(|ch| ch == 'x' || ch == 'X') {
                // Hex literal: 0x followed by hex digits
                self.advance(); // consume '0'
                self.advance(); // consume 'x'/'X'
                if self.position >= self.input.len() || !self.current_char().is_ascii_hexdigit() {
                    return Err(ParseError::at_position(
                        "Expected hex digit after 0x".to_string(),
                        start_pos,
                    ));
                }
                while self.position < self.input.len() && self.current_char().is_ascii_hexdigit() {
                    self.advance();
                }
                // Optional long suffix
                if self.position < self.input.len() && matches!(self.current_char(), 'L' | 'l') {
                    self.advance();
                }
                let image = self.input[start_pos..self.position].to_string();
                return Ok(Some(Token::new(
                    TokenType::HEX_LITERAL,
                    image,
                    start_pos,
                    self.position,
                )));
            }
            if self.peek(1).is_some_and(|ch| ('0'..='7').contains(&ch)) {
                // Octal literal: 0 followed by octal digits
                self.advance(); // consume leading '0'
                while self.position < self.input.len() && ('0'..='7').contains(&self.current_char()) {
                    self.advance();
                }
                // Optional long suffix
                if self.position < self.input.len() && matches!(self.current_char(), 'L' | 'l') {
                    self.advance();
                }
                let image = self.input[start_pos..self.position].to_string();
                return Ok(Some(Token::new(
                    TokenType::OCTAL_LITERAL,
                    image,
                    start_pos,
                    self.position,
                )));
            }
        }

        // Consume leading digits
        let mut is_float = false;
        while self.position < self.input.len() && self.current_char().is_ascii_digit() {
            self.advance();
        }
        // Check for decimal point followed by digits
        if self.position < self.input.len() && self.current_char() == '.'
            && self.peek(1).is_some_and(|ch| ch.is_ascii_digit())
        {
            is_float = true;
            self.advance(); // consume '.'
            while self.position < self.input.len() && self.current_char().is_ascii_digit() {
                self.advance();
            }
        }
        // Check for exponent (e/E followed by optional +/- and digits)
        if self.position < self.input.len() && matches!(self.current_char(), 'e' | 'E') {
            is_float = true;
            self.advance(); // consume 'e'/'E'
            if self.position < self.input.len() && matches!(self.current_char(), '+' | '-') {
                self.advance(); // consume sign
            }
            if self.position >= self.input.len() || !self.current_char().is_ascii_digit() {
                return Err(ParseError::at_position(
                    "Expected digit in exponent".to_string(),
                    start_pos,
                ));
            }
            while self.position < self.input.len() && self.current_char().is_ascii_digit() {
                self.advance();
            }
        }
        if is_float {
            let image = self.input[start_pos..self.position].to_string();
            return Ok(Some(Token::new(
                TokenType::FLOATING_POINT_LITERAL,
                image,
                start_pos,
                self.position,
            )));
        }
        // Optional long suffix for integer literals
        if self.position < self.input.len() && matches!(self.current_char(), 'L' | 'l') {
            self.advance();
        }
        let image = self.input[start_pos..self.position].to_string();
        Ok(Some(Token::new(
            TokenType::DECIMAL_LITERAL,
            image,
            start_pos,
            self.position,
        )))
    }

    /// Match an identifier or keyword
    fn match_identifier_or_keyword(&mut self, start_pos: usize) -> ParseResult<Option<Token>> {
        // Consume identifier characters
        while self.position < self.input.len() {
            let ch = self.current_char();
            if ch.is_ascii_alphanumeric() || ch == '_' || ch == '$' {
                self.advance();
            } else {
                break;
            }
        }
        let image = self.input[start_pos..self.position].to_string();
        let upper = image.to_ascii_uppercase();

        // Check against keywords (case-insensitive)
        let token_type = match upper.as_str() {
            "NOT" => TokenType::NOT,
            "AND" => TokenType::AND,
            "OR" => TokenType::OR,
            "BETWEEN" => TokenType::BETWEEN,
            "LIKE" => TokenType::LIKE,
            "ESCAPE" => TokenType::ESCAPE,
            "IN" => TokenType::IN,
            "IS" => TokenType::IS,
            "TRUE" => TokenType::TRUE,
            "FALSE" => TokenType::FALSE,
            "NULL" => TokenType::NULL,
            _ => TokenType::ID,
        };

        Ok(Some(Token::new(token_type, image, start_pos, self.position)))
    }

    /// Skip whitespace and ignored tokens
    fn skip_ignored(&mut self) -> ParseResult<()> {
        while self.position < self.input.len() {
            let ch = self.current_char();

            // Skip whitespace
            if ch.is_whitespace() {
                self.advance();
                continue;
            }

            // Skip line comments: -- to end of line
            if ch == '-' && self.peek(1) == Some('-') {
                self.advance(); // consume first -
                self.advance(); // consume second -
                while self.position < self.input.len() && self.current_char() != '\n' {
                    self.advance();
                }
                continue;
            }

            // Skip block comments: /* ... */
            if ch == '/' && self.peek(1) == Some('*') {
                let start_pos = self.position;
                self.advance(); // consume /
                self.advance(); // consume *
                loop {
                    if self.position >= self.input.len() {
                        return Err(ParseError::at_position(
                            "Unterminated block comment".to_string(),
                            start_pos,
                        ));
                    }
                    if self.current_char() == '*' && self.peek(1) == Some('/') {
                        self.advance(); // consume *
                        self.advance(); // consume /
                        break;
                    }
                    self.advance();
                }
                continue;
            }

            break;
        }
        Ok(())
    }

    /// Get the current character without consuming it
    fn current_char(&self) -> char {
        self.input[self.position..].chars().next().unwrap_or('\0')
    }

    /// Advance to the next character
    fn advance(&mut self) {
        if self.position < self.input.len() {
            let ch = self.current_char();
            self.position += ch.len_utf8();

            if ch == '\n' {
                self.current_line += 1;
                self.current_column = 1;
            } else {
                self.current_column += 1;
            }
        }
    }

    /// Peek ahead n characters without consuming
    fn peek(&self, n: usize) -> Option<char> {
        self.input[self.position..].chars().nth(n)
    }

    /// Check if current position matches a string
    fn matches_string(&self, s: &str) -> bool {
        self.input[self.position..].starts_with(s)
    }
}

impl TokenSource for Lexer {
    fn get_line_from_offset(&self, offset: usize) -> usize {
        // Binary search for the line containing this offset
        match self.line_starts.binary_search(&offset) {
            Ok(line) => line + 1,
            Err(line) => line,
        }
    }

    fn get_column_from_offset(&self, offset: usize) -> usize {
        let line_num = self.get_line_from_offset(offset);
        if line_num == 0 || line_num > self.line_starts.len() {
            return 1;
        }

        let line_start = self.line_starts[line_num - 1];
        offset.saturating_sub(line_start) + 1
    }
}