sqlexpr-congo-rust 1.0.0

//! Token types and definitions. Generated by CongoCC Parser Generator. Do not edit.

use std::fmt;

/// Token type enumeration
///
/// Represents all possible token types recognized by the lexer.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[allow(non_camel_case_types)]
pub enum TokenType {
    /// End of file
    EOF,
    /// Whitespace: space
    SPACE,
    /// Whitespace: tab
    TAB,
    /// Whitespace: newline
    NEWLINE,
    /// Whitespace: carriage return
    CR,
    /// Whitespace: form feed
    FORM_FEED,
    /// Keyword: NOT
    NOT,
    /// Keyword: AND
    AND,
    /// Keyword: OR
    OR,
    /// Keyword: BETWEEN
    BETWEEN,
    /// Keyword: LIKE
    LIKE,
    /// Keyword: ESCAPE
    ESCAPE,
    /// Keyword: IN
    IN,
    /// Keyword: IS
    IS,
    /// Keyword: TRUE
    TRUE,
    /// Keyword: FALSE
    FALSE,
    /// Keyword: NULL
    NULL,
    /// Operator: = (equals)
    EQ,
    /// Operator: <> or != (not equals)
    NE,
    /// Operator: > (greater than)
    GT,
    /// Operator: >= (greater than or equal)
    GE,
    /// Operator: < (less than)
    LT,
    /// Operator: <= (less than or equal)
    LE,
    /// Delimiter: ( (left parenthesis)
    LPAREN,
    /// Delimiter: , (comma)
    COMMA,
    /// Delimiter: ) (right parenthesis)
    RPAREN,
    /// Operator: + (plus)
    PLUS,
    /// Operator: - (minus)
    MINUS,
    /// Operator: * (multiply)
    STAR,
    /// Operator: / (divide)
    SLASH,
    /// Operator: % (modulo)
    PERCENT,
    /// Line comment
    LINE_COMMENT,
    /// Block comment
    BLOCK_COMMENT,
    /// Decimal numeric literal
    DECIMAL_LITERAL,
    /// Hexadecimal numeric literal
    HEX_LITERAL,
    /// Octal numeric literal
    OCTAL_LITERAL,
    /// Floating point numeric literal
    FLOATING_POINT_LITERAL,
    /// String literal (single-quoted)
    STRING_LITERAL,
    /// Identifier
    ID,
    /// Invalid token marker
    INVALID,
}

impl fmt::Display for TokenType {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            TokenType::EOF => write!(f, "EOF"),
            TokenType::SPACE => write!(f, "SPACE"),
            TokenType::TAB => write!(f, "TAB"),
            TokenType::NEWLINE => write!(f, "NEWLINE"),
            TokenType::CR => write!(f, "CR"),
            TokenType::FORM_FEED => write!(f, "FORM_FEED"),
            TokenType::NOT => write!(f, "NOT"),
            TokenType::AND => write!(f, "AND"),
            TokenType::OR => write!(f, "OR"),
            TokenType::BETWEEN => write!(f, "BETWEEN"),
            TokenType::LIKE => write!(f, "LIKE"),
            TokenType::ESCAPE => write!(f, "ESCAPE"),
            TokenType::IN => write!(f, "IN"),
            TokenType::IS => write!(f, "IS"),
            TokenType::TRUE => write!(f, "TRUE"),
            TokenType::FALSE => write!(f, "FALSE"),
            TokenType::NULL => write!(f, "NULL"),
            TokenType::EQ => write!(f, "="),
            TokenType::NE => write!(f, "<>"),
            TokenType::GT => write!(f, ">"),
            TokenType::GE => write!(f, ">="),
            TokenType::LT => write!(f, "<"),
            TokenType::LE => write!(f, "<="),
            TokenType::LPAREN => write!(f, "("),
            TokenType::COMMA => write!(f, ","),
            TokenType::RPAREN => write!(f, ")"),
            TokenType::PLUS => write!(f, "+"),
            TokenType::MINUS => write!(f, "-"),
            TokenType::STAR => write!(f, "*"),
            TokenType::SLASH => write!(f, "/"),
            TokenType::PERCENT => write!(f, "%"),
            TokenType::LINE_COMMENT => write!(f, "LINE_COMMENT"),
            TokenType::BLOCK_COMMENT => write!(f, "BLOCK_COMMENT"),
            TokenType::DECIMAL_LITERAL => write!(f, "DECIMAL_LITERAL"),
            TokenType::HEX_LITERAL => write!(f, "HEX_LITERAL"),
            TokenType::OCTAL_LITERAL => write!(f, "OCTAL_LITERAL"),
            TokenType::FLOATING_POINT_LITERAL => write!(f, "FLOATING_POINT_LITERAL"),
            TokenType::STRING_LITERAL => write!(f, "STRING_LITERAL"),
            TokenType::ID => write!(f, "ID"),
            TokenType::INVALID => write!(f, "INVALID"),
        }
    }
}

/// Lexical state enumeration
///
/// Represents different lexical states the lexer can be in.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum LexicalState {
    /// Lexical state: DEFAULT
    DEFAULT,
}

/// A single token in the input stream
///
/// Tokens are the atomic units produced by the lexer. Each token has:
/// - A type (e.g., INTEGER, PLUS, EOF)
/// - An image (the actual text matched)
/// - Position information (begin/end offsets)
/// - Optional links to previous/next tokens
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct Token {
    /// The type of this token
    pub token_type: TokenType,
    /// The actual text of this token
    pub image: String,
    /// Absolute offset in the input where this token begins
    pub begin_offset: usize,
    /// Absolute offset in the input where this token ends
    pub end_offset: usize,
    /// Index of the next token (if any)
    pub next: Option<usize>,
    /// Index of the previous token (if any)
    pub previous: Option<usize>,
}

impl Token {
    /// Create a new token
    pub fn new(
        token_type: TokenType,
        image: String,
        begin_offset: usize,
        end_offset: usize,
    ) -> Self {
        Token {
            token_type,
            image,
            begin_offset,
            end_offset,
            next: None,
            previous: None,
        }
    }

    /// Get the length of this token in characters
    pub fn len(&self) -> usize {
        self.end_offset.saturating_sub(self.begin_offset)
    }

    /// Check if this token is empty
    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }

    /// Check if this token is of a specific type
    pub fn is_type(&self, token_type: TokenType) -> bool {
        self.token_type == token_type
    }

    /// Check if this token is one of the specified types
    pub fn is_one_of(&self, types: &[TokenType]) -> bool {
        types.contains(&self.token_type)
    }
}

impl fmt::Display for Token {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}: \"{}\"", self.token_type, self.image)
    }
}

/// Token source trait for tracking token locations
pub trait TokenSource {
    /// Get the line number for a given offset
    fn get_line_from_offset(&self, offset: usize) -> usize;

    /// Get the column number for a given offset
    fn get_column_from_offset(&self, offset: usize) -> usize;
}