triviumdb 0.4.91

/// 手写词法分析器（Tokenizer / Lexer）
/// 零外部依赖，纯 Rust 迭代器实现。

#[derive(Debug, Clone, PartialEq)]
pub enum Token {
    // 关键字
    Match,
    Where,
    Return,
    And,
    Or,

    // 标识符 & 字面量
    Ident(String),
    IntLit(i64),
    FloatLit(f64),
    StringLit(String),
    BoolLit(bool),

    // 符号
    LParen,   // (
    RParen,   // )
    LBracket, // [
    RBracket, // ]
    LBrace,   // {
    RBrace,   // }
    Colon,    // :
    Dot,      // .
    Comma,    // ,
    Arrow,    // ->
    Dash,     // -

    // 运算符
    Eq,  // ==
    Ne,  // !=
    Gte, // >=
    Lte, // <=
    Gt,  // >
    Lt,  // <

    Eof,
}

pub struct Lexer {
    chars: Vec<char>,
    pos: usize,
}

impl Lexer {
    pub fn new(input: &str) -> Self {
        Self {
            chars: input.chars().collect(),
            pos: 0,
        }
    }

    fn peek(&self) -> Option<char> {
        self.chars.get(self.pos).copied()
    }

    fn advance(&mut self) -> Option<char> {
        let ch = self.chars.get(self.pos).copied();
        self.pos += 1;
        ch
    }

    fn skip_whitespace(&mut self) {
        while let Some(ch) = self.peek() {
            if ch.is_whitespace() {
                self.advance();
            } else {
                break;
            }
        }
    }

    pub fn tokenize(&mut self) -> Result<Vec<Token>, String> {
        let mut tokens = Vec::new();

        loop {
            self.skip_whitespace();

            match self.peek() {
                None => {
                    tokens.push(Token::Eof);
                    break;
                }
                Some(ch) => {
                    let tok = match ch {
                        '(' => {
                            self.advance();
                            Token::LParen
                        }
                        ')' => {
                            self.advance();
                            Token::RParen
                        }
                        '[' => {
                            self.advance();
                            Token::LBracket
                        }
                        ']' => {
                            self.advance();
                            Token::RBracket
                        }
                        '{' => {
                            self.advance();
                            Token::LBrace
                        }
                        '}' => {
                            self.advance();
                            Token::RBrace
                        }
                        ':' => {
                            self.advance();
                            Token::Colon
                        }
                        '.' => {
                            self.advance();
                            Token::Dot
                        }
                        ',' => {
                            self.advance();
                            Token::Comma
                        }

                        '-' => {
                            self.advance();
                            if self.peek() == Some('>') {
                                self.advance();
                                Token::Arrow
                            } else {
                                Token::Dash
                            }
                        }

                        '=' => {
                            self.advance();
                            if self.peek() == Some('=') {
                                self.advance();
                                Token::Eq
                            } else {
                                return Err("Expected '==' but got '='".into());
                            }
                        }

                        '!' => {
                            self.advance();
                            if self.peek() == Some('=') {
                                self.advance();
                                Token::Ne
                            } else {
                                return Err("Expected '!=' but got '!'".into());
                            }
                        }

                        '>' => {
                            self.advance();
                            if self.peek() == Some('=') {
                                self.advance();
                                Token::Gte
                            } else {
                                Token::Gt
                            }
                        }

                        '<' => {
                            self.advance();
                            if self.peek() == Some('=') {
                                self.advance();
                                Token::Lte
                            } else {
                                Token::Lt
                            }
                        }

                        '"' | '\'' => {
                            let quote = ch;
                            self.advance(); // skip opening quote
                            let mut s = String::new();
                            loop {
                                match self.advance() {
                                    Some(c) if c == quote => break,
                                    Some(c) => s.push(c),
                                    None => return Err("Unterminated string literal".into()),
                                }
                            }
                            Token::StringLit(s)
                        }

                        c if c.is_ascii_digit() => {
                            let mut num_str = String::new();
                            let mut is_float = false;
                            while let Some(c) = self.peek() {
                                if c.is_ascii_digit() {
                                    num_str.push(c);
                                    self.advance();
                                } else if c == '.' && !is_float {
                                    is_float = true;
                                    num_str.push(c);
                                    self.advance();
                                } else {
                                    break;
                                }
                            }
                            if is_float {
                                Token::FloatLit(
                                    num_str.parse().map_err(|e| format!("Bad float: {}", e))?,
                                )
                            } else {
                                Token::IntLit(
                                    num_str.parse().map_err(|e| format!("Bad int: {}", e))?,
                                )
                            }
                        }

                        c if c.is_ascii_alphabetic() || c == '_' => {
                            let mut ident = String::new();
                            while let Some(c) = self.peek() {
                                if c.is_ascii_alphanumeric() || c == '_' {
                                    ident.push(c);
                                    self.advance();
                                } else {
                                    break;
                                }
                            }
                            // 关键字匹配（大小写不敏感）
                            match ident.to_uppercase().as_str() {
                                "MATCH" => Token::Match,
                                "WHERE" => Token::Where,
                                "RETURN" => Token::Return,
                                "AND" => Token::And,
                                "OR" => Token::Or,
                                "TRUE" => Token::BoolLit(true),
                                "FALSE" => Token::BoolLit(false),
                                _ => Token::Ident(ident),
                            }
                        }

                        _ => return Err(format!("Unexpected character: '{}'", ch)),
                    };
                    tokens.push(tok);
                }
            }
        }

        Ok(tokens)
    }
}