pipeline_script/lexer/

mod.rs

pub mod iter;
pub mod position;
pub mod token;

use crate::lexer::position::Position;
use crate::lexer::token::Token;
use std::ops::Add;

/// 词法分析器状态
#[derive(Debug, Clone, PartialEq)]
enum State {
    /// 初始状态
    Initial,
    /// 识别标识符状态
    Identifier,
    /// 识别数字状态
    Number,
    /// 识别小数状态
    Decimal,
    /// 识别字符串状态
    String,
    /// 识别格式化字符串状态
    FormatString,
    /// 识别注释状态
    Comment,
    /// 识别多行注释状态
    MultilineComment,
    /// 完成状态，带有类型信息
    Done(TokenType),
}
/// 词法单元类型
#[derive(Debug, Clone, Copy, PartialEq)]
enum TokenType {
    /// 标识符
    Identifier,
    /// 整数
    Integer,
    /// 浮点数
    Float,
    /// 运算符或分隔符
    Symbol,
}

#[derive(Debug, Clone)]
pub struct Lexer {
    file_name: String,
    chars: Vec<char>,
    index: usize,
    col: usize,
    row: usize,
    keywords: Vec<&'static str>,
    /// 当前状态
    state: State,
    /// 当前词素缓冲区
    buffer: String,
    /// 开始位置
    start_pos: usize,
    start_col: usize,
    start_row: usize,
}

// impl IntoIterator for Lexer {
//     type Item = (Token, Position);
//     type IntoIter = TokenStream;
//
//     fn into_iter(self) -> Self::IntoIter {
//         TokenStream::new(self)
//     }
// }
impl Lexer {
    pub fn is_eof(&self) -> bool {
        self.index >= self.chars.len()
    }
    pub fn new(file_name: impl Into<String>) -> Self {
        Self {
            chars: vec![],
            index: 0,
            col: 1,
            row: 1,
            file_name: file_name.into(),
            keywords: vec![
                "let", "fn", "fun", "return", "if", "while", "import", "else", "val", "var",
                "break", "continue", "for", "in", "class", "static", "trait", "struct", "extern",
                "module", "const", "enum", "match",
            ],
            state: State::Initial,
            buffer: String::new(),
            start_pos: 0,
            start_col: 1,
            start_row: 1,
        }
    }
    #[allow(unused)]
    pub fn get_file_name(&self) -> String {
        self.file_name.clone()
    }
    #[allow(unused)]
    pub fn line(&self, line: usize) -> String {
        let mut s = String::new();
        let mut current_line = 1;
        for i in &self.chars {
            if i == &'\n' {
                current_line += 1;
                continue;
            }
            if current_line == line {
                s.push(*i);
            }
            if current_line > line {
                break;
            }
        }
        s
    }
    #[allow(unused)]
    pub fn set_chars(&mut self, chars: Vec<char>) {
        self.chars = chars;
    }

    /// 开始新的词法分析
    fn start_token(&mut self) {
        self.buffer.clear();
        self.start_pos = self.index;
        self.start_col = self.col;
        self.start_row = self.row;
        self.state = State::Initial;
    }

    /// 创建token位置
    fn create_position(&self) -> Position {
        Position::new(
            self.start_pos,
            self.buffer.len(),
            self.start_row,
            self.start_col,
            &self.file_name,
        )
    }

    pub fn next_token(&mut self) -> Option<(Token, Position)> {
        self.start_token();

        while self.index < self.chars.len() {
            let c = self.current_char().unwrap();
            let peek = self.peek_char().unwrap_or('\0');

            match &self.state {
                State::Initial => self.process_initial_state(c, peek),
                State::Identifier => self.process_identifier_state(c),
                State::Number => self.process_number_state(c, peek),
                State::Decimal => self.process_decimal_state(c),
                State::String => {
                    if let Some(token) = self.process_string_state(c) {
                        return Some(token);
                    }
                }
                State::FormatString => {
                    if let Some(token) = self.process_format_string_state(c) {
                        return Some(token);
                    }
                }
                State::Comment => {
                    if c == '\n' {
                        self.state = State::Initial;
                    }
                    self.increase_index();
                }
                State::MultilineComment => {
                    if c == '*' && peek == '/' {
                        self.increase_index();
                        self.increase_index();
                        self.state = State::Initial;
                    } else {
                        self.increase_index();
                    }
                }
                State::Done(token_type) => {
                    return self.finalize_token(*token_type);
                }
            }

            // 如果已经到达Done状态，返回结果
            if let State::Done(token_type) = &self.state {
                return self.finalize_token(*token_type);
            }
        }

        // 处理文件结束情况
        if !self.buffer.is_empty() {
            if let State::Identifier = self.state {
                return self.finalize_token(TokenType::Identifier);
            } else if let State::Number = self.state {
                return self.finalize_token(TokenType::Integer);
            } else if let State::Decimal = self.state {
                return self.finalize_token(TokenType::Float);
            } else {
                // 默认作为标识符处理未完成的token
                return self.finalize_token(TokenType::Identifier);
            }
        }

        None
    }

    /// 处理初始状态
    fn process_initial_state(&mut self, c: char, peek: char) {
        match (c, peek) {
            ('r', '"') => {
                self.buffer.push(c);
                self.buffer.push(peek);
                self.increase_index();
                self.increase_index();
                self.state = State::FormatString;
            }
            ('.', p) if !p.is_numeric() => {
                self.buffer.push(c);
                self.increase_index();
                self.state = State::Done(TokenType::Symbol);
            }
            ('0'..='9', _) => {
                self.buffer.push(c);
                self.increase_index();
                self.state = State::Number;
            }
            ('.', p) if p.is_numeric() => {
                self.buffer.push(c);
                self.increase_index();
                self.state = State::Decimal;
            }
            ('a'..='z' | 'A'..='Z' | '_', _) => {
                self.buffer.push(c);
                self.increase_index();
                self.state = State::Identifier;
            }
            ('"', _) => {
                self.buffer.push(c);
                self.increase_index();
                self.state = State::String;
            }
            ('\'', _) => {
                self.buffer.push(c);
                self.increase_index();
                self.state = State::String;
            }
            ('/', '/') => {
                self.increase_index();
                self.increase_index();
                self.state = State::Comment;
            }
            ('/', '*') => {
                self.increase_index();
                self.increase_index();
                self.state = State::MultilineComment;
            }
            (':', ':') => {
                self.buffer.push(':');
                self.buffer.push(':');
                self.increase_index();
                self.increase_index();
                self.state = State::Done(TokenType::Symbol);
            }
            ('!', '=') => {
                self.buffer.push('!');
                self.buffer.push('=');
                self.increase_index();
                self.increase_index();
                self.state = State::Done(TokenType::Symbol);
            }
            ('=', '=') => {
                self.buffer.push('=');
                self.buffer.push('=');
                self.increase_index();
                self.increase_index();
                self.state = State::Done(TokenType::Symbol);
            }
            ('&', '&') => {
                self.buffer.push('&');
                self.buffer.push('&');
                self.increase_index();
                self.increase_index();
                self.state = State::Done(TokenType::Symbol);
            }
            ('-', '>') => {
                self.buffer.push('-');
                self.buffer.push('>');
                self.increase_index();
                self.increase_index();
                self.state = State::Done(TokenType::Symbol);
            }
            ('>', '=') => {
                self.buffer.push('>');
                self.buffer.push('=');
                self.increase_index();
                self.increase_index();
                self.state = State::Done(TokenType::Symbol);
            }
            ('<', '=') => {
                self.buffer.push('<');
                self.buffer.push('=');
                self.increase_index();
                self.increase_index();
                self.state = State::Done(TokenType::Symbol);
            }
            ('(', _)
            | (')', _)
            | ('{', _)
            | ('}', _)
            | ('[', _)
            | (']', _)
            | (':', _)
            | (',', _)
            | ('|', _)
            | ('!', _)
            | ('@', _)
            | ('=', _)
            | ('&', _)
            | ('>', _)
            | ('<', _)
            | ('+', _)
            | ('-', _)
            | ('*', _)
            | ('%', _)
            | ('/', _) => {
                self.buffer.push(c);
                self.increase_index();
                self.state = State::Done(TokenType::Symbol);
            }
            (' ' | '\r' | ';' | '\t' | '\n', _) => {
                self.increase_index();
            }
            _ => {
                println!("Unexpected character: {:?}", c);
                self.increase_index();
            }
        }
    }

    /// 处理标识符状态
    fn process_identifier_state(&mut self, c: char) {
        if c.is_alphanumeric() || c == '_' {
            self.buffer.push(c);
            self.increase_index();
        } else {
            self.state = State::Done(TokenType::Identifier);
        }
    }

    /// 处理数字状态
    fn process_number_state(&mut self, c: char, peek: char) {
        if c.is_numeric() {
            self.buffer.push(c);
            self.increase_index();
        } else if c == '.' && peek.is_numeric() {
            self.buffer.push(c);
            self.increase_index();
            self.state = State::Decimal;
        } else {
            self.state = State::Done(TokenType::Integer);
        }
    }

    /// 处理小数状态
    fn process_decimal_state(&mut self, c: char) {
        if c.is_numeric() {
            self.buffer.push(c);
            self.increase_index();
        } else {
            self.state = State::Done(TokenType::Float);
        }
    }

    /// 处理字符串状态
    fn process_string_state(&mut self, c: char) -> Option<(Token, Position)> {
        let quote = self.buffer.chars().next().unwrap();
        if c == quote {
            self.buffer.push(c);
            self.increase_index();
            let pos = self.create_position();
            let content = self.buffer[1..self.buffer.len() - 1]
                .to_string()
                .replace("\\n", "\n");
            Some((Token::String(content), pos))
        } else {
            self.buffer.push(c);
            self.increase_index();
            None
        }
    }

    /// 处理格式化字符串状态
    fn process_format_string_state(&mut self, c: char) -> Option<(Token, Position)> {
        if c == '"' {
            self.buffer.push(c);
            self.increase_index();
            let pos = self.create_position();
            let content = self.buffer[2..self.buffer.len() - 1]
                .to_string()
                .replace("\\n", "\n");
            Some((Token::FormatString(content), pos))
        } else {
            self.buffer.push(c);
            self.increase_index();
            None
        }
    }

    /// 完成token处理
    fn finalize_token(&mut self, token_type: TokenType) -> Option<(Token, Position)> {
        let pos = self.create_position();

        // 根据缓冲区内容和当前状态生成token
        let token = match token_type {
            TokenType::Identifier => {
                let ident = self.buffer.clone();
                if self.keywords.contains(&ident.as_str()) {
                    Token::Keyword(ident)
                } else if ident == "true" {
                    Token::Boolean(true)
                } else if ident == "false" {
                    Token::Boolean(false)
                } else {
                    Token::Identifier(ident)
                }
            }
            TokenType::Integer => {
                let i: i64 = self.buffer.parse().unwrap();
                Token::Int(i)
            }
            TokenType::Float => {
                let f: f32 = self.buffer.parse().unwrap();
                Token::Float(f)
            }
            TokenType::Symbol => match self.buffer.as_str() {
                "(" => Token::BraceLeft,
                ")" => Token::BraceRight,
                "{" => Token::ParenLeft,
                "}" => Token::ParenRight,
                "[" => Token::BracketLeft,
                "]" => Token::BracketRight,
                "." => Token::Dot,
                ":" => Token::Colon,
                "::" => Token::ScopeSymbol,
                "=" => Token::Assign,
                "," => Token::Comma,
                "+" => Token::Plus,
                "-" => Token::Minus,
                "*" => Token::Star,
                "/" => Token::Slash,
                "%" => Token::Mod,
                ">" => Token::Greater,
                "<" => Token::Less,
                "<=" => Token::LessEqual,
                ">=" => Token::GreaterEqual,
                "==" => Token::Equal,
                "!=" => Token::NotEqual,
                "->" => Token::Arrow,
                "!" => Token::Not,
                "&&" => Token::And,
                "|" => Token::Vertical,
                "@" => Token::Annotation,
                "&" => Token::BitAnd,
                _ => {
                    println!("Unknown token: {}", self.buffer);
                    return None;
                }
            },
            // TokenType::String => {
            //     let content = self.buffer[1..self.buffer.len() - 1]
            //         .to_string()
            //         .replace("\\n", "\n");
            //     Token::String(content)
            // }
            // TokenType::FormatString => {
            //     let content = self.buffer[2..self.buffer.len() - 1]
            //         .to_string()
            //         .replace("\\n", "\n");
            //     Token::FormatString(content)
            // }
        };

        Some((token, pos))
    }

    fn peek_char(&self) -> Option<char> {
        self.chars.get(self.index + 1).copied()
    }

    fn current_char(&self) -> Option<char> {
        self.chars.get(self.index).copied()
    }

    fn increase_index(&mut self) {
        if self.index < self.chars.len() && self.chars[self.index] == '\n' {
            self.row += 1;
            self.col = 0;
        }
        self.index = self.index.add(1);
        self.col += 1;
    }

    #[allow(unused)]
    pub fn get_source(&self) -> Vec<char> {
        self.chars.clone()
    }

    pub fn from_script(module_name: impl Into<String>, script: impl AsRef<str>) -> Self {
        let script = script.as_ref().chars().collect();
        let mut lexer = Lexer::new(module_name);
        lexer.set_chars(script);
        lexer
    }
}