clex_gen 0.4.0

//! The `lexer` module provides lexical analysis capabilities for tokenizing input patterns in the `clex` language.
//! It defines the `Tokens` struct, which represents a collection of tokens produced by the lexer.
//! The lexer scans the input pattern and identifies different types of tokens, categorizing them according to the `TokenType` enum.
//!
//! # Types
//!
//! - `Tokens`: Represents a collection of tokens produced by the lexer, containing information about each token's type and lexeme.
//! - `Token`: Represents an individual token, consisting of a `TokenType` and the corresponding lexeme.
//! - `TokenType`: Enumerates different types of tokens that the lexer can identify, including metacharacters, character sets, literals, and end-of-file markers.
//!
//! The lexer is a crucial component in the `clex_language` module, providing the initial step in processing the `clex` language input patterns.
//! It tokenizes the input, making it easier for subsequent components, such as the parser, to understand and structure the code patterns.
//!
//! # Example
//!
//! ```rust
//! use clex_gen::clex_language::lexer::{TokenType, Token, Span};
//! use clex_gen::get_tokens;
//!
//! // Example input pattern
//! let src = "N";
//!
//! // Assert token types and lexemes in a test
//! assert_eq!(
//!     get_tokens(src.to_string()).unwrap(),
//!     vec![
//!         Token {
//!             token_type: TokenType::Integer,
//!             lexeme: "N".to_string(),
//!             span: Span { start: 0, end: 1 },
//!         },
//!         Token {
//!             token_type: TokenType::Eof,
//!             lexeme: "".to_string(),
//!             span: Span { start: 1, end: 1 },
//!         },
//!     ]
//! );
//! ```
//!
//! For more details on the types and methods provided by the lexer, refer to the documentation for each type.

use crate::clex_language::ast::{GraphKind, PositiveReferenceType};
use crate::clex_language::clex_error_type::{ClexErrorType, ParentErrorType};
use unicode_segmentation::UnicodeSegmentation;

/// Represents a span (position range) in the source code
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub struct Span {
    /// Starting position (byte offset) in the source
    pub start: usize,
    /// Ending position (byte offset) in the source
    pub end: usize,
}

/// Represents the different types of tokens in the lexer.
#[derive(Debug, PartialEq, Clone)]
pub enum TokenType {
    // Metacharacters
    LeftParens,
    RightParens,
    QuestionColon,
    Backslash,
    LeftSquareBracket,
    RightSquareBracket,
    LeftCurlyBrackets,
    RightCurlyBrackets,
    Comma,
    Pipe,

    // Character sets
    /// Integer token type
    Integer,
    /// Float token type
    Float,
    /// String token type.
    String,

    // Character Sets
    CharacterSetAlpha,
    CharacterSetAlnum,
    CharacterSetNewline,
    CharacterSetNumeric,
    CharacterSetUpper,
    CharacterSetLower,
    CharacterSetAll,

    // Literals
    /// Literal number token type with a specified value.
    LiteralNumber(i64),
    LiteralFloat(f64),
    LiteralString(String),
    LiteralText(String),
    /// Loop iteration index (%i, %i-1, %i+1, etc.) with offset.
    LoopIndex(i64),
    /// Graph or permutation topology primitive.
    GraphPrimitive(GraphKind),

    // End of file
    Eof,
}

/// Represents a token in the lexer, consisting of a token type and the corresponding lexeme.
#[derive(Debug, PartialEq, Clone)]
pub struct Token {
    /// The type of the token, indicating its classification.
    pub token_type: TokenType,
    /// The actual characters that form the token in the source code.
    pub lexeme: String,
    /// The span (position range) of this token in the source code
    pub span: Span,
}

/// Represents a collection of tokens produced by the lexer.
#[derive(Debug, Clone)]
pub struct Tokens {
    tokens: Vec<Token>,
    start: usize,
    current: usize,
    source_language: String,
    graphemes: Vec<(usize, usize, String)>,
}

impl Tokens {
    /// Creates a new Tokens instance for a given source language.
    pub fn new(source_language: String) -> Self {
        let graphemes = source_language
            .grapheme_indices(true)
            .map(|(idx, g)| (idx, idx + g.len(), g.to_string()))
            .collect();

        Self {
            tokens: Vec::new(),
            start: 0,
            current: 0,
            source_language,
            graphemes,
        }
    }

    pub fn get_tokens(&self) -> Vec<Token> {
        self.tokens.clone()
    }

    /// Scans tokens from the source language and adds them to the tokens vector.
    pub fn scan_tokens(&mut self) -> Result<(), ClexErrorType> {
        while !self.at_end() {
            self.start = self.current;
            self.scan_token()?;
        }

        let eof_pos = self.source_language.len();
        self.tokens.push(Token {
            token_type: TokenType::Eof,
            lexeme: String::new(),
            span: Span {
                start: eof_pos,
                end: eof_pos,
            },
        });

        Ok(())
    }

    /// Checks if the lexer has reached the end of the source language.
    fn at_end(&self) -> bool {
        self.current >= self.graphemes.len()
    }

    fn get_byte_span(&self, start_grapheme: usize, end_grapheme: usize) -> Span {
        let start = if start_grapheme < self.graphemes.len() {
            self.graphemes[start_grapheme].0
        } else {
            self.source_language.len()
        };
        let end = if end_grapheme > 0 && end_grapheme <= self.graphemes.len() {
            self.graphemes[end_grapheme - 1].1
        } else if end_grapheme == 0 && !self.graphemes.is_empty() {
            self.graphemes[0].0
        } else {
            self.source_language.len()
        };
        Span {
            start,
            end: end.max(start),
        }
    }

    /// Scans a single token from the source language.
    fn scan_token(&mut self) -> Result<(), ClexErrorType> {
        let c = self.advance().to_string(); // Convert to owned String to avoid borrow issues
        match c.as_str() {
            "(" => self.add_token(TokenType::LeftParens),
            ")" => self.add_token(TokenType::RightParens),
            "[" => self.add_token(TokenType::LeftSquareBracket),
            "]" => self.add_token(TokenType::RightSquareBracket),
            "{" => self.add_token(TokenType::LeftCurlyBrackets),
            "}" => self.add_token(TokenType::RightCurlyBrackets),
            "," => self.add_token(TokenType::Comma),
            "|" => self.add_token(TokenType::Pipe),
            "\\" => self.add_token(TokenType::Backslash),
            "%" => {
                let start_pos = self.start;
                if self.peek() == "i" {
                    self.advance(); // consume 'i'
                    let mut offset: i64 = 0;
                    if self.peek() == "+" || self.peek() == "-" {
                        let sign_str = self.advance().to_string();
                        let mut num_str = String::new();
                        while Self::is_digit(self.peek()) {
                            num_str.push_str(self.advance());
                        }
                        if let Ok(val) = num_str.parse::<i64>() {
                            offset = if sign_str == "-" { -val } else { val };
                        }
                    }
                    self.add_token_with_span(
                        TokenType::LoopIndex(offset),
                        self.get_byte_span(start_pos, self.current),
                    );
                } else {
                    return Err(ClexErrorType::UnknownCharacter(
                        ParentErrorType::LexerError,
                        self.get_byte_span(self.start, self.current),
                        c,
                    ));
                }
            }
            "N" => self.add_token(TokenType::Integer),
            "F" => self.add_token(TokenType::Float),
            "S" => self.add_token(TokenType::String),
            "@" => {
                let start_pos = self.start; // Store the starting position
                self.start += 1; // Skip first character in lexeme

                let mut literal = String::new();
                while self.peek() != "@" && !self.at_end() {
                    literal.push_str(self.advance());
                }

                // Unterminated string.
                if self.at_end() {
                    return Err(ClexErrorType::UnclosedAtSymbol(
                        ParentErrorType::LexerError,
                        self.get_byte_span(start_pos, self.current),
                    ));
                }

                let uppercase_literal = literal.trim().to_uppercase();
                let token_type = match uppercase_literal.as_str() {
                    "CH_ALPHA" => TokenType::CharacterSetAlpha,
                    "CH_NUM" => TokenType::CharacterSetNumeric,
                    "CH_NEWLINE" => TokenType::CharacterSetNewline,
                    "CH_ALNUM" => TokenType::CharacterSetAlnum,
                    "CH_UPPER" => TokenType::CharacterSetUpper,
                    "CH_LOWER" => TokenType::CharacterSetLower,
                    "CH_ALL" => TokenType::CharacterSetAll,
                    _ => {
                        if let Some(directive) = Self::parse_graph_primitive_directive(&literal) {
                            TokenType::GraphPrimitive(directive)
                        } else {
                            return Err(ClexErrorType::InvalidCharacterSet(
                                ParentErrorType::LexerError,
                                self.get_byte_span(start_pos, self.current),
                            ));
                        }
                    }
                };

                self.add_token(token_type);
                if !self.match_str("@") {
                    return Err(ClexErrorType::UnclosedAtSymbol(
                        ParentErrorType::LexerError,
                        self.get_byte_span(start_pos, self.current),
                    ));
                }
            }
            " " | "\r" | "\t" | "\n" => {
                // Do nothing, just ignore these spaces
            }
            "'" => {
                let start_pos = self.start; // Store the starting position
                self.start += 1; // Skip first character in lexeme

                let mut literal = String::new();
                while self.peek() != "'" && !self.at_end() {
                    let c = self.advance();
                    if c == "\\" {
                        let escaped = match self.peek() {
                            "n" => '\n',   // newline
                            "t" => '\t',   // tab
                            "r" => '\r',   // carriage return
                            "\\" => '\\',  // backslash
                            "'" => '\'',   // single quote
                            "\"" => '\"',  // double quote
                            "0" => '\0',   // null
                            "a" => '\x07', // bell
                            "b" => '\x08', // backspace
                            "f" => '\x0C', // form feed
                            "v" => '\x0B', // vertical tab
                            _ => '\\',
                        };
                        self.advance();
                        literal.push(escaped);
                    } else {
                        literal.push_str(c);
                    }
                }

                if self.at_end() {
                    return Err(ClexErrorType::UnclosedSingleQuotes(
                        ParentErrorType::LexerError,
                        self.get_byte_span(start_pos, self.current),
                    ));
                }

                self.add_token(TokenType::LiteralString(literal));
                if !self.match_str("'") {
                    return Err(ClexErrorType::UnclosedSingleQuotes(
                        ParentErrorType::LexerError,
                        self.get_byte_span(start_pos, self.current),
                    ));
                }
            }
            "\"" => {
                let start_pos = self.start;

                let mut literal = String::new();
                while self.peek() != "\"" && !self.at_end() {
                    let c = self.advance();
                    if c == "\\" {
                        let escaped = match self.peek() {
                            "n" => '\n',   // newline
                            "t" => '\t',   // tab
                            "r" => '\r',   // carriage return
                            "\\" => '\\',  // backslash
                            "'" => '\'',   // single quote
                            "\"" => '\"',  // double quote
                            "0" => '\0',   // null
                            "a" => '\x07', // bell
                            "b" => '\x08', // backspace
                            "f" => '\x0C', // form feed
                            "v" => '\x0B', // vertical tab
                            _ => '\\',
                        };
                        self.advance();
                        literal.push(escaped);
                    } else {
                        literal.push_str(c);
                    }
                }

                if self.at_end() {
                    return Err(ClexErrorType::UnclosedDoubleQuotes(
                        ParentErrorType::LexerError,
                        self.get_byte_span(start_pos, self.current),
                    ));
                }

                if !self.match_str("\"") {
                    return Err(ClexErrorType::UnclosedDoubleQuotes(
                        ParentErrorType::LexerError,
                        self.get_byte_span(start_pos, self.current),
                    ));
                }

                let span = self.get_byte_span(start_pos, self.current);
                self.add_token_with_span(TokenType::LiteralText(literal), span);
            }
            "?" => {
                let start_pos = self.start;
                if self.match_str(":") {
                    self.add_token(TokenType::QuestionColon);
                } else {
                    return Err(ClexErrorType::MissingColonAfterQuestionMark(
                        ParentErrorType::LexerError,
                        self.get_byte_span(start_pos, self.current),
                    ));
                }
            }
            _ => {
                if c.as_str() == "-" || Self::is_digit(c.as_str()) {
                    let start_pos = self.start;
                    if c.as_str() == "-" && !Self::is_digit(self.peek()) {
                        return Err(ClexErrorType::MissingNumberAfterNegativeSign(
                            ParentErrorType::LexerError,
                            self.get_byte_span(start_pos, self.current),
                        ));
                    }

                    while Self::is_digit(self.peek()) {
                        self.current += 1;
                    }

                    if self.peek() == "."
                        && self.current + 1 < self.graphemes.len()
                        && Self::is_digit(&self.graphemes[self.current + 1].2)
                    {
                        self.current += 1; // consume '.'
                        while Self::is_digit(self.peek()) {
                            self.current += 1;
                        }

                        let span = self.get_byte_span(start_pos, self.current);
                        let float_val =
                            match self.source_language[span.start..span.end].parse::<f64>() {
                                Ok(num) => num,
                                Err(_err) => {
                                    return Err(ClexErrorType::NumericParsingError(
                                        ParentErrorType::LexerError,
                                        span,
                                    ));
                                }
                            };
                        self.add_token(TokenType::LiteralFloat(float_val));
                    } else {
                        let span = self.get_byte_span(start_pos, self.current);
                        let number = match self.source_language[span.start..span.end].parse::<i64>()
                        {
                            Ok(num) => num,
                            Err(_err) => {
                                return Err(ClexErrorType::NumericParsingError(
                                    ParentErrorType::LexerError,
                                    span,
                                ));
                            }
                        };

                        self.add_token(TokenType::LiteralNumber(number));
                    }
                } else {
                    let span = self.get_byte_span(self.start, self.current);
                    return Err(ClexErrorType::UnknownCharacter(
                        ParentErrorType::LexerError,
                        span,
                        c,
                    ));
                }
            }
        }
        Ok(())
    }

    /// Adds a token to the tokens vector.
    fn add_token(&mut self, token_type: TokenType) {
        let span = self.get_byte_span(self.start, self.current);
        self.tokens.push(Token {
            token_type,
            lexeme: self.source_language[span.start..span.end].to_string(),
            span,
        });
    }

    /// Adds a token with a specific span to the tokens vector.
    fn add_token_with_span(&mut self, token_type: TokenType, span: Span) {
        self.tokens.push(Token {
            token_type,
            lexeme: self.source_language[span.start..span.end].to_string(),
            span,
        });
    }

    /// Advances the current index and returns the character at the new index.
    fn advance(&mut self) -> &str {
        self.current += 1;
        self.char_at(self.current - 1)
    }

    /// Retrieves the character at the specified index.
    fn char_at(&self, index: usize) -> &str {
        &self.graphemes[index].2
    }

    /// Checks if the current characters match the expected string if yes then traverse as well.
    fn match_str(&mut self, expected: &str) -> bool {
        if self.at_end() || self.char_at(self.current) != expected {
            false
        } else {
            self.current += 1;
            true
        }
    }

    /// Checks if a string is a digit.
    fn is_digit(c: &str) -> bool {
        c.chars().all(|ch| ch.is_ascii_digit())
    }

    fn parse_pos_ref_str(s: &str) -> Option<PositiveReferenceType> {
        let s = s.trim();
        if s.starts_with('\\') {
            let num: u64 = s[1..].parse().ok()?;
            Some(PositiveReferenceType::ByGroup { group_number: num })
        } else if let Some(rest) = s.strip_prefix("%i") {
            let offset: i64 = if rest.is_empty() {
                0
            } else {
                rest.parse().ok()?
            };
            Some(PositiveReferenceType::ByLoopIndex { offset })
        } else {
            let val: u64 = s.parse().ok()?;
            Some(PositiveReferenceType::ByLiteral(val))
        }
    }

    fn parse_graph_primitive_directive(literal: &str) -> Option<GraphKind> {
        let trimmed = literal.trim();
        let open_bracket = trimmed.find('(')?;
        if !trimmed.ends_with(')') {
            return None;
        }
        let name = trimmed[..open_bracket].trim().to_uppercase();
        let args_str = &trimmed[open_bracket + 1..trimmed.len() - 1];
        let args: Vec<&str> = args_str.split(',').collect();

        match name.as_str() {
            "TREE" if args.len() == 1 => {
                let n_ref = Self::parse_pos_ref_str(args[0])?;
                Some(GraphKind::Tree(n_ref))
            }
            "GRAPH" if args.len() == 2 => {
                let n_ref = Self::parse_pos_ref_str(args[0])?;
                let m_ref = Self::parse_pos_ref_str(args[1])?;
                Some(GraphKind::Graph(n_ref, m_ref))
            }
            "DAG" if args.len() == 2 => {
                let n_ref = Self::parse_pos_ref_str(args[0])?;
                let m_ref = Self::parse_pos_ref_str(args[1])?;
                Some(GraphKind::Dag(n_ref, m_ref))
            }
            "PERM" | "PERMUTATION" if args.len() == 1 => {
                let n_ref = Self::parse_pos_ref_str(args[0])?;
                Some(GraphKind::Permutation(n_ref))
            }
            _ => None,
        }
    }

    /// Peeks at the character at the current index.
    fn peek(&self) -> &str {
        if self.at_end() {
            "\0"
        } else {
            self.char_at(self.current)
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn tokenization_works() {
        let src = "12N3";
        let mut tokens = Tokens::new(src.to_string());

        tokens.scan_tokens().unwrap();

        assert_eq!(
            tokens.tokens,
            vec![
                Token {
                    token_type: TokenType::LiteralNumber(12),
                    lexeme: "12".to_string(),
                    span: Span { start: 0, end: 2 },
                },
                Token {
                    token_type: TokenType::Integer,
                    lexeme: "N".to_string(),
                    span: Span { start: 2, end: 3 },
                },
                Token {
                    token_type: TokenType::LiteralNumber(3),
                    lexeme: "3".to_string(),
                    span: Span { start: 3, end: 4 },
                },
                Token {
                    token_type: TokenType::Eof,
                    lexeme: String::new(),
                    span: Span { start: 4, end: 4 },
                }
            ]
        );
    }
}