skullbrain-lexer 0.1.0

mod cursor;
mod rustc;
mod utils;

pub(crate) use cursor::*;
use std::iter::FusedIterator;
use utils::*;

include!(concat!(env!("OUT_DIR"), "/keyword_matcher.rs"));

use crate::{Token, TokenKind, MULTILINE_COMMENT_BEGIN_KEYWORD, MULTILINE_COMMENT_END_KEYWORD};

impl<I> LexerCursor<I>
where
    I: Iterator<Item = char> + FusedIterator,
{
    /// Returns the next token in the input stream, potentially bundled with an unknown token.
    ///
    /// # Return Value
    /// - `Some((token, Some(unknown_token)))`: A valid token preceded by one unknown token.
    /// - `Some((token, None))`: A valid token with no preceding unknown tokens.
    /// - `Some((unknown_token, None))`: Only when the remaining input consists entirely of unknown tokens.
    /// - `None`: When the input stream is exhausted.
    ///
    /// # Behavior
    /// This method consumes input characters until it either:
    /// 1. Encounters a valid (non-unknown) token, potentially after seeing unknown tokens
    /// 2. Reaches the end of input with only unknown tokens remaining
    pub(super) fn next(&mut self) -> Option<(Token, Option<Token>)> {
        // We group consecutive unknown tokens into a single span
        let mut unknown_token_offset = 0u16;

        loop {
            match self.advance_token() {
                Some(token) if token.kind().is_unknown() => {
                    debug_assert_eq!(self.current_token_offset(), 0);
                    unknown_token_offset += token.len_repr();
                }
                Some(token) => {
                    debug_assert_eq!(self.current_token_offset(), 0);

                    let unknown_token = (unknown_token_offset > 0)
                        .then(|| Token::new(TokenKind::Unknown, unknown_token_offset));

                    break Some((token, unknown_token));
                }
                None if unknown_token_offset > 0 => {
                    let unknown_token = Token::new(TokenKind::Unknown, unknown_token_offset);
                    break Some((unknown_token, None));
                }
                None => break None,
            }
        }
    }
}

impl<I> LexerCursor<I>
where
    I: Iterator<Item = char>,
{
    fn advance_token(&mut self) -> Option<Token> {
        debug_assert_eq!(self.current_token_offset(), 0);

        let keyword = match_keyword::<I>(&mut self.iter);
        let kind = self.require_seperator_after_and_then_consume(keyword);

        let token = match kind {
            Some(kind) => {
                #[allow(clippy::shadow_unrelated)]
                let kind = match kind {
                    TokenKind::SingleLineComment => {
                        self.finish_parsing_single_line_comment();
                        TokenKind::SingleLineComment
                    }
                    TokenKind::MultiLineComment { .. } => {
                        let is_terminated: bool = self.finish_parsing_multiline_comment();
                        TokenKind::MultiLineComment { is_terminated }
                    }
                    _ => kind,
                };

                Token::new(kind, self.current_token_offset())
            }
            None => {
                #[allow(clippy::shadow_unrelated)]
                let kind = match self.iter.next()? {
                    c if c.is_whitespace() => {
                        self.take_while(|ch| ch.is_whitespace());
                        TokenKind::Whitespace
                    }
                    c if is_id_start(c) => self.finish_ident(),
                    c @ '0'..='9' => TokenKind::Literial(self.iter.number(c)),
                    '\'' => TokenKind::Literial(self.iter.parse_single_quote_string()),
                    '"' => TokenKind::Literial(self.iter.parse_double_quoted_string()),
                    _ => TokenKind::Unknown,
                };

                Token::new(kind, self.current_token_offset())
            }
        };

        self.reset_current_token_offset();

        Some(token)
    }
}

impl<I> LexerCursor<I>
where
    I: Iterator<Item = char>,
{
    fn finish_parsing_single_line_comment(&mut self) {
        self.take_while(|ch| ch != &'\n');
        // To consume the newline as well
        let c = self.iter.next();
        debug_assert!(c.is_none_or(|c| c == '\n'))
    }

    fn finish_parsing_multiline_comment(&mut self) -> bool {
        debug_assert!(
            !MULTILINE_COMMENT_BEGIN_KEYWORD.contains(MULTILINE_COMMENT_END_KEYWORD)
            && !MULTILINE_COMMENT_END_KEYWORD.contains(MULTILINE_COMMENT_BEGIN_KEYWORD),
            concat!(
                "LexerCursor::finish_parsing_multiline_comment$loop needs adjustment. ",
                "One delimiter is a substring of the other, which can cause incorrect matching. ",
                "Consider either reordering the if statements or implementing a match function ",
                "similar to match_keyword in keyword_matcher.rs that matches both delimiters simultaneously."
            )
        );

        let mut depth = 1usize;
        // Since we require a whitespace after the opening delimiter
        let mut last_char_was_word_seperator = true;

        while let Some(ch) = self.iter.next() {
            if last_char_was_word_seperator
                || (MULTILINE_COMMENT_BEGIN_KEYWORD.starts_with(ch)
                    || MULTILINE_COMMENT_END_KEYWORD.starts_with(ch))
            {
                let match_res = match match_multiline_comment_begin_keyword::<I>(&mut self.iter) {
                    Some(v) => self
                        .require_seperator_after_and_then_consume(Some(v))
                        .map(|_| 1i8),
                    None => match match_multiline_comment_end_keyword::<I>(&mut self.iter) {
                        Some(v) => self
                            .require_seperator_after_and_then_consume(Some(v))
                            .map(|_| -1i8),
                        None => continue,
                    },
                };

                if let Some(offset) = match_res {
                    depth = ((depth as i8) + offset) as usize;

                    // If we have closed all nested comments, return success.
                    if depth == 0 {
                        return true;
                    }
                }
            }

            // Update word-boundary flag: if current character is whitespace or punctuation,
            // we mark it as a word separator.
            last_char_was_word_seperator = ch.is_whitespace() || ch.is_ascii_punctuation();
        }

        depth == 0
    }

    fn finish_ident(&mut self) -> TokenKind {
        // Start is already eaten, eat the rest of identifier.
        self.take_while(|ch| is_id_continue(*ch));

        TokenKind::Ident
    }
}