skullbrain-lexer 0.1.0

Lexer for the skullbrain programming language, responsible for tokenizing source code.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

use crate::{PunctuationKind, TokenKind};

use super::{
    rustc::{rustc_is_id_continue, rustc_is_id_start},
    LexerCursor,
};

/// True if `c` is valid as a non-first character of an identifier.
pub(super) fn is_id_continue(c: char) -> bool {
    // Numbers a valid to be part of a ident
    rustc_is_id_continue(c) || unicode_properties::UnicodeEmoji::is_emoji_char(c)
}

pub(super) fn is_id_start(c: char) -> bool {
    // For some fucking reason we need to ensure its not in 0..9 as other wise it is counting "42" as emoji
    rustc_is_id_start(c)
        || (unicode_properties::UnicodeEmoji::is_emoji_char(c) && !c.is_ascii_digit())
}

impl<I> LexerCursor<I>
where
    I: Iterator<Item = char>,
{
    pub(super) fn require_seperator_after_and_then_consume(
        &mut self,
        res: Option<(TokenKind, usize)>,
    ) -> Option<TokenKind> {
        match res {
            Some((kind, matched_len)) if self.is_followed_by_seperator(matched_len, &kind) => {
                for _ in 0..matched_len {
                    debug_assert!(self.iter.next().is_some())
                }

                Some(kind)
            }
            _ => None,
        }
    }

    fn is_followed_by_seperator(&mut self, n: usize, kind: &TokenKind) -> bool {
        // The '.' is counted as a seperator already
        match kind {
            TokenKind::Punctuation(PunctuationKind::Dot) => true,
            _ => self.is_followed_by_primitive_seperator(n),
        }
    }

    fn is_followed_by_primitive_seperator(&mut self, n: usize) -> bool {
        match self.iter.peek_nth(n) {
            Some(ch) if ch.is_whitespace() || ch.is_ascii_punctuation() => true,
            // If input as ended then we do not require a seperator
            None => true,
            _ => false,
        }
    }
}