only_syntax/
lex.rs

1use logos::Logos;
2use smol_str::SmolStr;
3use text_size::{TextRange, TextSize};
4
5use crate::builtin::GROUP_KEYWORD;
6use crate::{LexToken, SyntaxKind};
7
8#[derive(Logos, Debug, Clone, Copy, PartialEq, Eq)]
9enum RawTokenKind {
10    #[token("\u{feff}")]
11    Bom,
12    #[token("shell~=")]
13    ShellFallbackKw,
14    #[token("shell")]
15    ShellKw,
16    #[token("!")]
17    Bang,
18    #[regex(r"#[^\n]*", allow_greedy = true)]
19    HashComment,
20    #[token(":")]
21    Colon,
22    #[token("?")]
23    Question,
24    #[token("&")]
25    Amp,
26    #[token("=")]
27    Eq,
28    #[token("@")]
29    At,
30    #[token("(")]
31    LParen,
32    #[token(")")]
33    RParen,
34    #[token("[")]
35    LBracket,
36    #[token("]")]
37    RBracket,
38    #[token("{")]
39    LBrace,
40    #[token("}")]
41    RBrace,
42    #[token(",")]
43    Comma,
44    #[regex(r#""([^"\n]|\\.)*""#)]
45    String,
46    #[regex(r"[A-Za-z_-][A-Za-z0-9_-]*")]
47    Ident,
48    #[regex(r"[ \t]+")]
49    Spaces,
50    #[regex(r"\r\n|\n|\r")]
51    Newline,
52}
53
54/// Lexes source text into tokens while preserving trivia.
55///
56/// Args:
57/// source: Raw Onlyfile source text.
58///
59/// Returns:
60/// Token stream including whitespace, comments and EOF.
61pub fn lex(source: &str) -> Vec<LexToken> {
62    let mut lexer = RawTokenKind::lexer(source);
63    let mut tokens = Vec::new();
64    let mut line_start = true;
65
66    while let Some(result) = lexer.next() {
67        let span = lexer.span();
68        let text = &source[span.clone()];
69        let start = TextSize::from(span.start as u32);
70        let end = TextSize::from(span.end as u32);
71        let kind = match result {
72            Ok(RawTokenKind::Bom) if span.start == 0 => SyntaxKind::Bom,
73            Ok(RawTokenKind::Bom) => SyntaxKind::Unknown,
74            Ok(RawTokenKind::ShellFallbackKw) => SyntaxKind::ShellFallbackKw,
75            Ok(RawTokenKind::ShellKw) => SyntaxKind::ShellKw,
76            Ok(RawTokenKind::Bang) => SyntaxKind::Bang,
77            Ok(RawTokenKind::HashComment) => SyntaxKind::Comment,
78            Ok(RawTokenKind::Colon) => SyntaxKind::Colon,
79            Ok(RawTokenKind::Question) => SyntaxKind::Question,
80            Ok(RawTokenKind::Amp) => SyntaxKind::Amp,
81            Ok(RawTokenKind::Eq) => SyntaxKind::Eq,
82            Ok(RawTokenKind::At) => SyntaxKind::At,
83            Ok(RawTokenKind::LParen) => SyntaxKind::LParen,
84            Ok(RawTokenKind::RParen) => SyntaxKind::RParen,
85            Ok(RawTokenKind::LBracket) => SyntaxKind::LBracket,
86            Ok(RawTokenKind::RBracket) => SyntaxKind::RBracket,
87            Ok(RawTokenKind::LBrace) => SyntaxKind::LBrace,
88            Ok(RawTokenKind::RBrace) => SyntaxKind::RBrace,
89            Ok(RawTokenKind::Comma) => SyntaxKind::Comma,
90            Ok(RawTokenKind::String) => SyntaxKind::String,
91            Ok(RawTokenKind::Ident) if text == GROUP_KEYWORD => SyntaxKind::GroupKw,
92            Ok(RawTokenKind::Ident) => SyntaxKind::Ident,
93            Ok(RawTokenKind::Newline) => SyntaxKind::Newline,
94            Ok(RawTokenKind::Spaces) if line_start => SyntaxKind::Indent,
95            Ok(RawTokenKind::Spaces) => SyntaxKind::Whitespace,
96            Err(_) => SyntaxKind::Unknown,
97        };
98
99        tokens.push(LexToken {
100            kind,
101            text: SmolStr::new(text),
102            range: TextRange::new(start, end),
103        });
104
105        line_start = kind == SyntaxKind::Newline;
106    }
107
108    let eof = TextSize::from(source.len() as u32);
109    tokens.push(LexToken {
110        kind: SyntaxKind::Eof,
111        text: SmolStr::new(""),
112        range: TextRange::new(eof, eof),
113    });
114
115    tokens
116}
only_syntax/lex.rs

only_syntax/
lex.rs
