only_syntax/
lex.rs

1use logos::Logos;
2use smol_str::SmolStr;
3use text_size::{TextRange, TextSize};
4
5use crate::builtin::GROUP_KEYWORD;
6use crate::{LexToken, SyntaxKind};
7
8#[derive(Logos, Debug, Clone, Copy, PartialEq, Eq)]
9enum RawTokenKind {
10    #[token("\u{feff}")]
11    Bom,
12    #[token("shell~=")]
13    ShellFallbackKw,
14    #[token("shell")]
15    ShellKw,
16    #[token("!")]
17    Bang,
18    #[regex(r"#[^\n]*", allow_greedy = true)]
19    HashComment,
20    #[token(":")]
21    Colon,
22    #[token("?")]
23    Question,
24    #[token("&")]
25    Amp,
26    #[token("=")]
27    Eq,
28    #[token("@")]
29    At,
30    #[token("(")]
31    LParen,
32    #[token(")")]
33    RParen,
34    #[token("[")]
35    LBracket,
36    #[token("]")]
37    RBracket,
38    #[token("{")]
39    LBrace,
40    #[token("}")]
41    RBrace,
42    #[token(",")]
43    Comma,
44    #[regex(r#""([^"\n]|\\.)*""#)]
45    String,
46    #[regex(r"[A-Za-z_-][A-Za-z0-9_-]*")]
47    Ident,
48    #[regex(r"//[^\n]*", allow_greedy = true)]
49    Comment,
50    #[regex(r"[ \t]+")]
51    Spaces,
52    #[regex(r"\r\n|\n|\r")]
53    Newline,
54}
55
56/// Lexes source text into tokens while preserving trivia.
57///
58/// Args:
59/// source: Raw Onlyfile source text.
60///
61/// Returns:
62/// Token stream including whitespace, comments and EOF.
63pub fn lex(source: &str) -> Vec<LexToken> {
64    let mut lexer = RawTokenKind::lexer(source);
65    let mut tokens = Vec::new();
66    let mut line_start = true;
67
68    while let Some(result) = lexer.next() {
69        let span = lexer.span();
70        let text = &source[span.clone()];
71        let start = TextSize::from(span.start as u32);
72        let end = TextSize::from(span.end as u32);
73        let kind = match result {
74            Ok(RawTokenKind::Bom) if span.start == 0 => SyntaxKind::Bom,
75            Ok(RawTokenKind::Bom) => SyntaxKind::Unknown,
76            Ok(RawTokenKind::ShellFallbackKw) => SyntaxKind::ShellFallbackKw,
77            Ok(RawTokenKind::ShellKw) => SyntaxKind::ShellKw,
78            Ok(RawTokenKind::Bang) => SyntaxKind::Bang,
79            Ok(RawTokenKind::HashComment) => SyntaxKind::Comment,
80            Ok(RawTokenKind::Colon) => SyntaxKind::Colon,
81            Ok(RawTokenKind::Question) => SyntaxKind::Question,
82            Ok(RawTokenKind::Amp) => SyntaxKind::Amp,
83            Ok(RawTokenKind::Eq) => SyntaxKind::Eq,
84            Ok(RawTokenKind::At) => SyntaxKind::At,
85            Ok(RawTokenKind::LParen) => SyntaxKind::LParen,
86            Ok(RawTokenKind::RParen) => SyntaxKind::RParen,
87            Ok(RawTokenKind::LBracket) => SyntaxKind::LBracket,
88            Ok(RawTokenKind::RBracket) => SyntaxKind::RBracket,
89            Ok(RawTokenKind::LBrace) => SyntaxKind::LBrace,
90            Ok(RawTokenKind::RBrace) => SyntaxKind::RBrace,
91            Ok(RawTokenKind::Comma) => SyntaxKind::Comma,
92            Ok(RawTokenKind::String) => SyntaxKind::String,
93            Ok(RawTokenKind::Ident) if text == GROUP_KEYWORD => SyntaxKind::GroupKw,
94            Ok(RawTokenKind::Ident) => SyntaxKind::Ident,
95            Ok(RawTokenKind::Comment) => SyntaxKind::Comment,
96            Ok(RawTokenKind::Newline) => SyntaxKind::Newline,
97            Ok(RawTokenKind::Spaces) if line_start => SyntaxKind::Indent,
98            Ok(RawTokenKind::Spaces) => SyntaxKind::Whitespace,
99            Err(_) => SyntaxKind::Unknown,
100        };
101
102        tokens.push(LexToken {
103            kind,
104            text: SmolStr::new(text),
105            range: TextRange::new(start, end),
106        });
107
108        line_start = kind == SyntaxKind::Newline;
109    }
110
111    let eof = TextSize::from(source.len() as u32);
112    tokens.push(LexToken {
113        kind: SyntaxKind::Eof,
114        text: SmolStr::new(""),
115        range: TextRange::new(eof, eof),
116    });
117
118    tokens
119}
only_syntax/lex.rs

only_syntax/
lex.rs
