only_syntax/
lex.rs

1use logos::Logos;
2use smol_str::SmolStr;
3use text_size::{TextRange, TextSize};
4
5use crate::{LexToken, SyntaxKind};
6
7#[derive(Logos, Debug, Clone, Copy, PartialEq, Eq)]
8enum RawTokenKind {
9    #[token("\u{feff}")]
10    Bom,
11    #[token("shell~=")]
12    ShellFallbackKw,
13    #[token("shell")]
14    ShellKw,
15    #[token("!")]
16    Bang,
17    #[token("#")]
18    Percent,
19    #[token(":")]
20    Colon,
21    #[token("?")]
22    Question,
23    #[token("&")]
24    Amp,
25    #[token("=")]
26    Eq,
27    #[token("@")]
28    At,
29    #[token("(")]
30    LParen,
31    #[token(")")]
32    RParen,
33    #[token("[")]
34    LBracket,
35    #[token("]")]
36    RBracket,
37    #[token("{")]
38    LBrace,
39    #[token("}")]
40    RBrace,
41    #[regex(r#""([^"\n]|\\.)*""#)]
42    String,
43    #[regex(r"[A-Za-z_-][A-Za-z0-9_-]*")]
44    Ident,
45    #[regex(r"//[^\n]*", allow_greedy = true)]
46    Comment,
47    #[regex(r"[ \t]+")]
48    Spaces,
49    #[regex(r"\r\n|\n|\r")]
50    Newline,
51}
52
53/// Lexes source text into tokens while preserving trivia.
54///
55/// Args:
56/// source: Raw Onlyfile source text.
57///
58/// Returns:
59/// Token stream including whitespace, comments and EOF.
60pub fn lex(source: &str) -> Vec<LexToken> {
61    let mut lexer = RawTokenKind::lexer(source);
62    let mut tokens = Vec::new();
63    let mut line_start = true;
64
65    while let Some(result) = lexer.next() {
66        let span = lexer.span();
67        let text = &source[span.clone()];
68        let start = TextSize::from(span.start as u32);
69        let end = TextSize::from(span.end as u32);
70        let kind = match result {
71            Ok(RawTokenKind::Bom) if span.start == 0 => SyntaxKind::Bom,
72            Ok(RawTokenKind::Bom) => SyntaxKind::Unknown,
73            Ok(RawTokenKind::ShellFallbackKw) => SyntaxKind::ShellFallbackKw,
74            Ok(RawTokenKind::ShellKw) => SyntaxKind::ShellKw,
75            Ok(RawTokenKind::Bang) => SyntaxKind::Bang,
76            Ok(RawTokenKind::Percent) => SyntaxKind::Percent,
77            Ok(RawTokenKind::Colon) => SyntaxKind::Colon,
78            Ok(RawTokenKind::Question) => SyntaxKind::Question,
79            Ok(RawTokenKind::Amp) => SyntaxKind::Amp,
80            Ok(RawTokenKind::Eq) => SyntaxKind::Eq,
81            Ok(RawTokenKind::At) => SyntaxKind::At,
82            Ok(RawTokenKind::LParen) => SyntaxKind::LParen,
83            Ok(RawTokenKind::RParen) => SyntaxKind::RParen,
84            Ok(RawTokenKind::LBracket) => SyntaxKind::LBracket,
85            Ok(RawTokenKind::RBracket) => SyntaxKind::RBracket,
86            Ok(RawTokenKind::LBrace) => SyntaxKind::LBrace,
87            Ok(RawTokenKind::RBrace) => SyntaxKind::RBrace,
88            Ok(RawTokenKind::String) => SyntaxKind::String,
89            Ok(RawTokenKind::Ident) => SyntaxKind::Ident,
90            Ok(RawTokenKind::Comment) => SyntaxKind::Comment,
91            Ok(RawTokenKind::Newline) => SyntaxKind::Newline,
92            Ok(RawTokenKind::Spaces) if line_start => SyntaxKind::Indent,
93            Ok(RawTokenKind::Spaces) => SyntaxKind::Whitespace,
94            Err(_) => SyntaxKind::Unknown,
95        };
96
97        tokens.push(LexToken {
98            kind,
99            text: SmolStr::new(text),
100            range: TextRange::new(start, end),
101        });
102
103        line_start = kind == SyntaxKind::Newline;
104    }
105
106    let eof = TextSize::from(source.len() as u32);
107    tokens.push(LexToken {
108        kind: SyntaxKind::Eof,
109        text: SmolStr::new(""),
110        range: TextRange::new(eof, eof),
111    });
112
113    tokens
114}
only_syntax/lex.rs

only_syntax/
lex.rs
