only_syntax/
lex.rs

1use logos::Logos;
2use smol_str::SmolStr;
3use text_size::{TextRange, TextSize};
4
5use crate::{LexToken, SyntaxKind};
6
7#[derive(Logos, Debug, Clone, Copy, PartialEq, Eq)]
8enum RawTokenKind {
9    #[token("\u{feff}")]
10    Bom,
11    #[token("shell?=")]
12    ShellFallbackKw,
13    #[token("shell")]
14    ShellKw,
15    #[token("!")]
16    Bang,
17    #[token("#")]
18    Percent,
19    #[token(":")]
20    Colon,
21    #[token("?")]
22    Question,
23    #[token("&")]
24    Amp,
25    #[token("=")]
26    Eq,
27    #[token("@")]
28    At,
29    #[token("(")]
30    LParen,
31    #[token(")")]
32    RParen,
33    #[token("[")]
34    LBracket,
35    #[token("]")]
36    RBracket,
37    #[regex(r#""([^"\n]|\\.)*""#)]
38    String,
39    #[regex(r"[A-Za-z_-][A-Za-z0-9_-]*")]
40    Ident,
41    #[regex(r"//[^\n]*", allow_greedy = true)]
42    Comment,
43    #[regex(r"[ ]+")]
44    Spaces,
45    #[regex(r"\r\n|\n|\r")]
46    Newline,
47}
48
49/// Lexes source text into tokens while preserving trivia.
50///
51/// Args:
52/// source: Raw Onlyfile source text.
53///
54/// Returns:
55/// Token stream including whitespace, comments and EOF.
56pub fn lex(source: &str) -> Vec<LexToken> {
57    let mut lexer = RawTokenKind::lexer(source);
58    let mut tokens = Vec::new();
59    let mut line_start = true;
60
61    while let Some(result) = lexer.next() {
62        let span = lexer.span();
63        let text = &source[span.clone()];
64        let start = TextSize::from(span.start as u32);
65        let end = TextSize::from(span.end as u32);
66        let kind = match result {
67            Ok(RawTokenKind::Bom) if span.start == 0 => SyntaxKind::Bom,
68            Ok(RawTokenKind::Bom) => SyntaxKind::Unknown,
69            Ok(RawTokenKind::ShellFallbackKw) => SyntaxKind::ShellFallbackKw,
70            Ok(RawTokenKind::ShellKw) => SyntaxKind::ShellKw,
71            Ok(RawTokenKind::Bang) => SyntaxKind::Bang,
72            Ok(RawTokenKind::Percent) => SyntaxKind::Percent,
73            Ok(RawTokenKind::Colon) => SyntaxKind::Colon,
74            Ok(RawTokenKind::Question) => SyntaxKind::Question,
75            Ok(RawTokenKind::Amp) => SyntaxKind::Amp,
76            Ok(RawTokenKind::Eq) => SyntaxKind::Eq,
77            Ok(RawTokenKind::At) => SyntaxKind::At,
78            Ok(RawTokenKind::LParen) => SyntaxKind::LParen,
79            Ok(RawTokenKind::RParen) => SyntaxKind::RParen,
80            Ok(RawTokenKind::LBracket) => SyntaxKind::LBracket,
81            Ok(RawTokenKind::RBracket) => SyntaxKind::RBracket,
82            Ok(RawTokenKind::String) => SyntaxKind::String,
83            Ok(RawTokenKind::Ident) => SyntaxKind::Ident,
84            Ok(RawTokenKind::Comment) => SyntaxKind::Comment,
85            Ok(RawTokenKind::Newline) => SyntaxKind::Newline,
86            Ok(RawTokenKind::Spaces) if line_start => SyntaxKind::Indent,
87            Ok(RawTokenKind::Spaces) => SyntaxKind::Whitespace,
88            Err(_) => SyntaxKind::Unknown,
89        };
90
91        tokens.push(LexToken {
92            kind,
93            text: SmolStr::new(text),
94            range: TextRange::new(start, end),
95        });
96
97        line_start = kind == SyntaxKind::Newline;
98    }
99
100    let eof = TextSize::from(source.len() as u32);
101    tokens.push(LexToken {
102        kind: SyntaxKind::Eof,
103        text: SmolStr::new(""),
104        range: TextRange::new(eof, eof),
105    });
106
107    tokens
108}
only_syntax/lex.rs

only_syntax/
lex.rs
