device_driver_lexer/
lib.rs

1use std::{borrow::Cow, fmt::Display};
2
3use device_driver_common::{
4    span::{SpanExt, Spanned},
5    specifiers::{Access, AddressMode, BaseType, ByteOrder, Integer, VariantNames},
6};
7use logos::Logos;
8
9pub mod semantic_token_object;
10
11pub fn lex(source: &str) -> Vec<Spanned<Token<'_>>> {
12    Token::lexer(source)
13        .spanned()
14        .map(|(token, span)| match token {
15            Ok(token) => token.with_span(span),
16            Err(()) => Token::Error.with_span(span),
17        })
18        .collect()
19}
20
21// Don't forget to update the book when tokens are added, changed or removed!
22#[derive(Debug, Clone, Copy, PartialEq, Logos)]
23#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
24#[logos(skip r"[ \t\r\n]+")] // Skip (common) whitespace
25#[logos(skip(r"//[^\n]*", allow_greedy = true))] // Skip comments
26pub enum Token<'src> {
27    #[regex(r"///[^\n]*", allow_greedy = true, callback = |lex| lex.slice().trim_start_matches("///"))]
28    DocCommentLine(&'src str),
29    #[regex(r"\p{XID_Start}[\p{XID_Continue}-]*")]
30    Ident(&'src str),
31    #[token("{")]
32    CurlyOpen,
33    #[token("}")]
34    CurlyClose,
35    #[token("[")]
36    BracketOpen,
37    #[token("]")]
38    BracketClose,
39    #[token(",")]
40    Comma,
41    #[token(":")]
42    Colon,
43    #[token("_")]
44    Underscore,
45    #[token("->")]
46    Arrow,
47    #[token("*")]
48    Star,
49    #[token("try")]
50    Try,
51    #[token("as")]
52    As,
53    #[token("allow")]
54    Allow,
55    #[token("default")]
56    Default,
57    #[token("catch-all")]
58    CatchAll,
59    #[token("stride")]
60    Stride,
61    #[regex(r"-?[0-9][_0-9]*")] // Decimal
62    #[regex(r"-?0b[_0-1]+")] // Binary
63    #[regex(r"-?0o[_0-7]+")] // Octal
64    #[regex(r"-?0x[_0-9a-fA-F]+")] // Hex
65    Num(&'src str),
66    #[token("RW", |_| Access::RW)]
67    #[token("RO", |_| Access::RO)]
68    #[token("WO", |_| Access::WO)]
69    Access(Access),
70    #[token("BE", |_| ByteOrder::BE)]
71    #[token("LE", |_| ByteOrder::LE)]
72    ByteOrder(ByteOrder),
73    /// All the base types except fixed integers. Those are [`Self::Integer`].
74    #[token("uint", |_| BaseType::Uint)]
75    #[token("int", |_| BaseType::Int)]
76    #[token("bool", |_| BaseType::Bool)]
77    BaseType(BaseType),
78    #[token("u8", |_| Integer::U8)]
79    #[token("u16", |_| Integer::U16)]
80    #[token("u32", |_| Integer::U32)]
81    #[token("u64", |_| Integer::U64)]
82    #[token("i8", |_| Integer::I8)]
83    #[token("i16", |_| Integer::I16)]
84    #[token("i32", |_| Integer::I32)]
85    #[token("i64", |_| Integer::I64)]
86    Integer(Integer),
87    #[token("mapped", |_| AddressMode::Mapped)]
88    #[token("indexed", |_| AddressMode::Indexed)]
89    AddressMode(AddressMode),
90    // Very simple definition without string escaping
91    #[regex(r#""[^"]*""#, callback = |lex| lex.slice().strip_prefix('"').unwrap().strip_suffix('"').unwrap())]
92    String(&'src str),
93    #[regex(r"\S", priority = 0)] // Any non-whitespace character
94    Unexpected(&'src str),
95    Error, // Catch-all for errors
96}
97
98impl Display for Token<'_> {
99    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
100        match self {
101            Token::DocCommentLine(_) => write!(f, "doc comment"),
102            Token::Ident(_) => write!(f, "identifier"),
103            Token::CurlyOpen => write!(f, "{{"),
104            Token::CurlyClose => write!(f, "}}"),
105            Token::BracketOpen => write!(f, "["),
106            Token::BracketClose => write!(f, "]"),
107            Token::Colon => write!(f, ":"),
108            Token::Underscore => write!(f, "_"),
109            Token::Comma => write!(f, ","),
110            Token::Arrow => write!(f, "->"),
111            Token::Star => write!(f, "*"),
112            Token::Try => write!(f, "try"),
113            Token::As => write!(f, "as"),
114            Token::Allow => write!(f, "allow"),
115            Token::Default => write!(f, "default"),
116            Token::CatchAll => write!(f, "catch-all"),
117            Token::Stride => write!(f, "stride"),
118            Token::Num(_) => write!(f, "number"),
119            Token::Access(_) => write!(f, "access specifier"),
120            Token::ByteOrder(_) => write!(f, "byte order"),
121            Token::BaseType(_) => write!(f, "base type"),
122            Token::Integer(_) => write!(f, "integer type"),
123            Token::AddressMode(_) => write!(f, "address mode"),
124            Token::String(_) => write!(f, "string"),
125            Token::Unexpected(val) => write!(f, "{}", val.escape_debug()),
126            Token::Error => write!(f, "ERROR"),
127        }
128    }
129}
130
131impl<'src> Token<'src> {
132    fn get_human_string(&self) -> Cow<'static, str> {
133        match self {
134            Token::DocCommentLine(line) => format!("///{line}").into(),
135            Token::Ident(ident) => format!("#{ident}").into(),
136            Token::CurlyOpen => "{".into(),
137            Token::CurlyClose => "}".into(),
138            Token::BracketOpen => "[".into(),
139            Token::BracketClose => "]".into(),
140            Token::Colon => ":".into(),
141            Token::Underscore => "_".into(),
142            Token::Comma => ",".into(),
143            Token::Arrow => "->".into(),
144            Token::Try => "try".into(),
145            Token::Star => "*".into(),
146            Token::As => "as".into(),
147            Token::Num(n) => n.to_string().into(),
148            Token::Access(val) => val.to_string().into(),
149            Token::ByteOrder(val) => val.to_string().into(),
150            Token::BaseType(val) => val.to_string().into(),
151            Token::Integer(val) => val.to_string().into(),
152            Token::AddressMode(val) => val.to_string().into(),
153            Token::Allow => "allow".into(),
154            Token::Default => "default".into(),
155            Token::CatchAll => "catch-all".into(),
156            Token::Stride => "stride".into(),
157            Token::Unexpected(raw) => format!("!{raw}").into(),
158            Token::Error => "UNEXPECTED".into(),
159            Token::String(val) => format!("\"{val}\"").into(),
160        }
161    }
162
163    /// New line before, new line after, indent change
164    fn get_print_format(&self) -> (bool, bool, i32) {
165        match self {
166            Token::DocCommentLine(_) => (true, true, 0),
167            Token::Comma => (false, true, 0),
168            Token::CurlyOpen | Token::BracketOpen => (false, true, 1),
169            Token::CurlyClose | Token::BracketClose => (true, false, -1),
170            _ => (false, false, 0),
171        }
172    }
173
174    pub fn formatted_print<'a, I: Iterator<Item = &'a Token<'src>>>(
175        stream: &mut impl std::fmt::Write,
176        tokens: I,
177    ) -> Result<(), std::fmt::Error>
178    where
179        'src: 'a,
180    {
181        let mut indent = 0i32;
182        for token in tokens {
183            let (newline_before, newline_after, indent_change) = token.get_print_format();
184
185            indent += indent_change;
186            if newline_before {
187                write!(
188                    stream,
189                    "\n{:width$}",
190                    "",
191                    width = indent.max(0) as usize * 4
192                )?;
193            }
194
195            write!(stream, "{} ", token.get_human_string())?;
196
197            if newline_after {
198                write!(
199                    stream,
200                    "\n{:width$}",
201                    "",
202                    width = indent.max(0) as usize * 4
203                )?;
204            }
205        }
206
207        Ok(())
208    }
209}
210
211#[derive(Debug, Clone, Copy, PartialEq, Eq)]
212pub enum TokenType {
213    Namespace,
214    Type,
215    Class,
216    Enum,
217    Interface,
218    Struct,
219    TypeParameter,
220    Parameter,
221    Variable,
222    Property,
223    EnumMember,
224    Event,
225    Function,
226    Method,
227    Macro,
228    Keyword,
229    Modifier,
230    Comment,
231    String,
232    Number,
233    Regexp,
234    Operator,
235    Decorator,
236    Label,
237    // Custom:
238    NodeType,
239    Access,
240    ByteOrder,
241    AddressMode,
242}
243
244impl VariantNames for TokenType {
245    const VARIANTS: &'static [&'static str] = &[
246        "namespace",
247        "type",
248        "class",
249        "enum",
250        "interface",
251        "struct",
252        "typeparameter",
253        "parameter",
254        "variable",
255        "property",
256        "enummember",
257        "event",
258        "function",
259        "method",
260        "macro",
261        "keyword",
262        "modifier",
263        "comment",
264        "string",
265        "number",
266        "regexp",
267        "operator",
268        "decorator",
269        "label",
270        "nodetype",
271        "access",
272        "byteorder",
273        "addressmode",
274    ];
275    fn name(&self) -> &'static str {
276        Self::VARIANTS[*self as usize]
277    }
278}
279
280impl Display for TokenType {
281    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
282        write!(f, "{}", Self::VARIANTS[*self as usize])
283    }
284}
285
286#[derive(Debug, Clone, Copy, PartialEq, Eq)]
287pub enum TokenModifier {
288    Declaration = 0b1,
289    Definition = 0b10,
290    Readonly = 0b100,
291    Static = 0b1000,
292    Deprecated = 0b1_0000,
293    Abstract = 0b10_0000,
294    Async = 0b100_0000,
295    Modification = 0b1000_0000,
296    Documentation = 0b1_0000_0000,
297    DefaultLibrary = 0b10_0000_0000,
298    // Custom:
299    Expression = 0b100_0000_0000,
300}
301
302impl VariantNames for TokenModifier {
303    const VARIANTS: &'static [&'static str] = &[
304        "declaration",
305        "definition",
306        "readonly",
307        "static",
308        "deprecated",
309        "abstract",
310        "async",
311        "modification",
312        "documentation",
313        "defaultlibrary",
314        "expression",
315    ];
316    fn name(&self) -> &'static str {
317        Self::VARIANTS[*self as usize]
318    }
319}
320
321impl Display for TokenModifier {
322    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
323        write!(f, "{}", Self::VARIANTS[*self as usize])
324    }
325}
device_driver_lexer/lib.rs

device_driver_lexer/
lib.rs
