device_driver_lexer/
lib.rs

1use std::{borrow::Cow, fmt::Display};
2
3use device_driver_common::{
4    span::{SpanExt, Spanned},
5    specifiers::{Access, AddressMode, BaseType, ByteOrder, Integer},
6};
7use logos::Logos;
8
9pub fn lex(source: &str) -> Vec<Spanned<Token<'_>>> {
10    Token::lexer(source)
11        .spanned()
12        .map(|(token, span)| match token {
13            Ok(token) => token.with_span(span),
14            Err(()) => Token::Error.with_span(span),
15        })
16        .collect()
17}
18
19// Don't forget to update the book when tokens are added, changed or removed!
20#[derive(Debug, Clone, Copy, PartialEq, Logos)]
21#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
22#[logos(skip r"[ \t\r\n]+")] // Skip (common) whitespace
23#[logos(skip(r"//[^\n]*", allow_greedy = true))] // Skip comments
24pub enum Token<'src> {
25    #[regex(r"///[^\n]*", allow_greedy = true, callback = |lex| lex.slice().trim_start_matches("///"))]
26    DocCommentLine(&'src str),
27    #[regex(r"\p{XID_Start}[\p{XID_Continue}-]*")]
28    Ident(&'src str),
29    #[token("{")]
30    CurlyOpen,
31    #[token("}")]
32    CurlyClose,
33    #[token("[")]
34    BracketOpen,
35    #[token("]")]
36    BracketClose,
37    #[token(",")]
38    Comma,
39    #[token(":")]
40    Colon,
41    #[token("_")]
42    Underscore,
43    #[token("->")]
44    Arrow,
45    #[token("*")]
46    Star,
47    #[token("try")]
48    Try,
49    #[token("as")]
50    As,
51    #[token("allow")]
52    Allow,
53    #[token("default")]
54    Default,
55    #[token("catch-all")]
56    CatchAll,
57    #[token("stride")]
58    Stride,
59    #[regex(r"-?[0-9][_0-9]*")] // Decimal
60    #[regex(r"-?0b[_0-1]+")] // Binary
61    #[regex(r"-?0o[_0-7]+")] // Octal
62    #[regex(r"-?0x[_0-9a-fA-F]+")] // Hex
63    Num(&'src str),
64    #[token("RW", |_| Access::RW)]
65    #[token("RO", |_| Access::RO)]
66    #[token("WO", |_| Access::WO)]
67    Access(Access),
68    #[token("BE", |_| ByteOrder::BE)]
69    #[token("LE", |_| ByteOrder::LE)]
70    ByteOrder(ByteOrder),
71    /// All the base types except fixed integers. Those are [`Self::Integer`].
72    #[token("uint", |_| BaseType::Uint)]
73    #[token("int", |_| BaseType::Int)]
74    #[token("bool", |_| BaseType::Bool)]
75    BaseType(BaseType),
76    #[token("u8", |_| Integer::U8)]
77    #[token("u16", |_| Integer::U16)]
78    #[token("u32", |_| Integer::U32)]
79    #[token("u64", |_| Integer::U64)]
80    #[token("i8", |_| Integer::I8)]
81    #[token("i16", |_| Integer::I16)]
82    #[token("i32", |_| Integer::I32)]
83    #[token("i64", |_| Integer::I64)]
84    Integer(Integer),
85    #[token("mapped", |_| AddressMode::Mapped)]
86    #[token("indexed", |_| AddressMode::Indexed)]
87    AddressMode(AddressMode),
88    // Very simple definition without string escaping
89    #[regex(r#""[^"]*""#, callback = |lex| lex.slice().strip_prefix('"').unwrap().strip_suffix('"').unwrap())]
90    String(&'src str),
91    #[regex(r"\S", priority = 0)] // Any non-whitespace character
92    Unexpected(&'src str),
93    Error, // Catch-all for errors
94}
95
96impl Display for Token<'_> {
97    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
98        match self {
99            Token::DocCommentLine(_) => write!(f, "doc comment"),
100            Token::Ident(_) => write!(f, "identifier"),
101            Token::CurlyOpen => write!(f, "{{"),
102            Token::CurlyClose => write!(f, "}}"),
103            Token::BracketOpen => write!(f, "["),
104            Token::BracketClose => write!(f, "]"),
105            Token::Colon => write!(f, ":"),
106            Token::Underscore => write!(f, "_"),
107            Token::Comma => write!(f, ","),
108            Token::Arrow => write!(f, "->"),
109            Token::Star => write!(f, "*"),
110            Token::Try => write!(f, "try"),
111            Token::As => write!(f, "as"),
112            Token::Allow => write!(f, "allow"),
113            Token::Default => write!(f, "default"),
114            Token::CatchAll => write!(f, "catch-all"),
115            Token::Stride => write!(f, "stride"),
116            Token::Num(_) => write!(f, "number"),
117            Token::Access(_) => write!(f, "access specifier"),
118            Token::ByteOrder(_) => write!(f, "byte order"),
119            Token::BaseType(_) => write!(f, "base type"),
120            Token::Integer(_) => write!(f, "integer type"),
121            Token::AddressMode(_) => write!(f, "address mode"),
122            Token::String(_) => write!(f, "string"),
123            Token::Unexpected(val) => write!(f, "{}", val.escape_debug()),
124            Token::Error => write!(f, "ERROR"),
125        }
126    }
127}
128
129impl<'src> Token<'src> {
130    fn get_human_string(&self) -> Cow<'static, str> {
131        match self {
132            Token::DocCommentLine(line) => format!("///{line}").into(),
133            Token::Ident(ident) => format!("#{ident}").into(),
134            Token::CurlyOpen => "{".into(),
135            Token::CurlyClose => "}".into(),
136            Token::BracketOpen => "[".into(),
137            Token::BracketClose => "]".into(),
138            Token::Colon => ":".into(),
139            Token::Underscore => "_".into(),
140            Token::Comma => ",".into(),
141            Token::Arrow => "->".into(),
142            Token::Try => "try".into(),
143            Token::Star => "*".into(),
144            Token::As => "as".into(),
145            Token::Num(n) => n.to_string().into(),
146            Token::Access(val) => val.to_string().into(),
147            Token::ByteOrder(val) => val.to_string().into(),
148            Token::BaseType(val) => val.to_string().into(),
149            Token::Integer(val) => val.to_string().into(),
150            Token::AddressMode(val) => val.to_string().into(),
151            Token::Allow => "allow".into(),
152            Token::Default => "default".into(),
153            Token::CatchAll => "catch-all".into(),
154            Token::Stride => "stride".into(),
155            Token::Unexpected(raw) => format!("!{raw}").into(),
156            Token::Error => "UNEXPECTED".into(),
157            Token::String(val) => format!("\"{val}\"").into(),
158        }
159    }
160
161    /// New line before, new line after, indent change
162    fn get_print_format(&self) -> (bool, bool, i32) {
163        match self {
164            Token::DocCommentLine(_) => (true, true, 0),
165            Token::Comma => (false, true, 0),
166            Token::CurlyOpen | Token::BracketOpen => (false, true, 1),
167            Token::CurlyClose | Token::BracketClose => (true, false, -1),
168            _ => (false, false, 0),
169        }
170    }
171
172    pub fn formatted_print<'a, I: Iterator<Item = &'a Token<'src>>>(
173        stream: &mut impl std::fmt::Write,
174        tokens: I,
175    ) -> Result<(), std::fmt::Error>
176    where
177        'src: 'a,
178    {
179        let mut indent = 0i32;
180        for token in tokens {
181            let (newline_before, newline_after, indent_change) = token.get_print_format();
182
183            indent += indent_change;
184            if newline_before {
185                write!(
186                    stream,
187                    "\n{:width$}",
188                    "",
189                    width = indent.max(0) as usize * 4
190                )?;
191            }
192
193            write!(stream, "{} ", token.get_human_string())?;
194
195            if newline_after {
196                write!(
197                    stream,
198                    "\n{:width$}",
199                    "",
200                    width = indent.max(0) as usize * 4
201                )?;
202            }
203        }
204
205        Ok(())
206    }
207}
device_driver_lexer/lib.rs

device_driver_lexer/
lib.rs
