scarf_parser/lexer/
mod.rs

1// =======================================================================
2// mod.rs
3// =======================================================================
4//! Lexing a source file into semantic tokens
5
6pub(crate) mod callbacks;
7pub(crate) mod keywords;
8pub(crate) mod tokens;
9use crate::SpannedToken;
10use crate::report::{Report, ReportKind};
11pub use keywords::StandardVersion;
12use logos::Logos;
13use logos::Span as ByteSpan;
14use scarf_syntax::Span;
15use std::fs::{self, File};
16use std::io::{self, BufWriter, Write};
17use std::path::Path;
18pub use tokens::Token;
19
20/// A single result from the lexer
21///
22/// This is either a valid token, or a (possible) explanation
23/// of what went wrong, if identifiable, along with the associated
24/// [`Span`]
25pub type LexerResult<'a> = (Result<Token<'a>, String>, Span<'a>);
26
27impl<'a> TryFrom<&LexerResult<'a>> for Report {
28    type Error = ();
29    fn try_from(
30        value: &(Result<Token<'a>, String>, Span<'a>),
31    ) -> Result<Self, Self::Error> {
32        let (result, span) = value;
33        let Err(text) = result else {
34            return Err(());
35        };
36        if text.len() == 0 {
37            Ok(
38                Report::new(
39                    ReportKind::Error,
40                    span,
41                    "L1",
42                    "Unrecognized token",
43                )
44                .with_label(
45                    &span,
46                    ReportKind::Error,
47                    "Unrecognized token",
48                ),
49            )
50        } else {
51            Ok(Report::new(ReportKind::Error, span, "L2", text).with_label(
52                &span,
53                ReportKind::Error,
54                text,
55            ))
56        }
57    }
58}
59
60fn map_lex_result<'a>(lex_result: LexerResult<'a>) -> SpannedToken<'a> {
61    match lex_result.0 {
62        Ok(tok) => SpannedToken(tok, lex_result.1),
63        Err(_) => SpannedToken(Token::Error, lex_result.1),
64    }
65}
66
67/// An iterator over syntactical tokens for a SystemVerilog source
68pub trait LexedSource<'a>: Iterator<Item = LexerResult<'a>> + Clone {
69    /// Generate error reports for any errors encountered in lexing
70    fn report_errors(&self) -> impl Iterator<Item = Report> {
71        self.clone().into_iter().filter_map(|result| {
72            let report_result: Result<Report, ()> = Report::try_from(&result);
73            report_result.ok()
74        })
75    }
76
77    /// Dump a representation of the lexed source to a file, for debugging
78    fn dump(&self, file_path: &Path) -> io::Result<()> {
79        if let Some(parent_dir) = file_path.parent() {
80            fs::create_dir_all(parent_dir)?;
81        }
82        let file = File::create(file_path)?;
83        let mut writer = BufWriter::new(file);
84        for (result, span) in self.clone() {
85            let dump_str = format!(
86                "[{:>2}:{:>2}] {}\n",
87                span.bytes.start,
88                span.bytes.end,
89                match result {
90                    Ok(token) => token,
91                    Err(_) => Token::Error,
92                }
93            );
94            writer.write_all(dump_str.as_bytes())?;
95        }
96        writer.flush()?;
97        Ok(())
98    }
99
100    /// Process the lexing of the source, storing the result
101    ///
102    /// While this does incur memory overhead, it avoid processing
103    /// the source multiple times if cloned.
104    fn process(self) -> std::vec::IntoIter<LexerResult<'a>> {
105        self.collect::<Vec<_>>().into_iter()
106    }
107
108    /// Convert lexer results into tokens, turning errors into [`Token::Error`]
109    fn tokens(self) -> impl Iterator<Item = SpannedToken<'a>> {
110        self.into_iter().map(map_lex_result)
111    }
112}
113impl<'a, T> LexedSource<'a> for T where
114    T: Iterator<Item = LexerResult<'a>> + Clone
115{
116}
117
118fn token_span_mapper<'a>(
119    file_name: &'a str,
120    included_from: Option<&'a Span<'a>>,
121) -> impl Fn((Result<Token<'a>, String>, ByteSpan)) -> LexerResult<'a> + Clone {
122    move |(token_result, byte_span)| {
123        (
124            token_result,
125            Span {
126                file: file_name,
127                bytes: byte_span,
128                expanded_from: None,
129                included_from,
130            },
131        )
132    }
133}
134
135pub(crate) fn lex_helper<'a>(
136    src: &'a str,
137    file_name: &'a str,
138    included_from: Option<&'a Span<'a>>,
139) -> impl LexedSource<'a> {
140    let span_mapper = token_span_mapper(file_name, included_from);
141    Token::lexer(src).spanned().map(span_mapper)
142}
143
144/// Separate a source file into syntactic tokens
145///
146/// ```rust
147/// # use scarf_parser::*;
148/// let file_contents = "module test_module; endmodule";
149/// let mut tokens = lex(file_contents, "test_file.v");
150/// assert!(matches!(tokens.next().unwrap(), (Ok(Token::Module), _)));
151/// assert!(matches!(tokens.next().unwrap(), (Ok(Token::SimpleIdentifier("test_module")), _)));
152/// assert!(matches!(tokens.next().unwrap(), (Ok(Token::SColon), _)));
153/// assert!(matches!(tokens.next().unwrap(), (Ok(Token::Endmodule), _)));
154/// assert!(tokens.next().is_none());
155/// ```
156///
157/// If the lexer encounters an error, the resulting `Err(string)` may contain
158/// more information if possible to discern.
159pub fn lex<'a>(src: &'a str, file_name: &'a str) -> impl LexedSource<'a> {
160    lex_helper(src, file_name, None)
161}
scarf_parser/lexer/mod.rs

scarf_parser/lexer/
mod.rs
