forj_parser/preprocessor/
mod.rs

1// =======================================================================
2// mod.rs
3// =======================================================================
4//! Preprocessing a token stream, elaborating compiler directives
5
6pub mod cache;
7pub(crate) mod conditional_compilation;
8pub(crate) mod define;
9pub(crate) mod error;
10pub(crate) mod implicit_nettype;
11pub(crate) mod include;
12pub(crate) mod keywords;
13pub(crate) mod line;
14pub mod state;
15pub(crate) mod text_macro;
16pub(crate) mod timescale;
17pub(crate) mod unconnected;
18use crate::*;
19pub use cache::*;
20use conditional_compilation::*;
21use define::*;
22pub use error::*;
23pub(crate) use implicit_nettype::DefaultNettype;
24use implicit_nettype::*;
25use include::*;
26use keywords::*;
27use line::*;
28pub use state::*;
29use std::collections::VecDeque;
30use text_macro::*;
31use timescale::*;
32pub(crate) use timescale::{Timescale, TimescaleUnit, TimescaleValue};
33pub(crate) use unconnected::UnconnectedDrive;
34use unconnected::*;
35
36/// A peekable, extendable iterator over tokens.
37///
38/// This iterator extends `<T>` by keeping track of an additional
39/// stack of tokens at the front, allowing users to peek the next
40/// token, as well as push tokens to be iterated on next (such as
41/// when expanding a preprocessor definition)
42pub(crate) struct TokenIterator<'s, T: Iterator<Item = SpannedToken<'s>>> {
43    iter: T,
44    extras: VecDeque<SpannedToken<'s>>,
45}
46
47impl<'s, T: Iterator<Item = SpannedToken<'s>>> Iterator
48    for TokenIterator<'s, T>
49{
50    type Item = SpannedToken<'s>;
51    fn next(&mut self) -> Option<Self::Item> {
52        if let Some(extra_token) = self.extras.pop_front() {
53            Some(extra_token)
54        } else {
55            self.iter.next()
56        }
57    }
58}
59
60impl<'s, T: Iterator<Item = SpannedToken<'s>>> TokenIterator<'s, T> {
61    pub fn new(iter: T) -> Self {
62        Self {
63            iter,
64            extras: VecDeque::default(),
65        }
66    }
67
68    pub fn prepend_tokens<I>(&mut self, extra_tokens: I)
69    where
70        I: Iterator<Item = SpannedToken<'s>>
71            + ExactSizeIterator
72            + std::iter::DoubleEndedIterator,
73    {
74        self.extras.reserve(extra_tokens.len());
75        for extra_token in extra_tokens.rev() {
76            self.extras.push_front(extra_token);
77        }
78    }
79
80    pub fn peek(&mut self) -> Option<&SpannedToken<'s>> {
81        if self.extras.is_empty() {
82            if let Some(next_token) = self.iter.next() {
83                self.extras.push_back(next_token);
84            }
85        }
86        self.extras.front()
87    }
88}
89
90/// Attempt to recover from a preprocessor error by going to the next
91/// non-escaped newline, returning whether one was encountered
92pub(crate) fn recover_newline<'s>(
93    src: &mut TokenIterator<'s, impl Iterator<Item = SpannedToken<'s>>>,
94) -> bool {
95    loop {
96        let Some(SpannedToken(curr_token, _)) = src.next() else {
97            return false;
98        };
99        let next_token = src.peek();
100        match (curr_token, next_token) {
101            (Token::Bslash, Some(SpannedToken(Token::Newline, _))) => {
102                let _newline_token = src.next();
103                ()
104            }
105            (Token::Newline, _) => {
106                return true;
107            }
108            _ => (),
109        }
110    }
111}
112
113/// Attempt to recover from a preprocessor error, returning whether it
114/// was successful
115///
116/// Many of these are trivial, as they are removed from the token stream
117/// already
118pub(crate) fn recover<'s>(
119    src: &mut TokenIterator<'s, impl Iterator<Item = SpannedToken<'s>>>,
120    state: &mut PreprocessorState<'s>,
121    err: PreprocessorError<'s>,
122) -> bool {
123    let recovered = match err {
124        PreprocessorError::Endif { .. } => true,
125        PreprocessorError::NoEndif { .. } => false, // EOF
126        PreprocessorError::Elsif { .. } => true,
127        PreprocessorError::Else { .. } => true,
128        PreprocessorError::EndKeywords { .. } => true,
129        PreprocessorError::NoEndKeywords { .. } => false, // EOF
130        PreprocessorError::InvalidDefineParameter { .. } => {
131            recover_newline(src)
132        }
133        PreprocessorError::InvalidDefineArgument { .. } => recover_newline(src),
134        PreprocessorError::InvalidVersionSpecifier { .. } => true,
135        PreprocessorError::IncompleteDirective { .. } => recover_newline(src),
136        PreprocessorError::IncompleteDefine { .. } => recover_newline(src),
137        PreprocessorError::UndefinedMacro { .. } => true, // Don't worry about functions here
138        PreprocessorError::DuplicateMacroParameter { .. } => {
139            recover_newline(src)
140        }
141        PreprocessorError::NoDefaultAfterDefault { .. } => recover_newline(src),
142        PreprocessorError::NoMacroArguments { .. } => true,
143        PreprocessorError::TooManyMacroArguments { .. } => true,
144        PreprocessorError::MissingMacroArgument { .. } => true,
145        PreprocessorError::InvalidIdentifierFormation { .. } => true,
146        PreprocessorError::InvalidRelativeTimescales { .. } => true,
147        PreprocessorError::IncompleteMacroWithToken { .. } => true,
148        PreprocessorError::Include { .. } => true,
149        PreprocessorError::IncludeDepth { .. } => true,
150        PreprocessorError::VerboseError { .. } => recover_newline(src),
151        PreprocessorError::NotPreviouslyDefinedMacro { .. }
152        | PreprocessorError::RedefinedMacro { .. } => {
153            panic!("Shouldn't need to recover from warnings")
154        }
155        PreprocessorError::NewlineInDefine(_)
156        | PreprocessorError::EndOfFunctionArgument(_) => {
157            panic!("Tried to recover from an internal error")
158        }
159    };
160    state.err(err);
161    recovered
162}
163
164pub(crate) fn preprocess_helper<'s>(
165    src: &mut TokenIterator<'s, impl Iterator<Item = SpannedToken<'s>>>,
166    dest: &mut Vec<SpannedToken<'s>>,
167    state: &mut PreprocessorState<'s>,
168    cache: &'s PreprocessorCache<'s>,
169) -> Result<(), PreprocessorError<'s>> {
170    let mut enclosures: Vec<Token<'s>> = vec![];
171    if state.in_define() || state.in_define_arg() || state.in_text_macro_arg() {
172        while let Some(spanned_token) = src.next() {
173            match spanned_token.0 {
174                Token::Bslash if !state.in_text_macro_arg() => {
175                    match src.next() {
176                        None => dest.push(spanned_token),
177                        Some(next_token) => match next_token.0 {
178                            Token::Newline => (),
179                            Token::Bslash => {
180                                dest.push(spanned_token);
181                            }
182                            _ => {
183                                dest.push(spanned_token);
184                                dest.push(next_token)
185                            }
186                        },
187                    };
188                }
189                Token::Newline if !state.in_text_macro_arg() => {
190                    return Err(PreprocessorError::NewlineInDefine(
191                        spanned_token.1,
192                    ));
193                }
194                Token::Paren
195                    if state.in_define_arg() || state.in_text_macro_arg() =>
196                {
197                    enclosures.push(Token::Paren);
198                    dest.push(spanned_token);
199                }
200                Token::Bracket
201                    if state.in_define_arg() || state.in_text_macro_arg() =>
202                {
203                    enclosures.push(Token::Bracket);
204                    dest.push(spanned_token);
205                }
206                Token::Brace
207                    if state.in_define_arg() || state.in_text_macro_arg() =>
208                {
209                    enclosures.push(Token::Brace);
210                    dest.push(spanned_token);
211                }
212                Token::EParen
213                    if state.in_define_arg() || state.in_text_macro_arg() =>
214                {
215                    match enclosures.pop() {
216                        Some(Token::Paren) => dest.push(spanned_token),
217                        None => {
218                            return Err(
219                                PreprocessorError::EndOfFunctionArgument(
220                                    spanned_token,
221                                ),
222                            );
223                        }
224                        _ => {
225                            return Err(
226                                PreprocessorError::IncompleteMacroWithToken {
227                                    error_token: spanned_token.0,
228                                    error_span: spanned_token.1,
229                                },
230                            );
231                        }
232                    }
233                }
234                Token::EBracket
235                    if state.in_define_arg() || state.in_text_macro_arg() =>
236                {
237                    match enclosures.pop() {
238                        Some(Token::Bracket) => dest.push(spanned_token),
239                        _ => {
240                            return Err(
241                                PreprocessorError::IncompleteMacroWithToken {
242                                    error_token: spanned_token.0,
243                                    error_span: spanned_token.1,
244                                },
245                            );
246                        }
247                    }
248                }
249                Token::EBrace
250                    if state.in_define_arg() || state.in_text_macro_arg() =>
251                {
252                    match enclosures.pop() {
253                        Some(Token::Brace) => dest.push(spanned_token),
254                        _ => {
255                            return Err(
256                                PreprocessorError::IncompleteMacroWithToken {
257                                    error_token: spanned_token.0,
258                                    error_span: spanned_token.1,
259                                },
260                            );
261                        }
262                    }
263                }
264                Token::Comma
265                    if state.in_define_arg() || state.in_text_macro_arg() =>
266                {
267                    if enclosures.is_empty() {
268                        return Err(PreprocessorError::EndOfFunctionArgument(
269                            spanned_token,
270                        ));
271                    } else {
272                        dest.push(spanned_token)
273                    }
274                }
275                Token::BlockComment(_) => (),
276                Token::OnelineComment(comment_text) => {
277                    if comment_text.ends_with('\\') & !state.in_text_macro_arg()
278                    {
279                        // Counts as escaping a newline
280                        //
281                        // Can safely consume and disregard next token (the newline)
282                        let _ = src.next();
283                    };
284                }
285                Token::TextMacro(macro_name)
286                    if state.in_define_arg() || state.in_text_macro_arg() =>
287                {
288                    preprocess_macro(
289                        src,
290                        state,
291                        cache,
292                        (macro_name, spanned_token.1),
293                    )?;
294                }
295                _ => dest.push(spanned_token),
296            }
297        }
298        Ok(())
299    } else {
300        while let Some(spanned_token) = src.next() {
301            match spanned_token.0 {
302                Token::DirResetall => {
303                    state.reset_all(spanned_token.1);
304                }
305                Token::DirInclude => {
306                    let include_span = cache.retain_span(spanned_token.1);
307                    preprocess_include(src, dest, state, cache, include_span)?;
308                }
309                Token::DirUndefineall => {
310                    state.undefineall();
311                }
312                Token::DirBeginKeywords => {
313                    preprocess_keyword_standard(
314                        src,
315                        dest,
316                        state,
317                        cache,
318                        spanned_token.1,
319                    )?;
320                }
321                Token::DirDefine => {
322                    preprocess_define(src, state, cache, spanned_token.1)?;
323                }
324                Token::DirElse => {
325                    return Err(PreprocessorError::Else {
326                        else_span: spanned_token.1,
327                    });
328                }
329                Token::DirElsif => {
330                    return Err(PreprocessorError::Elsif {
331                        elsif_span: spanned_token.1,
332                    });
333                }
334                Token::DirEndKeywords => {
335                    return Err(PreprocessorError::EndKeywords {
336                        end_keywords_span: spanned_token.1,
337                    });
338                }
339                Token::DirEndif => {
340                    return Err(PreprocessorError::Endif {
341                        endif_span: spanned_token.1,
342                    });
343                }
344                Token::DirIfdef => {
345                    preprocess_ifdef(
346                        src,
347                        dest,
348                        state,
349                        cache,
350                        spanned_token.1,
351                        true,
352                    )?;
353                }
354                Token::DirIfndef => {
355                    preprocess_ifdef(
356                        src,
357                        dest,
358                        state,
359                        cache,
360                        spanned_token.1,
361                        false,
362                    )?;
363                }
364                Token::TextMacro(macro_name) => {
365                    preprocess_macro(
366                        src,
367                        state,
368                        cache,
369                        (macro_name, spanned_token.1),
370                    )?;
371                }
372                Token::DirUndef => {
373                    preprocess_undefine(src, state, spanned_token.1)?;
374                }
375                Token::DirTimescale => {
376                    preprocess_timescale(src, state, cache, spanned_token.1)?;
377                }
378                Token::DirDefaultNettype => {
379                    preprocess_default_nettype(
380                        src,
381                        state,
382                        cache,
383                        spanned_token.1,
384                    )?;
385                }
386                Token::DirUnconnectedDrive => {
387                    preprocess_unconnected_drive(src, state, spanned_token.1)?;
388                }
389                Token::DirNounconnectedDrive => {
390                    preprocess_nounconnected_drive(state, spanned_token.1)?;
391                }
392                Token::DirCelldefine => {
393                    state.add_cell_define(true, spanned_token.1);
394                }
395                Token::DirEndcelldefine => {
396                    state.add_cell_define(false, spanned_token.1);
397                }
398                Token::DirLine => {
399                    preprocess_line(src, state, cache, spanned_token.1)?;
400                }
401                Token::DirUnderscoreFile => dest.push(SpannedToken(
402                    Token::StringLiteral(
403                        state.get_line_directive_file(&spanned_token.1),
404                    ),
405                    spanned_token.1,
406                )),
407                Token::DirUnderscoreLine => dest.push(SpannedToken(
408                    Token::UnsignedNumber(
409                        state.get_line_directive_line(&spanned_token.1, cache),
410                    ),
411                    spanned_token.1,
412                )),
413                Token::BlockComment(_)
414                | Token::OnelineComment(_)
415                | Token::Newline => {
416                    #[cfg(feature = "parse_lossless")]
417                    {
418                        dest.push(spanned_token)
419                    }
420                }
421                token if token.keyword_replace(&state.curr_standard) => {
422                    let new_token = SpannedToken(
423                        Token::SimpleIdentifier(token.as_str()),
424                        spanned_token.1,
425                    );
426                    dest.push(new_token)
427                }
428                _ => dest.push(spanned_token),
429            }
430        }
431        Ok(())
432    }
433}
434
435pub(crate) fn preprocess_single<'s>(
436    src: &mut TokenIterator<'s, impl Iterator<Item = SpannedToken<'s>>>,
437    state: &mut PreprocessorState<'s>,
438    cache: &'s PreprocessorCache<'s>,
439) -> Result<Option<SpannedToken<'s>>, PreprocessorError<'s>> {
440    loop {
441        match src.next() {
442            None => {
443                break Ok(None);
444            }
445            Some(SpannedToken(Token::BlockComment(_), _)) => (),
446            Some(SpannedToken(Token::TextMacro(macro_name), macro_span)) => {
447                preprocess_macro(src, state, cache, (macro_name, macro_span))?;
448            }
449            other => break Ok(other),
450        }
451    }
452}
453
454/// Preprocess the given token stream, elaborating any compiler directives
455///
456/// `state` is augmented during preprocessing (and can be examined afterwards,
457/// likely to inspect any errors found), and `cache` is used to retain any new
458/// files/spans found during preprocessing
459///
460/// [`preprocess`] returns the elaborated stream, as well as whether the
461/// initial stream was consumed completely (`false` if an irrecoverable
462/// error was encountered)
463///
464/// ```rust
465/// # use forj_parser::*;
466/// # let mut state = PreprocessorState::new(vec![], vec![]);
467/// # let cache = PreprocessorCache::new();
468/// let file_contents = "
469/// `define TEST(a, b) a + b
470/// `TEST(1, 2)
471/// ";
472/// let tokens = lex(file_contents, "test_file.v").tokens();
473/// let mut pp_tokens = preprocess(tokens, &mut state, &cache).unwrap().into_iter();
474/// assert_eq!(pp_tokens.next().unwrap().0, Token::UnsignedNumber("1"));
475/// assert_eq!(pp_tokens.next().unwrap().0, Token::Plus);
476/// assert_eq!(pp_tokens.next().unwrap().0, Token::UnsignedNumber("2"));
477/// assert_eq!(pp_tokens.next(), None)
478/// ```
479pub fn preprocess<'s>(
480    src: impl Iterator<Item = SpannedToken<'s>>,
481    state: &mut PreprocessorState<'s>,
482    cache: &'s PreprocessorCache<'s>,
483) -> Result<Vec<SpannedToken<'s>>, ()> {
484    let mut token_iter = TokenIterator::new(src);
485    let mut dest = Vec::new();
486    loop {
487        match preprocess_helper(&mut token_iter, &mut dest, state, cache) {
488            Ok(()) => {
489                if state.errors.iter().all(|err| err.is_warning()) {
490                    return Ok(dest);
491                } else {
492                    return Err(());
493                }
494            }
495            Err(err) => {
496                if !recover(&mut token_iter, state, err) {
497                    return Err(());
498                }
499            }
500        }
501    }
502}
forj_parser/preprocessor/mod.rs

forj_parser/preprocessor/
mod.rs
