scarf_parser/preprocessor/
mod.rs

1// =======================================================================
2// mod.rs
3// =======================================================================
4//! Preprocessing a token stream, elaborating compiler directives
5
6pub mod cache;
7pub(crate) mod conditional_compilation;
8pub(crate) mod define;
9pub(crate) mod error;
10pub(crate) mod implicit_nettype;
11pub(crate) mod include;
12pub(crate) mod keywords;
13pub(crate) mod line;
14pub mod state;
15pub(crate) mod text_macro;
16pub(crate) mod timescale;
17pub(crate) mod unconnected;
18use crate::*;
19pub use cache::*;
20use conditional_compilation::*;
21use define::*;
22pub use error::*;
23pub(crate) use implicit_nettype::DefaultNettype;
24use implicit_nettype::*;
25use include::*;
26use keywords::*;
27use line::*;
28pub use state::*;
29use std::collections::VecDeque;
30use text_macro::*;
31use timescale::*;
32pub(crate) use timescale::{Timescale, TimescaleUnit, TimescaleValue};
33pub(crate) use unconnected::UnconnectedDrive;
34use unconnected::*;
35
36/// A peekable, extendable iterator over tokens.
37///
38/// This iterator extends `<T>` by keeping track of an additional
39/// stack of tokens at the front, allowing users to peek the next
40/// token, as well as push tokens to be iterated on next (such as
41/// when expanding a preprocessor definition)
42pub(crate) struct TokenIterator<'s, T: Iterator<Item = SpannedToken<'s>>> {
43    iter: T,
44    extras: VecDeque<SpannedToken<'s>>,
45}
46
47impl<'s, T: Iterator<Item = SpannedToken<'s>>> Iterator
48    for TokenIterator<'s, T>
49{
50    type Item = SpannedToken<'s>;
51    fn next(&mut self) -> Option<Self::Item> {
52        if let Some(extra_token) = self.extras.pop_front() {
53            Some(extra_token)
54        } else {
55            self.iter.next()
56        }
57    }
58}
59
60impl<'s, T: Iterator<Item = SpannedToken<'s>>> TokenIterator<'s, T> {
61    pub fn new(iter: T) -> Self {
62        Self {
63            iter,
64            extras: VecDeque::default(),
65        }
66    }
67
68    pub fn prepend_tokens<I>(&mut self, extra_tokens: I)
69    where
70        I: Iterator<Item = SpannedToken<'s>>
71            + ExactSizeIterator
72            + std::iter::DoubleEndedIterator,
73    {
74        self.extras.reserve(extra_tokens.len());
75        for extra_token in extra_tokens.rev() {
76            self.extras.push_front(extra_token);
77        }
78    }
79
80    pub fn peek(&mut self) -> Option<&SpannedToken<'s>> {
81        if self.extras.is_empty() {
82            if let Some(next_token) = self.iter.next() {
83                self.extras.push_back(next_token);
84            }
85        }
86        self.extras.front()
87    }
88}
89
90/// Attempt to recover from a preprocessor error by going to the next
91/// non-escaped newline, returning whether one was encountered
92pub(crate) fn recover_newline<'s>(
93    src: &mut TokenIterator<'s, impl Iterator<Item = SpannedToken<'s>>>,
94) -> bool {
95    loop {
96        let Some(SpannedToken(curr_token, _)) = src.next() else {
97            return false;
98        };
99        let next_token = src.peek();
100        match (curr_token, next_token) {
101            (Token::Bslash, Some(SpannedToken(Token::Newline, _))) => {
102                let _newline_token = src.next();
103                ()
104            }
105            (Token::Newline, _) => {
106                return true;
107            }
108            _ => (),
109        }
110    }
111}
112
113/// Attempt to recover from a preprocessor error, returning whether it
114/// was successful
115///
116/// Many of these are trivial, as they are removed from the token stream
117/// already
118pub(crate) fn recover<'s>(
119    src: &mut TokenIterator<'s, impl Iterator<Item = SpannedToken<'s>>>,
120    state: &mut PreprocessorState<'s>,
121    err: PreprocessorError<'s>,
122) -> bool {
123    let recovered = match err {
124        PreprocessorError::Endif { .. } => true,
125        PreprocessorError::NoEndif { .. } => false, // EOF
126        PreprocessorError::Elsif { .. } => true,
127        PreprocessorError::Else { .. } => true,
128        PreprocessorError::EndKeywords { .. } => true,
129        PreprocessorError::NoEndKeywords { .. } => false, // EOF
130        PreprocessorError::InvalidDefineParameter { .. } => {
131            recover_newline(src)
132        }
133        PreprocessorError::InvalidDefineArgument { .. } => recover_newline(src),
134        PreprocessorError::InvalidVersionSpecifier { .. } => true,
135        PreprocessorError::IncompleteDirective { .. } => recover_newline(src),
136        PreprocessorError::IncompleteDefine { .. } => recover_newline(src),
137        PreprocessorError::UndefinedMacro { .. } => true, // Don't worry about functions here
138        PreprocessorError::DuplicateMacroParameter { .. } => {
139            recover_newline(src)
140        }
141        PreprocessorError::NoDefaultAfterDefault { .. } => recover_newline(src),
142        PreprocessorError::NoMacroArguments { .. } => true,
143        PreprocessorError::TooManyMacroArguments { .. } => true,
144        PreprocessorError::MissingMacroArgument { .. } => true,
145        PreprocessorError::InvalidIdentifierFormation { .. } => true,
146        PreprocessorError::InvalidRelativeTimescales { .. } => true,
147        PreprocessorError::IncompleteMacroWithToken { .. } => true,
148        PreprocessorError::Include { .. } => true,
149        PreprocessorError::IncludeDepth { .. } => true,
150        PreprocessorError::VerboseError { .. } => recover_newline(src),
151        PreprocessorError::NotPreviouslyDefinedMacro { .. }
152        | PreprocessorError::RedefinedMacro { .. } => {
153            panic!("Shouldn't need to recover from warnings")
154        }
155        PreprocessorError::NewlineInDefine(_)
156        | PreprocessorError::EndOfFunctionArgument(_) => {
157            panic!("Tried to recover from an internal error")
158        }
159    };
160    state.err(err);
161    recovered
162}
163
164pub(crate) fn preprocess_helper<'s>(
165    src: &mut TokenIterator<'s, impl Iterator<Item = SpannedToken<'s>>>,
166    dest: &mut Vec<SpannedToken<'s>>,
167    state: &mut PreprocessorState<'s>,
168    cache: &'s PreprocessorCache<'s>,
169) -> Result<(), PreprocessorError<'s>> {
170    let mut enclosures: Vec<Token<'s>> = vec![];
171    if state.in_define() || state.in_define_arg() {
172        while let Some(mut spanned_token) = src.next() {
173            match spanned_token.0 {
174                Token::Bslash => loop {
175                    match src.next() {
176                        None => dest.push(spanned_token),
177                        Some(next_token) => match next_token.0 {
178                            Token::Newline => (),
179                            Token::Bslash => {
180                                dest.push(spanned_token);
181                                spanned_token = next_token;
182                                continue;
183                            }
184                            _ => {
185                                dest.push(spanned_token);
186                                dest.push(next_token)
187                            }
188                        },
189                    };
190                    break;
191                },
192                Token::Newline => {
193                    return Err(PreprocessorError::NewlineInDefine(
194                        spanned_token.1,
195                    ));
196                }
197                Token::Paren if state.in_define_arg() => {
198                    enclosures.push(Token::Paren);
199                    dest.push(spanned_token);
200                }
201                Token::Bracket if state.in_define_arg() => {
202                    enclosures.push(Token::Bracket);
203                    dest.push(spanned_token);
204                }
205                Token::Brace if state.in_define_arg() => {
206                    enclosures.push(Token::Brace);
207                    dest.push(spanned_token);
208                }
209                Token::EParen if state.in_define_arg() => {
210                    match enclosures.pop() {
211                        Some(Token::Paren) => dest.push(spanned_token),
212                        None => {
213                            return Err(
214                                PreprocessorError::EndOfFunctionArgument(
215                                    spanned_token,
216                                ),
217                            );
218                        }
219                        _ => {
220                            return Err(
221                                PreprocessorError::IncompleteMacroWithToken {
222                                    error_token: spanned_token.0,
223                                    error_span: spanned_token.1,
224                                },
225                            );
226                        }
227                    }
228                }
229                Token::EBracket if state.in_define_arg() => {
230                    match enclosures.pop() {
231                        Some(Token::Bracket) => dest.push(spanned_token),
232                        _ => {
233                            return Err(
234                                PreprocessorError::IncompleteMacroWithToken {
235                                    error_token: spanned_token.0,
236                                    error_span: spanned_token.1,
237                                },
238                            );
239                        }
240                    }
241                }
242                Token::EBrace if state.in_define_arg() => {
243                    match enclosures.pop() {
244                        Some(Token::Brace) => dest.push(spanned_token),
245                        _ => {
246                            return Err(
247                                PreprocessorError::IncompleteMacroWithToken {
248                                    error_token: spanned_token.0,
249                                    error_span: spanned_token.1,
250                                },
251                            );
252                        }
253                    }
254                }
255                Token::Comma if state.in_define_arg() => {
256                    if enclosures.is_empty() {
257                        return Err(PreprocessorError::EndOfFunctionArgument(
258                            spanned_token,
259                        ));
260                    } else {
261                        dest.push(spanned_token)
262                    }
263                }
264                Token::BlockComment(_) | Token::OnelineComment(_) => (),
265                Token::TextMacro(macro_name) if state.in_define_arg() => {
266                    preprocess_macro(
267                        src,
268                        state,
269                        cache,
270                        (macro_name, spanned_token.1),
271                    )?;
272                }
273                _ => dest.push(spanned_token),
274            }
275        }
276        Ok(())
277    } else {
278        while let Some(spanned_token) = src.next() {
279            match spanned_token.0 {
280                Token::DirResetall => {
281                    state.reset_all(spanned_token.1);
282                }
283                Token::DirInclude => {
284                    let include_span = cache.retain_span(spanned_token.1);
285                    preprocess_include(src, dest, state, cache, include_span)?;
286                }
287                Token::DirUndefineall => {
288                    state.undefineall();
289                }
290                Token::DirBeginKeywords => {
291                    preprocess_keyword_standard(
292                        src,
293                        dest,
294                        state,
295                        cache,
296                        spanned_token.1,
297                    )?;
298                }
299                Token::DirDefine => {
300                    preprocess_define(src, state, cache, spanned_token.1)?;
301                }
302                Token::DirElse => {
303                    return Err(PreprocessorError::Else {
304                        else_span: spanned_token.1,
305                    });
306                }
307                Token::DirElsif => {
308                    return Err(PreprocessorError::Elsif {
309                        elsif_span: spanned_token.1,
310                    });
311                }
312                Token::DirEndKeywords => {
313                    return Err(PreprocessorError::EndKeywords {
314                        end_keywords_span: spanned_token.1,
315                    });
316                }
317                Token::DirEndif => {
318                    return Err(PreprocessorError::Endif {
319                        endif_span: spanned_token.1,
320                    });
321                }
322                Token::DirIfdef => {
323                    preprocess_ifdef(
324                        src,
325                        dest,
326                        state,
327                        cache,
328                        spanned_token.1,
329                        true,
330                    )?;
331                }
332                Token::DirIfndef => {
333                    preprocess_ifdef(
334                        src,
335                        dest,
336                        state,
337                        cache,
338                        spanned_token.1,
339                        false,
340                    )?;
341                }
342                Token::TextMacro(macro_name) => {
343                    preprocess_macro(
344                        src,
345                        state,
346                        cache,
347                        (macro_name, spanned_token.1),
348                    )?;
349                }
350                Token::DirUndef => {
351                    preprocess_undefine(src, state, spanned_token.1)?;
352                }
353                Token::DirTimescale => {
354                    preprocess_timescale(src, state, cache, spanned_token.1)?;
355                }
356                Token::DirDefaultNettype => {
357                    preprocess_default_nettype(
358                        src,
359                        state,
360                        cache,
361                        spanned_token.1,
362                    )?;
363                }
364                Token::DirUnconnectedDrive => {
365                    preprocess_unconnected_drive(src, state, spanned_token.1)?;
366                }
367                Token::DirNounconnectedDrive => {
368                    preprocess_nounconnected_drive(state, spanned_token.1)?;
369                }
370                Token::DirCelldefine => {
371                    state.add_cell_define(true, spanned_token.1);
372                }
373                Token::DirEndcelldefine => {
374                    state.add_cell_define(false, spanned_token.1);
375                }
376                Token::DirLine => {
377                    preprocess_line(src, state, cache, spanned_token.1)?;
378                }
379                Token::DirUnderscoreFile => dest.push(SpannedToken(
380                    Token::StringLiteral(
381                        state.get_line_directive_file(&spanned_token.1),
382                    ),
383                    spanned_token.1,
384                )),
385                Token::DirUnderscoreLine => dest.push(SpannedToken(
386                    Token::UnsignedNumber(
387                        state.get_line_directive_line(&spanned_token.1, cache),
388                    ),
389                    spanned_token.1,
390                )),
391                Token::BlockComment(_)
392                | Token::OnelineComment(_)
393                | Token::Newline => {
394                    #[cfg(feature = "parse_lossless")]
395                    {
396                        dest.push(spanned_token)
397                    }
398                }
399                token if token.keyword_replace(&state.curr_standard) => {
400                    let new_token = SpannedToken(
401                        Token::SimpleIdentifier(token.as_str()),
402                        spanned_token.1,
403                    );
404                    dest.push(new_token)
405                }
406                _ => dest.push(spanned_token),
407            }
408        }
409        Ok(())
410    }
411}
412
413pub(crate) fn preprocess_single<'s>(
414    src: &mut TokenIterator<'s, impl Iterator<Item = SpannedToken<'s>>>,
415    state: &mut PreprocessorState<'s>,
416    cache: &'s PreprocessorCache<'s>,
417) -> Result<Option<SpannedToken<'s>>, PreprocessorError<'s>> {
418    loop {
419        match src.next() {
420            None => {
421                break Ok(None);
422            }
423            Some(SpannedToken(Token::BlockComment(_), _)) => (),
424            Some(SpannedToken(Token::TextMacro(macro_name), macro_span)) => {
425                preprocess_macro(src, state, cache, (macro_name, macro_span))?;
426            }
427            other => break Ok(other),
428        }
429    }
430}
431
432/// Preprocess the given token stream, elaborating any compiler directives
433///
434/// `state` is augmented during preprocessing (and can be examined afterwards,
435/// likely to inspect any errors found), and `cache` is used to retain any new
436/// files/spans found during preprocessing
437///
438/// [`preprocess`] returns the elaborated stream, as well as whether the
439/// initial stream was consumed completely (`false` if an irrecoverable
440/// error was encountered)
441///
442/// ```rust
443/// # use scarf_parser::*;
444/// # let mut state = PreprocessorState::new(vec![], vec![]);
445/// # let cache = PreprocessorCache::new();
446/// let file_contents = "
447/// `define TEST(a, b) a + b
448/// `TEST(1, 2)
449/// ";
450/// let tokens = lex(file_contents, "test_file.v").tokens();
451/// let mut pp_tokens = preprocess(tokens, &mut state, &cache).unwrap().into_iter();
452/// assert_eq!(pp_tokens.next().unwrap().0, Token::UnsignedNumber("1"));
453/// assert_eq!(pp_tokens.next().unwrap().0, Token::Plus);
454/// assert_eq!(pp_tokens.next().unwrap().0, Token::UnsignedNumber("2"));
455/// assert_eq!(pp_tokens.next(), None)
456/// ```
457pub fn preprocess<'s>(
458    src: impl Iterator<Item = SpannedToken<'s>>,
459    state: &mut PreprocessorState<'s>,
460    cache: &'s PreprocessorCache<'s>,
461) -> Result<Vec<SpannedToken<'s>>, ()> {
462    let mut token_iter = TokenIterator::new(src);
463    let mut dest = Vec::new();
464    loop {
465        match preprocess_helper(&mut token_iter, &mut dest, state, cache) {
466            Ok(()) => {
467                if state.errors.iter().all(|err| err.is_warning()) {
468                    return Ok(dest);
469                } else {
470                    return Err(());
471                }
472            }
473            Err(err) => {
474                if !recover(&mut token_iter, state, err) {
475                    return Err(());
476                }
477            }
478        }
479    }
480}
scarf_parser/preprocessor/mod.rs

scarf_parser/preprocessor/
mod.rs
