scarf_parser/preprocessor/
mod.rs

1// =======================================================================
2// mod.rs
3// =======================================================================
4//! Preprocessing a token stream, elaborating compiler directives
5
6pub mod cache;
7pub(crate) mod conditional_compilation;
8pub(crate) mod define;
9pub(crate) mod error;
10pub(crate) mod implicit_nettype;
11pub(crate) mod include;
12pub(crate) mod keywords;
13pub(crate) mod line;
14pub mod state;
15pub(crate) mod text_macro;
16pub(crate) mod timescale;
17pub(crate) mod unconnected;
18use crate::*;
19pub use cache::*;
20use conditional_compilation::*;
21use define::*;
22pub use error::*;
23pub(crate) use implicit_nettype::DefaultNettype;
24use implicit_nettype::*;
25use include::*;
26use keywords::*;
27use line::*;
28pub use state::*;
29use std::collections::VecDeque;
30use text_macro::*;
31use timescale::*;
32pub(crate) use timescale::{Timescale, TimescaleUnit, TimescaleValue};
33pub(crate) use unconnected::UnconnectedDrive;
34use unconnected::*;
35
36/// A peekable, extendable iterator over tokens.
37///
38/// This iterator extends `<T>` by keeping track of an additional
39/// stack of tokens at the front, allowing users to peek the next
40/// token, as well as push tokens to be iterated on next (such as
41/// when expanding a preprocessor definition)
42pub struct TokenIterator<'s, T: Iterator<Item = SpannedToken<'s>>> {
43    iter: T,
44    extras: VecDeque<SpannedToken<'s>>,
45}
46
47impl<'s, T: Iterator<Item = SpannedToken<'s>>> Iterator
48    for TokenIterator<'s, T>
49{
50    type Item = SpannedToken<'s>;
51    fn next(&mut self) -> Option<Self::Item> {
52        if let Some(extra_token) = self.extras.pop_front() {
53            Some(extra_token)
54        } else {
55            self.iter.next()
56        }
57    }
58}
59
60impl<'s, T: Iterator<Item = SpannedToken<'s>>> TokenIterator<'s, T> {
61    pub fn new(iter: T) -> Self {
62        Self {
63            iter,
64            extras: VecDeque::default(),
65        }
66    }
67
68    pub fn prepend_tokens<I>(&mut self, extra_tokens: I)
69    where
70        I: Iterator<Item = SpannedToken<'s>>
71            + ExactSizeIterator
72            + std::iter::DoubleEndedIterator,
73    {
74        self.extras.reserve(extra_tokens.len());
75        for extra_token in extra_tokens.rev() {
76            self.extras.push_front(extra_token);
77        }
78    }
79
80    pub fn peek(&mut self) -> Option<&SpannedToken<'s>> {
81        if self.extras.is_empty() {
82            if let Some(next_token) = self.iter.next() {
83                self.extras.push_back(next_token);
84            }
85        }
86        self.extras.front()
87    }
88}
89
90pub(crate) fn preprocess_helper<'s>(
91    src: &mut TokenIterator<'s, impl Iterator<Item = SpannedToken<'s>>>,
92    dest: &mut Vec<SpannedToken<'s>>,
93    state: &mut PreprocessorState<'s>,
94    cache: &'s PreprocessorCache<'s>,
95) -> Result<(), PreprocessorError<'s>> {
96    let mut enclosures: Vec<Token<'s>> = vec![];
97    if state.in_define() || state.in_define_arg() {
98        while let Some(mut spanned_token) = src.next() {
99            match spanned_token.0 {
100                Token::Bslash => loop {
101                    match src.next() {
102                        None => dest.push(spanned_token),
103                        Some(next_token) => match next_token.0 {
104                            Token::Newline => (),
105                            Token::Bslash => {
106                                dest.push(spanned_token);
107                                spanned_token = next_token;
108                                continue;
109                            }
110                            _ => {
111                                dest.push(spanned_token);
112                                dest.push(next_token)
113                            }
114                        },
115                    };
116                    break;
117                },
118                Token::Newline => {
119                    return Err(PreprocessorError::NewlineInDefine(
120                        spanned_token.1,
121                    ));
122                }
123                Token::Paren if state.in_define_arg() => {
124                    enclosures.push(Token::Paren);
125                    dest.push(spanned_token);
126                }
127                Token::Bracket if state.in_define_arg() => {
128                    enclosures.push(Token::Bracket);
129                    dest.push(spanned_token);
130                }
131                Token::Brace if state.in_define_arg() => {
132                    enclosures.push(Token::Brace);
133                    dest.push(spanned_token);
134                }
135                Token::EParen if state.in_define_arg() => {
136                    match enclosures.pop() {
137                        Some(Token::Paren) => dest.push(spanned_token),
138                        None => {
139                            return Err(
140                                PreprocessorError::EndOfFunctionArgument(
141                                    spanned_token,
142                                ),
143                            );
144                        }
145                        _ => {
146                            return Err(
147                                PreprocessorError::IncompleteMacroWithToken(
148                                    spanned_token,
149                                ),
150                            );
151                        }
152                    }
153                }
154                Token::EBracket if state.in_define_arg() => {
155                    match enclosures.pop() {
156                        Some(Token::Bracket) => dest.push(spanned_token),
157                        _ => {
158                            return Err(
159                                PreprocessorError::IncompleteMacroWithToken(
160                                    spanned_token,
161                                ),
162                            );
163                        }
164                    }
165                }
166                Token::EBrace if state.in_define_arg() => {
167                    match enclosures.pop() {
168                        Some(Token::Brace) => dest.push(spanned_token),
169                        _ => {
170                            return Err(
171                                PreprocessorError::IncompleteMacroWithToken(
172                                    spanned_token,
173                                ),
174                            );
175                        }
176                    }
177                }
178                Token::Comma if state.in_define_arg() => {
179                    if enclosures.is_empty() {
180                        return Err(PreprocessorError::EndOfFunctionArgument(
181                            spanned_token,
182                        ));
183                    } else {
184                        dest.push(spanned_token)
185                    }
186                }
187                Token::BlockComment(_) | Token::OnelineComment(_) => (),
188                Token::TextMacro(macro_name) if state.in_define_arg() => {
189                    preprocess_macro(
190                        src,
191                        state,
192                        cache,
193                        (macro_name, spanned_token.1),
194                    )?;
195                }
196                _ => dest.push(spanned_token),
197            }
198        }
199        Ok(())
200    } else {
201        while let Some(spanned_token) = src.next() {
202            match spanned_token.0 {
203                Token::DirResetall => {
204                    state.reset_all(spanned_token.1);
205                }
206                Token::DirInclude => {
207                    let include_span = cache.retain_span(spanned_token.1);
208                    preprocess_include(src, dest, state, cache, include_span)?;
209                }
210                Token::DirUndefineall => {
211                    state.undefineall();
212                }
213                Token::DirBeginKeywords => {
214                    preprocess_keyword_standard(
215                        src,
216                        dest,
217                        state,
218                        cache,
219                        spanned_token.1,
220                    )?;
221                }
222                Token::DirDefine => {
223                    preprocess_define(src, state, cache, spanned_token.1)?;
224                }
225                Token::DirElse => {
226                    return Err(PreprocessorError::Else(spanned_token.1));
227                }
228                Token::DirElsif => {
229                    return Err(PreprocessorError::Elsif(spanned_token.1));
230                }
231                Token::DirEndKeywords => {
232                    return Err(PreprocessorError::EndKeywords(
233                        spanned_token.1,
234                    ));
235                }
236                Token::DirEndif => {
237                    return Err(PreprocessorError::Endif(spanned_token.1));
238                }
239                Token::DirIfdef => {
240                    preprocess_ifdef(
241                        src,
242                        dest,
243                        state,
244                        cache,
245                        spanned_token.1,
246                        true,
247                    )?;
248                }
249                Token::DirIfndef => {
250                    preprocess_ifdef(
251                        src,
252                        dest,
253                        state,
254                        cache,
255                        spanned_token.1,
256                        false,
257                    )?;
258                }
259                Token::TextMacro(macro_name) => {
260                    preprocess_macro(
261                        src,
262                        state,
263                        cache,
264                        (macro_name, spanned_token.1),
265                    )?;
266                }
267                Token::DirUndef => {
268                    preprocess_undefine(src, state, spanned_token.1)?;
269                }
270                Token::DirTimescale => {
271                    preprocess_timescale(src, state, cache, spanned_token.1)?;
272                }
273                Token::DirDefaultNettype => {
274                    preprocess_default_nettype(
275                        src,
276                        state,
277                        cache,
278                        spanned_token.1,
279                    )?;
280                }
281                Token::DirUnconnectedDrive => {
282                    preprocess_unconnected_drive(src, state, spanned_token.1)?;
283                }
284                Token::DirNounconnectedDrive => {
285                    preprocess_nounconnected_drive(state, spanned_token.1)?;
286                }
287                Token::DirCelldefine => {
288                    state.add_cell_define(true, spanned_token.1);
289                }
290                Token::DirEndcelldefine => {
291                    state.add_cell_define(false, spanned_token.1);
292                }
293                Token::DirLine => {
294                    preprocess_line(src, state, cache, spanned_token.1)?;
295                }
296                Token::DirUnderscoreFile => dest.push(SpannedToken(
297                    Token::StringLiteral(
298                        state.get_line_directive_file(&spanned_token.1),
299                    ),
300                    spanned_token.1,
301                )),
302                Token::DirUnderscoreLine => dest.push(SpannedToken(
303                    Token::UnsignedNumber(
304                        state.get_line_directive_line(&spanned_token.1, cache),
305                    ),
306                    spanned_token.1,
307                )),
308                Token::BlockComment(_)
309                | Token::OnelineComment(_)
310                | Token::Newline => {
311                    #[cfg(feature = "parse_lossless")]
312                    {
313                        dest.push(spanned_token)
314                    }
315                }
316                token if token.keyword_replace(&state.curr_standard) => {
317                    let new_token = SpannedToken(
318                        Token::SimpleIdentifier(token.as_str()),
319                        spanned_token.1,
320                    );
321                    dest.push(new_token)
322                }
323                _ => dest.push(spanned_token),
324            }
325        }
326        Ok(())
327    }
328}
329
330pub(crate) fn preprocess_single<'s>(
331    src: &mut TokenIterator<'s, impl Iterator<Item = SpannedToken<'s>>>,
332    state: &mut PreprocessorState<'s>,
333    cache: &'s PreprocessorCache<'s>,
334) -> Result<Option<SpannedToken<'s>>, PreprocessorError<'s>> {
335    loop {
336        match src.next() {
337            None => {
338                break Ok(None);
339            }
340            Some(SpannedToken(Token::BlockComment(_), _)) => (),
341            Some(SpannedToken(Token::TextMacro(macro_name), macro_span)) => {
342                preprocess_macro(src, state, cache, (macro_name, macro_span))?;
343            }
344            other => break Ok(other),
345        }
346    }
347}
348
349pub fn preprocess<'s>(
350    src: impl Iterator<Item = SpannedToken<'s>>,
351    state: &mut PreprocessorState<'s>,
352    cache: &'s PreprocessorCache<'s>,
353) -> Result<Vec<SpannedToken<'s>>, PreprocessorError<'s>> {
354    let mut token_iter = TokenIterator::new(src);
355    let mut dest = Vec::new();
356    preprocess_helper(&mut token_iter, &mut dest, state, cache)?;
357    Ok(dest)
358}
scarf_parser/preprocessor/mod.rs

scarf_parser/preprocessor/
mod.rs
