// Copyright (c) 2016-2021 Fabian Schuiki
//! A preprocessor for SystemVerilog files that takes the raw stream of
//! tokens generated by a lexer and performs include and macro
//! resolution.
use crate::cat::*;
use moore_common::errors::{DiagBuilder2, DiagResult2};
use moore_common::source::*;
use std::{collections::HashMap, fmt, path::Path, rc::Rc};
use once_cell::sync::Lazy;
type TokenAndSpan = (CatTokenKind, Span);
pub struct Preprocessor<'a> {
/// The stack of input files. Tokens are taken from the topmost stream until
/// the end of input, at which point the stream is popped and the process
/// continues with the next stream. Used to handle include files.
stack: Vec<Stream<'a>>,
/// References to the source contents that were touched by the preprocessor.
/// Keeping these around ensures that all emitted tokens remain valid (and
/// point to valid memory locations) at least until the preprocessor is
/// dropped.
contents: Vec<Rc<dyn SourceContent>>,
/// The current token, or None if either the end of the stream has been
/// encountered, or at the beginning when no token has been read yet.
token: Option<TokenAndSpan>,
/// The defined macros.
macro_defs: HashMap<String, Macro>,
/// The stack used to inject expanded macros into the token stream.
macro_stack: Vec<TokenAndSpan>,
/// The paths that are searched for included files, besides the current
/// file's directory.
include_paths: &'a [&'a Path],
/// The define conditional stack. Whenever a `ifdef, `ifndef, `else, `elsif,
/// or `endif directive is encountered, the stack is expanded, modified, or
/// reduced to reflect the kind of conditional block we're in.
defcond_stack: Vec<Defcond>,
/// Currently enabled directives.
dirs: Directives,
}
impl<'a> Preprocessor<'a> {
/// Create a new preprocessor for the given source file.
pub fn new(
source: Source,
include_paths: &'a [&'a Path],
macro_defs: &'a [(&'a str, Option<&'a str>)],
) -> Preprocessor<'a> {
let content = source.get_content();
let content_unbound = unsafe { &*(content.as_ref() as *const dyn SourceContent) };
let iter = content_unbound.iter();
let macro_defs = macro_defs
.into_iter()
.map(|(name, value)| {
let body = match value {
Some(value) => {
// Create dummy sources for each user defined macro.
let src = get_source_manager().add_anonymous(*value);
let span = Span::new(src, 0, value.len());
Cat::new(Box::new(value.char_indices()))
.map(|x| (x.0, span))
.collect()
}
None => Vec::new(),
};
(
name.to_string(),
Macro {
name: name.to_string(),
span: INVALID_SPAN,
args: Vec::new(),
body: body,
},
)
})
.collect();
Preprocessor {
stack: vec![Stream {
source: source,
iter: Cat::new(iter),
}],
contents: vec![content],
token: None,
macro_defs,
macro_stack: Vec::new(),
include_paths: include_paths,
defcond_stack: Vec::new(),
dirs: Default::default(),
}
}
/// Advance to the next token in the input stream.
fn bump(&mut self) {
self.token = self.macro_stack.pop();
if self.token.is_some() {
return;
}
loop {
self.token = match self.stack.last_mut() {
Some(stream) => stream
.iter
.next()
.map(|tkn| (tkn.0, Span::new(stream.source, tkn.1, tkn.2))),
None => return,
};
if self.token.is_none() {
self.stack.pop();
} else {
break;
}
}
}
/// Called whenever we have encountered a backtick followed by a text token.
/// This function handles all compiler directives and performs file
/// inclusion and macro expansion.
fn handle_directive<S: AsRef<str>>(&mut self, dir_name: S, span: Span) -> DiagResult2<()> {
let dir_name = dir_name.as_ref();
let dir = DIRECTIVES_TABLE
.get(dir_name)
.map(|x| *x)
.unwrap_or(Directive::Unknown);
match dir {
Directive::Include => {
if self.is_inactive() {
return Ok(());
}
// Skip leading whitespace.
match self.token {
Some((Whitespace, _)) => self.bump(),
_ => (),
}
// Match the opening double quotes or angular bracket.
let name_p;
let name_q;
let closing =
match self.token {
Some((Symbol('"'), sp)) => {
name_p = sp.end();
self.bump();
'"'
}
Some((Symbol('<'), sp)) => {
name_p = sp.end();
self.bump();
'>'
}
_ => return Err(DiagBuilder2::fatal(
"expected filename inside double quotes (\"...\") or angular brackets \
(<...>) after `include",
)
.span(span)),
};
// Accumulate the include path until the closing symbol.
let mut filename = String::new();
loop {
match self.token {
Some((Symbol(c), sp)) if c == closing => {
name_q = sp.begin();
break;
}
Some((Newline, sp)) => {
return Err(DiagBuilder2::fatal(
"expected end of included file's name before line break",
)
.span(sp));
}
Some((_, sp)) => {
filename.push_str(&sp.extract());
self.bump();
}
None => {
return Err(DiagBuilder2::fatal(
"expected filename after `include directive before the end of the \
input",
)
.span(span));
}
}
}
// Create a new lexer for the included filename and push it onto the
// stream stack.
// TODO: Search only system location if `include <...> is used
let included_source = match self.open_include(&filename, &span.source.get_path()) {
Some(src) => src,
None => {
// TODO: Add notes to the message indicating which files have been tried.
return Err(DiagBuilder2::fatal(format!(
"cannot open included file \"{}\"",
filename
))
.span(Span::union(name_p, name_q)));
}
};
let content = included_source.get_content();
let content_unbound = unsafe { &*(content.as_ref() as *const dyn SourceContent) };
let iter = content_unbound.iter();
self.contents.push(content);
self.stack.push(Stream {
source: included_source,
iter: Cat::new(iter),
});
self.bump();
return Ok(());
}
Directive::Define => {
if self.is_inactive() {
return Ok(());
}
// Skip leading whitespace.
match self.token {
Some((Whitespace, _)) => self.bump(),
_ => (),
}
let makro = self.handle_macro_definition(span)?;
self.macro_defs.insert(makro.name.clone(), makro);
return Ok(());
}
Directive::Undef => {
if self.is_inactive() {
return Ok(());
}
// Skip leading whitespace.
match self.token {
Some((Whitespace, _)) => self.bump(),
_ => (),
}
// Consume the macro name.
let (name, _) = match self.try_eat_name() {
Some(x) => x,
None => {
return Err(
DiagBuilder2::fatal("expected macro name after \"`undef\"").span(span)
);
}
};
// Remove the macro definition.
self.macro_defs.remove(&name);
return Ok(());
}
Directive::Undefineall => {
if self.is_inactive() {
return Ok(());
}
self.macro_defs.clear();
}
Directive::Ifdef | Directive::Ifndef | Directive::Elsif => {
// Skip leading whitespace.
match self.token {
Some((Whitespace, _)) => self.bump(),
_ => (),
}
// Consume the macro name.
let name = match self.try_eat_name() {
Some((x, _)) => x,
_ => {
return Err(DiagBuilder2::fatal(format!(
"expected macro name after {}",
dir_name
))
.span(span));
}
};
let exists = self.macro_defs.contains_key(&name);
// Depending on the directive, modify the define conditional
// stack.
match dir {
Directive::Ifdef => self.defcond_stack.push(if self.is_inactive() {
Defcond::Done
} else if exists {
Defcond::Enabled
} else {
Defcond::Disabled
}),
Directive::Ifndef => self.defcond_stack.push(if self.is_inactive() {
Defcond::Done
} else if exists {
Defcond::Disabled
} else {
Defcond::Enabled
}),
Directive::Elsif => {
match self.defcond_stack.pop() {
Some(Defcond::Done) | Some(Defcond::Enabled) => {
self.defcond_stack.push(Defcond::Done)
}
Some(Defcond::Disabled) => {
self.defcond_stack.push(if self.is_inactive() {
Defcond::Done
} else if exists {
Defcond::Enabled
} else {
Defcond::Disabled
})
}
None => {
return Err(DiagBuilder2::fatal(
"found `elsif without any preceeding `ifdef, `ifndef, or \
`elsif directive",
)
.span(span))
}
};
}
_ => unreachable!(),
}
return Ok(());
}
Directive::Else => {
match self.defcond_stack.pop() {
Some(Defcond::Disabled) => self.defcond_stack.push(Defcond::Enabled),
Some(Defcond::Enabled) | Some(Defcond::Done) => {
self.defcond_stack.push(Defcond::Done)
}
None => {
return Err(DiagBuilder2::fatal(
"found `else without any preceeding `ifdef, `ifndef, or `elsif \
directive",
)
.span(span))
}
}
return Ok(());
}
Directive::Endif => {
if self.defcond_stack.pop().is_none() {
return Err(DiagBuilder2::fatal(
"found `endif without any preceeding `ifdef, `ifndef, `else, or `elsif \
directive",
)
.span(span));
}
return Ok(());
}
// Perform macro substitution. If we're currently inside the
// inactive region of a define conditional (i.e. disabled or done),
// don't bother expanding the macro.
Directive::Unknown => {
if self.is_inactive() {
return Ok(());
}
// TODO(fschuiki): This unsafe here is ugly. Why is this needed
// again?
if let Some(ref makro) = unsafe { &*(self as *const Preprocessor) }
.macro_defs
.get(dir_name)
{
let args = self.handle_macro_expansion_args(makro, span)?;
// Now we have a problem. All the tokens of the macro name
// have been parsed and we would like to continue by
// injecting the tokens of the macro body, such as to
// perform substitution. The token just after the macro use,
// e.g. the whitespace in "`foo ", is already in the buffer.
// However, we don't want this token to be the next, but
// rather have it follow after the macro expansion. To do
// this, we need to push the token onto the macro stack and
// then call `self.bump()` once the expansion has been added
// to the stack.
match self.token {
Some((x, sp)) => self.macro_stack.push((x, sp)),
None => (),
}
// Push the tokens of the macro onto the stack, potentially
// substituting any macro parameters as necessary.
if args.is_empty() {
self.macro_stack
.extend(makro.body.iter().rev().map(|&(tkn, sp)| (tkn, sp)));
} else {
let mut replacement = Vec::<TokenAndSpan>::new();
// TODO: Make this work for argument names that contain
// underscores.
for tkn in &makro.body {
match *tkn {
(Text, sp) => match args.get(&sp.extract()) {
Some(substitute) => {
replacement.extend(substitute);
}
None => replacement.push(*tkn),
},
x => replacement.push(x),
}
}
self.macro_stack
.extend(replacement.iter().rev().map(|&(tkn, sp)| (tkn, sp)));
}
self.bump();
return Ok(());
}
}
// Ignore the "`timescale" directive for now.
Directive::Timescale => {
while let Some((tkn, _)) = self.token {
if tkn == Newline {
break;
}
self.bump();
}
return Ok(());
}
Directive::CurrentFile => {
if !self.is_inactive() {
self.macro_stack.push((CatTokenKind::Text, span));
}
return Ok(());
}
Directive::CurrentLine => {
if !self.is_inactive() {
self.macro_stack.push((CatTokenKind::Digits, span));
}
return Ok(());
}
Directive::Resetall => {
if !self.is_inactive() {
self.dirs = Default::default();
}
return Ok(());
}
Directive::Celldefine => {
if !self.is_inactive() {
self.dirs.celldefine = true;
}
return Ok(());
}
Directive::Endcelldefine => {
if !self.is_inactive() {
self.dirs.celldefine = false;
}
return Ok(());
}
Directive::DefaultNettype => {
if !self.is_inactive() {
// Skip leading whitespace.
match self.token {
Some((Whitespace, _)) => self.bump(),
_ => (),
}
// Parse the nettype.
let tkn = match self.token {
Some(tkn @ (Text, _)) => {
self.bump();
tkn
}
_ => {
return Err(DiagBuilder2::fatal(
"expected nettype after `default_nettype",
)
.span(span));
}
};
// Store the nettype in the directive set.
self.dirs.default_nettype = if tkn.1.extract() == "none" {
None
} else {
Some(tkn)
};
debug!(
"Set default_nettype to `{}`",
self.dirs
.default_nettype
.map(|(_, sp)| sp.extract())
.unwrap_or_else(|| "none".to_string())
);
}
return Ok(());
}
Directive::BeginKeywords => {
if !self.is_inactive() {
// Skip leading whitespace.
match self.token {
Some((Whitespace, _)) => self.bump(),
_ => (),
}
// Consume the opening symbol.
match self.token {
Some((Symbol('"'), _)) => self.bump(),
_ => {
return Err(DiagBuilder2::fatal("expected `\"` after `begin_keywords")
.span(span));
}
};
// Consume the version specifier.
let mut spec = String::new();
while let Some(tkn) = self.token {
if tkn.0 == Symbol('"') {
break;
}
spec.push_str(&tkn.1.extract());
self.bump();
}
// Consume the closing symbol.
match self.token {
Some((Symbol('"'), _)) => self.bump(),
_ => {
return Err(DiagBuilder2::fatal(
"expected `\"` after version specifier",
)
.span(span));
}
};
// Parse the version.
let spec = match KeywordsDirective::from_str(&spec) {
Some(spec) => spec,
_ => {
return Err(DiagBuilder2::fatal(format!(
"unknown `begin_keywords version specifier `{}`",
spec
))
.span(span));
}
};
self.dirs.keywords.push(spec);
debug!("Push keywords; now `{:?}`", self.dirs.keywords.last());
}
return Ok(());
}
Directive::EndKeywords => {
if !self.is_inactive() {
if self.dirs.keywords.pop().is_none() {
return Err(DiagBuilder2::fatal(
"`end_keywords without earlier `begin_keywords",
)
.span(span));
}
debug!("Pop keywords; now `{:?}`", self.dirs.keywords.last());
}
return Ok(());
}
Directive::Line => {
if !self.is_inactive() {
// Skip whitespace.
match self.token {
Some((Whitespace, _)) => self.bump(),
_ => (),
}
// Consume line number.
// TODO: Actually make use of this.
let _line = match self.token {
Some((Digits, sp)) => {
self.bump();
sp
}
_ => {
return Err(
DiagBuilder2::fatal("expected line number after `line").span(span)
);
}
};
// Skip whitespace.
match self.token {
Some((Whitespace, _)) => self.bump(),
_ => (),
}
// Consume the opening symbol.
match self.token {
Some((Symbol('"'), _)) => self.bump(),
_ => {
return Err(DiagBuilder2::fatal(
"expected `\"` after line number in `line",
)
.span(span));
}
};
// Consume the file name.
let mut filename = String::new();
while let Some(tkn) = self.token {
if tkn.0 == Symbol('"') {
break;
}
filename.push_str(&tkn.1.extract());
self.bump();
}
// Consume the closing symbol.
match self.token {
Some((Symbol('"'), _)) => self.bump(),
_ => {
return Err(DiagBuilder2::fatal(
"expected `\"` after filename in `line",
)
.span(span));
}
};
// Skip whitespace.
match self.token {
Some((Whitespace, _)) => self.bump(),
_ => (),
}
// Consume level.
// TODO: Actually make use of this.
let _level = match self.token {
Some((Digits, sp)) => {
self.bump();
sp
}
_ => {
return Err(
DiagBuilder2::fatal("expected level after `line").span(span)
);
}
};
debug!("Ignoring `line directive");
}
return Ok(());
}
Directive::UnconnectedDrive => {
if !self.is_inactive() {
// Skip leading whitespace.
match self.token {
Some((Whitespace, _)) => self.bump(),
_ => (),
}
// Parse the pull type.
let tkn = match self.token {
Some((Text, _)) => self.token,
_ => None,
};
let pull = match tkn.map(|(_, sp)| sp.extract()).as_ref().map(|s| s.as_str()) {
Some("pull0") => UnconnectedDrive::Pull0,
Some("pull1") => UnconnectedDrive::Pull1,
_ => {
return Err(DiagBuilder2::fatal(
"expected `pull0` or `pull1` after `unconnected_drive",
)
.span(span));
}
};
self.bump(); // consume the pull
// Store the directive.
self.dirs.unconnected_drive = Some(pull);
debug!("Set unconnected_drive to {:?}", self.dirs.unconnected_drive);
}
return Ok(());
}
Directive::NoUnconnectedDrive => {
if !self.is_inactive() {
self.dirs.unconnected_drive = None;
debug!("Set unconnected_drive to {:?}", self.dirs.unconnected_drive);
}
return Ok(());
}
}
return Err(
DiagBuilder2::fatal(format!("unknown compiler directive '`{}'", dir_name)).span(span),
);
}
fn open_include(&mut self, filename: &str, current_file: &str) -> Option<Source> {
// println!("Resolving include '{}' from '{}'", filename, current_file);
let first = [Path::new(current_file)
.parent()
.expect("current file path must have a valid parent")];
let prefices = first.iter().chain(self.include_paths.iter());
let sm = get_source_manager();
for prefix in prefices {
let mut buf = prefix.to_path_buf();
buf.push(filename);
// println!(" trying {}", buf.to_str().unwrap());
let src = sm.open(buf.to_str().unwrap());
if src.is_some() {
return src;
}
}
return None;
}
/// Check whether we are inside a disabled define conditional. That is,
/// whether a preceeding `ifdef, `ifndef, `else, or `elsif directive have
/// disabled the subsequent code.
fn is_inactive(&self) -> bool {
match self.defcond_stack.last() {
Some(&Defcond::Enabled) | None => false,
_ => true,
}
}
fn try_eat_name(&mut self) -> Option<(String, Span)> {
// Eat the first token of the name, which may either be a letter or an
// underscore.
let (mut name, mut span) = match self.token {
Some((Text, sp)) | Some((Symbol('_'), sp)) => (sp.extract(), sp),
_ => return None,
};
self.bump();
// Eat the remaining tokens of the name, which may be letters, digits,
// or underscores.
loop {
match self.token {
Some((Text, sp)) | Some((Digits, sp)) | Some((Symbol('_'), sp)) => {
name.push_str(&sp.extract());
span.expand(sp);
self.bump();
}
_ => break,
}
}
Some((name, span))
}
// Skip over any white space characters.
fn skip_whitespace(&mut self) -> bool {
match self.token {
Some((Whitespace, _)) => {
self.bump();
true
}
_ => false,
}
}
// Parse the macro definition following a '`define' directive.
fn handle_macro_definition(&mut self, define_span: Span) -> Result<Macro, DiagBuilder2> {
let mut all_span = define_span;
// Consume the macro name.
let (name, name_span) = match self.try_eat_name() {
Some(x) => x,
None => {
return Err(
DiagBuilder2::fatal("expected macro name after \"`define\"").span(define_span)
);
}
};
all_span.expand(define_span);
let mut makro = Macro::new(name, name_span);
// NOTE: No whitespace is allowed after the macro name such that
// the preprocessor does not mistake the a in "`define FOO (a)"
// for a macro argument.
makro.args = self.handle_macro_definition_args()?;
// Skip whitespace between the macro parameters and definition.
self.skip_whitespace();
// Consume the macro definition up to the next newline not preceded
// by a backslash, ignoring comments, whitespace and newlines.
loop {
match self.token {
Some((Newline, _)) => {
self.bump();
break;
}
// Some((Whitespace, _)) => self.bump(),
// Some((Comment, _)) => self.bump(),
Some((Symbol('\\'), _)) => {
self.bump();
match self.token {
Some((Newline, _)) => self.bump(),
_ => (),
};
}
Some(x) => {
makro.body.push(x);
self.bump();
}
None => break,
}
}
Ok(makro)
}
fn handle_macro_definition_args(&mut self) -> Result<Vec<MacroArg>, DiagBuilder2> {
// Consume the opening parenthesis.
let mut all_span = match self.token {
Some((Symbol('('), sp)) => {
self.bump();
sp
}
_ => return Ok(vec![]),
};
// Consume the arguments.
let mut args = vec![];
loop {
// Check if we ran out of arguments.
self.skip_whitespace();
match self.token {
Some((Symbol(')'), _)) => {
self.bump();
break;
}
None => {
return Err(
DiagBuilder2::fatal("expected `)` after macro arguments").span(all_span)
)
}
_ => (),
}
// Consume the argument name.
let (name, name_span) = match self.try_eat_name() {
Some(x) => x,
_ => {
return Err(
DiagBuilder2::fatal("expected macro argument name").span(all_span.end())
);
}
};
self.skip_whitespace();
// Consume the optional default parameter.
let default = match self.token {
Some((Symbol('='), sp)) => {
all_span.expand(sp);
self.bump();
self.skip_whitespace();
let mut tokens = vec![];
let mut nesting = 0;
loop {
match self.token {
Some((Symbol(','), _)) | Some((Symbol(')'), _)) if nesting == 0 => {
match tokens.last() {
Some((Whitespace, _)) => {
tokens.pop();
}
_ => (),
}
break;
}
Some(x @ (Symbol('('), _))
| Some(x @ (Symbol('{'), _))
| Some(x @ (Symbol('['), _)) => {
nesting += 1;
all_span.expand(x.1);
tokens.push(x);
self.bump();
}
Some(x @ (Symbol(')'), _))
| Some(x @ (Symbol('}'), _))
| Some(x @ (Symbol(']'), _)) => {
nesting -= 1;
all_span.expand(x.1);
tokens.push(x);
self.bump();
}
Some(x) => {
all_span.expand(x.1);
tokens.push(x);
self.bump();
}
_ => break,
}
}
Some(tokens)
}
_ => None,
};
args.push(MacroArg {
name,
span: name_span,
default,
});
// Consume the trailing comma.
self.skip_whitespace();
match self.token {
Some((Symbol(','), _)) => self.bump(),
Some((Symbol(')'), _)) => (),
Some((_, sp)) => {
return Err(
DiagBuilder2::fatal("expected `,` or `)` after macro argument").span(sp),
)
}
None => (),
}
}
Ok(args)
}
fn handle_macro_expansion_args(
&mut self,
makro: &Macro,
span: Span,
) -> Result<MacroExpansionParams, DiagBuilder2> {
// If the macro does not define any arguments, we're done.
if makro.args.is_empty() {
return Ok(Default::default());
}
let mut all_span = span;
// Skip whitespace between '`foo' and `(`.
self.skip_whitespace();
// Consume the opening paranthesis.
match self.token {
Some((Symbol('('), sp)) => {
self.bump();
all_span.expand(sp);
}
_ => {
return Err(DiagBuilder2::fatal(format!(
"expected macro arguments for `{}`",
makro.name
))
.span(all_span)
.add_note(format!(
"At least parenthesis are needed: `{}()`",
makro.name
))
.add_note(format!(
"Macro `{}` defines {} arguments:",
makro.name,
makro.args.len()
))
.span(makro.span));
}
}
self.skip_whitespace();
// Consume the macro arguments. Be careful about the fact that it is
// allowed to have parentheses as macro arguments, which requires
// bookkeeping of the parentheses nesting level. If a comma is
// encountered, we break out of the inner loop such that the next
// parameter will be read. If a closing parenthesis is encountered, we
// break out of the outer loop to finish parameter parsing.
let mut args = vec![];
'outer: loop {
let mut arg_tokens = Vec::<TokenAndSpan>::new();
let mut nesting = 0;
loop {
match self.token {
Some((Symbol(','), sp)) if nesting == 0 => {
args.push(arg_tokens);
all_span.expand(sp);
self.bump();
self.skip_whitespace();
break;
}
Some((Symbol(')'), sp)) if nesting == 0 => {
args.push(arg_tokens);
all_span.expand(sp);
self.bump();
break 'outer;
}
Some(x @ (Symbol('('), _))
| Some(x @ (Symbol('{'), _))
| Some(x @ (Symbol('['), _)) => {
arg_tokens.push(x);
nesting += 1;
self.bump();
all_span.expand(x.1);
}
Some(x @ (Symbol(')'), _))
| Some(x @ (Symbol('}'), _))
| Some(x @ (Symbol(']'), _)) => {
arg_tokens.push(x);
nesting -= 1;
self.bump();
all_span.expand(x.1);
}
Some(x) => {
arg_tokens.push(x);
self.bump();
all_span.expand(x.1);
}
None => {
return Err(DiagBuilder2::fatal("expected `)` after macro arguments")
.span(all_span));
}
}
}
}
// Remove trailing whitespace from macro arguments.
for arg in &mut args {
match arg.last() {
Some((Whitespace, _)) => {
arg.pop();
}
_ => (),
}
}
// Align the expansion arguments with those in the macro definition.
if makro.args.len() < args.len() {
let d = DiagBuilder2::fatal(format!(
"macro expansion with {} arguments, but `{}` expects {} arguments",
args.len(),
makro.name,
makro.args.len()
))
.span(all_span)
.add_note(format!("Definition of `{}` was here:", makro.name))
.span(makro.span);
return Err(d);
}
let args = makro
.args
.iter()
.zip(args.into_iter().map(Some).chain(std::iter::repeat(None)))
.map(|(def, exp)| {
let is_empty = !exp.iter().flatten().any(|(t, _)| match t {
Whitespace => false,
_ => true,
});
let value = match (exp, def.default.as_ref()) {
// Expansion arguments include this argument, but may be
// empty.
(Some(ref _exp), Some(default)) if is_empty => default.clone(),
(Some(ref _exp), None) if is_empty => vec![],
(Some(exp), _) => exp,
// Expansion arguments ended before this argument.
(None, Some(default)) => default.clone(),
(None, None) => {
let d = DiagBuilder2::fatal(format!(
"macro expansion missing value for `{}`",
def.name
))
.span(all_span)
.add_note(format!(
"Macro argument `{}` needs a value because it has no default:",
def.name
))
.span(def.span);
return Err(d);
}
};
Ok((def.name.clone(), value))
})
.collect::<Result<HashMap<_, _>, _>>()?;
Ok(args)
}
}
type MacroExpansionParams = HashMap<String, Vec<TokenAndSpan>>;
impl<'a> Iterator for Preprocessor<'a> {
type Item = DiagResult2<TokenAndSpan>;
fn next(&mut self) -> Option<DiagResult2<TokenAndSpan>> {
// In case this is the first call to next(), the token has not been
// populated yet. In this case we need to artificially bump the lexer.
if self.token.is_none() {
self.bump();
}
loop {
// This is the main loop of the lexer. Upon each iteration the next
// token is inspected and the lexer decides whether to emit it or
// not. If no token was emitted (e.g. because it was a preprocessor
// directive or we're inside an inactive `ifdef block), the loop
// continues with the next token.
match self.token {
Some((Symbol('`'), sp_backtick)) => {
self.bump(); // consume the backtick
if let Some((name, sp)) = self.try_eat_name() {
// We arrive here if the sequence a backtick
// followed by text was encountered. In this case we
// call upon the handle_directive function to
// perform the necessary actions.
let dir_span = Span::union(sp_backtick, sp);
match self.handle_directive(name, dir_span) {
Err(x) => return Some(Err(x)),
_ => (),
}
continue;
} else if let Some(tkn @ (Symbol('"'), _)) = self.token {
// emit the '"'
self.bump();
if !self.is_inactive() {
return Some(Ok(tkn));
}
} else if let Some(tkn @ (Symbol('\\'), _)) = self.token {
// emit the '\'
self.bump();
if !self.is_inactive() {
return Some(Ok(tkn));
}
} else if let Some((Symbol('`'), _)) = self.token {
self.bump(); // consume the second backtick and ignore
} else {
return Some(Err(DiagBuilder2::fatal(
"expected compiler directive after '`', or '``', '`\"', or '`\\'",
)
.span(sp_backtick)));
}
}
_ => {
// All tokens other than preprocessor directives are
// emitted, unless we're currently inside a disabled define
// conditional.
if self.is_inactive() {
self.bump();
} else {
let tkn = self.token.map(|x| Ok(x));
self.bump();
return tkn;
}
}
}
}
}
}
struct Stream<'a> {
source: Source,
iter: Cat<'a>,
}
/// The different compiler directives recognized by the preprocessor.
#[derive(Debug, Clone, Copy)]
enum Directive {
Include,
Define,
Undef,
Undefineall,
Ifdef,
Ifndef,
Else,
Elsif,
Endif,
Timescale,
CurrentFile,
CurrentLine,
Resetall,
Celldefine,
Endcelldefine,
DefaultNettype,
BeginKeywords,
EndKeywords,
Line,
UnconnectedDrive,
NoUnconnectedDrive,
Unknown,
}
impl fmt::Display for Directive {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Directive::Include => write!(f, "`include"),
Directive::Define => write!(f, "`define"),
Directive::Undef => write!(f, "`undef"),
Directive::Undefineall => write!(f, "`undefineall"),
Directive::Ifdef => write!(f, "`ifdef"),
Directive::Ifndef => write!(f, "`ifndef"),
Directive::Else => write!(f, "`else"),
Directive::Elsif => write!(f, "`elsif"),
Directive::Endif => write!(f, "`endif"),
Directive::Timescale => write!(f, "`timescale"),
Directive::CurrentFile => write!(f, "`__FILE__"),
Directive::CurrentLine => write!(f, "`__LINE__"),
Directive::Resetall => write!(f, "`resetall"),
Directive::Celldefine => write!(f, "`celldefine"),
Directive::Endcelldefine => write!(f, "`endcelldefine"),
Directive::DefaultNettype => write!(f, "`default_nettype"),
Directive::BeginKeywords => write!(f, "`begin_keywords"),
Directive::EndKeywords => write!(f, "`end_keywords"),
Directive::Line => write!(f, "`line"),
Directive::UnconnectedDrive => write!(f, "`unconnected_drive"),
Directive::NoUnconnectedDrive => write!(f, "`nounconnected_drive"),
Directive::Unknown => write!(f, "unknown"),
}
}
}
static DIRECTIVES_TABLE: Lazy<HashMap<&'static str, Directive>> = Lazy::new(|| {
let mut table = HashMap::new();
table.insert("include", Directive::Include);
table.insert("define", Directive::Define);
table.insert("undef", Directive::Undef);
table.insert("undefineall", Directive::Undefineall);
table.insert("ifdef", Directive::Ifdef);
table.insert("ifndef", Directive::Ifndef);
table.insert("else", Directive::Else);
table.insert("elsif", Directive::Elsif);
table.insert("endif", Directive::Endif);
table.insert("__FILE__", Directive::CurrentFile);
table.insert("__LINE__", Directive::CurrentLine);
table.insert("resetall", Directive::Resetall);
table.insert("celldefine", Directive::Celldefine);
table.insert("endcelldefine", Directive::Endcelldefine);
table.insert("default_nettype", Directive::DefaultNettype);
table.insert("begin_keywords", Directive::BeginKeywords);
table.insert("end_keywords", Directive::EndKeywords);
table.insert("line", Directive::Line);
table.insert("unconnected_drive", Directive::UnconnectedDrive);
table.insert("nounconnected_drive", Directive::NoUnconnectedDrive);
table.insert("timescale", Directive::Timescale);
table
});
#[derive(Debug)]
struct Macro {
name: String,
span: Span,
args: Vec<MacroArg>,
body: Vec<TokenAndSpan>,
}
impl Macro {
fn new(name: String, span: Span) -> Macro {
Macro {
name: name,
span: span,
args: Vec::new(),
body: Vec::new(),
}
}
}
#[derive(Debug)]
struct MacroArg {
name: String,
span: Span,
default: Option<Vec<TokenAndSpan>>,
}
enum Defcond {
Done,
Enabled,
Disabled,
}
#[derive(Default)]
struct Directives {
celldefine: bool,
default_nettype: Option<TokenAndSpan>,
keywords: Vec<KeywordsDirective>,
unconnected_drive: Option<UnconnectedDrive>,
}
#[allow(non_camel_case_types)]
#[derive(Debug)]
enum KeywordsDirective {
Ieee1800_2009,
Ieee1800_2005,
Ieee1364_2005,
Ieee1364_2001,
Ieee1364_2001_Noconfig,
Ieee1364_1995,
}
#[derive(Debug)]
enum UnconnectedDrive {
Pull0,
Pull1,
}
impl KeywordsDirective {
pub fn from_str(s: &str) -> Option<Self> {
match s {
"1800-2009" => Some(Self::Ieee1800_2009),
"1800-2005" => Some(Self::Ieee1800_2005),
"1364-2005" => Some(Self::Ieee1364_2005),
"1364-2001" => Some(Self::Ieee1364_2001),
"1364-2001-noconfig" => Some(Self::Ieee1364_2001_Noconfig),
"1364-1995" => Some(Self::Ieee1364_1995),
_ => None,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn preproc(input: &str) -> Preprocessor {
use std::cell::Cell;
thread_local!(static INDEX: Cell<usize> = Cell::new(0));
let sm = get_source_manager();
let idx = INDEX.with(|i| {
let v = i.get();
i.set(v + 1);
v
});
let source = sm.add(&format!("test_{}.sv", idx), input);
Preprocessor::new(source, &[], &[])
}
fn check_str(input: &str, expected: &str) {
let pp = preproc(input);
let actual: String = pp.map(|x| x.unwrap().1.extract()).collect();
assert_eq!(actual, expected);
}
#[test]
fn include() {
let sm = get_source_manager();
sm.add("other.sv", "bar\n");
sm.add("test.sv", "foo\n`include \"other.sv\"\nbaz");
let pp = Preprocessor::new(sm.open("test.sv").unwrap(), &[], &[]);
let actual: Vec<_> = pp.map(|x| x.unwrap().0).collect();
assert_eq!(actual, &[Text, Newline, Text, Newline, Newline, Text,]);
}
#[test]
fn include_and_define() {
let sm = get_source_manager();
sm.add("other.sv", "/* World */\n`define foo 42\nbar");
sm.add(
"test.sv",
"// Hello\n`include \"other.sv\"\n`foo something\n",
);
let pp = Preprocessor::new(sm.open("test.sv").unwrap(), &[], &[]);
let actual: String = pp
.map(|x| {
let x = x.unwrap();
println!("{:?}", x);
x.1.extract()
})
.collect();
assert_eq!(actual, "// Hello\n/* World */\nbar\n42 something\n");
}
#[test]
#[should_panic(expected = "unknown compiler directive")]
fn conditional_define() {
let sm = get_source_manager();
let source = sm.add("test.sv", "`ifdef FOO\n`define BAR\n`endif\n`BAR");
let mut pp = Preprocessor::new(source, &[], &[]);
while let Some(tkn) = pp.next() {
tkn.unwrap();
}
}
#[test]
fn macro_args() {
check_str(
"`define foo(x,y) {x + y _bar}\n`foo(12, foo)\n",
"{12 + foo _bar}\n",
);
}
/// Verify that macros that take no arguments but have parantheses around
/// their body parse properly.
#[test]
fn macro_noargs_parentheses() {
check_str(
"`define FOO 4\n`define BAR (`FOO+$clog2(2))\n`BAR",
"(4+$clog2(2))",
);
}
#[test]
fn macro_name_with_digits_and_underscores() {
check_str("`define AXI_BUS21_SV 42\n`AXI_BUS21_SV", "42");
}
}