#![doc(html_root_url = "https://docs.rs/rucc-pp/0.2.17")]
mod cond;
mod directive;
mod dump;
mod embed;
mod expand;
mod hide;
mod include;
mod macros;
mod predef;
mod print;
mod token;
mod trace;
pub use crate::directive::{LineDirective, Preprocessor};
pub use crate::dump::macros as dump_macros;
pub use crate::expand::Expander;
pub use crate::hide::{HideSet, HideSets};
pub use crate::include::Context;
pub use crate::macros::{Builtin, MacroDef, MacroTable, parse_define};
pub use crate::predef::{BUILT_IN, COMMAND_LINE, Predef, Timestamp};
pub use crate::print::{PrintOptions, print};
pub use crate::token::Tok;
pub use crate::trace::{Step, TraceId, Traces};
pub use rucc_session::GnucVersion;
pub const MILESTONE: &str = "M1";
#[cfg(test)]
mod tests {
use rucc_base::Interner;
use rucc_diag::{Diagnostic, SourceMap, Span};
use rucc_lex::{Options, PpToken, PpTokenKind, TokenFlags, tokenize};
use super::*;
struct Pp {
interner: Interner,
macros: MacroTable,
expander: Expander,
sources: SourceMap,
}
impl Pp {
fn new() -> Pp {
Pp {
interner: Interner::new(),
macros: MacroTable::new(),
expander: Expander::new(),
sources: SourceMap::new(),
}
}
fn lex(&mut self, src: &str) -> Vec<PpToken> {
let (tokens, errors) = tokenize(src.as_bytes(), 0, Options::new(), &mut self.interner);
assert!(errors.is_empty(), "test input should lex cleanly: {errors:?}");
tokens.into_iter().filter(|t| t.kind != PpTokenKind::Eof).collect()
}
fn define(&mut self, line: &str) {
let tokens = self.lex(line);
let (def, errors) = parse_define(&tokens, &mut self.interner);
assert!(errors.is_empty(), "definition should be clean: {errors:?}");
self.macros.define(def.expect("should parse"), &self.interner);
}
fn undef(&mut self, name: &str) {
let sym = self.interner.intern(name);
self.macros.undef(sym);
}
fn expand(&mut self, src: &str) -> Vec<Tok> {
let tokens = self.lex(src);
self.expander.expand(&tokens, &self.macros, &mut self.interner, &self.sources)
}
fn text(&mut self, src: &str) -> String {
let out = self.expand(src);
let mut text = String::new();
for (at, tok) in out.iter().enumerate() {
if at > 0 && tok.flags.has(TokenFlags::LEADING_SPACE) {
text.push(' ');
}
match tok.kind {
PpTokenKind::Punct(p) => text.push_str(p.as_str()),
_ => text.push_str(
self.interner.resolve(tok.value.expect("every non-punctuator interns")),
),
}
}
text
}
fn errors(&mut self) -> Vec<Diagnostic> {
self.expander.take_diagnostics()
}
}
#[test]
fn an_object_like_macro_is_replaced_by_its_body() {
let mut pp = Pp::new();
pp.define("N 42");
assert_eq!(pp.text("int a = N;"), "int a = 42;");
}
#[test]
fn an_undefined_identifier_is_left_alone() {
let mut pp = Pp::new();
assert_eq!(pp.text("int a = N;"), "int a = N;");
}
#[test]
fn a_function_like_macro_needs_a_parenthesis_to_be_invoked() {
let mut pp = Pp::new();
pp.define("f(x) x");
assert_eq!(pp.text("f"), "f", "a bare name is an ordinary identifier");
assert_eq!(pp.text("f (1)"), "1", "whitespace before the parenthesis is fine");
}
#[test]
fn arguments_are_expanded_before_they_are_substituted() {
let mut pp = Pp::new();
pp.define("ONE 1");
pp.define("f(x) (x + x)");
assert_eq!(pp.text("f(ONE)"), "(1 + 1)");
}
#[test]
fn an_empty_argument_is_an_argument() {
let mut pp = Pp::new();
pp.define("f(x) [x]");
assert_eq!(pp.text("f()"), "[]");
}
#[test]
fn a_macro_with_no_parameters_takes_no_arguments() {
let mut pp = Pp::new();
pp.define("f() nothing");
assert_eq!(pp.text("f()"), "nothing");
}
#[test]
fn a_comma_inside_parentheses_does_not_split_an_argument() {
let mut pp = Pp::new();
pp.define("f(x) [x]");
assert_eq!(pp.text("f((1, 2))"), "[(1, 2)]");
}
#[test]
fn an_invocation_may_span_lines() {
let mut pp = Pp::new();
pp.define("f(a, b) a b");
assert_eq!(pp.text("f(1,\n 2)"), "1 2");
}
#[test]
fn a_replacement_may_consume_tokens_that_follow_the_invocation() {
let mut pp = Pp::new();
pp.define("f(x) [x]");
pp.define("g f");
assert_eq!(pp.text("g(1)"), "[1]");
}
#[test]
fn a_parenthesis_that_has_not_expanded_yet_does_not_count() {
let mut pp = Pp::new();
pp.define("lparen (");
pp.define("f(x) [x]");
pp.define("g f lparen 1 )");
assert_eq!(pp.text("g"), "f ( 1 )");
}
#[test]
fn a_macro_does_not_expand_inside_its_own_expansion() {
let mut pp = Pp::new();
pp.define("A A + 1");
assert_eq!(pp.text("A"), "A + 1");
}
#[test]
fn mutually_recursive_macros_terminate_with_both_names_left() {
let mut pp = Pp::new();
pp.define("A B");
pp.define("B A");
assert_eq!(pp.text("A"), "A");
assert_eq!(pp.text("B"), "B");
}
#[test]
fn a_hidden_name_stays_hidden_when_it_is_carried_outwards() {
let mut pp = Pp::new();
pp.define("f(x) x");
pp.define("g(y) [y]");
pp.define("h f(h)");
assert_eq!(pp.text("g(h)"), "[h]");
}
#[test]
fn stringify_puts_the_argument_in_quotes() {
let mut pp = Pp::new();
pp.define("str(x) #x");
assert_eq!(pp.text("str(hello)"), "\"hello\"");
}
#[test]
fn stringify_uses_the_unexpanded_argument() {
let mut pp = Pp::new();
pp.define("N 42");
pp.define("str(x) #x");
pp.define("xstr(x) str(x)");
assert_eq!(pp.text("str(N)"), "\"N\"");
assert_eq!(pp.text("xstr(N)"), "\"42\"", "one level of indirection expands first");
}
#[test]
fn stringify_collapses_whitespace_and_drops_it_at_the_edges() {
let mut pp = Pp::new();
pp.define("str(x) #x");
assert_eq!(pp.text("str( a + b )"), "\"a + b\"");
}
#[test]
fn stringify_escapes_quotes_and_backslashes_inside_literals() {
let mut pp = Pp::new();
pp.define("str(x) #x");
assert_eq!(pp.text(r#"str("a\n")"#), r#""\"a\\n\"""#);
}
#[test]
fn paste_joins_two_tokens_into_one() {
let mut pp = Pp::new();
pp.define("cat(a, b) a ## b");
assert_eq!(pp.text("cat(foo, bar)"), "foobar");
assert_eq!(pp.text("cat(1, 2)"), "12");
assert_eq!(pp.text("cat(+, =)"), "+=");
}
#[test]
fn paste_uses_the_unexpanded_arguments() {
let mut pp = Pp::new();
pp.define("N 42");
pp.define("cat(a, b) a ## b");
assert_eq!(pp.text("cat(N, N)"), "NN");
}
#[test]
fn the_result_of_a_paste_is_rescanned() {
let mut pp = Pp::new();
pp.define("foobar yes");
pp.define("cat(a, b) a ## b");
assert_eq!(pp.text("cat(foo, bar)"), "yes");
}
#[test]
fn pasting_an_empty_argument_leaves_the_other_side() {
let mut pp = Pp::new();
pp.define("cat(a, b) a ## b");
assert_eq!(pp.text("cat(foo,)"), "foo");
assert_eq!(pp.text("cat(, bar)"), "bar");
assert_eq!(pp.text("cat(,)"), "");
}
#[test]
fn a_paste_that_does_not_make_a_token_is_an_error_and_both_tokens_survive() {
let mut pp = Pp::new();
pp.define("cat(a, b) a ## b");
assert_eq!(pp.text("cat(+, foo)"), "+foo");
let errors = pp.errors();
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].code, Some("E0313"));
assert!(errors[0].message.contains("pasting `+` and `foo`"));
}
fn note_texts(d: &Diagnostic) -> Vec<&str> {
d.children.iter().map(|c| c.message.as_str()).collect()
}
#[test]
fn a_paste_error_says_which_macro_wrote_the_paste() {
let mut pp = Pp::new();
pp.define("cat(a, b) a ## b");
assert_eq!(pp.text("cat(+, foo)"), "+foo");
let errors = pp.errors();
let notes = note_texts(&errors[0]);
assert_eq!(notes[2], "expanded from macro `cat`");
assert_eq!(errors[0].children[2].span, Span::new(12, 14));
}
#[test]
fn a_paste_error_names_every_macro_it_came_out_of_outermost_first() {
let mut pp = Pp::new();
pp.define("cat(a, b) a ## b");
pp.define("outer(y) cat(y, +)");
assert_eq!(pp.text("outer(z)"), "z+");
let errors = pp.errors();
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].code, Some("E0313"));
let notes = note_texts(&errors[0]);
assert_eq!(¬es[2..], ["expanded from macro `outer`", "expanded from macro `cat`"]);
assert_eq!(errors[0].children[2].span, Span::new(9, 12));
assert_eq!(errors[0].children[3].span, Span::new(12, 14));
}
#[test]
fn a_macro_called_wrongly_from_another_macro_says_where_it_was_called() {
let mut pp = Pp::new();
pp.define("two(a, b) a b");
pp.define("wrap(x) two(x)");
pp.text("wrap(1)");
let errors = pp.errors();
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].code, Some("E0312"));
assert_eq!(note_texts(&errors[0])[1], "expanded from macro `wrap`");
assert_eq!(errors[0].children[1].span, Span::new(8, 11));
}
#[test]
fn a_macro_used_in_an_argument_is_not_blamed_on_the_macro_it_is_passed_to() {
let mut pp = Pp::new();
pp.define("cat(a, b) a ## b");
pp.define("id(x) x");
pp.text("id(cat(+, foo))");
let errors = pp.errors();
assert_eq!(errors.len(), 1);
assert_eq!(note_texts(&errors[0])[2..], ["expanded from macro `cat`"]);
}
#[test]
fn variadic_arguments_arrive_as_one_argument_with_the_commas_intact() {
let mut pp = Pp::new();
pp.define("f(fmt, ...) g(fmt, __VA_ARGS__)");
assert_eq!(pp.text("f(\"%d %d\", 1, 2)"), "g(\"%d %d\", 1, 2)");
}
#[test]
fn the_gnu_named_variadic_form_works_the_same_way() {
let mut pp = Pp::new();
pp.define("f(fmt, rest...) g(fmt, rest)");
assert_eq!(pp.text("f(a, b, c)"), "g(a, b, c)");
}
#[test]
fn a_variadic_macro_may_be_called_with_nothing_for_the_variadic_part() {
let mut pp = Pp::new();
pp.define("f(a, ...) [a __VA_ARGS__]");
assert_eq!(pp.text("f(1)"), "[1 ]");
}
#[test]
fn the_gnu_comma_swallowing_extension_drops_the_comma() {
let mut pp = Pp::new();
pp.define("log(fmt, ...) printf(fmt, ## __VA_ARGS__)");
assert_eq!(pp.text("log(\"hi\")"), "printf(\"hi\")");
assert_eq!(pp.text("log(\"%d\", 1)"), "printf(\"%d\", 1)");
}
#[test]
fn va_opt_appears_only_when_there_are_variable_arguments() {
let mut pp = Pp::new();
pp.define("log(fmt, ...) printf(fmt __VA_OPT__(,) __VA_ARGS__)");
assert_eq!(pp.text("log(\"hi\")"), "printf(\"hi\" )");
assert_eq!(pp.text("log(\"%d\", 1)"), "printf(\"%d\" , 1)");
}
#[test]
fn va_opt_contents_are_substituted_like_any_other_replacement() {
let mut pp = Pp::new();
pp.define("f(a, ...) [a __VA_OPT__(and __VA_ARGS__ done)]");
assert_eq!(pp.text("f(1)"), "[1 ]");
assert_eq!(pp.text("f(1, 2)"), "[1 and 2 done]");
}
#[test]
fn va_opt_pastes_as_a_unit() {
let mut pp = Pp::new();
pp.define("f(a, ...) a ## __VA_OPT__(x)");
assert_eq!(pp.text("f(y)"), "y", "with no variable arguments it is a placemarker");
assert_eq!(pp.text("f(y, 1)"), "yx");
}
#[test]
fn too_few_arguments_are_reported_against_the_definition() {
let mut pp = Pp::new();
pp.define("f(a, b) a b");
assert_eq!(pp.text("f(1)"), "f", "the arguments are consumed, as GCC and Clang do");
let errors = pp.errors();
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].code, Some("E0312"));
assert_eq!(errors[0].children.len(), 1, "the definition is worth pointing at");
}
#[test]
fn an_unterminated_argument_list_is_reported_at_the_parenthesis() {
let mut pp = Pp::new();
pp.define("f(a) a");
assert_eq!(pp.text("f(1"), "f");
let errors = pp.errors();
assert_eq!(errors[0].code, Some("E0311"));
}
#[test]
fn a_token_from_a_macro_is_reported_at_the_invocation() {
let mut pp = Pp::new();
pp.define("N 42");
let out = pp.expand(" N");
assert_eq!(out.len(), 1);
assert_eq!(out[0].expansion, out[0].report_span());
assert_eq!(out[0].expansion.lo, 2, "the invocation is at offset 2 in the input");
assert_ne!(out[0].span, out[0].expansion, "the spelling is in the macro body");
}
#[test]
fn hide_sets_are_shared_rather_than_rebuilt() {
let mut pp = Pp::new();
pp.define("A 1");
pp.define("B 2");
for _ in 0..100 {
pp.text("A B A B");
}
assert!(
pp.expander.hide_sets() <= 3,
"the empty set plus one per macro, however many times they are used"
);
}
#[test]
fn expansion_is_the_same_every_time() {
let mut first = Pp::new();
let mut second = Pp::new();
for pp in [&mut first, &mut second] {
pp.define("N 42");
pp.define("cat(a, b) a ## b");
pp.define("log(fmt, ...) printf(fmt, ## __VA_ARGS__)");
}
let src = "cat(x, y) N log(\"a\") log(\"b\", 1)";
assert_eq!(first.text(src), second.text(src));
}
#[test]
fn the_standards_rescanning_example() {
let mut pp = Pp::new();
pp.define("x 3");
pp.define("f(a) f(x * (a))");
pp.undef("x");
pp.define("x 2");
pp.define("g f");
pp.define("z z[0]");
pp.define("h g(~");
pp.define("m(a) a(w)");
pp.define("w 0,1");
pp.define("t(a) a");
pp.define("p() int");
pp.define("q(x) x");
pp.define("r(x,y) x ## y");
pp.define("str(x) # x");
assert_eq!(
pp.text("f(y+1) + f(f(z)) % t(t(g)(0) + t)(1);"),
"f(2 * (y+1)) + f(2 * (f(2 * (z[0])))) % f(2 * (0)) + t(1);"
);
assert_eq!(
pp.text("g(x+(3,4)-w) | h 5) & m\n(f)^m(m);"),
"f(2 * (2+(3,4)-0,1)) | f(2 * (~ 5)) & f(2 * (0,1))^m(0,1);"
);
assert_eq!(
pp.text("p() i[q()] = { q(1), r(2,3), r(4,), r(,5), r(,) };"),
"int i[] = { 1, 23, 4, 5, };"
);
assert_eq!(
pp.text("char c[2][6] = { str(hello), str() };"),
"char c[2][6] = { \"hello\", \"\" };"
);
assert!(pp.errors().is_empty(), "the standard's example is well formed");
}
#[test]
fn the_standards_variadic_example() {
let mut pp = Pp::new();
pp.define("debug(...) fprintf(stderr, __VA_ARGS__)");
pp.define("showlist(...) puts(#__VA_ARGS__)");
pp.define("report(test, ...) ((test)?puts(#test): printf(__VA_ARGS__))");
assert_eq!(pp.text("debug(\"Flag\");"), "fprintf(stderr, \"Flag\");");
assert_eq!(pp.text("debug(\"X = %d\\n\", x);"), "fprintf(stderr, \"X = %d\\n\", x);");
assert_eq!(
pp.text("showlist(The first, second, and third items.);"),
"puts(\"The first, second, and third items.\");"
);
assert_eq!(
pp.text("report(x>y, \"x is %d but y is %d\", x, y);"),
"((x>y)?puts(\"x>y\"): printf(\"x is %d but y is %d\", x, y));"
);
assert!(pp.errors().is_empty(), "the standard's example is well formed");
}
#[test]
fn milestone_is_recorded() {
assert!(MILESTONE.starts_with('M'));
}
}