use cpp_code_analysis::{
ExpandedToken, MacroTable, Origin, Token, expand, expand_with_budget, preprocess::preprocess,
};
use cpp_parser::{CppParser, CppTokenKind, ParserConfig};
fn lex(source: &str) -> Vec<Token> {
let mut errors = Vec::new();
let mut lexer =
cpp_parser::CppLexer::new(source, cpp_parser::LexerConfig::default(), &mut errors);
lexer
.tokenize()
.into_iter()
.map(|token| {
Token::new(
token.kind,
&source[token.range.start_offset..token.range.end_offset()],
token.range,
)
})
.collect()
}
fn tokens_of(source: &str) -> Vec<Token> {
lex(source)
.into_iter()
.filter(|token| !cpp_code_analysis::token::is_trivia(token.kind))
.collect()
}
fn table(defines: &str) -> MacroTable {
let tree = CppParser::parse(defines, ParserConfig::default());
preprocess(defines, tree.get_tokens()).macros
}
fn expand_use(defines: &str, use_site: &str) -> cpp_code_analysis::Expansion {
let macros = table(defines);
expand(&lex(use_site), ¯os)
}
fn text(expansion: &cpp_code_analysis::Expansion) -> Vec<String> {
expansion
.tokens
.iter()
.filter(|token| !cpp_code_analysis::token::is_trivia(token.kind()))
.map(|token| token.text().to_string())
.collect()
}
fn joined(expansion: &cpp_code_analysis::Expansion) -> String {
rendered(expansion)
}
fn rendered(expansion: &cpp_code_analysis::Expansion) -> String {
let mut out = String::new();
for token in &expansion.tokens {
if token.space_before && !out.is_empty() {
out.push(' ');
}
out.push_str(token.text());
}
out
}
#[test]
fn an_object_like_macro_is_replaced_by_its_body() {
let expansion = expand_use("#define VERSION 3\n", "VERSION");
assert_eq!(joined(&expansion), "3");
assert!(
expansion.diagnostics.is_empty(),
"{:?}",
expansion.diagnostics
);
}
#[test]
fn a_macro_with_an_empty_body_expands_to_nothing() {
let expansion = expand_use("#define FEATURE\n", "FEATURE");
assert!(expansion.is_empty(), "{:?}", text(&expansion));
}
#[test]
fn an_identifier_that_is_not_a_macro_is_left_alone() {
let expansion = expand_use("#define A 1\n", "B");
assert_eq!(joined(&expansion), "B");
assert!(
expansion.diagnostics.is_empty(),
"not being a macro is not a note: {:?}",
expansion.diagnostics
);
}
#[test]
fn a_macro_body_containing_another_macro_is_expanded_too() {
let expansion = expand_use("#define A 1\n#define B A\n", "B");
assert_eq!(joined(&expansion), "1");
}
#[test]
fn a_function_like_macro_takes_arguments() {
let expansion = expand_use("#define MAX(a, b) ((a) > (b) ? (a) : (b))\n", "MAX(1, 2)");
assert_eq!(joined(&expansion), "((1) > (2) ? (1) : (2))");
}
#[test]
fn a_space_before_the_parenthesis_is_still_a_call() {
let expansion = expand_use("#define F(x) (x)\n", "F (1)");
assert_eq!(joined(&expansion), "(1)");
assert!(expansion.diagnostics.is_empty());
}
#[test]
fn a_function_like_name_ending_a_body_takes_the_arguments_after_the_call() {
let expansion = expand_use("#define F(x) (x)\n#define A F\n", "A(1) + 2");
assert_eq!(joined(&expansion), "(1) + 2");
}
#[test]
fn a_chain_of_object_like_macros_ends_in_a_call() {
let expansion = expand_use("#define F(x) [x]\n#define A F\n#define B A\n", "B (7) ;");
assert_eq!(joined(&expansion), "[7] ;");
}
#[test]
fn a_function_like_name_ending_a_body_without_arguments_is_left_alone() {
let expansion = expand_use("#define F(x) (x)\n#define A F\n", "A + 2");
assert_eq!(joined(&expansion), "F + 2");
}
#[test]
fn an_argument_is_expanded_before_a_nested_stringize_sees_it() {
let expansion = expand_use(
"#define LEVEL 0\n#define S_(x) #x\n#define S(x) S_(x)\n",
"S(LEVEL) S_(LEVEL)",
);
assert_eq!(
text(&expansion),
["\"0\"", "\"LEVEL\""],
"`#x` directly on the argument stringizes the name"
);
}
#[test]
fn a_pasted_argument_is_not_expanded_first() {
let expansion = expand_use(
"#define X 1\n#define CAT(a, b) a ## b\n",
"CAT(X, Y) CAT(1, X)",
);
assert_eq!(text(&expansion), ["XY", "1X"]);
}
#[test]
fn a_function_like_macro_without_arguments_is_not_expanded() {
let expansion = expand_use("#define F(x) (x)\n", "F + 1");
assert_eq!(joined(&expansion), "F + 1");
assert!(matches!(
expansion.diagnostics.first().map(|it| &it.note),
Some(cpp_code_analysis::ExpansionNote::NoArgumentList { .. })
));
}
#[test]
fn an_unterminated_argument_list_is_not_expanded() {
let expansion = expand_use("#define F(x) (x)\n", "F(1, 2");
assert_eq!(joined(&expansion), "F(1, 2");
assert!(matches!(
expansion.diagnostics.first().map(|it| &it.note),
Some(cpp_code_analysis::ExpansionNote::UnterminatedArgumentList { .. })
));
}
#[test]
fn a_comma_inside_brackets_does_not_split_an_argument() {
let expansion = expand_use("#define ID(x) x\n", "ID((1, 2))");
assert_eq!(joined(&expansion), "(1, 2)");
assert!(
expansion.diagnostics.is_empty(),
"{:?}",
expansion.diagnostics
);
}
#[test]
fn an_argument_is_expanded_before_substitution() {
let expansion = expand_use("#define INNER 7\n#define ID(x) x\n", "ID(INNER)");
assert_eq!(joined(&expansion), "7");
}
#[test]
fn the_wrong_number_of_arguments_is_not_expanded() {
let expansion = expand_use("#define MAX(a, b) 0\n", "MAX(1)");
assert_eq!(joined(&expansion), "MAX(1)");
assert!(matches!(
expansion.diagnostics.first().map(|it| &it.note),
Some(cpp_code_analysis::ExpansionNote::WrongArgumentCount {
expected: 2,
found: 1,
..
})
));
}
#[test]
fn an_empty_argument_list_is_zero_arguments() {
let expansion = expand_use("#define NOTHING() 42\n", "NOTHING()");
assert_eq!(joined(&expansion), "42");
assert!(
expansion.diagnostics.is_empty(),
"{:?}",
expansion.diagnostics
);
}
#[test]
fn a_variadic_macro_collects_the_rest_of_its_arguments() {
let expansion = expand_use("#define SUM(f, ...) f(__VA_ARGS__)\n", "SUM(1, 2, 3)");
assert_eq!(joined(&expansion), "1(2 , 3)");
}
#[test]
fn a_variadic_macro_with_no_extra_arguments_substitutes_nothing() {
let expansion = expand_use("#define LOG(f, ...) f(__VA_ARGS__)\n", "LOG(\"x\")");
assert_eq!(joined(&expansion), "\"x\"( )");
}
#[test]
fn stringize_turns_an_argument_into_a_string_literal() {
let expansion = expand_use("#define STR(x) #x\n", "STR(hello)");
assert_eq!(text(&expansion), vec!["\"hello\""]);
assert_eq!(expansion.tokens[0].kind(), CppTokenKind::StringLiteral);
}
#[test]
fn stringize_produces_the_spelling_of_the_argument() {
let expansion = expand_use("#define STR(x) #x\n", "STR(a + b)");
assert_eq!(joined(&expansion), "\"a + b\"");
let expansion = expand_use("#define STR(x) #x\n", "STR(\"quoted\")");
assert_eq!(joined(&expansion), "\"\\\"quoted\\\"\"");
}
#[test]
fn paste_re_lexes_the_result() {
let expansion = expand_use("#define CAT(a, b) a ## b\n", "CAT(+, =)");
assert_eq!(text(&expansion), vec!["+="]);
assert_eq!(expansion.tokens[0].kind(), CppTokenKind::PlusAssign);
}
#[test]
fn paste_builds_a_longer_identifier() {
let expansion = expand_use("#define GLUE(a, b) a ## b\n", "GLUE(foo, bar)");
assert_eq!(text(&expansion), vec!["foobar"]);
assert_eq!(expansion.tokens[0].kind(), CppTokenKind::Identifier);
}
#[test]
fn paste_builds_a_number() {
let expansion = expand_use("#define NUM(a, b) a ## b\n", "NUM(1, 2)");
assert_eq!(text(&expansion), vec!["12"]);
assert_eq!(expansion.tokens[0].kind(), CppTokenKind::IntegerLiteral);
}
#[test]
fn a_paste_that_produces_nothing_readable_keeps_both_halves() {
let expansion = expand_use("#define BAD(a, b) a ## b\n", "BAD(+, *)");
assert_eq!(joined(&expansion), "+ *");
}
#[test]
fn stringize_does_not_expand_its_argument_first() {
let expansion = expand_use("#define INNER 7\n#define STR(x) #x\n", "STR(INNER)");
assert_eq!(joined(&expansion), "\"INNER\"");
}
#[test]
fn a_token_from_a_macro_body_is_marked_as_expanded() {
let expansion = expand_use("#define VERSION 3\n", "int x = VERSION;");
let kinds: Vec<(String, bool)> = expansion
.tokens
.iter()
.filter(|token| !cpp_code_analysis::token::is_trivia(token.kind()))
.map(|token| (token.text().to_string(), token.is_expanded()))
.collect();
assert_eq!(
kinds,
vec![
("int".to_string(), false),
("x".to_string(), false),
("=".to_string(), false),
("3".to_string(), true),
(";".to_string(), false),
]
);
}
#[test]
fn navigation_reaches_the_macro_and_a_diagnostic_reaches_the_call_site() {
let defines = "#define VERSION 3\n";
let expansion = expand_use(defines, "int x = VERSION;");
let expanded: &ExpandedToken = expansion
.tokens
.iter()
.find(|token| token.is_expanded())
.expect("an expanded token");
let (name_file, name_range) = expanded.navigation_at().expect("a definition in a file");
assert_eq!(name_file, cpp_code_analysis::macros::MacroFile::Here);
assert_eq!(
&defines[name_range.start_offset..name_range.end_offset()],
"VERSION"
);
let range = expanded.diagnostic_range();
assert!(
range.start_offset < "int x = VERSION;".len(),
"the diagnostic range is in the use site: {range:?}"
);
assert!(
range.start_offset >= "int x = ".len(),
"and specifically on the macro's name: {range:?}"
);
}
#[test]
fn navigation_finds_the_name_however_the_head_is_spelled() {
for defines in [
"#define VERSION 3\n",
"# define VERSION 3\n",
"#define\tVERSION 3\n",
] {
let expansion = expand_use(defines, "VERSION");
let (file, range) = expansion.tokens[0]
.navigation_at()
.expect("a definition in a file");
assert_eq!(file, cpp_code_analysis::macros::MacroFile::Here);
assert_eq!(
&defines[range.start_offset..range.end_offset()],
"VERSION",
"{defines:?}"
);
}
}
#[test]
fn the_call_site_covers_the_whole_invocation() {
let expansion = expand_use("#define MAX(a, b) 0\n", "MAX(1, 2)");
let range = match &expansion.tokens[0].origin {
Origin::Expanded { invocations } => invocations.first().expect("an invocation").call_site,
other => panic!("expected an expansion, got {other:?}"),
};
assert_eq!(range.start_offset, 0);
assert_eq!(
range.length,
"MAX(1, 2)".len(),
"the arguments are part of the call"
);
}
#[test]
fn a_pasted_token_reports_against_the_call_site() {
let expansion = expand_use("#define CAT(a, b) a ## b\n", "CAT(+, =)");
assert!(matches!(expansion.tokens[0].origin, Origin::Pasted { .. }));
assert!(expansion.tokens[0].is_expanded());
}
#[test]
fn a_stringized_token_is_marked_as_stringized() {
let expansion = expand_use("#define STR(x) #x\n", "STR(hi)");
assert!(matches!(
expansion.tokens[0].origin,
Origin::Stringized { .. }
));
}
#[test]
fn an_inner_expansion_reports_against_the_outer_call() {
let expansion = expand_use("#define A 1\n#define B A\n", "B");
let range = expansion.tokens[0].diagnostic_range();
assert_eq!(
range.start_offset, 0,
"the outermost invocation, which is the `B` on this line"
);
assert_eq!(range.length, 1);
}
#[test]
fn a_self_referential_macro_terminates() {
let expansion = expand_use("#define A A\n", "A");
assert_eq!(joined(&expansion), "A");
assert!(matches!(
expansion.diagnostics.first().map(|it| &it.note),
Some(cpp_code_analysis::ExpansionNote::Recursive { .. })
));
}
#[test]
fn mutually_recursive_macros_terminate() {
let expansion = expand_use("#define A B\n#define B A\n", "A");
assert!(
expansion
.diagnostics
.iter()
.any(|it| matches!(it.note, cpp_code_analysis::ExpansionNote::Recursive { .. })),
"{:?}",
expansion.diagnostics
);
assert!(
expansion.tokens.len() < 16,
"it stopped: {:?}",
text(&expansion)
);
}
#[test]
fn a_macro_that_mentions_itself_after_other_tokens_terminates() {
let expansion = expand_use("#define A 1 + A\n", "A");
assert!(
expansion.tokens.len() < 16,
"it stopped: {:?}",
text(&expansion)
);
}
#[test]
fn the_token_budget_stops_a_runaway_expansion() {
let macros =
table("#define A B B\n#define B C C\n#define C D D\n#define D E E\n#define E F F\n");
let tokens = tokens_of("A");
let expansion = expand_with_budget(&tokens, ¯os, 20);
assert!(
expansion.tokens.len() <= 20,
"the budget is an upper bound: {} tokens",
expansion.tokens.len()
);
}
#[test]
fn expansion_never_panics() {
let defines = concat!(
"#define EMPTY\n",
"#define ONE 1\n",
"#define F(x) (x)\n",
"#define TWO(a, b) a b\n",
"#define STR(x) #x\n",
"#define CAT(a, b) a ## b\n",
"#define VAR(f, ...) f(__VA_ARGS__)\n",
"#define REC A\n",
"#define A REC\n",
"#define true 1\n",
);
let macros = table(defines);
let cases = [
"",
"ONE",
"ONE ONE ONE",
"F",
"F(",
"F()",
"F(1",
"F(1)",
"F((((1))))",
"F(,)",
"F(,,)",
"TWO(1)",
"TWO(1, 2, 3)",
"STR()",
"STR(#)",
"STR(\"a\\b\")",
"CAT(,)",
"CAT(+, *)",
"CAT(CAT(a, b), c)",
"VAR()",
"VAR(1)",
"VAR(1, 2, 3)",
"__VA_ARGS__",
"true",
"REC",
"A",
"EMPTY EMPTY EMPTY",
"#",
"##",
"\\",
"((((((((((",
"\"unterminated",
"'",
"1'000",
"\u{1F600}",
"MAX(1, 2)",
];
for source in cases {
let tokens = tokens_of(source);
let expansion = expand(&tokens, ¯os);
for token in &expansion.tokens {
let _ = token.diagnostic_range();
let _ = token.navigation_at();
}
}
}
#[test]
fn expansion_work_is_bounded_by_the_budget() {
let defines = "#define A B B\n#define B C C\n#define C D D\n#define D E E\n#define E F F\n\
#define F G G\n#define G H H\n#define H I I\n#define I J J\n#define J K K\n";
let macros = table(defines);
let tokens = tokens_of("A");
let expansion = expand(&tokens, ¯os);
assert!(
expansion.tokens.len() <= cpp_code_analysis::expand::MAX_TOKENS,
"the cap held: {} tokens",
expansion.tokens.len()
);
}
#[test]
fn a_macro_named_after_a_keyword_expands() {
let expansion = expand_use("#define true 1\n", "true");
assert_eq!(joined(&expansion), "1");
}
#[test]
fn every_expanded_token_has_a_usable_range() {
let defines = "#define VERSION 3\n#define CAT(a, b) a ## b\n#define STR(x) #x\n";
let expansion = expand_use(defines, "VERSION CAT(+, =) STR(hi)");
let use_site = "VERSION CAT(+, =) STR(hi)";
for token in &expansion.tokens {
let range = token.token.range;
assert!(
range.end_offset() <= use_site.len() || range.end_offset() <= defines.len(),
"a token's range is inside one of the two files: {token:?}"
);
}
}
#[test]
fn a_realistic_macro_expands_correctly() {
let defines = "#define MAX(a, b) ((a) > (b) ? (a) : (b))\n";
let expansion = expand_use(defines, "int m = MAX(x + 1, y);");
assert_eq!(
joined(&expansion),
"int m = ((x + 1) > (y) ? (x + 1) : (y));"
);
assert!(
expansion.diagnostics.is_empty(),
"{:?}",
expansion.diagnostics
);
let expanded = expansion
.tokens
.iter()
.filter(|token| token.is_expanded())
.count();
assert!(expanded > 8, "most of the expression came from the macro");
}