1#![doc(html_root_url = "https://docs.rs/rucc-pp/0.2.12")]
82
83mod cond;
84mod directive;
85mod dump;
86mod embed;
87mod expand;
88mod hide;
89mod include;
90mod macros;
91mod predef;
92mod print;
93mod token;
94mod trace;
95
96pub use crate::directive::{LineDirective, Preprocessor};
97pub use crate::dump::macros as dump_macros;
98pub use crate::expand::Expander;
99pub use crate::hide::{HideSet, HideSets};
100pub use crate::include::Context;
101pub use crate::macros::{Builtin, MacroDef, MacroTable, parse_define};
102pub use crate::predef::{BUILT_IN, COMMAND_LINE, Predef, Timestamp};
103pub use crate::print::{PrintOptions, print};
106pub use crate::token::Tok;
107pub use crate::trace::{Step, TraceId, Traces};
108pub use rucc_session::GnucVersion;
109
110pub const MILESTONE: &str = "M1";
112
113#[cfg(test)]
114mod tests {
115 use rucc_base::Interner;
116 use rucc_diag::{Diagnostic, SourceMap, Span};
117 use rucc_lex::{Options, PpToken, PpTokenKind, TokenFlags, tokenize};
118
119 use super::*;
120
121 struct Pp {
123 interner: Interner,
124 macros: MacroTable,
125 expander: Expander,
126 sources: SourceMap,
130 }
131
132 impl Pp {
133 fn new() -> Pp {
134 Pp {
135 interner: Interner::new(),
136 macros: MacroTable::new(),
137 expander: Expander::new(),
138 sources: SourceMap::new(),
139 }
140 }
141
142 fn lex(&mut self, src: &str) -> Vec<PpToken> {
143 let (tokens, errors) = tokenize(src.as_bytes(), 0, Options::new(), &mut self.interner);
144 assert!(errors.is_empty(), "test input should lex cleanly: {errors:?}");
145 tokens.into_iter().filter(|t| t.kind != PpTokenKind::Eof).collect()
146 }
147
148 fn define(&mut self, line: &str) {
150 let tokens = self.lex(line);
151 let (def, errors) = parse_define(&tokens, &mut self.interner);
152 assert!(errors.is_empty(), "definition should be clean: {errors:?}");
153 self.macros.define(def.expect("should parse"), &self.interner);
154 }
155
156 fn undef(&mut self, name: &str) {
157 let sym = self.interner.intern(name);
158 self.macros.undef(sym);
159 }
160
161 fn expand(&mut self, src: &str) -> Vec<Tok> {
162 let tokens = self.lex(src);
163 self.expander.expand(&tokens, &self.macros, &mut self.interner, &self.sources)
164 }
165
166 fn text(&mut self, src: &str) -> String {
170 let out = self.expand(src);
171 let mut text = String::new();
172 for (at, tok) in out.iter().enumerate() {
173 if at > 0 && tok.flags.has(TokenFlags::LEADING_SPACE) {
174 text.push(' ');
175 }
176 match tok.kind {
177 PpTokenKind::Punct(p) => text.push_str(p.as_str()),
178 _ => text.push_str(
179 self.interner.resolve(tok.value.expect("every non-punctuator interns")),
180 ),
181 }
182 }
183 text
184 }
185
186 fn errors(&mut self) -> Vec<Diagnostic> {
187 self.expander.take_diagnostics()
188 }
189 }
190
191 #[test]
192 fn an_object_like_macro_is_replaced_by_its_body() {
193 let mut pp = Pp::new();
194 pp.define("N 42");
195 assert_eq!(pp.text("int a = N;"), "int a = 42;");
196 }
197
198 #[test]
199 fn an_undefined_identifier_is_left_alone() {
200 let mut pp = Pp::new();
201 assert_eq!(pp.text("int a = N;"), "int a = N;");
202 }
203
204 #[test]
205 fn a_function_like_macro_needs_a_parenthesis_to_be_invoked() {
206 let mut pp = Pp::new();
207 pp.define("f(x) x");
208 assert_eq!(pp.text("f"), "f", "a bare name is an ordinary identifier");
209 assert_eq!(pp.text("f (1)"), "1", "whitespace before the parenthesis is fine");
210 }
211
212 #[test]
213 fn arguments_are_expanded_before_they_are_substituted() {
214 let mut pp = Pp::new();
215 pp.define("ONE 1");
216 pp.define("f(x) (x + x)");
217 assert_eq!(pp.text("f(ONE)"), "(1 + 1)");
218 }
219
220 #[test]
221 fn an_empty_argument_is_an_argument() {
222 let mut pp = Pp::new();
223 pp.define("f(x) [x]");
224 assert_eq!(pp.text("f()"), "[]");
225 }
226
227 #[test]
228 fn a_macro_with_no_parameters_takes_no_arguments() {
229 let mut pp = Pp::new();
230 pp.define("f() nothing");
231 assert_eq!(pp.text("f()"), "nothing");
232 }
233
234 #[test]
235 fn a_comma_inside_parentheses_does_not_split_an_argument() {
236 let mut pp = Pp::new();
237 pp.define("f(x) [x]");
238 assert_eq!(pp.text("f((1, 2))"), "[(1, 2)]");
239 }
240
241 #[test]
242 fn an_invocation_may_span_lines() {
243 let mut pp = Pp::new();
244 pp.define("f(a, b) a b");
245 assert_eq!(pp.text("f(1,\n 2)"), "1 2");
246 }
247
248 #[test]
249 fn a_replacement_may_consume_tokens_that_follow_the_invocation() {
250 let mut pp = Pp::new();
251 pp.define("f(x) [x]");
252 pp.define("g f");
253 assert_eq!(pp.text("g(1)"), "[1]");
257 }
258
259 #[test]
260 fn a_parenthesis_that_has_not_expanded_yet_does_not_count() {
261 let mut pp = Pp::new();
262 pp.define("lparen (");
263 pp.define("f(x) [x]");
264 pp.define("g f lparen 1 )");
265 assert_eq!(pp.text("g"), "f ( 1 )");
270 }
271
272 #[test]
273 fn a_macro_does_not_expand_inside_its_own_expansion() {
274 let mut pp = Pp::new();
275 pp.define("A A + 1");
276 assert_eq!(pp.text("A"), "A + 1");
277 }
278
279 #[test]
280 fn mutually_recursive_macros_terminate_with_both_names_left() {
281 let mut pp = Pp::new();
282 pp.define("A B");
283 pp.define("B A");
284 assert_eq!(pp.text("A"), "A");
285 assert_eq!(pp.text("B"), "B");
286 }
287
288 #[test]
289 fn a_hidden_name_stays_hidden_when_it_is_carried_outwards() {
290 let mut pp = Pp::new();
294 pp.define("f(x) x");
295 pp.define("g(y) [y]");
296 pp.define("h f(h)");
297 assert_eq!(pp.text("g(h)"), "[h]");
298 }
299
300 #[test]
301 fn stringify_puts_the_argument_in_quotes() {
302 let mut pp = Pp::new();
303 pp.define("str(x) #x");
304 assert_eq!(pp.text("str(hello)"), "\"hello\"");
305 }
306
307 #[test]
308 fn stringify_uses_the_unexpanded_argument() {
309 let mut pp = Pp::new();
310 pp.define("N 42");
311 pp.define("str(x) #x");
312 pp.define("xstr(x) str(x)");
313 assert_eq!(pp.text("str(N)"), "\"N\"");
314 assert_eq!(pp.text("xstr(N)"), "\"42\"", "one level of indirection expands first");
315 }
316
317 #[test]
318 fn stringify_collapses_whitespace_and_drops_it_at_the_edges() {
319 let mut pp = Pp::new();
320 pp.define("str(x) #x");
321 assert_eq!(pp.text("str( a + b )"), "\"a + b\"");
322 }
323
324 #[test]
325 fn stringify_escapes_quotes_and_backslashes_inside_literals() {
326 let mut pp = Pp::new();
327 pp.define("str(x) #x");
328 assert_eq!(pp.text(r#"str("a\n")"#), r#""\"a\\n\"""#);
329 }
330
331 #[test]
332 fn paste_joins_two_tokens_into_one() {
333 let mut pp = Pp::new();
334 pp.define("cat(a, b) a ## b");
335 assert_eq!(pp.text("cat(foo, bar)"), "foobar");
336 assert_eq!(pp.text("cat(1, 2)"), "12");
337 assert_eq!(pp.text("cat(+, =)"), "+=");
338 }
339
340 #[test]
341 fn paste_uses_the_unexpanded_arguments() {
342 let mut pp = Pp::new();
343 pp.define("N 42");
344 pp.define("cat(a, b) a ## b");
345 assert_eq!(pp.text("cat(N, N)"), "NN");
346 }
347
348 #[test]
349 fn the_result_of_a_paste_is_rescanned() {
350 let mut pp = Pp::new();
351 pp.define("foobar yes");
352 pp.define("cat(a, b) a ## b");
353 assert_eq!(pp.text("cat(foo, bar)"), "yes");
354 }
355
356 #[test]
357 fn pasting_an_empty_argument_leaves_the_other_side() {
358 let mut pp = Pp::new();
359 pp.define("cat(a, b) a ## b");
360 assert_eq!(pp.text("cat(foo,)"), "foo");
361 assert_eq!(pp.text("cat(, bar)"), "bar");
362 assert_eq!(pp.text("cat(,)"), "");
363 }
364
365 #[test]
366 fn a_paste_that_does_not_make_a_token_is_an_error_and_both_tokens_survive() {
367 let mut pp = Pp::new();
368 pp.define("cat(a, b) a ## b");
369 assert_eq!(pp.text("cat(+, foo)"), "+foo");
370 let errors = pp.errors();
371 assert_eq!(errors.len(), 1);
372 assert_eq!(errors[0].code, Some("E0313"));
373 assert!(errors[0].message.contains("pasting `+` and `foo`"));
374 }
375
376 fn note_texts(d: &Diagnostic) -> Vec<&str> {
378 d.children.iter().map(|c| c.message.as_str()).collect()
379 }
380
381 #[test]
382 fn a_paste_error_says_which_macro_wrote_the_paste() {
383 let mut pp = Pp::new();
386 pp.define("cat(a, b) a ## b");
387 assert_eq!(pp.text("cat(+, foo)"), "+foo");
388 let errors = pp.errors();
389 let notes = note_texts(&errors[0]);
390 assert_eq!(notes[2], "expanded from macro `cat`");
391 assert_eq!(errors[0].children[2].span, Span::new(12, 14));
394 }
395
396 #[test]
397 fn a_paste_error_names_every_macro_it_came_out_of_outermost_first() {
398 let mut pp = Pp::new();
401 pp.define("cat(a, b) a ## b");
402 pp.define("outer(y) cat(y, +)");
403 assert_eq!(pp.text("outer(z)"), "z+");
404 let errors = pp.errors();
405 assert_eq!(errors.len(), 1);
406 assert_eq!(errors[0].code, Some("E0313"));
407 let notes = note_texts(&errors[0]);
408 assert_eq!(¬es[2..], ["expanded from macro `outer`", "expanded from macro `cat`"]);
409 assert_eq!(errors[0].children[2].span, Span::new(9, 12));
411 assert_eq!(errors[0].children[3].span, Span::new(12, 14));
412 }
413
414 #[test]
415 fn a_macro_called_wrongly_from_another_macro_says_where_it_was_called() {
416 let mut pp = Pp::new();
417 pp.define("two(a, b) a b");
418 pp.define("wrap(x) two(x)");
419 pp.text("wrap(1)");
420 let errors = pp.errors();
421 assert_eq!(errors.len(), 1);
422 assert_eq!(errors[0].code, Some("E0312"));
423 assert_eq!(note_texts(&errors[0])[1], "expanded from macro `wrap`");
424 assert_eq!(errors[0].children[1].span, Span::new(8, 11));
425 }
426
427 #[test]
428 fn a_macro_used_in_an_argument_is_not_blamed_on_the_macro_it_is_passed_to() {
429 let mut pp = Pp::new();
432 pp.define("cat(a, b) a ## b");
433 pp.define("id(x) x");
434 pp.text("id(cat(+, foo))");
435 let errors = pp.errors();
436 assert_eq!(errors.len(), 1);
437 assert_eq!(note_texts(&errors[0])[2..], ["expanded from macro `cat`"]);
438 }
439
440 #[test]
441 fn variadic_arguments_arrive_as_one_argument_with_the_commas_intact() {
442 let mut pp = Pp::new();
443 pp.define("f(fmt, ...) g(fmt, __VA_ARGS__)");
444 assert_eq!(pp.text("f(\"%d %d\", 1, 2)"), "g(\"%d %d\", 1, 2)");
445 }
446
447 #[test]
448 fn the_gnu_named_variadic_form_works_the_same_way() {
449 let mut pp = Pp::new();
450 pp.define("f(fmt, rest...) g(fmt, rest)");
451 assert_eq!(pp.text("f(a, b, c)"), "g(a, b, c)");
452 }
453
454 #[test]
455 fn a_variadic_macro_may_be_called_with_nothing_for_the_variadic_part() {
456 let mut pp = Pp::new();
457 pp.define("f(a, ...) [a __VA_ARGS__]");
458 assert_eq!(pp.text("f(1)"), "[1 ]");
459 }
460
461 #[test]
462 fn the_gnu_comma_swallowing_extension_drops_the_comma() {
463 let mut pp = Pp::new();
464 pp.define("log(fmt, ...) printf(fmt, ## __VA_ARGS__)");
465 assert_eq!(pp.text("log(\"hi\")"), "printf(\"hi\")");
466 assert_eq!(pp.text("log(\"%d\", 1)"), "printf(\"%d\", 1)");
467 }
468
469 #[test]
470 fn va_opt_appears_only_when_there_are_variable_arguments() {
471 let mut pp = Pp::new();
472 pp.define("log(fmt, ...) printf(fmt __VA_OPT__(,) __VA_ARGS__)");
473 assert_eq!(pp.text("log(\"hi\")"), "printf(\"hi\" )");
474 assert_eq!(pp.text("log(\"%d\", 1)"), "printf(\"%d\" , 1)");
475 }
476
477 #[test]
478 fn va_opt_contents_are_substituted_like_any_other_replacement() {
479 let mut pp = Pp::new();
480 pp.define("f(a, ...) [a __VA_OPT__(and __VA_ARGS__ done)]");
481 assert_eq!(pp.text("f(1)"), "[1 ]");
482 assert_eq!(pp.text("f(1, 2)"), "[1 and 2 done]");
483 }
484
485 #[test]
486 fn va_opt_pastes_as_a_unit() {
487 let mut pp = Pp::new();
488 pp.define("f(a, ...) a ## __VA_OPT__(x)");
489 assert_eq!(pp.text("f(y)"), "y", "with no variable arguments it is a placemarker");
490 assert_eq!(pp.text("f(y, 1)"), "yx");
491 }
492
493 #[test]
494 fn too_few_arguments_are_reported_against_the_definition() {
495 let mut pp = Pp::new();
496 pp.define("f(a, b) a b");
497 assert_eq!(pp.text("f(1)"), "f", "the arguments are consumed, as GCC and Clang do");
498 let errors = pp.errors();
499 assert_eq!(errors.len(), 1);
500 assert_eq!(errors[0].code, Some("E0312"));
501 assert_eq!(errors[0].children.len(), 1, "the definition is worth pointing at");
502 }
503
504 #[test]
505 fn an_unterminated_argument_list_is_reported_at_the_parenthesis() {
506 let mut pp = Pp::new();
507 pp.define("f(a) a");
508 assert_eq!(pp.text("f(1"), "f");
509 let errors = pp.errors();
510 assert_eq!(errors[0].code, Some("E0311"));
511 }
512
513 #[test]
514 fn a_token_from_a_macro_is_reported_at_the_invocation() {
515 let mut pp = Pp::new();
516 pp.define("N 42");
517 let out = pp.expand(" N");
518 assert_eq!(out.len(), 1);
519 assert_eq!(out[0].expansion, out[0].report_span());
520 assert_eq!(out[0].expansion.lo, 2, "the invocation is at offset 2 in the input");
521 assert_ne!(out[0].span, out[0].expansion, "the spelling is in the macro body");
522 }
523
524 #[test]
525 fn hide_sets_are_shared_rather_than_rebuilt() {
526 let mut pp = Pp::new();
527 pp.define("A 1");
528 pp.define("B 2");
529 for _ in 0..100 {
530 pp.text("A B A B");
531 }
532 assert!(
533 pp.expander.hide_sets() <= 3,
534 "the empty set plus one per macro, however many times they are used"
535 );
536 }
537
538 #[test]
539 fn expansion_is_the_same_every_time() {
540 let mut first = Pp::new();
541 let mut second = Pp::new();
542 for pp in [&mut first, &mut second] {
543 pp.define("N 42");
544 pp.define("cat(a, b) a ## b");
545 pp.define("log(fmt, ...) printf(fmt, ## __VA_ARGS__)");
546 }
547 let src = "cat(x, y) N log(\"a\") log(\"b\", 1)";
548 assert_eq!(first.text(src), second.text(src));
549 }
550
551 #[test]
556 fn the_standards_rescanning_example() {
557 let mut pp = Pp::new();
558 pp.define("x 3");
559 pp.define("f(a) f(x * (a))");
560 pp.undef("x");
561 pp.define("x 2");
562 pp.define("g f");
563 pp.define("z z[0]");
564 pp.define("h g(~");
565 pp.define("m(a) a(w)");
566 pp.define("w 0,1");
567 pp.define("t(a) a");
568 pp.define("p() int");
569 pp.define("q(x) x");
570 pp.define("r(x,y) x ## y");
571 pp.define("str(x) # x");
572
573 assert_eq!(
574 pp.text("f(y+1) + f(f(z)) % t(t(g)(0) + t)(1);"),
575 "f(2 * (y+1)) + f(2 * (f(2 * (z[0])))) % f(2 * (0)) + t(1);"
576 );
577 assert_eq!(
581 pp.text("g(x+(3,4)-w) | h 5) & m\n(f)^m(m);"),
582 "f(2 * (2+(3,4)-0,1)) | f(2 * (~ 5)) & f(2 * (0,1))^m(0,1);"
583 );
584 assert_eq!(
585 pp.text("p() i[q()] = { q(1), r(2,3), r(4,), r(,5), r(,) };"),
586 "int i[] = { 1, 23, 4, 5, };"
587 );
588 assert_eq!(
589 pp.text("char c[2][6] = { str(hello), str() };"),
590 "char c[2][6] = { \"hello\", \"\" };"
591 );
592 assert!(pp.errors().is_empty(), "the standard's example is well formed");
593 }
594
595 #[test]
597 fn the_standards_variadic_example() {
598 let mut pp = Pp::new();
599 pp.define("debug(...) fprintf(stderr, __VA_ARGS__)");
600 pp.define("showlist(...) puts(#__VA_ARGS__)");
601 pp.define("report(test, ...) ((test)?puts(#test): printf(__VA_ARGS__))");
602
603 assert_eq!(pp.text("debug(\"Flag\");"), "fprintf(stderr, \"Flag\");");
604 assert_eq!(pp.text("debug(\"X = %d\\n\", x);"), "fprintf(stderr, \"X = %d\\n\", x);");
605 assert_eq!(
606 pp.text("showlist(The first, second, and third items.);"),
607 "puts(\"The first, second, and third items.\");"
608 );
609 assert_eq!(
610 pp.text("report(x>y, \"x is %d but y is %d\", x, y);"),
611 "((x>y)?puts(\"x>y\"): printf(\"x is %d but y is %d\", x, y));"
612 );
613 assert!(pp.errors().is_empty(), "the standard's example is well formed");
614 }
615
616 #[test]
617 fn milestone_is_recorded() {
618 assert!(MILESTONE.starts_with('M'));
619 }
620}