use super::lexer::{
CharsAndLine, Lexer, LexerItem, ReplaceComments, SkipBackslashNewline, Token, TokenValue,
COMMENT_SENTINEL_VALUE,
};
use super::token::{Float, Integer, Location, PreprocessorError, Punct};
use std::ops::Range;
fn c(c: char, line: u32) -> Option<(char, u32)> {
Some((c, line))
}
fn l(line: u32, pos: Range<u32>) -> Location {
Location {
line,
start: pos.start,
end: pos.end,
}
}
fn unwrap_token(item: Option<LexerItem>) -> Token {
item.unwrap().unwrap()
}
fn unwrap_token_value(item: Option<LexerItem>) -> TokenValue {
unwrap_token(item).value
}
fn unwrap_error(item: Option<LexerItem>) -> PreprocessorError {
item.unwrap().unwrap_err().0
}
fn expect_lexer_end(lexer: &mut Lexer) {
assert_eq!(unwrap_token_value(lexer.next()), TokenValue::NewLine);
assert_eq!(lexer.next(), None);
}
impl From<i32> for TokenValue {
fn from(value: i32) -> Self {
TokenValue::Integer(Integer {
value: value as u64,
signed: true,
width: 32,
})
}
}
impl From<u32> for TokenValue {
fn from(value: u32) -> Self {
TokenValue::Integer(Integer {
value: value as u64,
signed: false,
width: 32,
})
}
}
impl From<f32> for TokenValue {
fn from(value: f32) -> Self {
TokenValue::Float(Float { value, width: 32 })
}
}
#[test]
fn chars_and_location() {
let mut it = CharsAndLine::new("abc");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c('b', 1));
assert_eq!(it.next(), c('c', 1));
assert_eq!(it.next(), None);
let mut it = CharsAndLine::new("a\nb");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c('\n', 1));
assert_eq!(it.next(), c('b', 2));
assert_eq!(it.next(), None);
let mut it = CharsAndLine::new("a\rb");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c('\n', 1));
assert_eq!(it.next(), c('b', 2));
assert_eq!(it.next(), None);
let mut it = CharsAndLine::new("a\n\rb");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c('\n', 1));
assert_eq!(it.next(), c('b', 2));
assert_eq!(it.next(), None);
let mut it = CharsAndLine::new("a\r\nb");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c('\n', 1));
assert_eq!(it.next(), c('b', 2));
assert_eq!(it.next(), None);
let mut it = CharsAndLine::new("\n\r\n\r\r\r\n");
assert_eq!(it.next(), c('\n', 1));
assert_eq!(it.next(), c('\n', 2));
assert_eq!(it.next(), c('\n', 3));
assert_eq!(it.next(), c('\n', 4));
assert_eq!(it.next(), None);
let mut it = CharsAndLine::new("a→üs🦀");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c('→', 1));
assert_eq!(it.next(), c('ü', 1));
assert_eq!(it.next(), c('s', 1));
assert_eq!(it.next(), c('🦀', 1));
assert_eq!(it.next(), None);
}
#[test]
fn skip_backslash_newline() {
let mut it = SkipBackslashNewline::new("a\\\nb");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c('b', 2));
assert_eq!(it.next(), None);
let mut it = SkipBackslashNewline::new("a\\\n\\\nb");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c('b', 3));
assert_eq!(it.next(), None);
let mut it = SkipBackslashNewline::new("a\\b");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c('\\', 1));
assert_eq!(it.next(), c('b', 1));
assert_eq!(it.next(), None);
let mut it = SkipBackslashNewline::new("\\\n");
assert_eq!(it.next(), None);
}
#[test]
fn replace_comments() {
let mut it = ReplaceComments::new("a/b");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c('/', 1));
assert_eq!(it.next(), c('b', 1));
assert_eq!(it.next(), None);
let mut it = ReplaceComments::new("a/");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c('/', 1));
assert_eq!(it.next(), None);
let mut it = ReplaceComments::new("a//foo\nb");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c(COMMENT_SENTINEL_VALUE, 1));
assert_eq!(it.next(), c('\n', 1));
assert_eq!(it.next(), c('b', 2));
assert_eq!(it.next(), None);
let mut it = ReplaceComments::new("//foo");
assert_eq!(it.next(), c(COMMENT_SENTINEL_VALUE, 1));
assert_eq!(it.next(), None);
let mut it = ReplaceComments::new("//");
assert_eq!(it.next(), c(COMMENT_SENTINEL_VALUE, 1));
assert_eq!(it.next(), None);
let mut it = ReplaceComments::new("//foo\\\na");
assert_eq!(it.next(), c(COMMENT_SENTINEL_VALUE, 1));
assert_eq!(it.next(), None);
let mut it = ReplaceComments::new("a/*fo\n\no*/b");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c(COMMENT_SENTINEL_VALUE, 1));
assert_eq!(it.next(), c('b', 3));
assert_eq!(it.next(), None);
let mut it = ReplaceComments::new("a/*fo\n\no*");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c(COMMENT_SENTINEL_VALUE, 1));
assert_eq!(it.next(), None);
let mut it = ReplaceComments::new("a/*fo\n\no");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c(COMMENT_SENTINEL_VALUE, 1));
assert_eq!(it.next(), None);
let mut it = ReplaceComments::new("a/*/b");
assert_eq!(it.next(), c('a', 1));
assert_eq!(it.next(), c(COMMENT_SENTINEL_VALUE, 1));
assert_eq!(it.next(), None);
}
#[test]
fn lex_whitespace() {
let mut it = Lexer::new("");
assert_eq!(it.next(), None);
let mut it = Lexer::new("/**/\t //a");
assert_eq!(it.next(), None);
}
#[test]
fn lex_newline() {
let mut it = Lexer::new("\r\n\n");
assert_eq!(unwrap_token_value(it.next()), TokenValue::NewLine);
assert_eq!(unwrap_token_value(it.next()), TokenValue::NewLine);
assert_eq!(it.next(), None);
let mut it = Lexer::new("\r\n\n\t/**/ //");
assert_eq!(unwrap_token_value(it.next()), TokenValue::NewLine);
expect_lexer_end(&mut it);
let mut it = Lexer::new("\r\n\n#");
assert_eq!(unwrap_token_value(it.next()), TokenValue::NewLine);
assert_eq!(unwrap_token_value(it.next()), TokenValue::NewLine);
assert_eq!(unwrap_token_value(it.next()), TokenValue::Hash);
assert_eq!(unwrap_token_value(it.next()), TokenValue::NewLine);
assert_eq!(it.next(), None);
}
#[test]
fn lex_hash() {
let mut it = Lexer::new("a#b");
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("a".to_string())
);
let token = unwrap_token(it.next());
assert_eq!(token.value, TokenValue::Hash);
assert_eq!(token.location, l(1, 1..2));
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("b".to_string())
);
expect_lexer_end(&mut it);
let mut it = Lexer::new("\nvoid #");
assert_eq!(unwrap_token_value(it.next()), TokenValue::NewLine);
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("void".into())
);
let token = unwrap_token(it.next());
assert_eq!(token.value, TokenValue::Hash);
assert_eq!(token.location, l(2, 6..7));
expect_lexer_end(&mut it);
}
#[test]
fn lex_metadata() {
let mut it = Lexer::new("1");
assert_eq!(
unwrap_token(it.next()),
Token {
value: 1.into(),
location: l(1, 0..1),
leading_whitespace: true,
start_of_line: true
}
);
expect_lexer_end(&mut it);
let mut it = Lexer::new(" 1/*\n*/2\t3+\n4");
assert_eq!(
unwrap_token(it.next()),
Token {
value: 1.into(),
location: l(1, 1..2),
leading_whitespace: true,
start_of_line: true
}
);
assert_eq!(
unwrap_token(it.next()),
Token {
value: 2.into(),
location: l(2, 7..8),
leading_whitespace: true,
start_of_line: false
}
);
assert_eq!(
unwrap_token(it.next()),
Token {
value: 3.into(),
location: l(2, 9..10),
leading_whitespace: true,
start_of_line: false
}
);
assert_eq!(
unwrap_token(it.next()),
Token {
value: Punct::Plus.into(),
location: l(2, 10..11),
leading_whitespace: false,
start_of_line: false
}
);
assert_eq!(
unwrap_token(it.next()),
Token {
value: TokenValue::NewLine,
location: l(2, 11..12),
leading_whitespace: false,
start_of_line: false
}
);
assert_eq!(
unwrap_token(it.next()),
Token {
value: 4.into(),
location: l(3, 12..13),
leading_whitespace: true,
start_of_line: true
}
);
assert_eq!(
unwrap_token(it.next()),
Token {
value: TokenValue::NewLine,
location: l(3, 13..13),
leading_whitespace: false,
start_of_line: false
}
);
assert_eq!(it.next(), None);
}
#[test]
fn lex_identifiers() {
let mut it = Lexer::new("foo BA_R baz0");
let token = unwrap_token(it.next());
assert_eq!(token.value, TokenValue::Ident("foo".to_string()));
assert_eq!(token.location, l(1, 0..3),);
let token = unwrap_token(it.next());
assert_eq!(token.value, TokenValue::Ident("BA_R".to_string()));
assert_eq!(token.location, l(1, 4..8),);
let token = unwrap_token(it.next());
assert_eq!(token.value, TokenValue::Ident("baz0".to_string()));
assert_eq!(token.location, l(1, 9..13),);
expect_lexer_end(&mut it);
let mut it = Lexer::new("_");
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("_".to_string())
);
expect_lexer_end(&mut it);
let mut it = Lexer::new("a\\\nb");
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("ab".to_string())
);
expect_lexer_end(&mut it);
let mut it = Lexer::new("a/**/b");
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("a".to_string())
);
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("b".to_string())
);
expect_lexer_end(&mut it);
}
#[test]
fn lex_decimal() {
let mut it = Lexer::new("1 0u 42 65536U");
assert_eq!(unwrap_token_value(it.next()), 1.into());
let token = unwrap_token(it.next());
assert_eq!(token.value, 0u32.into());
assert_eq!(token.location, l(1, 2..4),);
let token = unwrap_token(it.next());
assert_eq!(token.value, 42.into());
assert_eq!(token.location, l(1, 5..7),);
let token = unwrap_token(it.next());
assert_eq!(token.value, 65536u32.into());
assert_eq!(token.location, l(1, 8..14),);
expect_lexer_end(&mut it);
let mut it = Lexer::new("31ab");
assert_eq!(unwrap_token_value(it.next()), 31.into());
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("ab".to_string())
);
expect_lexer_end(&mut it);
let mut it = Lexer::new("31/**/32");
assert_eq!(unwrap_token_value(it.next()), 31.into());
assert_eq!(unwrap_token_value(it.next()), 32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("31+32");
assert_eq!(unwrap_token_value(it.next()), 31.into());
assert_eq!(unwrap_token_value(it.next()), Punct::Plus.into());
assert_eq!(unwrap_token_value(it.next()), 32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("18446744073709551616");
assert_eq!(unwrap_error(it.next()), PreprocessorError::IntegerOverflow);
let mut it = Lexer::new("18446744073709551615");
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Integer(Integer {
value: 18446744073709551615,
signed: true,
width: 32
})
);
expect_lexer_end(&mut it);
let mut it = Lexer::new("13s");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::NotSupported16BitLiteral
);
let mut it = Lexer::new("13S");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::NotSupported16BitLiteral
);
let mut it = Lexer::new("13l");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::NotSupported64BitLiteral
);
let mut it = Lexer::new("13L");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::NotSupported64BitLiteral
);
let mut it = Lexer::new("13uS");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::NotSupported16BitLiteral
);
let mut it = Lexer::new("13Ul");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::NotSupported64BitLiteral
);
}
#[test]
fn lex_hexadecimal() {
let mut it = Lexer::new("0x1 0X0u 0xBaFfe 0XcaFeU");
assert_eq!(unwrap_token_value(it.next()), 1.into());
assert_eq!(unwrap_token_value(it.next()), 0u32.into());
let token = unwrap_token(it.next());
assert_eq!(token.value, 0xBAFFE.into());
assert_eq!(token.location, l(1, 9..16),);
let token = unwrap_token(it.next());
assert_eq!(token.value, 0xCAFEu32.into());
assert_eq!(token.location, l(1, 17..24),);
expect_lexer_end(&mut it);
let mut it = Lexer::new("0x000 0x000000000000001");
assert_eq!(unwrap_token_value(it.next()), 0.into());
assert_eq!(unwrap_token_value(it.next()), 1.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("0x31zb");
assert_eq!(unwrap_token_value(it.next()), 0x31.into());
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("zb".to_string())
);
expect_lexer_end(&mut it);
let mut it = Lexer::new("0x31/**/32");
assert_eq!(unwrap_token_value(it.next()), 0x31.into());
assert_eq!(unwrap_token_value(it.next()), 32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("0x31+32");
assert_eq!(unwrap_token_value(it.next()), 0x31.into());
assert_eq!(unwrap_token_value(it.next()), Punct::Plus.into());
assert_eq!(unwrap_token_value(it.next()), 32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("0x10000000000000000");
assert_eq!(unwrap_error(it.next()), PreprocessorError::IntegerOverflow);
let mut it = Lexer::new("0xFFFFFFFFFFFFFFFF");
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Integer(Integer {
value: 18446744073709551615,
signed: true,
width: 32
})
);
expect_lexer_end(&mut it);
}
#[test]
fn lex_octal() {
let mut it = Lexer::new("01 00u 07654 01234u");
assert_eq!(unwrap_token_value(it.next()), 1.into());
assert_eq!(unwrap_token_value(it.next()), 0u32.into());
assert_eq!(unwrap_token_value(it.next()), 4012.into());
assert_eq!(unwrap_token_value(it.next()), 668u32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("0000 0000000000000001");
assert_eq!(unwrap_token_value(it.next()), 0.into());
assert_eq!(unwrap_token_value(it.next()), 1.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("031zb");
assert_eq!(unwrap_token_value(it.next()), 25.into());
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("zb".to_string())
);
expect_lexer_end(&mut it);
let mut it = Lexer::new("031/**/32");
assert_eq!(unwrap_token_value(it.next()), 25.into());
assert_eq!(unwrap_token_value(it.next()), 32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("031+32");
assert_eq!(unwrap_token_value(it.next()), 25.into());
assert_eq!(unwrap_token_value(it.next()), Punct::Plus.into());
assert_eq!(unwrap_token_value(it.next()), 32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("02000000000000000000000");
assert_eq!(unwrap_error(it.next()), PreprocessorError::IntegerOverflow);
let mut it = Lexer::new("01777777777777777777777");
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Integer(Integer {
value: 18446744073709551615,
signed: true,
width: 32
})
);
expect_lexer_end(&mut it);
}
#[test]
fn lex_float() {
let mut it = Lexer::new("1.0 0.0");
assert_eq!(unwrap_token_value(it.next()), 1.0f32.into());
assert_eq!(unwrap_token_value(it.next()), 0.0f32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new(".99 0.01 .00000000");
assert_eq!(unwrap_token_value(it.next()), 0.99f32.into());
assert_eq!(unwrap_token_value(it.next()), 0.01f32.into());
assert_eq!(unwrap_token_value(it.next()), 0.0f32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("42. 0.");
assert_eq!(unwrap_token_value(it.next()), 42.0f32.into());
assert_eq!(unwrap_token_value(it.next()), 0.0f32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("1000.f 1.f .2f");
assert_eq!(unwrap_token_value(it.next()), 1000.0f32.into());
assert_eq!(unwrap_token_value(it.next()), 1.0f32.into());
assert_eq!(unwrap_token_value(it.next()), 0.2f32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("3e10 4.1e-10f .01e12F 4.1e+10f");
assert_eq!(unwrap_token_value(it.next()), 3e10f32.into());
assert_eq!(unwrap_token_value(it.next()), 4.1e-10f32.into());
assert_eq!(unwrap_token_value(it.next()), 0.01e12f32.into());
assert_eq!(unwrap_token_value(it.next()), 4.1e+10f32.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("05e2 0x1e-2");
assert_eq!(unwrap_token_value(it.next()), 5e2f32.into());
assert_eq!(unwrap_token_value(it.next()), 0x1Ei32.into());
assert_eq!(unwrap_token_value(it.next()), Punct::Minus.into());
assert_eq!(unwrap_token_value(it.next()), 2i32.into());
let mut it = Lexer::new("1.0e");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::FloatParsingError
);
let mut it = Lexer::new("1.0l");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::NotSupported64BitLiteral
);
let mut it = Lexer::new("1.0L");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::NotSupported64BitLiteral
);
let mut it = Lexer::new("1.0h");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::NotSupported16BitLiteral
);
let mut it = Lexer::new("1.0H");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::NotSupported16BitLiteral
);
}
#[test]
fn lex_punctuation() {
let mut it = Lexer::new("+ != <<=");
assert_eq!(unwrap_token_value(it.next()), Punct::Plus.into());
let token = unwrap_token(it.next());
assert_eq!(token.value, Punct::NotEqual.into());
assert_eq!(token.location, l(1, 2..4),);
let token = unwrap_token(it.next());
assert_eq!(token.value, Punct::LeftShiftAssign.into());
assert_eq!(token.location, l(1, 5..8),);
expect_lexer_end(&mut it);
let mut it = Lexer::new("<");
assert_eq!(unwrap_token_value(it.next()), Punct::LeftAngle.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("=\\\n=");
assert_eq!(unwrap_token_value(it.next()), Punct::EqualEqual.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("+/**/=");
assert_eq!(unwrap_token_value(it.next()), Punct::Plus.into());
assert_eq!(unwrap_token_value(it.next()), Punct::Equal.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("!1");
assert_eq!(unwrap_token_value(it.next()), Punct::Bang.into());
assert_eq!(unwrap_token_value(it.next()), 1.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("&a");
assert_eq!(unwrap_token_value(it.next()), Punct::Ampersand.into());
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("a".to_string())
);
expect_lexer_end(&mut it);
let mut it = Lexer::new(">/**/>");
assert_eq!(unwrap_token_value(it.next()), Punct::RightAngle.into());
assert_eq!(unwrap_token_value(it.next()), Punct::RightAngle.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("+++++");
assert_eq!(unwrap_token_value(it.next()), Punct::Increment.into());
assert_eq!(unwrap_token_value(it.next()), Punct::Increment.into());
assert_eq!(unwrap_token_value(it.next()), Punct::Plus.into());
expect_lexer_end(&mut it);
let mut it = Lexer::new("@");
assert_eq!(
unwrap_error(it.next()),
PreprocessorError::UnexpectedCharacter
);
let mut it = Lexer::new("<= >= += -= &= || |= | ^= { } ] ? .");
assert_eq!(unwrap_token_value(it.next()), Punct::LessEqual.into());
assert_eq!(unwrap_token_value(it.next()), Punct::GreaterEqual.into());
assert_eq!(unwrap_token_value(it.next()), Punct::AddAssign.into());
assert_eq!(unwrap_token_value(it.next()), Punct::SubAssign.into());
assert_eq!(unwrap_token_value(it.next()), Punct::AndAssign.into());
assert_eq!(unwrap_token_value(it.next()), Punct::LogicalOr.into());
assert_eq!(unwrap_token_value(it.next()), Punct::OrAssign.into());
assert_eq!(unwrap_token_value(it.next()), Punct::Pipe.into());
assert_eq!(unwrap_token_value(it.next()), Punct::XorAssign.into());
assert_eq!(unwrap_token_value(it.next()), Punct::LeftBrace.into());
assert_eq!(unwrap_token_value(it.next()), Punct::RightBrace.into());
assert_eq!(unwrap_token_value(it.next()), Punct::RightBracket.into());
assert_eq!(unwrap_token_value(it.next()), Punct::Question.into());
assert_eq!(unwrap_token_value(it.next()), Punct::Dot.into());
expect_lexer_end(&mut it);
}
#[test]
fn lex_had_comments() {
let mut it = Lexer::new("#version");
assert!(!it.had_comments());
assert_eq!(unwrap_token_value(it.next()), TokenValue::Hash);
assert!(!it.had_comments());
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("version".to_string())
);
assert!(!it.had_comments());
expect_lexer_end(&mut it);
let mut it = Lexer::new("\r!");
assert!(!it.had_comments());
assert_eq!(unwrap_token_value(it.next()), TokenValue::NewLine);
assert_eq!(unwrap_token_value(it.next()), Punct::Bang.into());
assert!(!it.had_comments());
expect_lexer_end(&mut it);
let mut it = Lexer::new("//\n!");
assert!(!it.had_comments());
assert_eq!(unwrap_token_value(it.next()), TokenValue::NewLine);
assert!(it.had_comments());
assert_eq!(unwrap_token_value(it.next()), Punct::Bang.into());
assert!(it.had_comments());
expect_lexer_end(&mut it);
let mut it = Lexer::new("/**/#version");
assert!(!it.had_comments());
assert_eq!(unwrap_token_value(it.next()), TokenValue::Hash);
assert!(it.had_comments());
assert_eq!(
unwrap_token_value(it.next()),
TokenValue::Ident("version".to_string())
);
assert!(it.had_comments());
expect_lexer_end(&mut it);
}