use cpp_parser::{
DocCommentStyle, DocToken, DocTokenKind, is_block_comment, is_documentation_comment,
lex_comment,
};
fn lex(source: &str) -> (Vec<DocToken>, String) {
let style = if is_block_comment(source) {
DocCommentStyle::Block
} else {
DocCommentStyle::Line
};
let _ = style;
let tokens = lex_comment(source, 0);
let text: String = tokens
.iter()
.map(|token| &source[token.range.start_offset..token.range.end_offset()])
.collect();
(tokens, text)
}
fn pairs(source: &str) -> Vec<(DocTokenKind, String)> {
lex(source)
.0
.iter()
.map(|token| {
(
token.kind,
source[token.range.start_offset..token.range.end_offset()].to_string(),
)
})
.collect()
}
const CORPUS: &[&str] = &[
"// plain",
"/// doc",
"//! doc",
"//// banner, not a doc comment",
"//",
"///",
"/** block doc */",
"/*! block doc */",
"/* plain block */",
"/**/",
"/* */",
"/**\n * @brief x\n */",
"/**\n * @param[in] a the a\n * @param[out] b the b\n */",
"/** @brief one line */",
"/// @brief x\n/// @param y desc",
"/// email user@example.com here",
"/// a \\ b and 1 < 2 and a::b",
"/// @code\n/// int x = 1; // @notacommand\n/// @endcode",
"/// @ref Foo<T>",
"/// @param x : described",
"/* unterminated",
"/**\n * **bold** and *em*\n */",
"/// @param[1,3] range",
"/// trailing @",
"/// backslash \\param x",
];
#[test]
fn tokens_tile_the_comment_exactly() {
for source in CORPUS {
let (tokens, reconstructed) = lex(source);
assert_eq!(
reconstructed, *source,
"{source:?} was not tiled by its tokens: {tokens:?}"
);
}
}
#[test]
fn ranges_are_in_file_coordinates() {
let prefix = "int x; ";
let comment = "/// hi";
let source = format!("{prefix}{comment}");
let tokens = lex_comment(comment, prefix.len());
assert!(!tokens.is_empty());
assert_eq!(tokens[0].range.start_offset, prefix.len());
assert_eq!(
&source[tokens[0].range.start_offset..tokens[0].range.end_offset()],
"///"
);
let last = tokens.last().unwrap();
assert_eq!(
last.range.end_offset(),
prefix.len() + comment.len(),
"the last token must end at the end of the comment, not at the end of the file"
);
}
#[test]
fn the_third_character_decides_documentation() {
let documented = ["/// x", "//! x", "/** x */", "/*! x */", "/*** x */"];
let not_documented = ["// x", "//// x", "/* x */", "/*!*/", "/**/", "", "x"];
for source in documented {
assert!(
is_documentation_comment(source),
"{source:?} should be documentation"
);
}
for source in not_documented {
assert!(
!is_documentation_comment(source),
"{source:?} should not be documentation"
);
}
}
#[test]
fn commands_are_recognised_and_split_from_their_introducer() {
let lexed = pairs("/// @param x");
let name = lexed
.iter()
.find(|(kind, _)| *kind == DocTokenKind::DocCommandName)
.expect("a command name");
assert_eq!(name.1, "param", "the name is the name, without the `@`");
assert!(
lexed
.iter()
.any(|(kind, text)| *kind == DocTokenKind::DocIntroducer && text == "@"),
"the introducer is its own token: {lexed:?}"
);
}
#[test]
fn backslash_introduces_commands_too() {
let lexed = pairs("/// \\param x");
assert!(
lexed
.iter()
.any(|(kind, text)| *kind == DocTokenKind::DocCommandName && text == "param"),
"{lexed:?}"
);
}
#[test]
fn a_bare_at_sign_is_prose() {
let lexed = pairs("/// mail user@example.com now");
assert!(
!lexed
.iter()
.any(|(kind, _)| *kind == DocTokenKind::DocCommandName),
"no command should be found in prose: {lexed:?}"
);
assert!(
lexed
.iter()
.any(|(_, text)| text.contains("user@example.com")),
"the address should stay one run of text: {lexed:?}"
);
}
#[test]
fn block_comment_continuation_markers_are_layout() {
let source = "/**\n * @brief x\n */";
let lexed = pairs(source);
let name = lexed
.iter()
.find(|(kind, _)| *kind == DocTokenKind::DocCommandName)
.expect("a command name");
assert_eq!(name.1, "brief");
assert!(
!lexed.iter().any(|(_, text)| text == "*"),
"the continuation marker should have been absorbed into whitespace: {lexed:?}"
);
assert!(
lexed
.iter()
.any(|(kind, text)| *kind == DocTokenKind::BlockCommentEnd && text == "*/"),
"the closer is a token of its own: {lexed:?}"
);
}
#[test]
fn the_opening_line_is_not_a_continuation() {
let lexed = pairs("/** @brief x */");
assert_eq!(
lexed.first().map(|(kind, _)| *kind),
Some(DocTokenKind::DocBlockStart)
);
assert!(
!lexed
.iter()
.any(|(kind, text)| *kind == DocTokenKind::DocWhitespace && text.contains('*')),
"{lexed:?}"
);
}
#[test]
fn emphasis_is_not_mistaken_for_a_line_marker() {
let lexed = pairs("/**\n * **bold**\n */");
assert!(
lexed
.iter()
.any(|(kind, text)| *kind == DocTokenKind::DocText && text.contains("**bold**")),
"{lexed:?}"
);
}
#[test]
fn an_unterminated_block_comment_still_tiles() {
let source = "/**\n * @brief x";
let (_, reconstructed) = lex(source);
assert_eq!(reconstructed, source);
}
#[test]
fn punctuation_is_structural_only_after_a_command() {
let lexed = pairs("/// @param[in] x");
let punctuation: Vec<&str> = lexed
.iter()
.filter(|(kind, _)| {
matches!(
kind,
DocTokenKind::DocLeftBracket | DocTokenKind::DocRightBracket
)
})
.map(|(_, text)| text.as_str())
.collect();
assert_eq!(punctuation, vec!["[", "]"], "{lexed:?}");
}
#[test]
fn prose_punctuation_stays_in_the_text() {
let cases = [
("/// see std::vector<int> for details", "std::vector<int>"),
("/// mail user@example.com now", "user@example.com"),
("/// 1 < 2 and 3 > 2", "1 < 2 and 3 > 2"),
];
for (source, expected) in cases {
let lexed = pairs(source);
assert!(
lexed
.iter()
.any(|(kind, text)| *kind == DocTokenKind::DocText && text.contains(expected)),
"{source:?} should keep {expected:?} as text: {lexed:?}"
);
}
}
#[test]
fn a_code_block_keeps_its_content() {
let source = "/// @code\n/// int x = 1; // @notacommand\n/// @endcode";
let (_, reconstructed) = lex(source);
assert_eq!(reconstructed, source);
}
#[test]
fn the_space_before_a_block_closer_is_not_text() {
let lexed = pairs("/** @brief B */");
let text = lexed
.iter()
.find(|(kind, _)| *kind == DocTokenKind::DocText)
.expect("the brief's text");
assert_eq!(
text.1, "B",
"the text run stops before the closer: {lexed:?}"
);
assert!(
lexed
.iter()
.any(|(kind, text)| *kind == DocTokenKind::DocWhitespace && text == " "),
"the space is emitted as layout: {lexed:?}"
);
}
#[test]
fn a_space_inside_a_line_stays_in_the_text() {
let lexed = pairs("/// @param const T& x");
assert!(
lexed
.iter()
.any(|(kind, text)| *kind == DocTokenKind::DocText && text == "const T& x"),
"an inner space is text, not a separator: {lexed:?}"
);
}
#[test]
fn lexing_always_terminates_and_tiles() {
let cases = [
"@",
"\\",
"////",
"/***/",
"/**/",
"/*/",
"/// @",
"/// \\",
"/// @param[",
"/// ])(",
"@@@@",
"\\\\\\\\",
"/// \u{1F600} emoji",
"/** \u{4e2d}\u{6587} */",
"/** x */",
"/** x */",
"/** x\n */",
"/**\n * x */",
"/** */",
"/** ",
];
for source in cases {
let (tokens, reconstructed) = lex(source);
assert_eq!(
reconstructed, source,
"{source:?} was not tiled: {tokens:?}"
);
}
}