use crate::{kind::CppTokenKind, lexer::doc_token_kind::DocTokenKind, text::SourceRange};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DocToken {
pub kind: DocTokenKind,
pub range: SourceRange,
}
impl DocToken {
pub fn new(kind: DocTokenKind, range: SourceRange) -> Self {
DocToken { kind, range }
}
pub fn cpp_kind(&self) -> CppTokenKind {
self.kind.to_cpp_token()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DocCommentStyle {
Line,
Block,
}
pub fn is_documentation_comment(text: &str) -> bool {
if text.starts_with("////") {
return false;
}
if text.starts_with("///") || text.starts_with("//!") {
return true;
}
(text.starts_with("/**") && !text.starts_with("/**/"))
|| text.starts_with("/*!") && !text.starts_with("/*!*/")
}
pub fn is_block_comment(text: &str) -> bool {
text.starts_with("/*")
}
pub fn lex_comment(text: &str, offset: usize) -> Vec<DocToken> {
if is_block_comment(text) {
DocLexer::new(text, offset, DocCommentStyle::Block).tokenize()
} else {
DocLexer::new(text, offset, DocCommentStyle::Line).tokenize()
}
}
struct DocLexer<'a> {
text: &'a str,
offset: usize,
style: DocCommentStyle,
pos: usize,
body_end: usize,
at_comment_start: bool,
inside_argument_list: bool,
pending: Vec<DocToken>,
}
impl<'a> DocLexer<'a> {
fn new(text: &'a str, offset: usize, style: DocCommentStyle) -> Self {
let body_end = match style {
DocCommentStyle::Block => {
let opener_len = if (text.starts_with("/**") && !text.starts_with("/**/"))
|| text.starts_with("/*!")
{
3
} else {
2
};
text[opener_len..]
.find("*/")
.map_or(text.len(), |at| opener_len + at)
}
DocCommentStyle::Line => text.len(),
};
DocLexer {
text,
offset,
style,
pos: 0,
body_end,
at_comment_start: true,
inside_argument_list: false,
pending: Vec::new(),
}
}
fn range(&self, start: usize, end: usize) -> SourceRange {
SourceRange::new(self.offset + start, end - start)
}
fn token(&self, kind: DocTokenKind, start: usize, end: usize) -> DocToken {
DocToken::new(kind, self.range(start, end))
}
fn rest(&self) -> &'a str {
&self.text[self.pos.min(self.text.len())..]
}
fn char_at(&self, pos: usize) -> Option<char> {
self.text.get(pos..).and_then(|s| s.chars().next())
}
fn tokenize(mut self) -> Vec<DocToken> {
let mut tokens = Vec::new();
if let Some(token) = self.lex_opening() {
tokens.push(token);
}
while self.pos < self.body_end || !self.pending.is_empty() {
let produced = self.lex_one();
if produced.is_empty() {
break;
}
tokens.extend(produced);
}
tokens.extend(std::mem::take(&mut self.pending));
if let Some(token) = self.lex_closing() {
tokens.push(token);
}
tokens
}
fn lex_opening(&mut self) -> Option<DocToken> {
let rest = self.rest();
let (kind, len) = if rest.starts_with("///") && !rest.starts_with("////")
|| rest.starts_with("//!")
{
(DocTokenKind::DocLineStart, 3)
} else if rest.starts_with("//") {
(DocTokenKind::LineCommentStart, 2)
} else if rest.starts_with("/**") && !rest.starts_with("/**/") || rest.starts_with("/*!") {
(DocTokenKind::DocBlockStart, 3)
} else if rest.starts_with("/*") {
(DocTokenKind::BlockCommentStart, 2)
} else {
return None;
};
let start = self.pos;
self.pos += len;
self.at_comment_start = false;
Some(self.token(kind, start, self.pos))
}
fn lex_closing(&mut self) -> Option<DocToken> {
if self.style != DocCommentStyle::Block {
return None;
}
let start = self.body_end;
if !self.text[start..].starts_with("*/") {
return None;
}
self.pos = start + 2;
Some(self.token(DocTokenKind::BlockCommentEnd, start, self.pos))
}
fn lex_one(&mut self) -> Vec<DocToken> {
if !self.pending.is_empty() {
return std::mem::take(&mut self.pending);
}
let start = self.pos;
if let Some(len) = self.line_ending_len(start) {
self.pos = start + len;
return vec![self.token(DocTokenKind::DocNewline, start, self.pos)];
}
if self.at_line_start()
&& let Some(end) = self.skip_block_line_prefix(start)
&& end > start
{
self.pos = end;
let asterisk = end - 1;
let mut tokens = Vec::with_capacity(2);
if asterisk > start {
tokens.push(self.token(DocTokenKind::DocWhitespace, start, asterisk));
}
tokens.push(self.token(DocTokenKind::DocDecoration, asterisk, end));
return tokens;
}
if self.char_at(start).is_some_and(is_doc_whitespace) {
while self.char_at(self.pos).is_some_and(is_doc_whitespace) {
self.pos += self.char_at(self.pos).map_or(0, char::len_utf8);
}
return vec![self.token(DocTokenKind::DocWhitespace, start, self.pos)];
}
if let Some(tokens) = self.lex_command() {
return tokens;
}
if let Some((kind, len)) = self.punctuation_at(start) {
self.pos = start + len;
match kind {
DocTokenKind::DocLeftBracket => self.inside_argument_list = true,
DocTokenKind::DocRightBracket => self.inside_argument_list = false,
DocTokenKind::DocNewline => {}
_ => {}
}
return vec![self.token(kind, start, self.pos)];
}
vec![self.lex_text()]
}
fn at_line_start(&self) -> bool {
if self.at_comment_start {
return true;
}
self.pos == 0
|| self.text[..self.pos].ends_with('\n')
|| self.text[..self.pos].ends_with('\r')
}
fn line_ending_len(&self, pos: usize) -> Option<usize> {
let rest = &self.text[pos..];
match rest.as_bytes() {
[b'\r', b'\n', ..] => Some(2),
[b'\r', ..] | [b'\n', ..] => Some(1),
_ => None,
}
}
fn skip_block_line_prefix(&self, start: usize) -> Option<usize> {
if self.style != DocCommentStyle::Block {
return None;
}
let mut pos = start;
while self.char_at(pos) == Some(' ') || self.char_at(pos) == Some('\t') {
pos += 1;
}
if self.text[pos..].starts_with("*/") {
return None;
}
if self.char_at(pos) != Some('*') {
return None;
}
pos += 1;
if matches!(self.char_at(pos), Some('*') | Some('/')) {
return None;
}
Some(pos)
}
fn lex_command(&mut self) -> Option<Vec<DocToken>> {
let start = self.pos;
let introducer = self.char_at(start)?;
if introducer != '@' && introducer != '\\' {
return None;
}
let name_start = start + 1;
let mut pos = name_start;
while self
.char_at(pos)
.is_some_and(|c| c.is_ascii_alphanumeric() || c == '_')
{
pos += 1;
}
if pos == name_start {
return None;
}
self.pos = pos;
Some(vec![
self.token(DocTokenKind::DocIntroducer, start, name_start),
self.token(DocTokenKind::DocCommandName, name_start, pos),
])
}
fn lex_text(&mut self) -> DocToken {
let start = self.pos;
let mut pos = start;
while pos < self.body_end {
if self.line_ending_len(pos).is_some() || self.text[pos..].starts_with("*/") {
break;
}
let Some(ch) = self.char_at(pos) else {
break;
};
if (ch == '@' || ch == '\\') && self.starts_a_command(pos) {
break;
}
if ch == ']' && self.inside_argument_list {
break;
}
if ch == ' ' && (self.spaces_start_the_body(pos) || self.spaces_end_the_comment(pos)) {
break;
}
pos += ch.len_utf8();
}
if pos == start {
if let Some(ch) = self.char_at(pos) {
pos += ch.len_utf8();
}
}
self.pos = pos;
self.token(DocTokenKind::DocText, start, self.pos)
}
fn spaces_end_the_comment(&self, pos: usize) -> bool {
let rest = &self.text[pos..self.body_end];
rest.trim_start_matches([' ', '\t']).is_empty()
|| rest.trim_start_matches([' ', '\t']).starts_with("*/")
}
fn spaces_start_the_body(&self, pos: usize) -> bool {
let rest = &self.text[pos..self.body_end];
let run = rest.len() - rest.trim_start_matches(' ').len();
run >= 2
}
fn starts_a_command(&self, pos: usize) -> bool {
let ch = self.char_at(pos);
if ch != Some('@') && ch != Some('\\') {
return false;
}
let mut after = pos + 1;
while self
.char_at(after)
.is_some_and(|c| c.is_ascii_alphanumeric() || c == '_')
{
after += 1;
}
if after == pos + 1 {
return false;
}
if ch == Some('@')
&& let Some(before) = self.previous_char(pos)
{
return !(before.is_alphanumeric() || before == '_');
}
true
}
fn previous_char(&self, pos: usize) -> Option<char> {
self.text[..pos].chars().next_back()
}
fn punctuation_at(&self, pos: usize) -> Option<(DocTokenKind, usize)> {
let rest = &self.text[pos..];
let bytes = rest.as_bytes();
Some(match bytes {
[b':', b':', ..] => (DocTokenKind::DocScope, 2),
[b'[', ..] => (DocTokenKind::DocLeftBracket, 1),
[b']', ..] => (DocTokenKind::DocRightBracket, 1),
[b',', ..] => (DocTokenKind::DocComma, 1),
[b'(', ..] => (DocTokenKind::DocLeftParen, 1),
[b')', ..] => (DocTokenKind::DocRightParen, 1),
[b'.', ..] => (DocTokenKind::DocDot, 1),
[b':', ..] => (DocTokenKind::DocColon, 1),
[b'<', ..] => (DocTokenKind::DocLess, 1),
[b'>', ..] => (DocTokenKind::DocGreater, 1),
[b'=', ..] => (DocTokenKind::DocEquals, 1),
_ => return None,
})
}
}
pub fn is_doc_whitespace(ch: char) -> bool {
ch == ' ' || ch == '\t' || ch == '\u{0b}' || ch == '\u{0c}'
}