use cpp_parser::{CppTokenKind, LexerConfig};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Token<'a> {
pub kind: CppTokenKind,
pub text: &'a str,
pub start: usize,
pub end: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Comment<'a> {
pub kind: CppTokenKind,
pub text: &'a str,
pub start: usize,
pub end: usize,
pub own_line: bool,
pub ends_line: bool,
}
impl Comment<'_> {
pub fn is_line_comment(&self) -> bool {
self.kind == CppTokenKind::LineComment
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Gap<'a> {
pub start: usize,
pub end: usize,
pub newlines: usize,
pub comments: Vec<Comment<'a>>,
pub at_end: bool,
}
impl Gap<'_> {
pub fn is_empty(&self) -> bool {
self.start == self.end
}
}
pub struct Stream<'a> {
pub source: &'a str,
pub tokens: Vec<Token<'a>>,
pub gaps: Vec<Gap<'a>>,
}
pub fn line_ending(source: &str) -> &'static str {
if source.contains("\r\n") {
"\r\n"
} else {
"\n"
}
}
pub fn stream(source: &str) -> Stream<'_> {
let (lexed, _errors) = cpp_parser::lex(source, &LexerConfig::default());
let mut tokens: Vec<Token<'_>> = Vec::new();
let mut gaps: Vec<Gap<'_>> = Vec::new();
let mut gap_start = 0usize;
let mut gap_end = 0usize;
let mut newlines = 0usize;
let mut comments: Vec<Comment<'_>> = Vec::new();
for token in &lexed {
let start = token.range.start_offset;
let end = token.range.start_offset + token.range.length;
let text = source.get(start..end).unwrap_or("");
match token.kind {
CppTokenKind::Whitespace => gap_end = end,
CppTokenKind::Newline => {
gap_end = end;
newlines += 1;
}
CppTokenKind::LineContinuation => {
gap_end = end;
}
CppTokenKind::LineComment | CppTokenKind::BlockComment => {
if let Some(previous) = comments.last_mut() {
previous.ends_line = newlines > 0;
}
let own_line = newlines > 0 || (start == 0 && comments.is_empty());
comments.push(Comment {
kind: token.kind,
text,
start,
end,
own_line,
ends_line: false,
});
gap_end = end;
newlines = 0;
}
_ => {
if let Some(last) = comments.last_mut() {
last.ends_line = newlines > 0;
}
gaps.push(Gap {
start: gap_start,
end: gap_end,
newlines,
comments: std::mem::take(&mut comments),
at_end: false,
});
tokens.push(Token {
kind: token.kind,
text,
start,
end,
});
gap_start = end;
gap_end = end;
newlines = 0;
}
}
}
if let Some(last) = comments.last_mut() {
last.ends_line = true;
}
gaps.push(Gap {
start: gap_start,
end: gap_end,
newlines,
comments,
at_end: true,
});
Stream {
source,
tokens,
gaps,
}
}
impl<'a> Comment<'a> {
pub fn text(&self) -> &'a str {
self.text
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_gap_carries_its_comments_in_order_and_whether_they_start_a_line() {
let source = "int x; // trailing\n// own line\nint y;\n";
let stream = stream(source);
assert_eq!(stream.tokens.len(), 6, "int x ; int y ; -> 6 tokens");
let gap = &stream.gaps[3];
assert_eq!(gap.comments.len(), 2);
assert!(
!gap.comments[0].own_line,
"the first comment trails `int x;`"
);
assert!(gap.comments[0].ends_line);
assert!(gap.comments[1].own_line);
assert!(gap.comments[1].ends_line);
assert_eq!(gap.comments[0].text(), "// trailing");
}
#[test]
fn the_last_gap_is_the_end_of_the_file() {
let stream = stream("int x;\n");
assert!(stream.gaps.last().unwrap().at_end);
assert_eq!(stream.gaps.len(), stream.tokens.len() + 1);
}
#[test]
fn a_line_continuation_is_not_a_newline() {
let stream = stream("#define A \\\n 1\n");
let first = &stream.tokens[0];
assert_eq!(first.text, "#");
assert!(stream.gaps[1].comments.is_empty());
}
}