use crate::syntax::{BlockRule, Dialect, LineRule, StringRule, Syntax};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SpanKind {
Line,
Block,
Str,
DocString,
}
impl SpanKind {
pub fn is_comment(self) -> bool {
matches!(self, SpanKind::Line | SpanKind::Block | SpanKind::DocString)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Span {
pub kind: SpanKind,
pub start: usize,
pub end: usize,
pub inner_start: usize,
pub inner_end: usize,
}
impl Span {
pub fn text<'a>(&self, src: &'a str) -> &'a str {
&src[self.start..self.end]
}
pub fn inner<'a>(&self, src: &'a str) -> &'a str {
&src[self.inner_start..self.inner_end]
}
}
#[derive(Clone, Copy, Debug)]
pub struct ScanConfig {
pub record_strings: bool,
pub docstrings: bool,
pub keep_shebang: bool,
}
impl Default for ScanConfig {
fn default() -> Self {
ScanConfig {
record_strings: false,
docstrings: false,
keep_shebang: true,
}
}
}
pub fn scan(src: &str, syn: &Syntax, cfg: ScanConfig) -> Vec<Span> {
let mut out = Vec::new();
if syn.is_empty() && !cfg.record_strings {
return out;
}
let mut start = 0;
if cfg.keep_shebang && syn.shebang_is_code && src.as_bytes().starts_with(b"#!") {
start = line_end(src.as_bytes(), 0);
}
scan_range(src, syn, cfg, start, src.len(), &mut out);
out
}
fn scan_range(
src: &str,
syn: &Syntax,
cfg: ScanConfig,
from: usize,
to: usize,
out: &mut Vec<Span>,
) {
let b = src.as_bytes();
let interesting = syn.interesting();
let mut i = from;
let mut at_ls = from == 0 || (from > 0 && b[from - 1] == b'\n');
while i < to {
let c = b[i];
if c == b'\n' {
i += 1;
at_ls = true;
continue;
}
if !interesting[c as usize] {
if c != b' ' && c != b'\t' && c != b'\r' {
at_ls = false;
}
i += 1;
continue;
}
if let Some(next) = try_block(b, i, to, syn, at_ls, out) {
at_ls = false;
i = next;
continue;
}
if let Some(next) = try_line(b, i, to, syn, at_ls, out) {
i = next;
at_ls = true; continue;
}
if let Some(next) = try_char_prefix(b, i, to, syn) {
at_ls = false;
i = next;
continue;
}
if let Some(next) = try_long_bracket_string(b, i, to, syn, cfg, out) {
at_ls = false;
i = next;
continue;
}
if let Some(next) = try_string(b, i, to, syn, cfg, at_ls, out) {
at_ls = false;
i = next;
continue;
}
if let Some(next) = try_embedded(src, i, to, syn, cfg, out) {
at_ls = false;
i = next;
continue;
}
at_ls = false;
i += 1;
}
}
fn try_block(
b: &[u8],
i: usize,
to: usize,
syn: &Syntax,
at_ls: bool,
out: &mut Vec<Span>,
) -> Option<usize> {
for r in syn.blocks {
if r.anchored && !at_ls {
continue;
}
if !matches_at(b, i, to, r.open, false) {
continue;
}
let (end, closed) = block_end(b, i + r.open.len(), to, r);
let inner_end = if closed { end - r.close.len() } else { end };
out.push(Span {
kind: SpanKind::Block,
start: i,
end,
inner_start: i + r.open.len(),
inner_end,
});
return Some(end);
}
if syn.long_brackets {
for r in syn.lines {
if !matches_at(b, i, to, r.marker, false) {
continue;
}
let after = i + r.marker.len();
if let Some((body, level)) = long_bracket_open(b, after, to) {
let (end, closed) = long_bracket_end(b, body, to, level);
let inner_end = if closed { end - (level + 2) } else { end };
out.push(Span {
kind: SpanKind::Block,
start: i,
end,
inner_start: body,
inner_end,
});
return Some(end);
}
}
}
None
}
fn try_line(
b: &[u8],
i: usize,
to: usize,
syn: &Syntax,
at_ls: bool,
out: &mut Vec<Span>,
) -> Option<usize> {
for r in syn.lines {
if !line_rule_applies(b, i, to, r, at_ls) {
continue;
}
let end = line_end(b, i).min(to);
let end = if end > i && b[end - 1] == b'\r' {
end - 1
} else {
end
};
out.push(Span {
kind: SpanKind::Line,
start: i,
end,
inner_start: i + r.marker.len(),
inner_end: end,
});
return Some(end);
}
None
}
fn line_rule_applies(b: &[u8], i: usize, to: usize, r: &LineRule, at_ls: bool) -> bool {
if r.anchored && !at_ls {
return false;
}
if r.boundary_before && i > 0 && !matches!(b[i - 1], b' ' | b'\t' | b'\r' | b'\n') {
return false;
}
if !matches_at(b, i, to, r.marker, r.ci) {
return false;
}
if i > 0 && r.not_after.contains(&b[i - 1]) {
return false;
}
if let Some(allowed) = r.require_after {
let j = i + r.marker.len();
if j < to && b[j] != b'\n' && !allowed.contains(&b[j]) {
return false;
}
}
true
}
fn try_char_prefix(b: &[u8], i: usize, to: usize, syn: &Syntax) -> Option<usize> {
let p = syn.char_prefix?;
if !matches_at(b, i, to, p, false) {
return None;
}
let mut j = i + p.len();
if j < to {
j += char_len(b[j]);
}
while j < to && (b[j].is_ascii_alphanumeric() || b[j] == b'-') {
j += 1;
}
Some(j.min(to))
}
fn try_string(
b: &[u8],
i: usize,
to: usize,
syn: &Syntax,
cfg: ScanConfig,
at_ls: bool,
out: &mut Vec<Span>,
) -> Option<usize> {
for r in syn.strings {
if !matches_at(b, i, to, r.open, false) {
continue;
}
let raw = if r.raw_prefix {
raw_prefix_hashes(b, i)
} else {
None
};
let end = match raw {
Some(hashes) => raw_string_end(b, i + r.open.len(), to, r.close, hashes),
None if r.char_like => match char_literal_end(b, i, to, r) {
Some(end) => end,
None => continue,
},
None => string_end(b, i + r.open.len(), to, r),
};
let kind = if cfg.docstrings && r.doc_capable && at_ls {
SpanKind::DocString
} else {
SpanKind::Str
};
if kind == SpanKind::DocString || cfg.record_strings {
let inner_end = end.saturating_sub(r.close.len()).max(i + r.open.len());
out.push(Span {
kind,
start: i,
end,
inner_start: i + r.open.len(),
inner_end,
});
}
return Some(end);
}
None
}
fn try_long_bracket_string(
b: &[u8],
i: usize,
to: usize,
syn: &Syntax,
cfg: ScanConfig,
out: &mut Vec<Span>,
) -> Option<usize> {
if !syn.long_brackets || b[i] != b'[' {
return None;
}
let (body, level) = long_bracket_open(b, i, to)?;
let (end, closed) = long_bracket_end(b, body, to, level);
if cfg.record_strings {
let inner_end = if closed { end - (level + 2) } else { end };
out.push(Span {
kind: SpanKind::Str,
start: i,
end,
inner_start: body,
inner_end,
});
}
Some(end)
}
fn try_embedded(
src: &str,
i: usize,
to: usize,
syn: &Syntax,
cfg: ScanConfig,
out: &mut Vec<Span>,
) -> Option<usize> {
if !syn.html_embedded {
return None;
}
let b = src.as_bytes();
let (open, close) = if matches_at(b, i, to, "<script", true) {
("<script", "</script")
} else if matches_at(b, i, to, "<style", true) {
("<style", "</style")
} else {
return None;
};
let after = i + open.len();
if after < to && !matches!(b[after], b'>' | b' ' | b'\t' | b'\n' | b'\r') {
return None;
}
let body = find(b, after, to, ">")? + 1;
let body_end = find(b, body, to, close)?;
if body_end > body {
let inner = Syntax::of(Dialect::CFamily);
scan_range(src, &inner, cfg, body, body_end, out);
}
Some(body_end)
}
fn matches_at(b: &[u8], i: usize, to: usize, pat: &str, ci: bool) -> bool {
let p = pat.as_bytes();
if i + p.len() > to {
return false;
}
if ci {
b[i..i + p.len()].eq_ignore_ascii_case(p)
} else {
&b[i..i + p.len()] == p
}
}
fn find(b: &[u8], from: usize, to: usize, pat: &str) -> Option<usize> {
let p = pat.as_bytes();
if p.is_empty() || to < p.len() {
return None;
}
let first = p[0];
let ci = p.iter().any(|c| c.is_ascii_alphabetic());
let mut i = from;
while i + p.len() <= to {
if (b[i] == first || (ci && b[i].eq_ignore_ascii_case(&first)))
&& matches_at(b, i, to, pat, ci)
{
return Some(i);
}
i += 1;
}
None
}
fn line_end(b: &[u8], i: usize) -> usize {
let mut j = i;
while j < b.len() && b[j] != b'\n' {
j += 1;
}
j
}
fn char_len(first: u8) -> usize {
match first {
0x00..=0x7f => 1,
0xc0..=0xdf => 2,
0xe0..=0xef => 3,
_ => 4,
}
}
fn block_end(b: &[u8], from: usize, to: usize, r: &BlockRule) -> (usize, bool) {
let mut depth = 1usize;
let mut j = from;
let mut at_ls = false;
while j < to {
if b[j] == b'\n' {
j += 1;
at_ls = true;
continue;
}
if (!r.close_anchored || at_ls) && matches_at(b, j, to, r.close, false) {
depth -= 1;
j += r.close.len();
if depth == 0 {
return (j, true);
}
continue;
}
if r.nestable && (!r.anchored || at_ls) && matches_at(b, j, to, r.open, false) {
depth += 1;
j += r.open.len();
continue;
}
if b[j] != b' ' && b[j] != b'\t' && b[j] != b'\r' {
at_ls = false;
}
j += 1;
}
(to, false)
}
fn long_bracket_open(b: &[u8], i: usize, to: usize) -> Option<(usize, usize)> {
if i >= to || b[i] != b'[' {
return None;
}
let mut j = i + 1;
while j < to && b[j] == b'=' {
j += 1;
}
if j < to && b[j] == b'[' {
Some((j + 1, j - i - 1))
} else {
None
}
}
fn long_bracket_end(b: &[u8], from: usize, to: usize, level: usize) -> (usize, bool) {
let mut j = from;
while j < to {
if b[j] == b']' {
let mut k = j + 1;
while k < to && b[k] == b'=' {
k += 1;
}
if k - j - 1 == level && k < to && b[k] == b']' {
return (k + 1, true);
}
}
j += 1;
}
(to, false)
}
fn string_end(b: &[u8], from: usize, to: usize, r: &StringRule) -> usize {
let close = r.close.as_bytes();
let mut j = from;
while j < to {
let c = b[j];
if let Some(esc) = r.escape {
if c == esc {
j += 2;
continue;
}
}
if c == close[0] && matches_at(b, j, to, r.close, false) {
let after = j + close.len();
if r.doubling && matches_at(b, after, to, r.close, false) {
j = after + close.len();
continue;
}
return after;
}
if c == b'\n' && !r.multiline {
return j;
}
j += 1;
}
to
}
fn raw_string_end(b: &[u8], from: usize, to: usize, close: &str, hashes: usize) -> usize {
let mut j = from;
while j < to {
if matches_at(b, j, to, close, false) {
let mut k = j + close.len();
let mut seen = 0;
while k < to && b[k] == b'#' && seen < hashes {
k += 1;
seen += 1;
}
if seen == hashes {
return k;
}
}
j += 1;
}
to
}
fn raw_prefix_hashes(b: &[u8], i: usize) -> Option<usize> {
let mut p = i;
while p > 0 && b[p - 1] == b'#' {
p -= 1;
}
let hashes = i - p;
let mut q = p;
while q > 0 && b[q - 1].is_ascii_alphabetic() && p - q < 3 {
q -= 1;
}
if q == p {
return None;
}
let prefix = &b[q..p];
if !prefix.iter().any(|c| c.eq_ignore_ascii_case(&b'r')) {
return None;
}
if q > 0 && (b[q - 1].is_ascii_alphanumeric() || b[q - 1] == b'_') {
return None;
}
Some(hashes)
}
fn char_literal_end(b: &[u8], i: usize, to: usize, r: &StringRule) -> Option<usize> {
let close = r.close.as_bytes()[0];
let mut j = i + r.open.len();
if j >= to {
return None;
}
if Some(b[j]) == r.escape {
j += 1;
let limit = (j + 12).min(to);
while j < limit {
if b[j] == b'\n' {
return None;
}
if b[j] == close {
return Some(j + 1);
}
j += 1;
}
return None;
}
if b[j] == b'\n' || b[j] == close {
return None;
}
let end = j + char_len(b[j]);
if end < to && b[end] == close {
Some(end + 1)
} else {
None
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::syntax::Dialect;
fn comments(src: &str, d: Dialect) -> Vec<&str> {
scan(src, &d.syntax(), ScanConfig::default())
.iter()
.filter(|s| s.kind.is_comment())
.map(|s| s.text(src))
.collect()
}
#[test]
fn urls_inside_strings_are_not_comments() {
assert!(comments("let u = \"https://example.com/#x\";", Dialect::CFamily).is_empty());
assert!(comments("u = 'https://example.com/#x'", Dialect::Hash).is_empty());
}
#[test]
fn nested_block_comments() {
let src = "a /* x /* y */ z */ b";
assert_eq!(comments(src, Dialect::CFamily), ["/* x /* y */"]);
assert_eq!(comments(src, Dialect::Sql), ["/* x /* y */ z */"]);
}
#[test]
fn rust_lifetimes_do_not_open_a_string() {
let src = "fn f<'a>(x: &'a str) { // note\n}";
assert_eq!(comments(src, Dialect::CFamily), ["// note"]);
}
#[test]
fn rust_raw_strings_swallow_quotes_and_slashes() {
let src = "let s = r#\"a \" // b\"#; // real\n";
assert_eq!(comments(src, Dialect::CFamily), ["// real"]);
}
#[test]
fn unterminated_quote_only_damages_its_own_line() {
let src = "echo don't\n# real comment\n";
assert_eq!(comments(src, Dialect::Hash), ["# real comment"]);
}
#[test]
fn lua_long_bracket_comment() {
let src = "--[==[ hi\nstill ]==]\nx = 1 -- tail\n";
assert_eq!(
comments(src, Dialect::Lua),
["--[==[ hi\nstill ]==]", "-- tail"]
);
}
#[test]
fn html_script_bodies_use_c_rules() {
let src = "<!-- x -->\n<script>\n// js\nvar a = \"<!-- not -->\";\n</script>\n";
assert_eq!(comments(src, Dialect::Markup), ["<!-- x -->", "// js"]);
}
#[test]
fn shebang_is_kept_as_code() {
assert!(comments("#!/bin/sh\n", Dialect::Hash).is_empty());
assert_eq!(comments("#!/bin/sh\n# c\n", Dialect::Hash), ["# c"]);
}
#[test]
fn tex_escaped_percent_is_not_a_comment() {
assert_eq!(comments("50\\% off % note\n", Dialect::Percent), ["% note"]);
}
#[test]
fn word_markers_need_a_boundary() {
assert!(comments("REMOVE x\n", Dialect::Batch).is_empty());
assert_eq!(comments("rem gone\n", Dialect::Batch), ["rem gone"]);
}
#[test]
fn spans_are_ordered_and_disjoint() {
let src = "/* a */ x // b\n\"s\" /* c */\n";
let spans = scan(
src,
&Dialect::CFamily.syntax(),
ScanConfig {
record_strings: true,
..Default::default()
},
);
let mut prev = 0;
for s in &spans {
assert!(s.start >= prev, "{:?}", spans);
assert!(s.end >= s.start);
prev = s.end;
}
}
#[test]
fn unterminated_block_comment_reaches_end_of_input() {
let src = "x /* never closed\nmore\n";
assert_eq!(comments(src, Dialect::CFamily), ["/* never closed\nmore\n"]);
}
}